From aa844899c937bde5d2b24f276b59997e5b668bde Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Thu, 8 Aug 2019 10:56:58 -0700 Subject: Creation of LTS branch "lts_2019_08_08" - 9ee91d3e430fb33a4590486573792eb0fa146c2d Export of internal Abseil changes by Abseil Team - 8efba58a3b656e9b41fb0471ae6453425a61c520 Export of internal Abseil changes by Abseil Team - b49b8d16b67ec6912899684b732e6367f258cfdb Export of internal Abseil changes by Abseil Team - 67222ffc4c83d918ce8395aa61769eeb77df4c4d Export of internal Abseil changes by Abseil Team - c5c4db4f5191fe5e76cbf68dcc71fb28702f7d2b Export of internal Abseil changes by Abseil Team - 14550beb3b7b97195e483fb74b5efb906395c31e Export of internal Abseil changes. by Abseil Team - 52e88ee56b72cf32bc66534d942c7398ce481331 Export of internal Abseil changes. by Abseil Team - 36d37ab992038f52276ca66b9da80c1cf0f57dc2 Export of internal Abseil changes. by Abseil Team - ad1485c8986246b2ae9105e512738d0e97aec887 Export of internal Abseil changes. by Abseil Team - f3840bc5e33ce4932e35986cf3718450c6f02af2 Export of internal Abseil changes. by Abseil Team - 278b26058c036833a4f7f3047d3f4d9296527f87 Export of internal Abseil changes. by Abseil Team - c6c3c1b498e4ee939b24be59cae29d59c3863be8 Export of internal Abseil changes. by Abseil Team - 44efe96dfca674a17b45ca53fc77fb69f1e29bf4 Export of internal Abseil changes. by Abseil Team - 3c98fcc0461bd2a4b9c149d4748a7373a225cf4b Merge pull request #340 from jtsylve/macos_cxx17_fix by Matt Calabrese <38107210+mattcalabrese-google@users.noreply.github.com> - 74d91756c11bc22f9b0108b94da9326f7f9e376f Export of internal Abseil changes. by Abseil Team - e6b050212c859fbaf67abac76105da10ec348274 Export of internal Abseil changes. by Abseil Team - c964fcffac27bd4a9ff67fe393410dd1146ef8b8 Export of internal Abseil changes. by Abseil Team - 72e09a54d993b192db32be14c65adf7e9bd08c31 Export of internal Abseil changes. by Abseil Team - d65e19dfcd8697076f68598c0131c6930cdcd74d Export of internal Abseil changes. by Abseil Team - 5162fc83d2f3b79a9753ed59594c43966afdd37a Merge pull request #336 from shields/patch-2 by Shaindel Schwartz <31392632+shaindelschwartz@users.noreply.github.com> - 0389f7bf58fa41f35b3ad60be61d32f31e4f8ed6 Merge pull request #335 from shields/patch-1 by Shaindel Schwartz <31392632+shaindelschwartz@users.noreply.github.com> - e9324d926a9189e222741fce6e676f0944661a72 Export of internal Abseil changes. by Abseil Team - 43ef2148c0936ebf7cb4be6b19927a9d9d145b8f Export of internal Abseil changes. by Abseil Team - a13d3df2b3ba68aeead92e2d078fba0510d55024 Merge pull request #323 from gosnik/master by Gennadiy Rozental - 310a11865c97c5cdcc42a4ee2c2e3578423afe69 Merge pull request #324 from RasPat1/patch-1 by Gennadiy Rozental - 8f11724067248acc330b4d1f12f0c76d03f2cfb1 Export of internal Abseil changes. by Abseil Team - b1dd425423380126f6441ce4fbb6f8f6c75b793a Export of internal Abseil changes. by Abseil Team - 361cb8a9db2f2130442389fd80593255be26d681 Export of internal Abseil changes. by Abseil Team - 0238ab0a831f179518c1a814f9584e99da2d75a3 Merge pull request #321 from christoph-cullmann/c4245_fix... by Xiaoyi Zhang - 61c9bf3e3e1c28a4aa6d7f1be4b37fd473bb5529 Export of internal Abseil changes. by Abseil Team - bc9101f9982391019521161a36179b52555ed212 Merge pull request #320 from christoph-cullmann/master by Xiaoyi Zhang - 2f76a9bf50046e396138cc8eeb3cdc17b7a5ac24 Export of internal Abseil changes. by Abseil Team - 4adaf5490921f13028b55018c9f550277de5aebb Export of internal Abseil changes. by Abseil Team - 27c30ec671cb7b5ba84c4e79feff7fd0b0ac6338 Avoid undefined behavior when nullptr is passed to memcpy... by Roman Gershman - ce65f5ac3cbf897bb5e3de1a51d80fd00866abaa Export of internal Abseil changes. by Abseil Team - a18fc7461e7409c2ad64e28537261db1e02e76fa Export of internal Abseil changes. by Abseil Team - 8a394b19c149cab50534b04c5e21d42bc2217a7d Export of internal Abseil changes. by Abseil Team - daf381e8535a1f1f1b8a75966a74e7cca63dee89 Export of internal Abseil changes. by Abseil Team - fa00c321073c7ea40a4fc3dfc8a06309eae3d025 Export of internal Abseil changes. by Abseil Team - 436ba6c4a0ea3a06eca6e055f9c8d296bf3bae12 Export of internal Abseil changes. by Abseil Team - 0cbdc774b97f7e80ab60dbe2ed4eaca3b2e33fc8 Export of internal Abseil changes. by Abseil Team - 27c2f6e2f3b5929fbd322b0f0ca392eb02efd9f8 Export of internal Abseil changes. by Abseil Team - aa468ad75539619b47979911297efbb629c52e44 Export of internal Abseil changes. by Abseil Team - cd86d0d20ab167c33b23d3875db68d1d4bad3a3b Export of internal Abseil changes. by Abseil Team - 33841c5c963aa9c3f096ef8e6c1e71624b941940 Export of internal Abseil changes. by Abseil Team - ca3f87560a0eef716195cadf66dc6b938a579ec6 Export of internal Abseil changes. by Abseil Team - d902eb869bcfacc1bad14933ed9af4bed006d481 Export of internal Abseil changes. by Abseil Team - a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a Export of internal Abseil changes. by Abseil Team - 0b545b460141b882b244a1efcef7621d59278160 Export of internal Abseil changes. by Abseil Team - dbae8764fbd429bf7d7745e24bcf73962177a7c0 Export of internal Abseil changes. by Abseil Team - 044da8a29c923506af0f0b46bc46f43c1e1300b5 Export of internal Abseil changes. by Abseil Team - 6cc6ac44e065b9e8975fadfd6ccb99cbcf89aac4 Export of internal Abseil changes. by Abseil Team - 666fc1266bccfd8e6eaaa084e7b42580bb8eb199 Export of internal Abseil changes. by Abseil Team - 93dfcf74cb5fccae3da07897d8613ae6cab958a0 Export of internal Abseil changes. by Abseil Team - 2c8421e1c6cef0da9e8a20b01c15256ec9ec116d Export of internal Abseil changes. by Abseil Team - 5b65c4af5107176555b23a638e5947686410ac1f Export of internal Abseil changes. by Abseil Team - eab2078b53c9e3d9d240135c09d27e3393acb50a Export of internal Abseil changes. by Abseil Team - 253eb7416421661873afbaa33828a850db978541 [CMake] Set correct flags for clang-cl (#278) by Loo Rong Jie - e75672f6afc7e8f23ee7b532e86d1b3b9be3984e Export of internal Abseil changes. by Abseil Team - bf29470384a101b307873b26d358433138c857fc Export of internal Abseil changes. by Abseil Team - 6fd827124facd8336981e73218997f9e73029b4f Merge pull request #280 from chiumichael/master by Derek Mauro <761129+derekmauro@users.noreply.github.com> - 7c7754fb3ed9ffb57d35fe8658f3ba4d73a31e72 Export of internal Abseil changes. by Abseil Team - 256be563447a315f2a7993ec669460ba475fa86a Export of internal Abseil changes. by Abseil Team - 88a152ae747c3c42dc9167d46c590929b048d436 Export of internal Abseil changes. by Abseil Team - c1cecb25a94c075725e9d2640f6b978a8f61957b Implement Span::first and Span::last from C++20 (#274) by Girts - 38b704384cd2f17590b3922b97744be0b43622c9 Changed HTTP URLs to HTTPS where possible (#270) by nik7273 - febc5ee6a92d0eb7dac1fceaa6c648cf6521b4dc Export of internal Abseil changes. by Abseil Team - 9fdf5e5b805412cb2a2e624d3e9a11588120465f Export of internal Abseil changes. by Abseil Team - 419f3184f8ebcdb23105295eadd2a569f3351eb9 Export of internal Abseil changes. by Abseil Team - b312c3cb53a0aad75a85ac2bf57c4a614fbd48d4 Export of internal Abseil changes. by Abseil Team - 308ce31528a7edfa39f5f6d36142278a0ae1bf45 Export of internal Abseil changes. by Abseil Team - 93d155bc4414f6c121bb1f19dba9fdb27c8943bc Export of internal Abseil changes. by Abseil Team - 426eaa4aa44e4580418bee46c1bd13911151bfb1 Export of internal Abseil changes. by Abseil Team - 2901ec32a919311384d6ad4194e2d927c06831f7 Export of internal Abseil changes. by Abseil Team - d78310fe5a82f2e0e6e16509ef8079c8d7e4674e Export of internal Abseil changes. by Abseil Team - a4cb1c8ba61531a63f9d309eea01ac1d43d8371d Export of internal Abseil changes. by Abseil Team - 540e2537b92cd4abfae6ceddfe24304345461f32 Export of internal Abseil changes. by Abseil Team - 89ea0c5ff34aaa5855cfc7aa41f323b8a0ef0ede Merge pull request #255 from uilianries/hotfix/conan by ahedberg - 5e0dcf72c64fae912184d2e0de87195fe8f0a425 Export of internal Abseil changes. by Abseil Team - 0dffca4e36791c7beeda04da10460b534283948a Export of internal Abseil changes. by Abseil Team - 6b4201f9ef650637510a21b8d6cbcc3bee4a606f Fix GCC8 warnings by Boris Staletic - 0b1e6d417b414aad9282e32e8c49c719edeb63c1 Export of internal Abseil changes. by Abseil Team - efccc502606bed768e50a6cd5806d8eb13e4e935 Export of internal Abseil changes. by Abseil Team - 5e6a78131f7bd5940218462c07d88cdefdd75dbe Export of internal Abseil changes. by Abseil Team - 5eea0f713c14ac17788b83e496f11903f8e2bbb0 Export of internal Abseil changes. by Abseil Team - 66f9becbb98ecc083f4db349b4b1e0ca9de93b15 Export of internal Abseil changes. by Abseil Team - 018b4db1d73ec8238e6dc4b17fd9e1fd7468d0ed Export of internal Abseil changes. by Abseil Team - 9449ae94397f2fd683851348e25ed8c93f75b3b9 Merge pull request #243 from ThomsonTan/FixIntrinsic by Alex Strelnikov - b16aeb6756bdab08cdf12d40baab5b51f7d15b16 Export of internal Abseil changes. by Abseil Team - 7ffbe09f3d85504bd018783bbe1e2c12992fe47c Export of internal Abseil changes. by Abseil Team - 01b471d9f3ebef27f5aaca14b66509099fa8cd6c Export of internal Abseil changes. by Abseil Team - 7bd8f36c741c7cbe311611d7981bf38ba04c6fef Export of internal Abseil changes. by Abseil Team - 968a34ffdaadd7db062a9621dfbdf8b2d16e05af Export of internal Abseil changes. by Abseil Team - 3e2e9b5557e76d098de4b8a2a659125b98ca519b Merge pull request #231 from uilianries/feature/conan by Mark Barolak - 111ca7060a6ff50115ca85b59f6b5d8c8c5e9105 Export of internal Abseil changes. by Abseil Team - 389ec3f906f018661a5308458d623d01f96d7b23 Export of internal Abseil changes. by Abseil Team - 8fbcdb90952c57828c4a9c2f6d79fcd7cae9088f Export of internal Abseil changes. by Abseil Team - 455dc17ba1af9635f0b60155bc565bc572a1e722 Export of internal Abseil changes. by Abseil Team - f197d7c72a54064cfde5a2058f1513a4a0ee36fb Export of internal Abseil changes. by Abseil Team - 284378a71b32dfb3af4e3661f585e671d1b603a3 Export of internal Abseil changes. by Abseil Team GitOrigin-RevId: 9ee91d3e430fb33a4590486573792eb0fa146c2d Change-Id: Ia06e548bc106cc9d136f6c65714be6645317aced --- absl/container/internal/raw_hash_set.h | 392 ++++++++++++--------------------- 1 file changed, 145 insertions(+), 247 deletions(-) (limited to 'absl/container/internal/raw_hash_set.h') diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 0c42e4ae..7b379d4f 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// https://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -91,36 +91,6 @@ #ifndef ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ #define ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ -#ifndef SWISSTABLE_HAVE_SSE2 -#if defined(__SSE2__) || \ - (defined(_MSC_VER) && \ - (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2))) -#define SWISSTABLE_HAVE_SSE2 1 -#else -#define SWISSTABLE_HAVE_SSE2 0 -#endif -#endif - -#ifndef SWISSTABLE_HAVE_SSSE3 -#ifdef __SSSE3__ -#define SWISSTABLE_HAVE_SSSE3 1 -#else -#define SWISSTABLE_HAVE_SSSE3 0 -#endif -#endif - -#if SWISSTABLE_HAVE_SSSE3 && !SWISSTABLE_HAVE_SSE2 -#error "Bad configuration!" -#endif - -#if SWISSTABLE_HAVE_SSE2 -#include -#endif - -#if SWISSTABLE_HAVE_SSSE3 -#include -#endif - #include #include #include @@ -135,18 +105,20 @@ #include "absl/base/internal/bits.h" #include "absl/base/internal/endian.h" #include "absl/base/port.h" +#include "absl/container/internal/common.h" #include "absl/container/internal/compressed_tuple.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_policy_traits.h" #include "absl/container/internal/hashtable_debug_hooks.h" +#include "absl/container/internal/hashtablez_sampler.h" +#include "absl/container/internal/have_sse.h" #include "absl/container/internal/layout.h" #include "absl/memory/memory.h" #include "absl/meta/type_traits.h" -#include "absl/types/optional.h" #include "absl/utility/utility.h" namespace absl { -inline namespace lts_2018_12_18 { +inline namespace lts_2019_08_08 { namespace container_internal { template @@ -194,12 +166,6 @@ struct IsDecomposable< std::declval()...))>, Policy, Hash, Eq, Ts...> : std::true_type {}; -template -struct IsTransparent : std::false_type {}; -template -struct IsTransparent> - : std::true_type {}; - // TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it. template constexpr bool IsNoThrowSwappable() { @@ -385,7 +351,7 @@ struct GroupSse2Impl { return BitMask( _mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl))); #else - return Match(kEmpty); + return Match(static_cast(kEmpty)); #endif } @@ -481,9 +447,7 @@ using Group = GroupPortableImpl; template class raw_hash_set; -inline bool IsValidCapacity(size_t n) { - return ((n + 1) & n) == 0 && n >= Group::kWidth - 1; -} +inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } // PRECONDITION: // IsValidCapacity(capacity) @@ -505,152 +469,32 @@ inline void ConvertDeletedToEmptyAndFullToDeleted( ctrl[capacity] = kSentinel; } -// Rounds up the capacity to the next power of 2 minus 1 and ensures it is -// greater or equal to Group::kWidth - 1. +// Rounds up the capacity to the next power of 2 minus 1, with a minimum of 1. inline size_t NormalizeCapacity(size_t n) { - constexpr size_t kMinCapacity = Group::kWidth - 1; - return n <= kMinCapacity - ? kMinCapacity - : (std::numeric_limits::max)() >> LeadingZeros(n); + return n ? ~size_t{} >> LeadingZeros(n) : 1; } -// The node_handle concept from C++17. -// We specialize node_handle for sets and maps. node_handle_base holds the -// common API of both. -template -class node_handle_base { - protected: - using PolicyTraits = hash_policy_traits; - using slot_type = typename PolicyTraits::slot_type; - - public: - using allocator_type = Alloc; - - constexpr node_handle_base() {} - node_handle_base(node_handle_base&& other) noexcept { - *this = std::move(other); - } - ~node_handle_base() { destroy(); } - node_handle_base& operator=(node_handle_base&& other) { - destroy(); - if (!other.empty()) { - alloc_ = other.alloc_; - PolicyTraits::transfer(alloc(), slot(), other.slot()); - other.reset(); - } - return *this; - } - - bool empty() const noexcept { return !alloc_; } - explicit operator bool() const noexcept { return !empty(); } - allocator_type get_allocator() const { return *alloc_; } - - protected: - template - friend class raw_hash_set; - - node_handle_base(const allocator_type& a, slot_type* s) : alloc_(a) { - PolicyTraits::transfer(alloc(), slot(), s); - } - - void destroy() { - if (!empty()) { - PolicyTraits::destroy(alloc(), slot()); - reset(); - } - } - - void reset() { - assert(alloc_.has_value()); - alloc_ = absl::nullopt; - } - - slot_type* slot() const { - assert(!empty()); - return reinterpret_cast(std::addressof(slot_space_)); - } - allocator_type* alloc() { return std::addressof(*alloc_); } - - private: - absl::optional alloc_; - mutable absl::aligned_storage_t - slot_space_; -}; - -// For sets. -template -class node_handle : public node_handle_base { - using Base = typename node_handle::node_handle_base; - - public: - using value_type = typename Base::PolicyTraits::value_type; - - constexpr node_handle() {} - - value_type& value() const { - return Base::PolicyTraits::element(this->slot()); - } - - private: - template - friend class raw_hash_set; - - node_handle(const Alloc& a, typename Base::slot_type* s) : Base(a, s) {} -}; - -// For maps. -template -class node_handle> - : public node_handle_base { - using Base = typename node_handle::node_handle_base; - - public: - using key_type = typename Policy::key_type; - using mapped_type = typename Policy::mapped_type; - - constexpr node_handle() {} - - auto key() const -> decltype(Base::PolicyTraits::key(this->slot())) { - return Base::PolicyTraits::key(this->slot()); - } - - mapped_type& mapped() const { - return Base::PolicyTraits::value( - &Base::PolicyTraits::element(this->slot())); +// We use 7/8th as maximum load factor. +// For 16-wide groups, that gives an average of two empty slots per group. +inline size_t CapacityToGrowth(size_t capacity) { + assert(IsValidCapacity(capacity)); + // `capacity*7/8` + if (Group::kWidth == 8 && capacity == 7) { + // x-x/8 does not work when x==7. + return 6; } - - private: - template - friend class raw_hash_set; - - node_handle(const Alloc& a, typename Base::slot_type* s) : Base(a, s) {} -}; - -// Implement the insert_return_type<> concept of C++17. -template -struct insert_return_type { - Iterator position; - bool inserted; - NodeType node; -}; - -// Helper trait to allow or disallow arbitrary keys when the hash and -// eq functions are transparent. -// It is very important that the inner template is an alias and that the type it -// produces is not a dependent type. Otherwise, type deduction would fail. -template -struct KeyArg { - // Transparent. Forward `K`. - template - using type = K; -}; - -template <> -struct KeyArg { - // Not transparent. Always use `key_type`. - template - using type = key_type; -}; + return capacity - capacity / 8; +} +// From desired "growth" to a lowerbound of the necessary capacity. +// Might not be a valid one and required NormalizeCapacity(). +inline size_t GrowthToLowerboundCapacity(size_t growth) { + // `growth*8/7` + if (Group::kWidth == 8 && growth == 7) { + // x+(x-1)/7 does not work when x==7. + return 8; + } + return growth + static_cast((static_cast(growth) - 1) / 7); +} // Policy: a policy defines how to perform different operations on // the slots of the hashtable (see hash_policy_traits.h for the full interface @@ -666,14 +510,14 @@ struct KeyArg { // if they are equal, false if they are not. If two keys compare equal, then // their hash values as defined by Hash MUST be equal. // -// Allocator: an Allocator [http://devdocs.io/cpp/concept/allocator] with which +// Allocator: an Allocator [https://devdocs.io/cpp/concept/allocator] with which // the storage of the hashtable will be allocated and the elements will be // constructed and destroyed. template class raw_hash_set { using PolicyTraits = hash_policy_traits; - using KeyArgImpl = container_internal::KeyArg::value && - IsTransparent::value>; + using KeyArgImpl = + KeyArg::value && IsTransparent::value>; public: using init_type = typename PolicyTraits::init_type; @@ -814,7 +658,11 @@ class raw_hash_set { } ctrl_t* ctrl_ = nullptr; - slot_type* slot_; + // To avoid uninitialized member warnigs, put slot_ in an anonymous union. + // The member is not initialized on singleton and end iterators. + union { + slot_type* slot_; + }; }; class const_iterator { @@ -854,7 +702,8 @@ class raw_hash_set { iterator inner_; }; - using node_type = container_internal::node_handle; + using node_type = node_handle, Alloc>; + using insert_return_type = InsertReturnType; raw_hash_set() noexcept( std::is_nothrow_default_constructible::value&& @@ -867,7 +716,7 @@ class raw_hash_set { : ctrl_(EmptyGroup()), settings_(0, hash, eq, alloc) { if (bucket_count) { capacity_ = NormalizeCapacity(bucket_count); - growth_left() = static_cast(capacity_ * kMaxLoadFactor); + reset_growth_left(); initialize_slots(); } } @@ -909,8 +758,8 @@ class raw_hash_set { // that accept std::initializer_list and std::initializer_list. // This is advantageous for performance. // - // // Turns {"abc", "def"} into std::initializer_list, then copies - // // the strings into the set. + // // Turns {"abc", "def"} into std::initializer_list, then + // // copies the strings into the set. // std::unordered_set s = {"abc", "def"}; // // // Turns {"abc", "def"} into std::initializer_list, then @@ -973,9 +822,10 @@ class raw_hash_set { // than a full `insert`. for (const auto& v : that) { const size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, v); - const size_t i = find_first_non_full(hash); - set_ctrl(i, H2(hash)); - emplace_at(i, v); + auto target = find_first_non_full(hash); + set_ctrl(target.offset, H2(hash)); + emplace_at(target.offset, v); + infoz_.RecordInsert(hash, target.probe_length); } size_ = that.size(); growth_left() -= that.size(); @@ -989,6 +839,7 @@ class raw_hash_set { slots_(absl::exchange(that.slots_, nullptr)), size_(absl::exchange(that.size_, 0)), capacity_(absl::exchange(that.capacity_, 0)), + infoz_(absl::exchange(that.infoz_, HashtablezInfoHandle())), // Hash, equality and allocator are copied instead of moved because // `that` must be left valid. If Hash is std::function, moving it // would create a nullptr functor that cannot be called. @@ -1009,6 +860,7 @@ class raw_hash_set { std::swap(size_, that.size_); std::swap(capacity_, that.capacity_); std::swap(growth_left(), that.growth_left()); + std::swap(infoz_, that.infoz_); } else { reserve(that.size()); // Note: this will copy elements of dense_set and unordered_set instead of @@ -1058,7 +910,7 @@ class raw_hash_set { size_t capacity() const { return capacity_; } size_t max_size() const { return (std::numeric_limits::max)(); } - void clear() { + ABSL_ATTRIBUTE_REINITIALIZES void clear() { // Iterating over this container is O(bucket_count()). When bucket_count() // is much greater than size(), iteration becomes prohibitively expensive. // For clear() it is more important to reuse the allocated array when the @@ -1076,9 +928,10 @@ class raw_hash_set { } size_ = 0; reset_ctrl(); - growth_left() = static_cast(capacity_ * kMaxLoadFactor); + reset_growth_left(); } assert(empty()); + infoz_.RecordStorageChanged(0, capacity_); } // This overload kicks in when the argument is an rvalue of insertable and @@ -1117,7 +970,7 @@ class raw_hash_set { // This overload kicks in when the argument is an rvalue of init_type. Its // purpose is to handle brace-init-list arguments. // - // flat_hash_set s; + // flat_hash_map s; // s.insert({"abc", 42}); std::pair insert(init_type&& value) { return emplace(std::move(value)); @@ -1158,13 +1011,14 @@ class raw_hash_set { insert(ilist.begin(), ilist.end()); } - insert_return_type insert(node_type&& node) { + insert_return_type insert(node_type&& node) { if (!node) return {end(), false, node_type()}; - const auto& elem = PolicyTraits::element(node.slot()); + const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); auto res = PolicyTraits::apply( - InsertSlot{*this, std::move(*node.slot())}, elem); + InsertSlot{*this, std::move(*CommonAccess::GetSlot(node))}, + elem); if (res.second) { - node.reset(); + CommonAccess::Reset(&node); return {res.first, true, node_type()}; } else { return {res.first, false, std::move(node)}; @@ -1328,7 +1182,8 @@ class raw_hash_set { } node_type extract(const_iterator position) { - node_type node(alloc_ref(), position.inner_.slot_); + auto node = + CommonAccess::Transfer(alloc_ref(), position.inner_.slot_); erase_meta_only(position); return node; } @@ -1353,6 +1208,7 @@ class raw_hash_set { swap(growth_left(), that.growth_left()); swap(hash_ref(), that.hash_ref()); swap(eq_ref(), that.eq_ref()); + swap(infoz_, that.infoz_); if (AllocTraits::propagate_on_container_swap::value) { swap(alloc_ref(), that.alloc_ref()); } else { @@ -1363,17 +1219,21 @@ class raw_hash_set { void rehash(size_t n) { if (n == 0 && capacity_ == 0) return; - if (n == 0 && size_ == 0) return destroy_slots(); - auto m = NormalizeCapacity(std::max(n, NumSlotsFast(size()))); + if (n == 0 && size_ == 0) { + destroy_slots(); + infoz_.RecordStorageChanged(0, 0); + return; + } + // bitor is a faster way of doing `max` here. We will round up to the next + // power-of-2-minus-1, so bitor is good enough. + auto m = NormalizeCapacity(n | GrowthToLowerboundCapacity(size())); // n == 0 unconditionally rehashes as per the standard. if (n == 0 || m > capacity_) { resize(m); } } - void reserve(size_t n) { - rehash(NumSlotsFast(n)); - } + void reserve(size_t n) { rehash(GrowthToLowerboundCapacity(n)); } // Extension API: support for heterogeneous keys. // @@ -1551,13 +1411,6 @@ class raw_hash_set { slot_type&& slot; }; - // Computes std::ceil(n / kMaxLoadFactor). Faster than calling std::ceil. - static inline size_t NumSlotsFast(size_t n) { - return static_cast( - (n * kMaxLoadFactorDenominator + (kMaxLoadFactorNumerator - 1)) / - kMaxLoadFactorNumerator); - } - // "erases" the object from the container, except that it doesn't actually // destroy the object. It only updates all the metadata of the class. // This can be used in conjunction with Policy::transfer to move the object to @@ -1580,17 +1433,34 @@ class raw_hash_set { set_ctrl(index, was_never_full ? kEmpty : kDeleted); growth_left() += was_never_full; + infoz_.RecordErase(); } void initialize_slots() { assert(capacity_); + // Folks with custom allocators often make unwarranted assumptions about the + // behavior of their classes vis-a-vis trivial destructability and what + // calls they will or wont make. Avoid sampling for people with custom + // allocators to get us out of this mess. This is not a hard guarantee but + // a workaround while we plan the exact guarantee we want to provide. + // + // People are often sloppy with the exact type of their allocator (sometimes + // it has an extra const or is missing the pair, but rebinds made it work + // anyway). To avoid the ambiguity, we work off SlotAlloc which we have + // bound more carefully. + if (std::is_same>::value && + slots_ == nullptr) { + infoz_ = Sample(); + } + auto layout = MakeLayout(capacity_); char* mem = static_cast( Allocate(&alloc_ref(), layout.AllocSize())); ctrl_ = reinterpret_cast(layout.template Pointer<0>(mem)); slots_ = layout.template Pointer<1>(mem); reset_ctrl(); - growth_left() = static_cast(capacity_ * kMaxLoadFactor) - size_; + reset_growth_left(); + infoz_.RecordStorageChanged(size_, capacity_); } void destroy_slots() { @@ -1619,11 +1489,14 @@ class raw_hash_set { capacity_ = new_capacity; initialize_slots(); + size_t total_probe_length = 0; for (size_t i = 0; i != old_capacity; ++i) { if (IsFull(old_ctrl[i])) { size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, PolicyTraits::element(old_slots + i)); - size_t new_i = find_first_non_full(hash); + auto target = find_first_non_full(hash); + size_t new_i = target.offset; + total_probe_length += target.probe_length; set_ctrl(new_i, H2(hash)); PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i); } @@ -1635,10 +1508,12 @@ class raw_hash_set { Deallocate(&alloc_ref(), old_ctrl, layout.AllocSize()); } + infoz_.RecordRehash(total_probe_length); } void drop_deletes_without_resize() ABSL_ATTRIBUTE_NOINLINE { assert(IsValidCapacity(capacity_)); + assert(!is_small()); // Algorithm: // - mark all DELETED slots as EMPTY // - mark all FULL slots as DELETED @@ -1658,12 +1533,15 @@ class raw_hash_set { ConvertDeletedToEmptyAndFullToDeleted(ctrl_, capacity_); typename std::aligned_storage::type raw; + size_t total_probe_length = 0; slot_type* slot = reinterpret_cast(&raw); for (size_t i = 0; i != capacity_; ++i) { if (!IsDeleted(ctrl_[i])) continue; size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, PolicyTraits::element(slots_ + i)); - size_t new_i = find_first_non_full(hash); + auto target = find_first_non_full(hash); + size_t new_i = target.offset; + total_probe_length += target.probe_length; // Verify if the old and new i fall within the same group wrt the hash. // If they do, we don't need to move the object as it falls already in the @@ -1695,13 +1573,14 @@ class raw_hash_set { --i; // repeat } } - growth_left() = static_cast(capacity_ * kMaxLoadFactor) - size_; + reset_growth_left(); + infoz_.RecordRehash(total_probe_length); } void rehash_and_grow_if_necessary() { if (capacity_ == 0) { - resize(Group::kWidth - 1); - } else if (size() <= kMaxLoadFactor / 2 * capacity_) { + resize(1); + } else if (size() <= CapacityToGrowth(capacity()) / 2) { // Squash DELETED without growing if there is enough capacity. drop_deletes_without_resize(); } else { @@ -1736,24 +1615,26 @@ class raw_hash_set { // - the input is already a set // - there are enough slots // - the element with the hash is not in the table - size_t find_first_non_full(size_t hash) { + struct FindInfo { + size_t offset; + size_t probe_length; + }; + FindInfo find_first_non_full(size_t hash) { auto seq = probe(hash); while (true) { Group g{ctrl_ + seq.offset()}; auto mask = g.MatchEmptyOrDeleted(); if (mask) { #if !defined(NDEBUG) - // We want to force small tables to have random entries too, so - // in debug build we will randomly insert in either the front or back of + // We want to add entropy even when ASLR is not enabled. + // In debug build we will randomly insert in either the front or back of // the group. // TODO(kfm,sbenza): revisit after we do unconditional mixing - if (ShouldInsertBackwards(hash, ctrl_)) - return seq.offset(mask.HighestBitSet()); - else - return seq.offset(mask.LowestBitSet()); -#else - return seq.offset(mask.LowestBitSet()); + if (!is_small() && ShouldInsertBackwards(hash, ctrl_)) { + return {seq.offset(mask.HighestBitSet()), seq.index()}; + } #endif + return {seq.offset(mask.LowestBitSet()), seq.index()}; } assert(seq.index() < capacity_ && "full table!"); seq.next(); @@ -1792,15 +1673,17 @@ class raw_hash_set { } size_t prepare_insert(size_t hash) ABSL_ATTRIBUTE_NOINLINE { - size_t target = find_first_non_full(hash); - if (ABSL_PREDICT_FALSE(growth_left() == 0 && !IsDeleted(ctrl_[target]))) { + auto target = find_first_non_full(hash); + if (ABSL_PREDICT_FALSE(growth_left() == 0 && + !IsDeleted(ctrl_[target.offset]))) { rehash_and_grow_if_necessary(); target = find_first_non_full(hash); } ++size_; - growth_left() -= IsEmpty(ctrl_[target]); - set_ctrl(target, H2(hash)); - return target; + growth_left() -= IsEmpty(ctrl_[target.offset]); + set_ctrl(target.offset, H2(hash)); + infoz_.RecordInsert(hash, target.probe_length); + return target.offset; } // Constructs the value in the space pointed by the iterator. This only works @@ -1838,6 +1721,10 @@ class raw_hash_set { SanitizerPoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); } + void reset_growth_left() { + growth_left() = CapacityToGrowth(capacity()) - size_; + } + // Sets the control byte, and if `i < Group::kWidth`, set the cloned byte at // the end too. void set_ctrl(size_t i, ctrl_t h) { @@ -1850,11 +1737,28 @@ class raw_hash_set { } ctrl_[i] = h; - ctrl_[((i - Group::kWidth) & capacity_) + Group::kWidth] = h; + ctrl_[((i - Group::kWidth) & capacity_) + 1 + + ((Group::kWidth - 1) & capacity_)] = h; } size_t& growth_left() { return settings_.template get<0>(); } + // The representation of the object has two modes: + // - small: For capacities < kWidth-1 + // - large: For the rest. + // + // Differences: + // - In small mode we are able to use the whole capacity. The extra control + // bytes give us at least one "empty" control byte to stop the iteration. + // This is important to make 1 a valid capacity. + // + // - In small mode only the first `capacity()` control bytes after the + // sentinel are valid. The rest contain dummy kEmpty values that do not + // represent a real slot. This is important to take into account on + // find_first_non_full(), where we never try ShouldInsertBackwards() for + // small tables. + bool is_small() const { return capacity_ < Group::kWidth - 1; } + hasher& hash_ref() { return settings_.template get<1>(); } const hasher& hash_ref() const { return settings_.template get<1>(); } key_equal& eq_ref() { return settings_.template get<2>(); } @@ -1864,12 +1768,6 @@ class raw_hash_set { return settings_.template get<3>(); } - // On average each group has 2 empty slot (for the vectorized case). - static constexpr int64_t kMaxLoadFactorNumerator = 14; - static constexpr int64_t kMaxLoadFactorDenominator = 16; - static constexpr float kMaxLoadFactor = - 1.0 * kMaxLoadFactorNumerator / kMaxLoadFactorDenominator; - // TODO(alkis): Investigate removing some of these fields: // - ctrl/slots can be derived from each other // - size can be moved into the slot array @@ -1877,6 +1775,7 @@ class raw_hash_set { slot_type* slots_ = nullptr; // [capacity * slot_type] size_t size_ = 0; // number of full slots size_t capacity_ = 0; // total number of slots + HashtablezInfoHandle infoz_; absl::container_internal::CompressedTuple settings_{0, hasher{}, key_equal{}, allocator_type{}}; @@ -1929,10 +1828,9 @@ struct HashtableDebugAccess> { } static size_t LowerBoundAllocatedByteSize(size_t size) { - size_t capacity = container_internal::NormalizeCapacity( - std::ceil(size / Set::kMaxLoadFactor)); + size_t capacity = GrowthToLowerboundCapacity(size); if (capacity == 0) return 0; - auto layout = Set::MakeLayout(capacity); + auto layout = Set::MakeLayout(NormalizeCapacity(capacity)); size_t m = layout.AllocSize(); size_t per_slot = Traits::space_used(static_cast(nullptr)); if (per_slot != ~size_t{}) { @@ -1944,7 +1842,7 @@ struct HashtableDebugAccess> { } // namespace hashtable_debug_internal } // namespace container_internal -} // inline namespace lts_2018_12_18 +} // inline namespace lts_2019_08_08 } // namespace absl #endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ -- cgit v1.2.3