diff options
Diffstat (limited to 'absl/container/internal')
19 files changed, 2173 insertions, 770 deletions
diff --git a/absl/container/internal/btree.h b/absl/container/internal/btree.h index 002ccc1e..0bb38366 100644 --- a/absl/container/internal/btree.h +++ b/absl/container/internal/btree.h @@ -182,15 +182,44 @@ struct key_compare_to_adapter<std::greater<absl::Cord>> { using type = StringBtreeDefaultGreater; }; +// Detects an 'absl_btree_prefer_linear_node_search' member. This is +// a protocol used as an opt-in or opt-out of linear search. +// +// For example, this would be useful for key types that wrap an integer +// and define their own cheap operator<(). For example: +// +// class K { +// public: +// using absl_btree_prefer_linear_node_search = std::true_type; +// ... +// private: +// friend bool operator<(K a, K b) { return a.k_ < b.k_; } +// int k_; +// }; +// +// btree_map<K, V> m; // Uses linear search +// +// If T has the preference tag, then it has a preference. +// Btree will use the tag's truth value. +template <typename T, typename = void> +struct has_linear_node_search_preference : std::false_type {}; +template <typename T, typename = void> +struct prefers_linear_node_search : std::false_type {}; +template <typename T> +struct has_linear_node_search_preference< + T, absl::void_t<typename T::absl_btree_prefer_linear_node_search>> + : std::true_type {}; +template <typename T> +struct prefers_linear_node_search< + T, absl::void_t<typename T::absl_btree_prefer_linear_node_search>> + : T::absl_btree_prefer_linear_node_search {}; + template <typename Key, typename Compare, typename Alloc, int TargetNodeSize, bool Multi, typename SlotPolicy> struct common_params { // If Compare is a common comparator for a string-like type, then we adapt it // to use heterogeneous lookup and to be a key-compare-to comparator. using key_compare = typename key_compare_to_adapter<Compare>::type; - // True when key_compare has been adapted to StringBtreeDefault{Less,Greater}. - using is_key_compare_adapted = - absl::negation<std::is_same<key_compare, Compare>>; // A type which indicates if we have a key-compare-to functor or a plain old // key-compare functor. using is_key_compare_to = btree_is_key_compare_to<key_compare, Key>; @@ -200,9 +229,6 @@ struct common_params { using size_type = std::make_signed<size_t>::type; using difference_type = ptrdiff_t; - // True if this is a multiset or multimap. - using is_multi_container = std::integral_constant<bool, Multi>; - using slot_policy = SlotPolicy; using slot_type = typename slot_policy::slot_type; using value_type = typename slot_policy::value_type; @@ -212,6 +238,23 @@ struct common_params { using reference = value_type &; using const_reference = const value_type &; + // For the given lookup key type, returns whether we can have multiple + // equivalent keys in the btree. If this is a multi-container, then we can. + // Otherwise, we can have multiple equivalent keys only if all of the + // following conditions are met: + // - The comparator is transparent. + // - The lookup key type is not the same as key_type. + // - The comparator is not a StringBtreeDefault{Less,Greater} comparator + // that we know has the same equivalence classes for all lookup types. + template <typename LookupKey> + constexpr static bool can_have_multiple_equivalent_keys() { + return Multi || + (IsTransparent<key_compare>::value && + !std::is_same<LookupKey, Key>::value && + !std::is_same<key_compare, StringBtreeDefaultLess>::value && + !std::is_same<key_compare, StringBtreeDefaultGreater>::value); + } + enum { kTargetNodeSize = TargetNodeSize, @@ -391,6 +434,10 @@ struct SearchResult { // useful information. template <typename V> struct SearchResult<V, false> { + SearchResult() {} + explicit SearchResult(V value) : value(value) {} + SearchResult(V value, MatchKind /*match*/) : value(value) {} + V value; static constexpr bool HasMatch() { return false; } @@ -403,7 +450,6 @@ struct SearchResult<V, false> { template <typename Params> class btree_node { using is_key_compare_to = typename Params::is_key_compare_to; - using is_multi_container = typename Params::is_multi_container; using field_type = typename Params::node_count_type; using allocator_type = typename Params::allocator_type; using slot_type = typename Params::slot_type; @@ -421,15 +467,22 @@ class btree_node { using difference_type = typename Params::difference_type; // Btree decides whether to use linear node search as follows: + // - If the comparator expresses a preference, use that. + // - If the key expresses a preference, use that. // - If the key is arithmetic and the comparator is std::less or // std::greater, choose linear. // - Otherwise, choose binary. // TODO(ezb): Might make sense to add condition(s) based on node-size. using use_linear_search = std::integral_constant< bool, - std::is_arithmetic<key_type>::value && - (std::is_same<std::less<key_type>, key_compare>::value || - std::is_same<std::greater<key_type>, key_compare>::value)>; + has_linear_node_search_preference<key_compare>::value + ? prefers_linear_node_search<key_compare>::value + : has_linear_node_search_preference<key_type>::value + ? prefers_linear_node_search<key_type>::value + : std::is_arithmetic<key_type>::value && + (std::is_same<std::less<key_type>, key_compare>::value || + std::is_same<std::greater<key_type>, + key_compare>::value)>; // This class is organized by gtl::Layout as if it had the following // structure: @@ -446,23 +499,23 @@ class btree_node { // // is the same as the count of values. // field_type finish; // // The maximum number of values the node can hold. This is an integer in - // // [1, kNodeValues] for root leaf nodes, kNodeValues for non-root leaf + // // [1, kNodeSlots] for root leaf nodes, kNodeSlots for non-root leaf // // nodes, and kInternalNodeMaxCount (as a sentinel value) for internal - // // nodes (even though there are still kNodeValues values in the node). + // // nodes (even though there are still kNodeSlots values in the node). // // TODO(ezb): make max_count use only 4 bits and record log2(capacity) // // to free extra bits for is_root, etc. // field_type max_count; // // // The array of values. The capacity is `max_count` for leaf nodes and - // // kNodeValues for internal nodes. Only the values in + // // kNodeSlots for internal nodes. Only the values in // // [start, finish) have been initialized and are valid. // slot_type values[max_count]; // // // The array of child pointers. The keys in children[i] are all less // // than key(i). The keys in children[i + 1] are all greater than key(i). - // // There are 0 children for leaf nodes and kNodeValues + 1 children for + // // There are 0 children for leaf nodes and kNodeSlots + 1 children for // // internal nodes. - // btree_node *children[kNodeValues + 1]; + // btree_node *children[kNodeSlots + 1]; // // This class is only constructed by EmptyNodeType. Normally, pointers to the // layout above are allocated, cast to btree_node*, and de-allocated within @@ -484,57 +537,62 @@ class btree_node { private: using layout_type = absl::container_internal::Layout<btree_node *, field_type, slot_type, btree_node *>; - constexpr static size_type SizeWithNValues(size_type n) { + constexpr static size_type SizeWithNSlots(size_type n) { return layout_type(/*parent*/ 1, /*position, start, finish, max_count*/ 4, - /*values*/ n, + /*slots*/ n, /*children*/ 0) .AllocSize(); } // A lower bound for the overhead of fields other than values in a leaf node. constexpr static size_type MinimumOverhead() { - return SizeWithNValues(1) - sizeof(value_type); + return SizeWithNSlots(1) - sizeof(value_type); } // Compute how many values we can fit onto a leaf node taking into account // padding. - constexpr static size_type NodeTargetValues(const int begin, const int end) { + constexpr static size_type NodeTargetSlots(const int begin, const int end) { return begin == end ? begin - : SizeWithNValues((begin + end) / 2 + 1) > + : SizeWithNSlots((begin + end) / 2 + 1) > params_type::kTargetNodeSize - ? NodeTargetValues(begin, (begin + end) / 2) - : NodeTargetValues((begin + end) / 2 + 1, end); + ? NodeTargetSlots(begin, (begin + end) / 2) + : NodeTargetSlots((begin + end) / 2 + 1, end); } enum { kTargetNodeSize = params_type::kTargetNodeSize, - kNodeTargetValues = NodeTargetValues(0, params_type::kTargetNodeSize), + kNodeTargetSlots = NodeTargetSlots(0, params_type::kTargetNodeSize), - // We need a minimum of 3 values per internal node in order to perform + // We need a minimum of 3 slots per internal node in order to perform // splitting (1 value for the two nodes involved in the split and 1 value - // propagated to the parent as the delimiter for the split). - kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3, + // propagated to the parent as the delimiter for the split). For performance + // reasons, we don't allow 3 slots-per-node due to bad worst case occupancy + // of 1/3 (for a node, not a b-tree). + kMinNodeSlots = 4, + + kNodeSlots = + kNodeTargetSlots >= kMinNodeSlots ? kNodeTargetSlots : kMinNodeSlots, // The node is internal (i.e. is not a leaf node) if and only if `max_count` // has this value. kInternalNodeMaxCount = 0, }; - // Leaves can have less than kNodeValues values. - constexpr static layout_type LeafLayout(const int max_values = kNodeValues) { + // Leaves can have less than kNodeSlots values. + constexpr static layout_type LeafLayout(const int slot_count = kNodeSlots) { return layout_type(/*parent*/ 1, /*position, start, finish, max_count*/ 4, - /*values*/ max_values, + /*slots*/ slot_count, /*children*/ 0); } constexpr static layout_type InternalLayout() { return layout_type(/*parent*/ 1, /*position, start, finish, max_count*/ 4, - /*values*/ kNodeValues, - /*children*/ kNodeValues + 1); + /*slots*/ kNodeSlots, + /*children*/ kNodeSlots + 1); } - constexpr static size_type LeafSize(const int max_values = kNodeValues) { - return LeafLayout(max_values).AllocSize(); + constexpr static size_type LeafSize(const int slot_count = kNodeSlots) { + return LeafLayout(slot_count).AllocSize(); } constexpr static size_type InternalSize() { return InternalLayout().AllocSize(); @@ -591,10 +649,10 @@ class btree_node { } field_type max_count() const { // Internal nodes have max_count==kInternalNodeMaxCount. - // Leaf nodes have max_count in [1, kNodeValues]. + // Leaf nodes have max_count in [1, kNodeSlots]. const field_type max_count = GetField<1>()[3]; return max_count == field_type{kInternalNodeMaxCount} - ? field_type{kNodeValues} + ? field_type{kNodeSlots} : max_count; } @@ -672,7 +730,7 @@ class btree_node { } ++s; } - return {s}; + return SearchResult<int, false>{s}; } // Returns the position of the first value whose key is not less than k using @@ -707,7 +765,7 @@ class btree_node { e = mid; } } - return {s}; + return SearchResult<int, false>{s}; } // Returns the position of the first value whose key is not less than k using @@ -716,7 +774,7 @@ class btree_node { SearchResult<int, true> binary_search_impl( const K &k, int s, int e, const CompareTo &comp, std::true_type /* IsCompareTo */) const { - if (is_multi_container::value) { + if (params_type::template can_have_multiple_equivalent_keys<K>()) { MatchKind exact_match = MatchKind::kNe; while (s != e) { const int mid = (s + e) >> 1; @@ -727,14 +785,14 @@ class btree_node { e = mid; if (c == 0) { // Need to return the first value whose key is not less than k, - // which requires continuing the binary search if this is a - // multi-container. + // which requires continuing the binary search if there could be + // multiple equivalent keys. exact_match = MatchKind::kEq; } } } return {s, exact_match}; - } else { // Not a multi-container. + } else { // Can't have multiple equivalent keys. while (s != e) { const int mid = (s + e) >> 1; const absl::weak_ordering c = comp(key(mid), k); @@ -784,12 +842,12 @@ class btree_node { start_slot(), max_count * sizeof(slot_type)); } void init_internal(btree_node *parent) { - init_leaf(parent, kNodeValues); + init_leaf(parent, kNodeSlots); // Set `max_count` to a sentinel value to indicate that this node is // internal. set_max_count(kInternalNodeMaxCount); absl::container_internal::SanitizerPoisonMemoryRegion( - &mutable_child(start()), (kNodeValues + 1) * sizeof(btree_node *)); + &mutable_child(start()), (kNodeSlots + 1) * sizeof(btree_node *)); } static void deallocate(const size_type size, btree_node *node, @@ -800,12 +858,6 @@ class btree_node { // Deletes a node and all of its children. static void clear_and_delete(btree_node *node, allocator_type *alloc); - public: - // Exposed only for tests. - static bool testonly_uses_linear_node_search() { - return use_linear_search::value; - } - private: template <typename... Args> void value_init(const field_type i, allocator_type *alloc, Args &&... args) { @@ -873,6 +925,7 @@ struct btree_iterator { using key_type = typename Node::key_type; using size_type = typename Node::size_type; using params_type = typename Node::params_type; + using is_map_container = typename params_type::is_map_container; using node_type = Node; using normal_node = typename std::remove_const<Node>::type; @@ -884,7 +937,7 @@ struct btree_iterator { using slot_type = typename params_type::slot_type; using iterator = - btree_iterator<normal_node, normal_reference, normal_pointer>; + btree_iterator<normal_node, normal_reference, normal_pointer>; using const_iterator = btree_iterator<const_node, const_reference, const_pointer>; @@ -901,20 +954,19 @@ struct btree_iterator { btree_iterator(Node *n, int p) : node(n), position(p) {} // NOTE: this SFINAE allows for implicit conversions from iterator to - // const_iterator, but it specifically avoids defining copy constructors so - // that btree_iterator can be trivially copyable. This is for performance and - // binary size reasons. + // const_iterator, but it specifically avoids hiding the copy constructor so + // that the trivial one will be used when possible. template <typename N, typename R, typename P, absl::enable_if_t< std::is_same<btree_iterator<N, R, P>, iterator>::value && std::is_same<btree_iterator, const_iterator>::value, int> = 0> - btree_iterator(const btree_iterator<N, R, P> &other) // NOLINT + btree_iterator(const btree_iterator<N, R, P> other) // NOLINT : node(other.node), position(other.position) {} private: // This SFINAE allows explicit conversions from const_iterator to - // iterator, but also avoids defining a copy constructor. + // iterator, but also avoids hiding the copy constructor. // NOTE: the const_cast is safe because this constructor is only called by // non-const methods and the container owns the nodes. template <typename N, typename R, typename P, @@ -922,7 +974,7 @@ struct btree_iterator { std::is_same<btree_iterator<N, R, P>, const_iterator>::value && std::is_same<btree_iterator, iterator>::value, int> = 0> - explicit btree_iterator(const btree_iterator<N, R, P> &other) + explicit btree_iterator(const btree_iterator<N, R, P> other) : node(const_cast<node_type *>(other.node)), position(other.position) {} // Increment/decrement the iterator. @@ -985,6 +1037,8 @@ struct btree_iterator { } private: + friend iterator; + friend const_iterator; template <typename Params> friend class btree; template <typename Tree> @@ -995,8 +1049,6 @@ struct btree_iterator { friend class btree_map_container; template <typename Tree> friend class btree_multiset_container; - template <typename N, typename R, typename P> - friend struct btree_iterator; template <typename TreeType, typename CheckerType> friend class base_checker; @@ -1017,8 +1069,6 @@ class btree { using is_key_compare_to = typename Params::is_key_compare_to; using init_type = typename Params::init_type; using field_type = typename node_type::field_type; - using is_multi_container = typename Params::is_multi_container; - using is_key_compare_adapted = typename Params::is_key_compare_adapted; // We use a static empty node for the root/leftmost/rightmost of empty btrees // in order to avoid branching in begin()/end(). @@ -1054,8 +1104,8 @@ class btree { } enum : uint32_t { - kNodeValues = node_type::kNodeValues, - kMinNodeValues = kNodeValues / 2, + kNodeSlots = node_type::kNodeSlots, + kMinNodeValues = kNodeSlots / 2, }; struct node_stats { @@ -1085,7 +1135,8 @@ class btree { using const_reference = typename Params::const_reference; using pointer = typename Params::pointer; using const_pointer = typename Params::const_pointer; - using iterator = btree_iterator<node_type, reference, pointer>; + using iterator = + typename btree_iterator<node_type, reference, pointer>::iterator; using const_iterator = typename iterator::const_iterator; using reverse_iterator = std::reverse_iterator<iterator>; using const_reverse_iterator = std::reverse_iterator<const_iterator>; @@ -1098,28 +1149,46 @@ class btree { private: // For use in copy_or_move_values_in_order. const value_type &maybe_move_from_iterator(const_iterator it) { return *it; } - value_type &&maybe_move_from_iterator(iterator it) { return std::move(*it); } + value_type &&maybe_move_from_iterator(iterator it) { + // This is a destructive operation on the other container so it's safe for + // us to const_cast and move from the keys here even if it's a set. + return std::move(const_cast<value_type &>(*it)); + } // Copies or moves (depending on the template parameter) the values in // other into this btree in their order in other. This btree must be empty // before this method is called. This method is used in copy construction, // copy assignment, and move assignment. template <typename Btree> - void copy_or_move_values_in_order(Btree *other); + void copy_or_move_values_in_order(Btree &other); // Validates that various assumptions/requirements are true at compile time. constexpr static bool static_assert_validation(); public: - btree(const key_compare &comp, const allocator_type &alloc); + btree(const key_compare &comp, const allocator_type &alloc) + : root_(comp, alloc, EmptyNode()), rightmost_(EmptyNode()), size_(0) {} - btree(const btree &other); + btree(const btree &other) : btree(other, other.allocator()) {} + btree(const btree &other, const allocator_type &alloc) + : btree(other.key_comp(), alloc) { + copy_or_move_values_in_order(other); + } btree(btree &&other) noexcept : root_(std::move(other.root_)), rightmost_(absl::exchange(other.rightmost_, EmptyNode())), size_(absl::exchange(other.size_, 0)) { other.mutable_root() = EmptyNode(); } + btree(btree &&other, const allocator_type &alloc) + : btree(other.key_comp(), alloc) { + if (alloc == other.allocator()) { + swap(other); + } else { + // Move values from `other` one at a time when allocators are different. + copy_or_move_values_in_order(other); + } + } ~btree() { // Put static_asserts in destructor to avoid triggering them before the type @@ -1147,17 +1216,22 @@ class btree { return const_reverse_iterator(begin()); } - // Finds the first element whose key is not less than key. + // Finds the first element whose key is not less than `key`. template <typename K> iterator lower_bound(const K &key) { - return internal_end(internal_lower_bound(key)); + return internal_end(internal_lower_bound(key).value); } template <typename K> const_iterator lower_bound(const K &key) const { - return internal_end(internal_lower_bound(key)); + return internal_end(internal_lower_bound(key).value); } - // Finds the first element whose key is greater than key. + // Finds the first element whose key is not less than `key` and also returns + // whether that element is equal to `key`. + template <typename K> + std::pair<iterator, bool> lower_bound_equal(const K &key) const; + + // Finds the first element whose key is greater than `key`. template <typename K> iterator upper_bound(const K &key) { return internal_end(internal_upper_bound(key)); @@ -1239,18 +1313,8 @@ class btree { // to the element after the last erased element. std::pair<size_type, iterator> erase_range(iterator begin, iterator end); - // Erases the specified key from the btree. Returns 1 if an element was - // erased and 0 otherwise. - template <typename K> - size_type erase_unique(const K &key); - - // Erases all of the entries matching the specified key from the - // btree. Returns the number of elements erased. - template <typename K> - size_type erase_multi(const K &key); - - // Finds the iterator corresponding to a key or returns end() if the key is - // not present. + // Finds an element with key equivalent to `key` or returns `end()` if `key` + // is not present. template <typename K> iterator find(const K &key) { return internal_end(internal_find(key)); @@ -1260,23 +1324,6 @@ class btree { return internal_end(internal_find(key)); } - // Returns a count of the number of times the key appears in the btree. - template <typename K> - size_type count_unique(const K &key) const { - const iterator begin = internal_find(key); - if (begin.node == nullptr) { - // The key doesn't exist in the tree. - return 0; - } - return 1; - } - // Returns a count of the number of times the key appears in the btree. - template <typename K> - size_type count_multi(const K &key) const { - const auto range = equal_range(key); - return std::distance(range.first, range.second); - } - // Clear the btree, deleting all of the values it contains. void clear(); @@ -1339,12 +1386,14 @@ class btree { } } - // The average number of bytes used per value stored in the btree. + // The average number of bytes used per value stored in the btree assuming + // random insertion order. static double average_bytes_per_value() { - // Returns the number of bytes per value on a leaf node that is 75% - // full. Experimentally, this matches up nicely with the computed number of - // bytes per value in trees that had their values inserted in random order. - return node_type::LeafSize() / (kNodeValues * 0.75); + // The expected number of values per node with random insertion order is the + // average of the maximum and minimum numbers of values per node. + const double expected_values_per_node = + (kNodeSlots + kMinNodeValues) / 2.0; + return node_type::LeafSize() / expected_values_per_node; } // The fullness of the btree. Computed as the number of elements in the btree @@ -1354,7 +1403,7 @@ class btree { // Returns 0 for empty trees. double fullness() const { if (empty()) return 0.0; - return static_cast<double>(size()) / (nodes() * kNodeValues); + return static_cast<double>(size()) / (nodes() * kNodeSlots); } // The overhead of the btree structure in bytes per node. Computed as the // total number of bytes used by the btree minus the number of bytes used for @@ -1404,7 +1453,7 @@ class btree { } node_type *new_leaf_node(node_type *parent) { node_type *n = allocate(node_type::LeafSize()); - n->init_leaf(parent, kNodeValues); + n->init_leaf(parent, kNodeSlots); return n; } node_type *new_leaf_root_node(const int max_count) { @@ -1453,28 +1502,19 @@ class btree { static IterType internal_last(IterType iter); // Returns an iterator pointing to the leaf position at which key would - // reside in the tree. We provide 2 versions of internal_locate. The first - // version uses a less-than comparator and is incapable of distinguishing when - // there is an exact match. The second version is for the key-compare-to - // specialization and distinguishes exact matches. The key-compare-to - // specialization allows the caller to avoid a subsequent comparison to - // determine if an exact match was made, which is important for keys with - // expensive comparison, such as strings. + // reside in the tree, unless there is an exact match - in which case, the + // result may not be on a leaf. When there's a three-way comparator, we can + // return whether there was an exact match. This allows the caller to avoid a + // subsequent comparison to determine if an exact match was made, which is + // important for keys with expensive comparison, such as strings. template <typename K> SearchResult<iterator, is_key_compare_to::value> internal_locate( const K &key) const; - template <typename K> - SearchResult<iterator, false> internal_locate_impl( - const K &key, std::false_type /* IsCompareTo */) const; - - template <typename K> - SearchResult<iterator, true> internal_locate_impl( - const K &key, std::true_type /* IsCompareTo */) const; - // Internal routine which implements lower_bound(). template <typename K> - iterator internal_lower_bound(const K &key) const; + SearchResult<iterator, is_key_compare_to::value> internal_lower_bound( + const K &key) const; // Internal routine which implements upper_bound(). template <typename K> @@ -1503,13 +1543,6 @@ class btree { return res; } - public: - // Exposed only for tests. - static bool testonly_uses_linear_node_search() { - return node_type::testonly_uses_linear_node_search(); - } - - private: // We use compressed tuple in order to save space because key_compare and // allocator_type are usually empty. absl::container_internal::CompressedTuple<key_compare, allocator_type, @@ -1665,7 +1698,7 @@ template <typename P> void btree_node<P>::split(const int insert_position, btree_node *dest, allocator_type *alloc) { assert(dest->count() == 0); - assert(max_count() == kNodeValues); + assert(max_count() == kNodeSlots); // We bias the split based on the position being inserted. If we're // inserting at the beginning of the left node then bias the split to put @@ -1673,7 +1706,7 @@ void btree_node<P>::split(const int insert_position, btree_node *dest, // right node then bias the split to put more values on the left node. if (insert_position == start()) { dest->set_finish(dest->start() + finish() - 1); - } else if (insert_position == kNodeValues) { + } else if (insert_position == kNodeSlots) { dest->set_finish(dest->start()); } else { dest->set_finish(dest->start() + count() / 2); @@ -1744,7 +1777,7 @@ void btree_node<P>::clear_and_delete(btree_node *node, allocator_type *alloc) { // Navigate to the leftmost leaf under node, and then delete upwards. while (!node->leaf()) node = node->start_child(); - // Use `int` because `pos` needs to be able to hold `kNodeValues+1`, which + // Use `int` because `pos` needs to be able to hold `kNodeSlots+1`, which // isn't guaranteed to be a valid `field_type`. int pos = node->position(); btree_node *parent = node->parent(); @@ -1832,7 +1865,7 @@ void btree_iterator<N, R, P>::decrement_slow() { // btree methods template <typename P> template <typename Btree> -void btree<P>::copy_or_move_values_in_order(Btree *other) { +void btree<P>::copy_or_move_values_in_order(Btree &other) { static_assert(std::is_same<btree, Btree>::value || std::is_same<const btree, Btree>::value, "Btree type must be same or const."); @@ -1840,11 +1873,11 @@ void btree<P>::copy_or_move_values_in_order(Btree *other) { // We can avoid key comparisons because we know the order of the // values is the same order we'll store them in. - auto iter = other->begin(); - if (iter == other->end()) return; + auto iter = other.begin(); + if (iter == other.end()) return; insert_multi(maybe_move_from_iterator(iter)); ++iter; - for (; iter != other->end(); ++iter) { + for (; iter != other.end(); ++iter) { // If the btree is not empty, we can just insert the new value at the end // of the tree. internal_emplace(end(), maybe_move_from_iterator(iter)); @@ -1863,7 +1896,7 @@ constexpr bool btree<P>::static_assert_validation() { // Note: We assert that kTargetValues, which is computed from // Params::kTargetNodeSize, must fit the node_type::field_type. static_assert( - kNodeValues < (1 << (8 * sizeof(typename node_type::field_type))), + kNodeSlots < (1 << (8 * sizeof(typename node_type::field_type))), "target node size too large"); // Verify that key_compare returns an absl::{weak,strong}_ordering or bool. @@ -1883,31 +1916,29 @@ constexpr bool btree<P>::static_assert_validation() { } template <typename P> -btree<P>::btree(const key_compare &comp, const allocator_type &alloc) - : root_(comp, alloc, EmptyNode()), rightmost_(EmptyNode()), size_(0) {} - -template <typename P> -btree<P>::btree(const btree &other) - : btree(other.key_comp(), other.allocator()) { - copy_or_move_values_in_order(&other); +template <typename K> +auto btree<P>::lower_bound_equal(const K &key) const + -> std::pair<iterator, bool> { + const SearchResult<iterator, is_key_compare_to::value> res = + internal_lower_bound(key); + const iterator lower = iterator(internal_end(res.value)); + const bool equal = res.HasMatch() + ? res.IsEq() + : lower != end() && !compare_keys(key, lower.key()); + return {lower, equal}; } template <typename P> template <typename K> auto btree<P>::equal_range(const K &key) -> std::pair<iterator, iterator> { - const iterator lower = lower_bound(key); - // TODO(ezb): we should be able to avoid this comparison when there's a - // three-way comparator. - if (lower == end() || compare_keys(key, lower.key())) return {lower, lower}; + const std::pair<iterator, bool> lower_and_equal = lower_bound_equal(key); + const iterator lower = lower_and_equal.first; + if (!lower_and_equal.second) { + return {lower, lower}; + } const iterator next = std::next(lower); - // When the comparator is heterogeneous, we can't assume that comparison with - // non-`key_type` will be equivalent to `key_type` comparisons so there - // could be multiple equivalent keys even in a unique-container. But for - // heterogeneous comparisons from the default string adapted comparators, we - // don't need to worry about this. - if (!is_multi_container::value && - (std::is_same<K, key_type>::value || is_key_compare_adapted::value)) { + if (!params_type::template can_have_multiple_equivalent_keys<K>()) { // The next iterator after lower must point to a key greater than `key`. // Note: if this assert fails, then it may indicate that the comparator does // not meet the equivalence requirements for Compare @@ -1918,7 +1949,7 @@ auto btree<P>::equal_range(const K &key) -> std::pair<iterator, iterator> { // Try once more to avoid the call to upper_bound() if there's only one // equivalent key. This should prevent all calls to upper_bound() in cases of // unique-containers with heterogeneous comparators in which all comparison - // operators are equivalent. + // operators have the same equivalence classes. if (next == end() || compare_keys(key, next.key())) return {lower, next}; // In this case, we need to call upper_bound() to avoid worst case O(N) @@ -1934,8 +1965,8 @@ auto btree<P>::insert_unique(const K &key, Args &&... args) mutable_root() = rightmost_ = new_leaf_root_node(1); } - auto res = internal_locate(key); - iterator &iter = res.value; + SearchResult<iterator, is_key_compare_to::value> res = internal_locate(key); + iterator iter = res.value; if (res.HasMatch()) { if (res.IsEq()) { @@ -2049,7 +2080,7 @@ auto btree<P>::operator=(const btree &other) -> btree & { *mutable_allocator() = other.allocator(); } - copy_or_move_values_in_order(&other); + copy_or_move_values_in_order(other); } return *this; } @@ -2079,7 +2110,7 @@ auto btree<P>::operator=(btree &&other) noexcept -> btree & { // comparator while moving the values so we can't swap the key // comparators. *mutable_key_comp() = other.key_comp(); - copy_or_move_values_in_order(&other); + copy_or_move_values_in_order(other); } } } @@ -2203,31 +2234,6 @@ auto btree<P>::erase_range(iterator begin, iterator end) } template <typename P> -template <typename K> -auto btree<P>::erase_unique(const K &key) -> size_type { - const iterator iter = internal_find(key); - if (iter.node == nullptr) { - // The key doesn't exist in the tree, return nothing done. - return 0; - } - erase(iter); - return 1; -} - -template <typename P> -template <typename K> -auto btree<P>::erase_multi(const K &key) -> size_type { - const iterator begin = internal_lower_bound(key); - if (begin.node == nullptr) { - // The key doesn't exist in the tree, return nothing done. - return 0; - } - // Delete all of the keys between begin and upper_bound(key). - const iterator end = internal_end(internal_upper_bound(key)); - return erase_range(begin, end).first; -} - -template <typename P> void btree<P>::clear() { if (!empty()) { node_type::clear_and_delete(root(), mutable_allocator()); @@ -2271,7 +2277,7 @@ void btree<P>::rebalance_or_split(iterator *iter) { node_type *&node = iter->node; int &insert_position = iter->position; assert(node->count() == node->max_count()); - assert(kNodeValues == node->max_count()); + assert(kNodeSlots == node->max_count()); // First try to make room on the node by rebalancing. node_type *parent = node->parent(); @@ -2279,17 +2285,17 @@ void btree<P>::rebalance_or_split(iterator *iter) { if (node->position() > parent->start()) { // Try rebalancing with our left sibling. node_type *left = parent->child(node->position() - 1); - assert(left->max_count() == kNodeValues); - if (left->count() < kNodeValues) { + assert(left->max_count() == kNodeSlots); + if (left->count() < kNodeSlots) { // We bias rebalancing based on the position being inserted. If we're // inserting at the end of the right node then we bias rebalancing to // fill up the left node. - int to_move = (kNodeValues - left->count()) / - (1 + (insert_position < kNodeValues)); + int to_move = (kNodeSlots - left->count()) / + (1 + (insert_position < static_cast<int>(kNodeSlots))); to_move = (std::max)(1, to_move); if (insert_position - to_move >= node->start() || - left->count() + to_move < kNodeValues) { + left->count() + to_move < static_cast<int>(kNodeSlots)) { left->rebalance_right_to_left(to_move, node, mutable_allocator()); assert(node->max_count() - node->count() == to_move); @@ -2308,17 +2314,17 @@ void btree<P>::rebalance_or_split(iterator *iter) { if (node->position() < parent->finish()) { // Try rebalancing with our right sibling. node_type *right = parent->child(node->position() + 1); - assert(right->max_count() == kNodeValues); - if (right->count() < kNodeValues) { + assert(right->max_count() == kNodeSlots); + if (right->count() < kNodeSlots) { // We bias rebalancing based on the position being inserted. If we're // inserting at the beginning of the left node then we bias rebalancing // to fill up the right node. - int to_move = (kNodeValues - right->count()) / + int to_move = (static_cast<int>(kNodeSlots) - right->count()) / (1 + (insert_position > node->start())); to_move = (std::max)(1, to_move); if (insert_position <= node->finish() - to_move || - right->count() + to_move < kNodeValues) { + right->count() + to_move < static_cast<int>(kNodeSlots)) { node->rebalance_left_to_right(to_move, right, mutable_allocator()); if (insert_position > node->finish()) { @@ -2334,8 +2340,8 @@ void btree<P>::rebalance_or_split(iterator *iter) { // Rebalancing failed, make sure there is room on the parent node for a new // value. - assert(parent->max_count() == kNodeValues); - if (parent->count() == kNodeValues) { + assert(parent->max_count() == kNodeSlots); + if (parent->count() == kNodeSlots) { iterator parent_iter(node->parent(), node->position()); rebalance_or_split(&parent_iter); } @@ -2380,8 +2386,8 @@ bool btree<P>::try_merge_or_rebalance(iterator *iter) { if (iter->node->position() > parent->start()) { // Try merging with our left sibling. node_type *left = parent->child(iter->node->position() - 1); - assert(left->max_count() == kNodeValues); - if (1 + left->count() + iter->node->count() <= kNodeValues) { + assert(left->max_count() == kNodeSlots); + if (1U + left->count() + iter->node->count() <= kNodeSlots) { iter->position += 1 + left->count(); merge_nodes(left, iter->node); iter->node = left; @@ -2391,8 +2397,8 @@ bool btree<P>::try_merge_or_rebalance(iterator *iter) { if (iter->node->position() < parent->finish()) { // Try merging with our right sibling. node_type *right = parent->child(iter->node->position() + 1); - assert(right->max_count() == kNodeValues); - if (1 + iter->node->count() + right->count() <= kNodeValues) { + assert(right->max_count() == kNodeSlots); + if (1U + iter->node->count() + right->count() <= kNodeSlots) { merge_nodes(iter->node, right); return true; } @@ -2473,12 +2479,12 @@ inline auto btree<P>::internal_emplace(iterator iter, Args &&... args) allocator_type *alloc = mutable_allocator(); if (iter.node->count() == max_count) { // Make room in the leaf for the new item. - if (max_count < kNodeValues) { + if (max_count < kNodeSlots) { // Insertion into the root where the root is smaller than the full node // size. Simply grow the size of the root node. assert(iter.node == root()); iter.node = - new_leaf_root_node((std::min<int>)(kNodeValues, 2 * max_count)); + new_leaf_root_node((std::min<int>)(kNodeSlots, 2 * max_count)); // Transfer the values from the old root to the new root. node_type *old_root = root(); node_type *new_root = iter.node; @@ -2501,61 +2507,51 @@ template <typename P> template <typename K> inline auto btree<P>::internal_locate(const K &key) const -> SearchResult<iterator, is_key_compare_to::value> { - return internal_locate_impl(key, is_key_compare_to()); -} - -template <typename P> -template <typename K> -inline auto btree<P>::internal_locate_impl( - const K &key, std::false_type /* IsCompareTo */) const - -> SearchResult<iterator, false> { - iterator iter(const_cast<node_type *>(root())); - for (;;) { - iter.position = iter.node->lower_bound(key, key_comp()).value; - // NOTE: we don't need to walk all the way down the tree if the keys are - // equal, but determining equality would require doing an extra comparison - // on each node on the way down, and we will need to go all the way to the - // leaf node in the expected case. - if (iter.node->leaf()) { - break; - } - iter.node = iter.node->child(iter.position); - } - return {iter}; -} - -template <typename P> -template <typename K> -inline auto btree<P>::internal_locate_impl( - const K &key, std::true_type /* IsCompareTo */) const - -> SearchResult<iterator, true> { iterator iter(const_cast<node_type *>(root())); for (;;) { - SearchResult<int, true> res = iter.node->lower_bound(key, key_comp()); + SearchResult<int, is_key_compare_to::value> res = + iter.node->lower_bound(key, key_comp()); iter.position = res.value; - if (res.match == MatchKind::kEq) { + if (res.IsEq()) { return {iter, MatchKind::kEq}; } + // Note: in the non-key-compare-to case, we don't need to walk all the way + // down the tree if the keys are equal, but determining equality would + // require doing an extra comparison on each node on the way down, and we + // will need to go all the way to the leaf node in the expected case. if (iter.node->leaf()) { break; } iter.node = iter.node->child(iter.position); } + // Note: in the non-key-compare-to case, the key may actually be equivalent + // here (and the MatchKind::kNe is ignored). return {iter, MatchKind::kNe}; } template <typename P> template <typename K> -auto btree<P>::internal_lower_bound(const K &key) const -> iterator { +auto btree<P>::internal_lower_bound(const K &key) const + -> SearchResult<iterator, is_key_compare_to::value> { + if (!params_type::template can_have_multiple_equivalent_keys<K>()) { + SearchResult<iterator, is_key_compare_to::value> ret = internal_locate(key); + ret.value = internal_last(ret.value); + return ret; + } iterator iter(const_cast<node_type *>(root())); + SearchResult<int, is_key_compare_to::value> res; + bool seen_eq = false; for (;;) { - iter.position = iter.node->lower_bound(key, key_comp()).value; + res = iter.node->lower_bound(key, key_comp()); + iter.position = res.value; if (iter.node->leaf()) { break; } + seen_eq = seen_eq || res.IsEq(); iter.node = iter.node->child(iter.position); } - return internal_last(iter); + if (res.IsEq()) return {iter, MatchKind::kEq}; + return {internal_last(iter), seen_eq ? MatchKind::kEq : MatchKind::kNe}; } template <typename P> @@ -2575,7 +2571,7 @@ auto btree<P>::internal_upper_bound(const K &key) const -> iterator { template <typename P> template <typename K> auto btree<P>::internal_find(const K &key) const -> iterator { - auto res = internal_locate(key); + SearchResult<iterator, is_key_compare_to::value> res = internal_locate(key); if (res.HasMatch()) { if (res.IsEq()) { return res.value; diff --git a/absl/container/internal/btree_container.h b/absl/container/internal/btree_container.h index 137614f8..03be708e 100644 --- a/absl/container/internal/btree_container.h +++ b/absl/container/internal/btree_container.h @@ -23,6 +23,7 @@ #include "absl/base/internal/throw_delegate.h" #include "absl/container/internal/btree.h" // IWYU pragma: export #include "absl/container/internal/common.h" +#include "absl/memory/memory.h" #include "absl/meta/type_traits.h" namespace absl { @@ -68,8 +69,21 @@ class btree_container { explicit btree_container(const key_compare &comp, const allocator_type &alloc = allocator_type()) : tree_(comp, alloc) {} - btree_container(const btree_container &other) = default; - btree_container(btree_container &&other) noexcept = default; + explicit btree_container(const allocator_type &alloc) + : tree_(key_compare(), alloc) {} + + btree_container(const btree_container &other) + : btree_container(other, absl::allocator_traits<allocator_type>:: + select_on_container_copy_construction( + other.get_allocator())) {} + btree_container(const btree_container &other, const allocator_type &alloc) + : tree_(other.tree_, alloc) {} + + btree_container(btree_container &&other) noexcept( + std::is_nothrow_move_constructible<Tree>::value) = default; + btree_container(btree_container &&other, const allocator_type &alloc) + : tree_(std::move(other.tree_), alloc) {} + btree_container &operator=(const btree_container &other) = default; btree_container &operator=(btree_container &&other) noexcept( std::is_nothrow_move_assignable<Tree>::value) = default; @@ -90,6 +104,11 @@ class btree_container { // Lookup routines. template <typename K = key_type> + size_type count(const key_arg<K> &key) const { + auto equal_range = this->equal_range(key); + return std::distance(equal_range.first, equal_range.second); + } + template <typename K = key_type> iterator find(const key_arg<K> &key) { return tree_.find(key); } @@ -138,6 +157,11 @@ class btree_container { iterator erase(const_iterator first, const_iterator last) { return tree_.erase_range(iterator(first), iterator(last)).second; } + template <typename K = key_type> + size_type erase(const key_arg<K> &key) { + auto equal_range = this->equal_range(key); + return tree_.erase_range(equal_range.first, equal_range.second).first; + } // Extract routines. node_type extract(iterator position) { @@ -151,7 +175,6 @@ class btree_container { return extract(iterator(position)); } - public: // Utility routines. void clear() { tree_.clear(); } void swap(btree_container &other) { tree_.swap(other.tree_); } @@ -235,7 +258,7 @@ class btree_set_container : public btree_container<Tree> { using super_type::super_type; btree_set_container() {} - // Range constructor. + // Range constructors. template <class InputIterator> btree_set_container(InputIterator b, InputIterator e, const key_compare &comp = key_compare(), @@ -243,18 +266,19 @@ class btree_set_container : public btree_container<Tree> { : super_type(comp, alloc) { insert(b, e); } + template <class InputIterator> + btree_set_container(InputIterator b, InputIterator e, + const allocator_type &alloc) + : btree_set_container(b, e, key_compare(), alloc) {} - // Initializer list constructor. + // Initializer list constructors. btree_set_container(std::initializer_list<init_type> init, const key_compare &comp = key_compare(), const allocator_type &alloc = allocator_type()) : btree_set_container(init.begin(), init.end(), comp, alloc) {} - - // Lookup routines. - template <typename K = key_type> - size_type count(const key_arg<K> &key) const { - return this->tree_.count_unique(key); - } + btree_set_container(std::initializer_list<init_type> init, + const allocator_type &alloc) + : btree_set_container(init.begin(), init.end(), alloc) {} // Insertion routines. std::pair<iterator, bool> insert(const value_type &v) { @@ -313,20 +337,13 @@ class btree_set_container : public btree_container<Tree> { return res.first; } - // Deletion routines. - // TODO(ezb): we should support heterogeneous comparators that have different - // behavior for K!=key_type. - template <typename K = key_type> - size_type erase(const key_arg<K> &key) { - return this->tree_.erase_unique(key); - } - using super_type::erase; - // Node extraction routines. template <typename K = key_type> node_type extract(const key_arg<K> &key) { - auto it = this->find(key); - return it == this->end() ? node_type() : extract(it); + const std::pair<iterator, bool> lower_and_equal = + this->tree_.lower_bound_equal(key); + return lower_and_equal.second ? extract(lower_and_equal.first) + : node_type(); } using super_type::extract; @@ -344,7 +361,7 @@ class btree_set_container : public btree_container<Tree> { int> = 0> void merge(btree_container<T> &src) { // NOLINT for (auto src_it = src.begin(); src_it != src.end();) { - if (insert(std::move(*src_it)).second) { + if (insert(std::move(params_type::element(src_it.slot()))).second) { src_it = src.erase(src_it); } else { ++src_it; @@ -371,6 +388,7 @@ template <typename Tree> class btree_map_container : public btree_set_container<Tree> { using super_type = btree_set_container<Tree>; using params_type = typename Tree::params_type; + friend class BtreeNodePeer; private: template <class K> @@ -535,7 +553,7 @@ class btree_multiset_container : public btree_container<Tree> { using super_type::super_type; btree_multiset_container() {} - // Range constructor. + // Range constructors. template <class InputIterator> btree_multiset_container(InputIterator b, InputIterator e, const key_compare &comp = key_compare(), @@ -543,18 +561,19 @@ class btree_multiset_container : public btree_container<Tree> { : super_type(comp, alloc) { insert(b, e); } + template <class InputIterator> + btree_multiset_container(InputIterator b, InputIterator e, + const allocator_type &alloc) + : btree_multiset_container(b, e, key_compare(), alloc) {} - // Initializer list constructor. + // Initializer list constructors. btree_multiset_container(std::initializer_list<init_type> init, const key_compare &comp = key_compare(), const allocator_type &alloc = allocator_type()) : btree_multiset_container(init.begin(), init.end(), comp, alloc) {} - - // Lookup routines. - template <typename K = key_type> - size_type count(const key_arg<K> &key) const { - return this->tree_.count_multi(key); - } + btree_multiset_container(std::initializer_list<init_type> init, + const allocator_type &alloc) + : btree_multiset_container(init.begin(), init.end(), alloc) {} // Insertion routines. iterator insert(const value_type &v) { return this->tree_.insert_multi(v); } @@ -600,18 +619,13 @@ class btree_multiset_container : public btree_container<Tree> { return res; } - // Deletion routines. - template <typename K = key_type> - size_type erase(const key_arg<K> &key) { - return this->tree_.erase_multi(key); - } - using super_type::erase; - // Node extraction routines. template <typename K = key_type> node_type extract(const key_arg<K> &key) { - auto it = this->find(key); - return it == this->end() ? node_type() : extract(it); + const std::pair<iterator, bool> lower_and_equal = + this->tree_.lower_bound_equal(key); + return lower_and_equal.second ? extract(lower_and_equal.first) + : node_type(); } using super_type::extract; @@ -627,8 +641,9 @@ class btree_multiset_container : public btree_container<Tree> { typename T::params_type::is_map_container>>::value, int> = 0> void merge(btree_container<T> &src) { // NOLINT - insert(std::make_move_iterator(src.begin()), - std::make_move_iterator(src.end())); + for (auto src_it = src.begin(), end = src.end(); src_it != end; ++src_it) { + insert(std::move(params_type::element(src_it.slot()))); + } src.clear(); } diff --git a/absl/container/internal/compressed_tuple.h b/absl/container/internal/compressed_tuple.h index 02bfd03f..5ebe1649 100644 --- a/absl/container/internal/compressed_tuple.h +++ b/absl/container/internal/compressed_tuple.h @@ -257,7 +257,7 @@ class ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple template <int I> ElemT<I>& get() & { - return internal_compressed_tuple::Storage<ElemT<I>, I>::get(); + return StorageT<I>::get(); } template <int I> diff --git a/absl/container/internal/container_memory_test.cc b/absl/container/internal/container_memory_test.cc index 6a7fcd29..fb9c4dde 100644 --- a/absl/container/internal/container_memory_test.cc +++ b/absl/container/internal/container_memory_test.cc @@ -166,7 +166,7 @@ TryDecomposeValue(F&& f, Arg&& arg) { } TEST(DecomposeValue, Decomposable) { - auto f = [](const int& x, int&& y) { + auto f = [](const int& x, int&& y) { // NOLINT EXPECT_EQ(&x, &y); EXPECT_EQ(42, x); return 'A'; @@ -200,7 +200,8 @@ TryDecomposePair(F&& f, Args&&... args) { } TEST(DecomposePair, Decomposable) { - auto f = [](const int& x, std::piecewise_construct_t, std::tuple<int&&> k, + auto f = [](const int& x, // NOLINT + std::piecewise_construct_t, std::tuple<int&&> k, std::tuple<double>&& v) { EXPECT_EQ(&x, &std::get<0>(k)); EXPECT_EQ(42, x); diff --git a/absl/container/internal/hashtablez_sampler.cc b/absl/container/internal/hashtablez_sampler.cc index e4484fbb..5a29bed7 100644 --- a/absl/container/internal/hashtablez_sampler.cc +++ b/absl/container/internal/hashtablez_sampler.cc @@ -72,6 +72,7 @@ void HashtablezInfo::PrepareForSampling() { total_probe_length.store(0, std::memory_order_relaxed); hashes_bitwise_or.store(0, std::memory_order_relaxed); hashes_bitwise_and.store(~size_t{}, std::memory_order_relaxed); + hashes_bitwise_xor.store(0, std::memory_order_relaxed); create_time = absl::Now(); // The inliner makes hardcoded skip_count difficult (especially when combined @@ -180,7 +181,9 @@ static bool ShouldForceSampling() { if (ABSL_PREDICT_TRUE(state == kDontForce)) return false; if (state == kUninitialized) { - state = AbslContainerInternalSampleEverything() ? kForce : kDontForce; + state = ABSL_INTERNAL_C_SYMBOL(AbslContainerInternalSampleEverything)() + ? kForce + : kDontForce; global_state.store(state, std::memory_order_relaxed); } return state == kForce; @@ -235,6 +238,7 @@ void RecordInsertSlow(HashtablezInfo* info, size_t hash, info->hashes_bitwise_and.fetch_and(hash, std::memory_order_relaxed); info->hashes_bitwise_or.fetch_or(hash, std::memory_order_relaxed); + info->hashes_bitwise_xor.fetch_xor(hash, std::memory_order_relaxed); info->max_probe_length.store( std::max(info->max_probe_length.load(std::memory_order_relaxed), probe_length), diff --git a/absl/container/internal/hashtablez_sampler.h b/absl/container/internal/hashtablez_sampler.h index 394348da..85685f72 100644 --- a/absl/container/internal/hashtablez_sampler.h +++ b/absl/container/internal/hashtablez_sampler.h @@ -78,6 +78,7 @@ struct HashtablezInfo { std::atomic<size_t> total_probe_length; std::atomic<size_t> hashes_bitwise_or; std::atomic<size_t> hashes_bitwise_and; + std::atomic<size_t> hashes_bitwise_xor; // `HashtablezSampler` maintains intrusive linked lists for all samples. See // comments on `HashtablezSampler::all_` for details on these. `init_mu` @@ -312,7 +313,7 @@ void SetHashtablezMaxSamples(int32_t max); // initialization of static storage duration objects. // The definition of this constant is weak, which allows us to inject a // different value for it at link time. -extern "C" bool AbslContainerInternalSampleEverything(); +extern "C" bool ABSL_INTERNAL_C_SYMBOL(AbslContainerInternalSampleEverything)(); } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/hashtablez_sampler_force_weak_definition.cc b/absl/container/internal/hashtablez_sampler_force_weak_definition.cc index 78b9d362..ed35a7ee 100644 --- a/absl/container/internal/hashtablez_sampler_force_weak_definition.cc +++ b/absl/container/internal/hashtablez_sampler_force_weak_definition.cc @@ -21,7 +21,8 @@ ABSL_NAMESPACE_BEGIN namespace container_internal { // See hashtablez_sampler.h for details. -extern "C" ABSL_ATTRIBUTE_WEAK bool AbslContainerInternalSampleEverything() { +extern "C" ABSL_ATTRIBUTE_WEAK bool ABSL_INTERNAL_C_SYMBOL( + AbslContainerInternalSampleEverything)() { return false; } diff --git a/absl/container/internal/hashtablez_sampler_test.cc b/absl/container/internal/hashtablez_sampler_test.cc index 8d10a1e9..5f4c83b7 100644 --- a/absl/container/internal/hashtablez_sampler_test.cc +++ b/absl/container/internal/hashtablez_sampler_test.cc @@ -89,6 +89,7 @@ TEST(HashtablezInfoTest, PrepareForSampling) { EXPECT_EQ(info.total_probe_length.load(), 0); EXPECT_EQ(info.hashes_bitwise_or.load(), 0); EXPECT_EQ(info.hashes_bitwise_and.load(), ~size_t{}); + EXPECT_EQ(info.hashes_bitwise_xor.load(), 0); EXPECT_GE(info.create_time, test_start); info.capacity.store(1, std::memory_order_relaxed); @@ -98,6 +99,7 @@ TEST(HashtablezInfoTest, PrepareForSampling) { info.total_probe_length.store(1, std::memory_order_relaxed); info.hashes_bitwise_or.store(1, std::memory_order_relaxed); info.hashes_bitwise_and.store(1, std::memory_order_relaxed); + info.hashes_bitwise_xor.store(1, std::memory_order_relaxed); info.create_time = test_start - absl::Hours(20); info.PrepareForSampling(); @@ -109,6 +111,7 @@ TEST(HashtablezInfoTest, PrepareForSampling) { EXPECT_EQ(info.total_probe_length.load(), 0); EXPECT_EQ(info.hashes_bitwise_or.load(), 0); EXPECT_EQ(info.hashes_bitwise_and.load(), ~size_t{}); + EXPECT_EQ(info.hashes_bitwise_xor.load(), 0); EXPECT_GE(info.create_time, test_start); } @@ -133,14 +136,17 @@ TEST(HashtablezInfoTest, RecordInsert) { EXPECT_EQ(info.max_probe_length.load(), 6); EXPECT_EQ(info.hashes_bitwise_and.load(), 0x0000FF00); EXPECT_EQ(info.hashes_bitwise_or.load(), 0x0000FF00); + EXPECT_EQ(info.hashes_bitwise_xor.load(), 0x0000FF00); RecordInsertSlow(&info, 0x000FF000, 4 * kProbeLength); EXPECT_EQ(info.max_probe_length.load(), 6); EXPECT_EQ(info.hashes_bitwise_and.load(), 0x0000F000); EXPECT_EQ(info.hashes_bitwise_or.load(), 0x000FFF00); + EXPECT_EQ(info.hashes_bitwise_xor.load(), 0x000F0F00); RecordInsertSlow(&info, 0x00FF0000, 12 * kProbeLength); EXPECT_EQ(info.max_probe_length.load(), 12); EXPECT_EQ(info.hashes_bitwise_and.load(), 0x00000000); EXPECT_EQ(info.hashes_bitwise_or.load(), 0x00FFFF00); + EXPECT_EQ(info.hashes_bitwise_xor.load(), 0x00F00F00); } TEST(HashtablezInfoTest, RecordErase) { diff --git a/absl/container/internal/inlined_vector.h b/absl/container/internal/inlined_vector.h index 4d80b727..b8aec45b 100644 --- a/absl/container/internal/inlined_vector.h +++ b/absl/container/internal/inlined_vector.h @@ -33,6 +33,12 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace inlined_vector_internal { +// GCC does not deal very well with the below code +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + template <typename Iterator> using IsAtLeastForwardIterator = std::is_convertible< typename std::iterator_traits<Iterator>::iterator_category, @@ -75,6 +81,23 @@ void DestroyElements(AllocatorType* alloc_ptr, Pointer destroy_first, } } +// If kUseMemcpy is true, memcpy(dst, src, n); else do nothing. +// Useful to avoid compiler warnings when memcpy() is used for T values +// that are not trivially copyable in non-reachable code. +template <bool kUseMemcpy> +inline void MemcpyIfAllowed(void* dst, const void* src, size_t n); + +// memcpy when allowed. +template <> +inline void MemcpyIfAllowed<true>(void* dst, const void* src, size_t n) { + memcpy(dst, src, n); +} + +// Do nothing for types that are not memcpy-able. This function is only +// called from non-reachable branches. +template <> +inline void MemcpyIfAllowed<false>(void*, const void*, size_t) {} + template <typename AllocatorType, typename Pointer, typename ValueAdapter, typename SizeType> void ConstructElements(AllocatorType* alloc_ptr, Pointer construct_first, @@ -298,14 +321,20 @@ class Storage { // Storage Constructors and Destructor // --------------------------------------------------------------------------- - Storage() : metadata_() {} + Storage() : metadata_(allocator_type(), /* size and is_allocated */ 0) {} - explicit Storage(const allocator_type& alloc) : metadata_(alloc, {}) {} + explicit Storage(const allocator_type& alloc) + : metadata_(alloc, /* size and is_allocated */ 0) {} ~Storage() { - pointer data = GetIsAllocated() ? GetAllocatedData() : GetInlinedData(); - inlined_vector_internal::DestroyElements(GetAllocPtr(), data, GetSize()); - DeallocateIfAllocated(); + if (GetSizeAndIsAllocated() == 0) { + // Empty and not allocated; nothing to do. + } else if (IsMemcpyOk::value) { + // No destructors need to be run; just deallocate if necessary. + DeallocateIfAllocated(); + } else { + DestroyContents(); + } } // --------------------------------------------------------------------------- @@ -363,6 +392,8 @@ class Storage { // Storage Member Mutators // --------------------------------------------------------------------------- + ABSL_ATTRIBUTE_NOINLINE void InitFrom(const Storage& other); + template <typename ValueAdapter> void Initialize(ValueAdapter values, size_type new_size); @@ -445,6 +476,8 @@ class Storage { } private: + ABSL_ATTRIBUTE_NOINLINE void DestroyContents(); + using Metadata = container_internal::CompressedTuple<allocator_type, size_type>; @@ -462,11 +495,48 @@ class Storage { Inlined inlined; }; + template <typename... Args> + ABSL_ATTRIBUTE_NOINLINE reference EmplaceBackSlow(Args&&... args); + Metadata metadata_; Data data_; }; template <typename T, size_t N, typename A> +void Storage<T, N, A>::DestroyContents() { + pointer data = GetIsAllocated() ? GetAllocatedData() : GetInlinedData(); + inlined_vector_internal::DestroyElements(GetAllocPtr(), data, GetSize()); + DeallocateIfAllocated(); +} + +template <typename T, size_t N, typename A> +void Storage<T, N, A>::InitFrom(const Storage& other) { + const auto n = other.GetSize(); + assert(n > 0); // Empty sources handled handled in caller. + const_pointer src; + pointer dst; + if (!other.GetIsAllocated()) { + dst = GetInlinedData(); + src = other.GetInlinedData(); + } else { + // Because this is only called from the `InlinedVector` constructors, it's + // safe to take on the allocation with size `0`. If `ConstructElements(...)` + // throws, deallocation will be automatically handled by `~Storage()`. + size_type new_capacity = ComputeCapacity(GetInlinedCapacity(), n); + dst = AllocatorTraits::allocate(*GetAllocPtr(), new_capacity); + SetAllocatedData(dst, new_capacity); + src = other.GetAllocatedData(); + } + if (IsMemcpyOk::value) { + MemcpyIfAllowed<IsMemcpyOk::value>(dst, src, sizeof(dst[0]) * n); + } else { + auto values = IteratorValueAdapter<const_pointer>(src); + inlined_vector_internal::ConstructElements(GetAllocPtr(), dst, &values, n); + } + GetSizeAndIsAllocated() = other.GetSizeAndIsAllocated(); +} + +template <typename T, size_t N, typename A> template <typename ValueAdapter> auto Storage<T, N, A>::Initialize(ValueAdapter values, size_type new_size) -> void { @@ -542,48 +612,42 @@ template <typename T, size_t N, typename A> template <typename ValueAdapter> auto Storage<T, N, A>::Resize(ValueAdapter values, size_type new_size) -> void { StorageView storage_view = MakeStorageView(); - - IteratorValueAdapter<MoveIterator> move_values( - MoveIterator(storage_view.data)); - - AllocationTransaction allocation_tx(GetAllocPtr()); - ConstructionTransaction construction_tx(GetAllocPtr()); - - absl::Span<value_type> construct_loop; - absl::Span<value_type> move_construct_loop; - absl::Span<value_type> destroy_loop; - - if (new_size > storage_view.capacity) { + auto* const base = storage_view.data; + const size_type size = storage_view.size; + auto* alloc = GetAllocPtr(); + if (new_size <= size) { + // Destroy extra old elements. + inlined_vector_internal::DestroyElements(alloc, base + new_size, + size - new_size); + } else if (new_size <= storage_view.capacity) { + // Construct new elements in place. + inlined_vector_internal::ConstructElements(alloc, base + size, &values, + new_size - size); + } else { + // Steps: + // a. Allocate new backing store. + // b. Construct new elements in new backing store. + // c. Move existing elements from old backing store to now. + // d. Destroy all elements in old backing store. + // Use transactional wrappers for the first two steps so we can roll + // back if necessary due to exceptions. + AllocationTransaction allocation_tx(alloc); size_type new_capacity = ComputeCapacity(storage_view.capacity, new_size); pointer new_data = allocation_tx.Allocate(new_capacity); - construct_loop = {new_data + storage_view.size, - new_size - storage_view.size}; - move_construct_loop = {new_data, storage_view.size}; - destroy_loop = {storage_view.data, storage_view.size}; - } else if (new_size > storage_view.size) { - construct_loop = {storage_view.data + storage_view.size, - new_size - storage_view.size}; - } else { - destroy_loop = {storage_view.data + new_size, storage_view.size - new_size}; - } - construction_tx.Construct(construct_loop.data(), &values, - construct_loop.size()); + ConstructionTransaction construction_tx(alloc); + construction_tx.Construct(new_data + size, &values, new_size - size); - inlined_vector_internal::ConstructElements( - GetAllocPtr(), move_construct_loop.data(), &move_values, - move_construct_loop.size()); - - inlined_vector_internal::DestroyElements(GetAllocPtr(), destroy_loop.data(), - destroy_loop.size()); + IteratorValueAdapter<MoveIterator> move_values((MoveIterator(base))); + inlined_vector_internal::ConstructElements(alloc, new_data, &move_values, + size); - construction_tx.Commit(); - if (allocation_tx.DidAllocate()) { + inlined_vector_internal::DestroyElements(alloc, base, size); + construction_tx.Commit(); DeallocateIfAllocated(); AcquireAllocatedData(&allocation_tx); SetIsAllocated(); } - SetSize(new_size); } @@ -684,44 +748,50 @@ template <typename T, size_t N, typename A> template <typename... Args> auto Storage<T, N, A>::EmplaceBack(Args&&... args) -> reference { StorageView storage_view = MakeStorageView(); + const auto n = storage_view.size; + if (ABSL_PREDICT_TRUE(n != storage_view.capacity)) { + // Fast path; new element fits. + pointer last_ptr = storage_view.data + n; + AllocatorTraits::construct(*GetAllocPtr(), last_ptr, + std::forward<Args>(args)...); + AddSize(1); + return *last_ptr; + } + // TODO(b/173712035): Annotate with musttail attribute to prevent regression. + return EmplaceBackSlow(std::forward<Args>(args)...); +} +template <typename T, size_t N, typename A> +template <typename... Args> +auto Storage<T, N, A>::EmplaceBackSlow(Args&&... args) -> reference { + StorageView storage_view = MakeStorageView(); AllocationTransaction allocation_tx(GetAllocPtr()); - IteratorValueAdapter<MoveIterator> move_values( MoveIterator(storage_view.data)); - - pointer construct_data; - if (storage_view.size == storage_view.capacity) { - size_type new_capacity = NextCapacity(storage_view.capacity); - construct_data = allocation_tx.Allocate(new_capacity); - } else { - construct_data = storage_view.data; - } - + size_type new_capacity = NextCapacity(storage_view.capacity); + pointer construct_data = allocation_tx.Allocate(new_capacity); pointer last_ptr = construct_data + storage_view.size; + // Construct new element. AllocatorTraits::construct(*GetAllocPtr(), last_ptr, std::forward<Args>(args)...); - - if (allocation_tx.DidAllocate()) { - ABSL_INTERNAL_TRY { - inlined_vector_internal::ConstructElements( - GetAllocPtr(), allocation_tx.GetData(), &move_values, - storage_view.size); - } - ABSL_INTERNAL_CATCH_ANY { - AllocatorTraits::destroy(*GetAllocPtr(), last_ptr); - ABSL_INTERNAL_RETHROW; - } - - inlined_vector_internal::DestroyElements(GetAllocPtr(), storage_view.data, - storage_view.size); - - DeallocateIfAllocated(); - AcquireAllocatedData(&allocation_tx); - SetIsAllocated(); + // Move elements from old backing store to new backing store. + ABSL_INTERNAL_TRY { + inlined_vector_internal::ConstructElements( + GetAllocPtr(), allocation_tx.GetData(), &move_values, + storage_view.size); + } + ABSL_INTERNAL_CATCH_ANY { + AllocatorTraits::destroy(*GetAllocPtr(), last_ptr); + ABSL_INTERNAL_RETHROW; } + // Destroy elements in old backing store. + inlined_vector_internal::DestroyElements(GetAllocPtr(), storage_view.data, + storage_view.size); + DeallocateIfAllocated(); + AcquireAllocatedData(&allocation_tx); + SetIsAllocated(); AddSize(1); return *last_ptr; } @@ -885,6 +955,11 @@ auto Storage<T, N, A>::Swap(Storage* other_storage_ptr) -> void { swap(*GetAllocPtr(), *other_storage_ptr->GetAllocPtr()); } +// End ignore "maybe-uninitialized" +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + } // namespace inlined_vector_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/internal/layout.h b/absl/container/internal/layout.h index 23367833..a59a2430 100644 --- a/absl/container/internal/layout.h +++ b/absl/container/internal/layout.h @@ -404,7 +404,7 @@ class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, constexpr size_t Offset() const { static_assert(N < NumOffsets, "Index out of bounds"); return adl_barrier::Align( - Offset<N - 1>() + SizeOf<ElementType<N - 1>>() * size_[N - 1], + Offset<N - 1>() + SizeOf<ElementType<N - 1>>::value * size_[N - 1], ElementAlignment<N>::value); } @@ -597,7 +597,7 @@ class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, constexpr size_t AllocSize() const { static_assert(NumTypes == NumSizes, "You must specify sizes of all fields"); return Offset<NumTypes - 1>() + - SizeOf<ElementType<NumTypes - 1>>() * size_[NumTypes - 1]; + SizeOf<ElementType<NumTypes - 1>>::value * size_[NumTypes - 1]; } // If built with --config=asan, poisons padding bytes (if any) in the @@ -621,7 +621,7 @@ class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, // The `if` is an optimization. It doesn't affect the observable behaviour. if (ElementAlignment<N - 1>::value % ElementAlignment<N>::value) { size_t start = - Offset<N - 1>() + SizeOf<ElementType<N - 1>>() * size_[N - 1]; + Offset<N - 1>() + SizeOf<ElementType<N - 1>>::value * size_[N - 1]; ASAN_POISON_MEMORY_REGION(p + start, Offset<N>() - start); } #endif @@ -645,7 +645,7 @@ class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, // produce "unsigned*" where another produces "unsigned int *". std::string DebugString() const { const auto offsets = Offsets(); - const size_t sizes[] = {SizeOf<ElementType<OffsetSeq>>()...}; + const size_t sizes[] = {SizeOf<ElementType<OffsetSeq>>::value...}; const std::string types[] = { adl_barrier::TypeName<ElementType<OffsetSeq>>()...}; std::string res = absl::StrCat("@0", types[0], "(", sizes[0], ")"); diff --git a/absl/container/internal/layout_benchmark.cc b/absl/container/internal/layout_benchmark.cc new file mode 100644 index 00000000..d8636e8d --- /dev/null +++ b/absl/container/internal/layout_benchmark.cc @@ -0,0 +1,122 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Every benchmark should have the same performance as the corresponding +// headroom benchmark. + +#include "absl/base/internal/raw_logging.h" +#include "absl/container/internal/layout.h" +#include "benchmark/benchmark.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +using ::benchmark::DoNotOptimize; + +using Int128 = int64_t[2]; + +// This benchmark provides the upper bound on performance for BM_OffsetConstant. +template <size_t Offset, class... Ts> +void BM_OffsetConstantHeadroom(benchmark::State& state) { + for (auto _ : state) { + DoNotOptimize(Offset); + } +} + +template <size_t Offset, class... Ts> +void BM_OffsetConstant(benchmark::State& state) { + using L = Layout<Ts...>; + ABSL_RAW_CHECK(L::Partial(3, 5, 7).template Offset<3>() == Offset, + "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(L::Partial(3, 5, 7).template Offset<3>()); + } +} + +template <class... Ts> +size_t VariableOffset(size_t n, size_t m, size_t k); + +template <> +size_t VariableOffset<int8_t, int16_t, int32_t, Int128>(size_t n, size_t m, + size_t k) { + auto Align = [](size_t n, size_t m) { return (n + m - 1) & ~(m - 1); }; + return Align(Align(Align(n * 1, 2) + m * 2, 4) + k * 4, 8); +} + +template <> +size_t VariableOffset<Int128, int32_t, int16_t, int8_t>(size_t n, size_t m, + size_t k) { + // No alignment is necessary. + return n * 16 + m * 4 + k * 2; +} + +// This benchmark provides the upper bound on performance for BM_OffsetVariable. +template <size_t Offset, class... Ts> +void BM_OffsetVariableHeadroom(benchmark::State& state) { + size_t n = 3; + size_t m = 5; + size_t k = 7; + ABSL_RAW_CHECK(VariableOffset<Ts...>(n, m, k) == Offset, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(n); + DoNotOptimize(m); + DoNotOptimize(k); + DoNotOptimize(VariableOffset<Ts...>(n, m, k)); + } +} + +template <size_t Offset, class... Ts> +void BM_OffsetVariable(benchmark::State& state) { + using L = Layout<Ts...>; + size_t n = 3; + size_t m = 5; + size_t k = 7; + ABSL_RAW_CHECK(L::Partial(n, m, k).template Offset<3>() == Offset, + "Inavlid offset"); + for (auto _ : state) { + DoNotOptimize(n); + DoNotOptimize(m); + DoNotOptimize(k); + DoNotOptimize(L::Partial(n, m, k).template Offset<3>()); + } +} + +// Run all benchmarks in two modes: +// +// Layout with padding: int8_t[3], int16_t[5], int32_t[7], Int128[?]. +// Layout without padding: Int128[3], int32_t[5], int16_t[7], int8_t[?]. + +#define OFFSET_BENCHMARK(NAME, OFFSET, T1, T2, T3, T4) \ + auto& NAME##_##OFFSET##_##T1##_##T2##_##T3##_##T4 = \ + NAME<OFFSET, T1, T2, T3, T4>; \ + BENCHMARK(NAME##_##OFFSET##_##T1##_##T2##_##T3##_##T4) + +OFFSET_BENCHMARK(BM_OffsetConstantHeadroom, 48, int8_t, int16_t, int32_t, + Int128); +OFFSET_BENCHMARK(BM_OffsetConstant, 48, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_OffsetConstantHeadroom, 82, Int128, int32_t, int16_t, + int8_t); +OFFSET_BENCHMARK(BM_OffsetConstant, 82, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_OffsetVariableHeadroom, 48, int8_t, int16_t, int32_t, + Int128); +OFFSET_BENCHMARK(BM_OffsetVariable, 48, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_OffsetVariableHeadroom, 82, Int128, int32_t, int16_t, + int8_t); +OFFSET_BENCHMARK(BM_OffsetVariable, 82, Int128, int32_t, int16_t, int8_t); +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/container/internal/layout_test.cc b/absl/container/internal/layout_test.cc index 757272f1..1d7158ff 100644 --- a/absl/container/internal/layout_test.cc +++ b/absl/container/internal/layout_test.cc @@ -128,8 +128,10 @@ TEST(Layout, ElementTypes) { { using L = Layout<int32_t, int32_t>; SameType<std::tuple<int32_t, int32_t>, L::ElementTypes>(); - SameType<std::tuple<int32_t, int32_t>, decltype(L::Partial())::ElementTypes>(); - SameType<std::tuple<int32_t, int32_t>, decltype(L::Partial(0))::ElementTypes>(); + SameType<std::tuple<int32_t, int32_t>, + decltype(L::Partial())::ElementTypes>(); + SameType<std::tuple<int32_t, int32_t>, + decltype(L::Partial(0))::ElementTypes>(); } { using L = Layout<int8_t, int32_t, Int128>; @@ -368,18 +370,21 @@ TEST(Layout, PointerByIndex) { { using L = Layout<int32_t>; EXPECT_EQ(0, Distance(p, Type<const int32_t*>(L::Partial().Pointer<0>(p)))); - EXPECT_EQ(0, Distance(p, Type<const int32_t*>(L::Partial(3).Pointer<0>(p)))); + EXPECT_EQ(0, + Distance(p, Type<const int32_t*>(L::Partial(3).Pointer<0>(p)))); EXPECT_EQ(0, Distance(p, Type<const int32_t*>(L(3).Pointer<0>(p)))); } { using L = Layout<int32_t, int32_t>; EXPECT_EQ(0, Distance(p, Type<const int32_t*>(L::Partial().Pointer<0>(p)))); - EXPECT_EQ(0, Distance(p, Type<const int32_t*>(L::Partial(3).Pointer<0>(p)))); - EXPECT_EQ(12, Distance(p, Type<const int32_t*>(L::Partial(3).Pointer<1>(p)))); EXPECT_EQ(0, - Distance(p, Type<const int32_t*>(L::Partial(3, 5).Pointer<0>(p)))); + Distance(p, Type<const int32_t*>(L::Partial(3).Pointer<0>(p)))); EXPECT_EQ(12, - Distance(p, Type<const int32_t*>(L::Partial(3, 5).Pointer<1>(p)))); + Distance(p, Type<const int32_t*>(L::Partial(3).Pointer<1>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<const int32_t*>(L::Partial(3, 5).Pointer<0>(p)))); + EXPECT_EQ( + 12, Distance(p, Type<const int32_t*>(L::Partial(3, 5).Pointer<1>(p)))); EXPECT_EQ(0, Distance(p, Type<const int32_t*>(L(3, 5).Pointer<0>(p)))); EXPECT_EQ(12, Distance(p, Type<const int32_t*>(L(3, 5).Pointer<1>(p)))); } @@ -387,39 +392,44 @@ TEST(Layout, PointerByIndex) { using L = Layout<int8_t, int32_t, Int128>; EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L::Partial().Pointer<0>(p)))); EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L::Partial(0).Pointer<0>(p)))); - EXPECT_EQ(0, Distance(p, Type<const int32_t*>(L::Partial(0).Pointer<1>(p)))); + EXPECT_EQ(0, + Distance(p, Type<const int32_t*>(L::Partial(0).Pointer<1>(p)))); EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L::Partial(1).Pointer<0>(p)))); - EXPECT_EQ(4, Distance(p, Type<const int32_t*>(L::Partial(1).Pointer<1>(p)))); + EXPECT_EQ(4, + Distance(p, Type<const int32_t*>(L::Partial(1).Pointer<1>(p)))); EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L::Partial(5).Pointer<0>(p)))); - EXPECT_EQ(8, Distance(p, Type<const int32_t*>(L::Partial(5).Pointer<1>(p)))); + EXPECT_EQ(8, + Distance(p, Type<const int32_t*>(L::Partial(5).Pointer<1>(p)))); EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L::Partial(0, 0).Pointer<0>(p)))); - EXPECT_EQ(0, - Distance(p, Type<const int32_t*>(L::Partial(0, 0).Pointer<1>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<const int32_t*>(L::Partial(0, 0).Pointer<1>(p)))); EXPECT_EQ(0, Distance(p, Type<const Int128*>(L::Partial(0, 0).Pointer<2>(p)))); EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L::Partial(1, 0).Pointer<0>(p)))); - EXPECT_EQ(4, - Distance(p, Type<const int32_t*>(L::Partial(1, 0).Pointer<1>(p)))); + EXPECT_EQ( + 4, Distance(p, Type<const int32_t*>(L::Partial(1, 0).Pointer<1>(p)))); EXPECT_EQ(8, Distance(p, Type<const Int128*>(L::Partial(1, 0).Pointer<2>(p)))); EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L::Partial(5, 3).Pointer<0>(p)))); - EXPECT_EQ(8, - Distance(p, Type<const int32_t*>(L::Partial(5, 3).Pointer<1>(p)))); + EXPECT_EQ( + 8, Distance(p, Type<const int32_t*>(L::Partial(5, 3).Pointer<1>(p)))); EXPECT_EQ(24, Distance(p, Type<const Int128*>(L::Partial(5, 3).Pointer<2>(p)))); EXPECT_EQ( 0, Distance(p, Type<const int8_t*>(L::Partial(0, 0, 0).Pointer<0>(p)))); EXPECT_EQ( - 0, Distance(p, Type<const int32_t*>(L::Partial(0, 0, 0).Pointer<1>(p)))); + 0, + Distance(p, Type<const int32_t*>(L::Partial(0, 0, 0).Pointer<1>(p)))); EXPECT_EQ( 0, Distance(p, Type<const Int128*>(L::Partial(0, 0, 0).Pointer<2>(p)))); EXPECT_EQ( 0, Distance(p, Type<const int8_t*>(L::Partial(1, 0, 0).Pointer<0>(p)))); EXPECT_EQ( - 4, Distance(p, Type<const int32_t*>(L::Partial(1, 0, 0).Pointer<1>(p)))); + 4, + Distance(p, Type<const int32_t*>(L::Partial(1, 0, 0).Pointer<1>(p)))); EXPECT_EQ( 8, Distance(p, Type<const Int128*>(L::Partial(1, 0, 0).Pointer<2>(p)))); EXPECT_EQ( @@ -428,7 +438,8 @@ TEST(Layout, PointerByIndex) { 24, Distance(p, Type<const Int128*>(L::Partial(5, 3, 1).Pointer<2>(p)))); EXPECT_EQ( - 8, Distance(p, Type<const int32_t*>(L::Partial(5, 3, 1).Pointer<1>(p)))); + 8, + Distance(p, Type<const int32_t*>(L::Partial(5, 3, 1).Pointer<1>(p)))); EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L(5, 3, 1).Pointer<0>(p)))); EXPECT_EQ(24, Distance(p, Type<const Int128*>(L(5, 3, 1).Pointer<2>(p)))); EXPECT_EQ(8, Distance(p, Type<const int32_t*>(L(5, 3, 1).Pointer<1>(p)))); @@ -439,75 +450,78 @@ TEST(Layout, PointerByType) { alignas(max_align_t) const unsigned char p[100] = {}; { using L = Layout<int32_t>; - EXPECT_EQ(0, - Distance(p, Type<const int32_t*>(L::Partial().Pointer<int32_t>(p)))); - EXPECT_EQ(0, - Distance(p, Type<const int32_t*>(L::Partial(3).Pointer<int32_t>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<const int32_t*>(L::Partial().Pointer<int32_t>(p)))); + EXPECT_EQ( + 0, + Distance(p, Type<const int32_t*>(L::Partial(3).Pointer<int32_t>(p)))); EXPECT_EQ(0, Distance(p, Type<const int32_t*>(L(3).Pointer<int32_t>(p)))); } { using L = Layout<int8_t, int32_t, Int128>; - EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L::Partial().Pointer<int8_t>(p)))); - EXPECT_EQ(0, - Distance(p, Type<const int8_t*>(L::Partial(0).Pointer<int8_t>(p)))); - EXPECT_EQ(0, - Distance(p, Type<const int32_t*>(L::Partial(0).Pointer<int32_t>(p)))); - EXPECT_EQ(0, - Distance(p, Type<const int8_t*>(L::Partial(1).Pointer<int8_t>(p)))); - EXPECT_EQ(4, - Distance(p, Type<const int32_t*>(L::Partial(1).Pointer<int32_t>(p)))); - EXPECT_EQ(0, - Distance(p, Type<const int8_t*>(L::Partial(5).Pointer<int8_t>(p)))); - EXPECT_EQ(8, - Distance(p, Type<const int32_t*>(L::Partial(5).Pointer<int32_t>(p)))); EXPECT_EQ( - 0, Distance(p, Type<const int8_t*>(L::Partial(0, 0).Pointer<int8_t>(p)))); + 0, Distance(p, Type<const int8_t*>(L::Partial().Pointer<int8_t>(p)))); EXPECT_EQ( - 0, Distance(p, Type<const int32_t*>(L::Partial(0, 0).Pointer<int32_t>(p)))); + 0, Distance(p, Type<const int8_t*>(L::Partial(0).Pointer<int8_t>(p)))); EXPECT_EQ( 0, - Distance(p, Type<const Int128*>(L::Partial(0, 0).Pointer<Int128>(p)))); - EXPECT_EQ( - 0, Distance(p, Type<const int8_t*>(L::Partial(1, 0).Pointer<int8_t>(p)))); + Distance(p, Type<const int32_t*>(L::Partial(0).Pointer<int32_t>(p)))); EXPECT_EQ( - 4, Distance(p, Type<const int32_t*>(L::Partial(1, 0).Pointer<int32_t>(p)))); + 0, Distance(p, Type<const int8_t*>(L::Partial(1).Pointer<int8_t>(p)))); EXPECT_EQ( - 8, - Distance(p, Type<const Int128*>(L::Partial(1, 0).Pointer<Int128>(p)))); + 4, + Distance(p, Type<const int32_t*>(L::Partial(1).Pointer<int32_t>(p)))); EXPECT_EQ( - 0, Distance(p, Type<const int8_t*>(L::Partial(5, 3).Pointer<int8_t>(p)))); + 0, Distance(p, Type<const int8_t*>(L::Partial(5).Pointer<int8_t>(p)))); EXPECT_EQ( - 8, Distance(p, Type<const int32_t*>(L::Partial(5, 3).Pointer<int32_t>(p)))); + 8, + Distance(p, Type<const int32_t*>(L::Partial(5).Pointer<int32_t>(p)))); EXPECT_EQ( - 24, - Distance(p, Type<const Int128*>(L::Partial(5, 3).Pointer<Int128>(p)))); + 0, + Distance(p, Type<const int8_t*>(L::Partial(0, 0).Pointer<int8_t>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int32_t*>( + L::Partial(0, 0).Pointer<int32_t>(p)))); EXPECT_EQ( 0, - Distance(p, Type<const int8_t*>(L::Partial(0, 0, 0).Pointer<int8_t>(p)))); + Distance(p, Type<const Int128*>(L::Partial(0, 0).Pointer<Int128>(p)))); EXPECT_EQ( 0, - Distance(p, Type<const int32_t*>(L::Partial(0, 0, 0).Pointer<int32_t>(p)))); - EXPECT_EQ(0, Distance(p, Type<const Int128*>( - L::Partial(0, 0, 0).Pointer<Int128>(p)))); + Distance(p, Type<const int8_t*>(L::Partial(1, 0).Pointer<int8_t>(p)))); + EXPECT_EQ(4, Distance(p, Type<const int32_t*>( + L::Partial(1, 0).Pointer<int32_t>(p)))); + EXPECT_EQ( + 8, + Distance(p, Type<const Int128*>(L::Partial(1, 0).Pointer<Int128>(p)))); EXPECT_EQ( 0, - Distance(p, Type<const int8_t*>(L::Partial(1, 0, 0).Pointer<int8_t>(p)))); + Distance(p, Type<const int8_t*>(L::Partial(5, 3).Pointer<int8_t>(p)))); + EXPECT_EQ(8, Distance(p, Type<const int32_t*>( + L::Partial(5, 3).Pointer<int32_t>(p)))); EXPECT_EQ( - 4, - Distance(p, Type<const int32_t*>(L::Partial(1, 0, 0).Pointer<int32_t>(p)))); + 24, + Distance(p, Type<const Int128*>(L::Partial(5, 3).Pointer<Int128>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int8_t*>( + L::Partial(0, 0, 0).Pointer<int8_t>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int32_t*>( + L::Partial(0, 0, 0).Pointer<int32_t>(p)))); + EXPECT_EQ(0, Distance(p, Type<const Int128*>( + L::Partial(0, 0, 0).Pointer<Int128>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int8_t*>( + L::Partial(1, 0, 0).Pointer<int8_t>(p)))); + EXPECT_EQ(4, Distance(p, Type<const int32_t*>( + L::Partial(1, 0, 0).Pointer<int32_t>(p)))); EXPECT_EQ(8, Distance(p, Type<const Int128*>( L::Partial(1, 0, 0).Pointer<Int128>(p)))); - EXPECT_EQ( - 0, - Distance(p, Type<const int8_t*>(L::Partial(5, 3, 1).Pointer<int8_t>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int8_t*>( + L::Partial(5, 3, 1).Pointer<int8_t>(p)))); EXPECT_EQ(24, Distance(p, Type<const Int128*>( L::Partial(5, 3, 1).Pointer<Int128>(p)))); - EXPECT_EQ( - 8, - Distance(p, Type<const int32_t*>(L::Partial(5, 3, 1).Pointer<int32_t>(p)))); + EXPECT_EQ(8, Distance(p, Type<const int32_t*>( + L::Partial(5, 3, 1).Pointer<int32_t>(p)))); EXPECT_EQ(24, Distance(p, Type<const Int128*>(L(5, 3, 1).Pointer<Int128>(p)))); - EXPECT_EQ(8, Distance(p, Type<const int32_t*>(L(5, 3, 1).Pointer<int32_t>(p)))); + EXPECT_EQ( + 8, Distance(p, Type<const int32_t*>(L(5, 3, 1).Pointer<int32_t>(p)))); } } @@ -548,15 +562,18 @@ TEST(Layout, MutablePointerByIndex) { EXPECT_EQ(8, Distance(p, Type<int32_t*>(L::Partial(5, 3).Pointer<1>(p)))); EXPECT_EQ(24, Distance(p, Type<Int128*>(L::Partial(5, 3).Pointer<2>(p)))); EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(0, 0, 0).Pointer<0>(p)))); - EXPECT_EQ(0, Distance(p, Type<int32_t*>(L::Partial(0, 0, 0).Pointer<1>(p)))); + EXPECT_EQ(0, + Distance(p, Type<int32_t*>(L::Partial(0, 0, 0).Pointer<1>(p)))); EXPECT_EQ(0, Distance(p, Type<Int128*>(L::Partial(0, 0, 0).Pointer<2>(p)))); EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(1, 0, 0).Pointer<0>(p)))); - EXPECT_EQ(4, Distance(p, Type<int32_t*>(L::Partial(1, 0, 0).Pointer<1>(p)))); + EXPECT_EQ(4, + Distance(p, Type<int32_t*>(L::Partial(1, 0, 0).Pointer<1>(p)))); EXPECT_EQ(8, Distance(p, Type<Int128*>(L::Partial(1, 0, 0).Pointer<2>(p)))); EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(5, 3, 1).Pointer<0>(p)))); EXPECT_EQ(24, Distance(p, Type<Int128*>(L::Partial(5, 3, 1).Pointer<2>(p)))); - EXPECT_EQ(8, Distance(p, Type<int32_t*>(L::Partial(5, 3, 1).Pointer<1>(p)))); + EXPECT_EQ(8, + Distance(p, Type<int32_t*>(L::Partial(5, 3, 1).Pointer<1>(p)))); EXPECT_EQ(0, Distance(p, Type<int8_t*>(L(5, 3, 1).Pointer<0>(p)))); EXPECT_EQ(24, Distance(p, Type<Int128*>(L(5, 3, 1).Pointer<2>(p)))); EXPECT_EQ(8, Distance(p, Type<int32_t*>(L(5, 3, 1).Pointer<1>(p)))); @@ -568,48 +585,61 @@ TEST(Layout, MutablePointerByType) { { using L = Layout<int32_t>; EXPECT_EQ(0, Distance(p, Type<int32_t*>(L::Partial().Pointer<int32_t>(p)))); - EXPECT_EQ(0, Distance(p, Type<int32_t*>(L::Partial(3).Pointer<int32_t>(p)))); + EXPECT_EQ(0, + Distance(p, Type<int32_t*>(L::Partial(3).Pointer<int32_t>(p)))); EXPECT_EQ(0, Distance(p, Type<int32_t*>(L(3).Pointer<int32_t>(p)))); } { using L = Layout<int8_t, int32_t, Int128>; EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial().Pointer<int8_t>(p)))); EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(0).Pointer<int8_t>(p)))); - EXPECT_EQ(0, Distance(p, Type<int32_t*>(L::Partial(0).Pointer<int32_t>(p)))); + EXPECT_EQ(0, + Distance(p, Type<int32_t*>(L::Partial(0).Pointer<int32_t>(p)))); EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(1).Pointer<int8_t>(p)))); - EXPECT_EQ(4, Distance(p, Type<int32_t*>(L::Partial(1).Pointer<int32_t>(p)))); + EXPECT_EQ(4, + Distance(p, Type<int32_t*>(L::Partial(1).Pointer<int32_t>(p)))); EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(5).Pointer<int8_t>(p)))); - EXPECT_EQ(8, Distance(p, Type<int32_t*>(L::Partial(5).Pointer<int32_t>(p)))); - EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(0, 0).Pointer<int8_t>(p)))); - EXPECT_EQ(0, Distance(p, Type<int32_t*>(L::Partial(0, 0).Pointer<int32_t>(p)))); + EXPECT_EQ(8, + Distance(p, Type<int32_t*>(L::Partial(5).Pointer<int32_t>(p)))); + EXPECT_EQ(0, + Distance(p, Type<int8_t*>(L::Partial(0, 0).Pointer<int8_t>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<int32_t*>(L::Partial(0, 0).Pointer<int32_t>(p)))); EXPECT_EQ(0, Distance(p, Type<Int128*>(L::Partial(0, 0).Pointer<Int128>(p)))); - EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(1, 0).Pointer<int8_t>(p)))); - EXPECT_EQ(4, Distance(p, Type<int32_t*>(L::Partial(1, 0).Pointer<int32_t>(p)))); + EXPECT_EQ(0, + Distance(p, Type<int8_t*>(L::Partial(1, 0).Pointer<int8_t>(p)))); + EXPECT_EQ( + 4, Distance(p, Type<int32_t*>(L::Partial(1, 0).Pointer<int32_t>(p)))); EXPECT_EQ(8, Distance(p, Type<Int128*>(L::Partial(1, 0).Pointer<Int128>(p)))); - EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(5, 3).Pointer<int8_t>(p)))); - EXPECT_EQ(8, Distance(p, Type<int32_t*>(L::Partial(5, 3).Pointer<int32_t>(p)))); + EXPECT_EQ(0, + Distance(p, Type<int8_t*>(L::Partial(5, 3).Pointer<int8_t>(p)))); + EXPECT_EQ( + 8, Distance(p, Type<int32_t*>(L::Partial(5, 3).Pointer<int32_t>(p)))); EXPECT_EQ(24, Distance(p, Type<Int128*>(L::Partial(5, 3).Pointer<Int128>(p)))); - EXPECT_EQ(0, - Distance(p, Type<int8_t*>(L::Partial(0, 0, 0).Pointer<int8_t>(p)))); - EXPECT_EQ(0, - Distance(p, Type<int32_t*>(L::Partial(0, 0, 0).Pointer<int32_t>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<int8_t*>(L::Partial(0, 0, 0).Pointer<int8_t>(p)))); + EXPECT_EQ( + 0, + Distance(p, Type<int32_t*>(L::Partial(0, 0, 0).Pointer<int32_t>(p)))); EXPECT_EQ( 0, Distance(p, Type<Int128*>(L::Partial(0, 0, 0).Pointer<Int128>(p)))); - EXPECT_EQ(0, - Distance(p, Type<int8_t*>(L::Partial(1, 0, 0).Pointer<int8_t>(p)))); - EXPECT_EQ(4, - Distance(p, Type<int32_t*>(L::Partial(1, 0, 0).Pointer<int32_t>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<int8_t*>(L::Partial(1, 0, 0).Pointer<int8_t>(p)))); + EXPECT_EQ( + 4, + Distance(p, Type<int32_t*>(L::Partial(1, 0, 0).Pointer<int32_t>(p)))); EXPECT_EQ( 8, Distance(p, Type<Int128*>(L::Partial(1, 0, 0).Pointer<Int128>(p)))); - EXPECT_EQ(0, - Distance(p, Type<int8_t*>(L::Partial(5, 3, 1).Pointer<int8_t>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<int8_t*>(L::Partial(5, 3, 1).Pointer<int8_t>(p)))); EXPECT_EQ( 24, Distance(p, Type<Int128*>(L::Partial(5, 3, 1).Pointer<Int128>(p)))); - EXPECT_EQ(8, - Distance(p, Type<int32_t*>(L::Partial(5, 3, 1).Pointer<int32_t>(p)))); + EXPECT_EQ( + 8, + Distance(p, Type<int32_t*>(L::Partial(5, 3, 1).Pointer<int32_t>(p)))); EXPECT_EQ(0, Distance(p, Type<int8_t*>(L(5, 3, 1).Pointer<int8_t>(p)))); EXPECT_EQ(24, Distance(p, Type<Int128*>(L(5, 3, 1).Pointer<Int128>(p)))); EXPECT_EQ(8, Distance(p, Type<int32_t*>(L(5, 3, 1).Pointer<int32_t>(p)))); @@ -790,67 +820,72 @@ TEST(Layout, SliceByIndexData) { { using L = Layout<int32_t>; EXPECT_EQ( - 0, - Distance(p, Type<Span<const int32_t>>(L::Partial(0).Slice<0>(p)).data())); + 0, Distance( + p, Type<Span<const int32_t>>(L::Partial(0).Slice<0>(p)).data())); EXPECT_EQ( - 0, - Distance(p, Type<Span<const int32_t>>(L::Partial(3).Slice<0>(p)).data())); - EXPECT_EQ(0, Distance(p, Type<Span<const int32_t>>(L(3).Slice<0>(p)).data())); + 0, Distance( + p, Type<Span<const int32_t>>(L::Partial(3).Slice<0>(p)).data())); + EXPECT_EQ(0, + Distance(p, Type<Span<const int32_t>>(L(3).Slice<0>(p)).data())); } { using L = Layout<int32_t, int32_t>; EXPECT_EQ( - 0, - Distance(p, Type<Span<const int32_t>>(L::Partial(3).Slice<0>(p)).data())); + 0, Distance( + p, Type<Span<const int32_t>>(L::Partial(3).Slice<0>(p)).data())); EXPECT_EQ( 0, - Distance(p, - Type<Span<const int32_t>>(L::Partial(3, 5).Slice<0>(p)).data())); + Distance( + p, Type<Span<const int32_t>>(L::Partial(3, 5).Slice<0>(p)).data())); EXPECT_EQ( 12, - Distance(p, - Type<Span<const int32_t>>(L::Partial(3, 5).Slice<1>(p)).data())); - EXPECT_EQ(0, - Distance(p, Type<Span<const int32_t>>(L(3, 5).Slice<0>(p)).data())); - EXPECT_EQ(12, - Distance(p, Type<Span<const int32_t>>(L(3, 5).Slice<1>(p)).data())); + Distance( + p, Type<Span<const int32_t>>(L::Partial(3, 5).Slice<1>(p)).data())); + EXPECT_EQ( + 0, Distance(p, Type<Span<const int32_t>>(L(3, 5).Slice<0>(p)).data())); + EXPECT_EQ( + 12, Distance(p, Type<Span<const int32_t>>(L(3, 5).Slice<1>(p)).data())); } { using L = Layout<int8_t, int32_t, Int128>; EXPECT_EQ( - 0, - Distance(p, Type<Span<const int8_t>>(L::Partial(0).Slice<0>(p)).data())); - EXPECT_EQ( - 0, - Distance(p, Type<Span<const int8_t>>(L::Partial(1).Slice<0>(p)).data())); + 0, Distance( + p, Type<Span<const int8_t>>(L::Partial(0).Slice<0>(p)).data())); EXPECT_EQ( - 0, - Distance(p, Type<Span<const int8_t>>(L::Partial(5).Slice<0>(p)).data())); + 0, Distance( + p, Type<Span<const int8_t>>(L::Partial(1).Slice<0>(p)).data())); EXPECT_EQ( 0, Distance( - p, Type<Span<const int8_t>>(L::Partial(0, 0).Slice<0>(p)).data())); + p, Type<Span<const int8_t>>(L::Partial(5).Slice<0>(p)).data())); EXPECT_EQ( 0, - Distance(p, - Type<Span<const int32_t>>(L::Partial(0, 0).Slice<1>(p)).data())); + Distance( + p, Type<Span<const int8_t>>(L::Partial(0, 0).Slice<0>(p)).data())); EXPECT_EQ( - 0, Distance( - p, Type<Span<const int8_t>>(L::Partial(1, 0).Slice<0>(p)).data())); + 0, + Distance( + p, Type<Span<const int32_t>>(L::Partial(0, 0).Slice<1>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, Type<Span<const int8_t>>(L::Partial(1, 0).Slice<0>(p)).data())); EXPECT_EQ( 4, - Distance(p, - Type<Span<const int32_t>>(L::Partial(1, 0).Slice<1>(p)).data())); + Distance( + p, Type<Span<const int32_t>>(L::Partial(1, 0).Slice<1>(p)).data())); EXPECT_EQ( - 0, Distance( - p, Type<Span<const int8_t>>(L::Partial(5, 3).Slice<0>(p)).data())); + 0, + Distance( + p, Type<Span<const int8_t>>(L::Partial(5, 3).Slice<0>(p)).data())); EXPECT_EQ( 8, - Distance(p, - Type<Span<const int32_t>>(L::Partial(5, 3).Slice<1>(p)).data())); + Distance( + p, Type<Span<const int32_t>>(L::Partial(5, 3).Slice<1>(p)).data())); EXPECT_EQ( 0, Distance( - p, Type<Span<const int8_t>>(L::Partial(0, 0, 0).Slice<0>(p)).data())); + p, + Type<Span<const int8_t>>(L::Partial(0, 0, 0).Slice<0>(p)).data())); EXPECT_EQ( 0, Distance( @@ -864,7 +899,8 @@ TEST(Layout, SliceByIndexData) { EXPECT_EQ( 0, Distance( - p, Type<Span<const int8_t>>(L::Partial(1, 0, 0).Slice<0>(p)).data())); + p, + Type<Span<const int8_t>>(L::Partial(1, 0, 0).Slice<0>(p)).data())); EXPECT_EQ( 4, Distance( @@ -878,7 +914,8 @@ TEST(Layout, SliceByIndexData) { EXPECT_EQ( 0, Distance( - p, Type<Span<const int8_t>>(L::Partial(5, 3, 1).Slice<0>(p)).data())); + p, + Type<Span<const int8_t>>(L::Partial(5, 3, 1).Slice<0>(p)).data())); EXPECT_EQ( 24, Distance( @@ -890,12 +927,14 @@ TEST(Layout, SliceByIndexData) { p, Type<Span<const int32_t>>(L::Partial(5, 3, 1).Slice<1>(p)).data())); EXPECT_EQ( - 0, Distance(p, Type<Span<const int8_t>>(L(5, 3, 1).Slice<0>(p)).data())); + 0, + Distance(p, Type<Span<const int8_t>>(L(5, 3, 1).Slice<0>(p)).data())); EXPECT_EQ( 24, Distance(p, Type<Span<const Int128>>(L(5, 3, 1).Slice<2>(p)).data())); EXPECT_EQ( - 8, Distance(p, Type<Span<const int32_t>>(L(5, 3, 1).Slice<1>(p)).data())); + 8, + Distance(p, Type<Span<const int32_t>>(L(5, 3, 1).Slice<1>(p)).data())); } } @@ -906,98 +945,94 @@ TEST(Layout, SliceByTypeData) { EXPECT_EQ( 0, Distance( - p, Type<Span<const int32_t>>(L::Partial(0).Slice<int32_t>(p)).data())); + p, + Type<Span<const int32_t>>(L::Partial(0).Slice<int32_t>(p)).data())); EXPECT_EQ( 0, Distance( - p, Type<Span<const int32_t>>(L::Partial(3).Slice<int32_t>(p)).data())); + p, + Type<Span<const int32_t>>(L::Partial(3).Slice<int32_t>(p)).data())); EXPECT_EQ( - 0, Distance(p, Type<Span<const int32_t>>(L(3).Slice<int32_t>(p)).data())); + 0, + Distance(p, Type<Span<const int32_t>>(L(3).Slice<int32_t>(p)).data())); } { using L = Layout<int8_t, int32_t, Int128>; EXPECT_EQ( - 0, Distance( - p, Type<Span<const int8_t>>(L::Partial(0).Slice<int8_t>(p)).data())); - EXPECT_EQ( - 0, Distance( - p, Type<Span<const int8_t>>(L::Partial(1).Slice<int8_t>(p)).data())); - EXPECT_EQ( - 0, Distance( - p, Type<Span<const int8_t>>(L::Partial(5).Slice<int8_t>(p)).data())); - EXPECT_EQ( - 0, - Distance( - p, Type<Span<const int8_t>>(L::Partial(0, 0).Slice<int8_t>(p)).data())); - EXPECT_EQ( 0, Distance( p, - Type<Span<const int32_t>>(L::Partial(0, 0).Slice<int32_t>(p)).data())); + Type<Span<const int8_t>>(L::Partial(0).Slice<int8_t>(p)).data())); EXPECT_EQ( 0, Distance( - p, Type<Span<const int8_t>>(L::Partial(1, 0).Slice<int8_t>(p)).data())); - EXPECT_EQ( - 4, - Distance( p, - Type<Span<const int32_t>>(L::Partial(1, 0).Slice<int32_t>(p)).data())); + Type<Span<const int8_t>>(L::Partial(1).Slice<int8_t>(p)).data())); EXPECT_EQ( 0, Distance( - p, Type<Span<const int8_t>>(L::Partial(5, 3).Slice<int8_t>(p)).data())); - EXPECT_EQ( - 8, - Distance( p, - Type<Span<const int32_t>>(L::Partial(5, 3).Slice<int32_t>(p)).data())); + Type<Span<const int8_t>>(L::Partial(5).Slice<int8_t>(p)).data())); EXPECT_EQ( 0, - Distance( - p, - Type<Span<const int8_t>>(L::Partial(0, 0, 0).Slice<int8_t>(p)).data())); + Distance(p, Type<Span<const int8_t>>(L::Partial(0, 0).Slice<int8_t>(p)) + .data())); + EXPECT_EQ(0, Distance(p, Type<Span<const int32_t>>( + L::Partial(0, 0).Slice<int32_t>(p)) + .data())); EXPECT_EQ( 0, - Distance(p, Type<Span<const int32_t>>(L::Partial(0, 0, 0).Slice<int32_t>(p)) + Distance(p, Type<Span<const int8_t>>(L::Partial(1, 0).Slice<int8_t>(p)) .data())); - EXPECT_EQ(0, Distance(p, Type<Span<const Int128>>( - L::Partial(0, 0, 0).Slice<Int128>(p)) + EXPECT_EQ(4, Distance(p, Type<Span<const int32_t>>( + L::Partial(1, 0).Slice<int32_t>(p)) .data())); EXPECT_EQ( 0, - Distance( - p, - Type<Span<const int8_t>>(L::Partial(1, 0, 0).Slice<int8_t>(p)).data())); - EXPECT_EQ( - 4, - Distance(p, Type<Span<const int32_t>>(L::Partial(1, 0, 0).Slice<int32_t>(p)) + Distance(p, Type<Span<const int8_t>>(L::Partial(5, 3).Slice<int8_t>(p)) .data())); + EXPECT_EQ(8, Distance(p, Type<Span<const int32_t>>( + L::Partial(5, 3).Slice<int32_t>(p)) + .data())); + EXPECT_EQ(0, Distance(p, Type<Span<const int8_t>>( + L::Partial(0, 0, 0).Slice<int8_t>(p)) + .data())); + EXPECT_EQ(0, Distance(p, Type<Span<const int32_t>>( + L::Partial(0, 0, 0).Slice<int32_t>(p)) + .data())); + EXPECT_EQ(0, Distance(p, Type<Span<const Int128>>( + L::Partial(0, 0, 0).Slice<Int128>(p)) + .data())); + EXPECT_EQ(0, Distance(p, Type<Span<const int8_t>>( + L::Partial(1, 0, 0).Slice<int8_t>(p)) + .data())); + EXPECT_EQ(4, Distance(p, Type<Span<const int32_t>>( + L::Partial(1, 0, 0).Slice<int32_t>(p)) + .data())); EXPECT_EQ(8, Distance(p, Type<Span<const Int128>>( L::Partial(1, 0, 0).Slice<Int128>(p)) .data())); - EXPECT_EQ( - 0, - Distance( - p, - Type<Span<const int8_t>>(L::Partial(5, 3, 1).Slice<int8_t>(p)).data())); + EXPECT_EQ(0, Distance(p, Type<Span<const int8_t>>( + L::Partial(5, 3, 1).Slice<int8_t>(p)) + .data())); EXPECT_EQ(24, Distance(p, Type<Span<const Int128>>( L::Partial(5, 3, 1).Slice<Int128>(p)) .data())); - EXPECT_EQ( - 8, - Distance(p, Type<Span<const int32_t>>(L::Partial(5, 3, 1).Slice<int32_t>(p)) - .data())); + EXPECT_EQ(8, Distance(p, Type<Span<const int32_t>>( + L::Partial(5, 3, 1).Slice<int32_t>(p)) + .data())); EXPECT_EQ( 0, - Distance(p, Type<Span<const int8_t>>(L(5, 3, 1).Slice<int8_t>(p)).data())); + Distance(p, + Type<Span<const int8_t>>(L(5, 3, 1).Slice<int8_t>(p)).data())); EXPECT_EQ( 24, Distance(p, Type<Span<const Int128>>(L(5, 3, 1).Slice<Int128>(p)).data())); EXPECT_EQ( - 8, Distance( - p, Type<Span<const int32_t>>(L(5, 3, 1).Slice<int32_t>(p)).data())); + 8, + Distance( + p, Type<Span<const int32_t>>(L(5, 3, 1).Slice<int32_t>(p)).data())); } } @@ -1005,18 +1040,19 @@ TEST(Layout, MutableSliceByIndexData) { alignas(max_align_t) unsigned char p[100]; { using L = Layout<int32_t>; - EXPECT_EQ(0, - Distance(p, Type<Span<int32_t>>(L::Partial(0).Slice<0>(p)).data())); - EXPECT_EQ(0, - Distance(p, Type<Span<int32_t>>(L::Partial(3).Slice<0>(p)).data())); + EXPECT_EQ( + 0, Distance(p, Type<Span<int32_t>>(L::Partial(0).Slice<0>(p)).data())); + EXPECT_EQ( + 0, Distance(p, Type<Span<int32_t>>(L::Partial(3).Slice<0>(p)).data())); EXPECT_EQ(0, Distance(p, Type<Span<int32_t>>(L(3).Slice<0>(p)).data())); } { using L = Layout<int32_t, int32_t>; - EXPECT_EQ(0, - Distance(p, Type<Span<int32_t>>(L::Partial(3).Slice<0>(p)).data())); EXPECT_EQ( - 0, Distance(p, Type<Span<int32_t>>(L::Partial(3, 5).Slice<0>(p)).data())); + 0, Distance(p, Type<Span<int32_t>>(L::Partial(3).Slice<0>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, Type<Span<int32_t>>(L::Partial(3, 5).Slice<0>(p)).data())); EXPECT_EQ( 12, Distance(p, Type<Span<int32_t>>(L::Partial(3, 5).Slice<1>(p)).data())); @@ -1025,55 +1061,63 @@ TEST(Layout, MutableSliceByIndexData) { } { using L = Layout<int8_t, int32_t, Int128>; - EXPECT_EQ(0, - Distance(p, Type<Span<int8_t>>(L::Partial(0).Slice<0>(p)).data())); - EXPECT_EQ(0, - Distance(p, Type<Span<int8_t>>(L::Partial(1).Slice<0>(p)).data())); - EXPECT_EQ(0, - Distance(p, Type<Span<int8_t>>(L::Partial(5).Slice<0>(p)).data())); - EXPECT_EQ( - 0, Distance(p, Type<Span<int8_t>>(L::Partial(0, 0).Slice<0>(p)).data())); EXPECT_EQ( - 0, Distance(p, Type<Span<int32_t>>(L::Partial(0, 0).Slice<1>(p)).data())); + 0, Distance(p, Type<Span<int8_t>>(L::Partial(0).Slice<0>(p)).data())); EXPECT_EQ( - 0, Distance(p, Type<Span<int8_t>>(L::Partial(1, 0).Slice<0>(p)).data())); + 0, Distance(p, Type<Span<int8_t>>(L::Partial(1).Slice<0>(p)).data())); EXPECT_EQ( - 4, Distance(p, Type<Span<int32_t>>(L::Partial(1, 0).Slice<1>(p)).data())); + 0, Distance(p, Type<Span<int8_t>>(L::Partial(5).Slice<0>(p)).data())); EXPECT_EQ( - 0, Distance(p, Type<Span<int8_t>>(L::Partial(5, 3).Slice<0>(p)).data())); + 0, + Distance(p, Type<Span<int8_t>>(L::Partial(0, 0).Slice<0>(p)).data())); EXPECT_EQ( - 8, Distance(p, Type<Span<int32_t>>(L::Partial(5, 3).Slice<1>(p)).data())); + 0, + Distance(p, Type<Span<int32_t>>(L::Partial(0, 0).Slice<1>(p)).data())); EXPECT_EQ( 0, - Distance(p, Type<Span<int8_t>>(L::Partial(0, 0, 0).Slice<0>(p)).data())); + Distance(p, Type<Span<int8_t>>(L::Partial(1, 0).Slice<0>(p)).data())); + EXPECT_EQ( + 4, + Distance(p, Type<Span<int32_t>>(L::Partial(1, 0).Slice<1>(p)).data())); EXPECT_EQ( 0, - Distance(p, Type<Span<int32_t>>(L::Partial(0, 0, 0).Slice<1>(p)).data())); + Distance(p, Type<Span<int8_t>>(L::Partial(5, 3).Slice<0>(p)).data())); + EXPECT_EQ( + 8, + Distance(p, Type<Span<int32_t>>(L::Partial(5, 3).Slice<1>(p)).data())); + EXPECT_EQ( + 0, Distance( + p, Type<Span<int8_t>>(L::Partial(0, 0, 0).Slice<0>(p)).data())); + EXPECT_EQ( + 0, Distance( + p, Type<Span<int32_t>>(L::Partial(0, 0, 0).Slice<1>(p)).data())); EXPECT_EQ( 0, Distance( p, Type<Span<Int128>>(L::Partial(0, 0, 0).Slice<2>(p)).data())); EXPECT_EQ( - 0, - Distance(p, Type<Span<int8_t>>(L::Partial(1, 0, 0).Slice<0>(p)).data())); + 0, Distance( + p, Type<Span<int8_t>>(L::Partial(1, 0, 0).Slice<0>(p)).data())); EXPECT_EQ( - 4, - Distance(p, Type<Span<int32_t>>(L::Partial(1, 0, 0).Slice<1>(p)).data())); + 4, Distance( + p, Type<Span<int32_t>>(L::Partial(1, 0, 0).Slice<1>(p)).data())); EXPECT_EQ( 8, Distance( p, Type<Span<Int128>>(L::Partial(1, 0, 0).Slice<2>(p)).data())); EXPECT_EQ( - 0, - Distance(p, Type<Span<int8_t>>(L::Partial(5, 3, 1).Slice<0>(p)).data())); + 0, Distance( + p, Type<Span<int8_t>>(L::Partial(5, 3, 1).Slice<0>(p)).data())); EXPECT_EQ( 24, Distance( p, Type<Span<Int128>>(L::Partial(5, 3, 1).Slice<2>(p)).data())); EXPECT_EQ( - 8, - Distance(p, Type<Span<int32_t>>(L::Partial(5, 3, 1).Slice<1>(p)).data())); - EXPECT_EQ(0, Distance(p, Type<Span<int8_t>>(L(5, 3, 1).Slice<0>(p)).data())); + 8, Distance( + p, Type<Span<int32_t>>(L::Partial(5, 3, 1).Slice<1>(p)).data())); + EXPECT_EQ(0, + Distance(p, Type<Span<int8_t>>(L(5, 3, 1).Slice<0>(p)).data())); EXPECT_EQ(24, Distance(p, Type<Span<Int128>>(L(5, 3, 1).Slice<2>(p)).data())); - EXPECT_EQ(8, Distance(p, Type<Span<int32_t>>(L(5, 3, 1).Slice<1>(p)).data())); + EXPECT_EQ(8, + Distance(p, Type<Span<int32_t>>(L(5, 3, 1).Slice<1>(p)).data())); } } @@ -1082,66 +1126,84 @@ TEST(Layout, MutableSliceByTypeData) { { using L = Layout<int32_t>; EXPECT_EQ( - 0, - Distance(p, Type<Span<int32_t>>(L::Partial(0).Slice<int32_t>(p)).data())); + 0, Distance( + p, Type<Span<int32_t>>(L::Partial(0).Slice<int32_t>(p)).data())); EXPECT_EQ( - 0, - Distance(p, Type<Span<int32_t>>(L::Partial(3).Slice<int32_t>(p)).data())); - EXPECT_EQ(0, Distance(p, Type<Span<int32_t>>(L(3).Slice<int32_t>(p)).data())); + 0, Distance( + p, Type<Span<int32_t>>(L::Partial(3).Slice<int32_t>(p)).data())); + EXPECT_EQ(0, + Distance(p, Type<Span<int32_t>>(L(3).Slice<int32_t>(p)).data())); } { using L = Layout<int8_t, int32_t, Int128>; EXPECT_EQ( - 0, Distance(p, Type<Span<int8_t>>(L::Partial(0).Slice<int8_t>(p)).data())); + 0, + Distance(p, Type<Span<int8_t>>(L::Partial(0).Slice<int8_t>(p)).data())); EXPECT_EQ( - 0, Distance(p, Type<Span<int8_t>>(L::Partial(1).Slice<int8_t>(p)).data())); + 0, + Distance(p, Type<Span<int8_t>>(L::Partial(1).Slice<int8_t>(p)).data())); EXPECT_EQ( - 0, Distance(p, Type<Span<int8_t>>(L::Partial(5).Slice<int8_t>(p)).data())); + 0, + Distance(p, Type<Span<int8_t>>(L::Partial(5).Slice<int8_t>(p)).data())); EXPECT_EQ( 0, - Distance(p, Type<Span<int8_t>>(L::Partial(0, 0).Slice<int8_t>(p)).data())); + Distance(p, + Type<Span<int8_t>>(L::Partial(0, 0).Slice<int8_t>(p)).data())); EXPECT_EQ( - 0, Distance( - p, Type<Span<int32_t>>(L::Partial(0, 0).Slice<int32_t>(p)).data())); + 0, + Distance( + p, Type<Span<int32_t>>(L::Partial(0, 0).Slice<int32_t>(p)).data())); EXPECT_EQ( 0, - Distance(p, Type<Span<int8_t>>(L::Partial(1, 0).Slice<int8_t>(p)).data())); + Distance(p, + Type<Span<int8_t>>(L::Partial(1, 0).Slice<int8_t>(p)).data())); EXPECT_EQ( - 4, Distance( - p, Type<Span<int32_t>>(L::Partial(1, 0).Slice<int32_t>(p)).data())); + 4, + Distance( + p, Type<Span<int32_t>>(L::Partial(1, 0).Slice<int32_t>(p)).data())); EXPECT_EQ( 0, - Distance(p, Type<Span<int8_t>>(L::Partial(5, 3).Slice<int8_t>(p)).data())); + Distance(p, + Type<Span<int8_t>>(L::Partial(5, 3).Slice<int8_t>(p)).data())); EXPECT_EQ( - 8, Distance( - p, Type<Span<int32_t>>(L::Partial(5, 3).Slice<int32_t>(p)).data())); + 8, + Distance( + p, Type<Span<int32_t>>(L::Partial(5, 3).Slice<int32_t>(p)).data())); EXPECT_EQ( - 0, Distance( - p, Type<Span<int8_t>>(L::Partial(0, 0, 0).Slice<int8_t>(p)).data())); + 0, + Distance( + p, + Type<Span<int8_t>>(L::Partial(0, 0, 0).Slice<int8_t>(p)).data())); EXPECT_EQ( 0, Distance( - p, Type<Span<int32_t>>(L::Partial(0, 0, 0).Slice<int32_t>(p)).data())); + p, + Type<Span<int32_t>>(L::Partial(0, 0, 0).Slice<int32_t>(p)).data())); EXPECT_EQ( 0, Distance( p, Type<Span<Int128>>(L::Partial(0, 0, 0).Slice<Int128>(p)).data())); EXPECT_EQ( - 0, Distance( - p, Type<Span<int8_t>>(L::Partial(1, 0, 0).Slice<int8_t>(p)).data())); + 0, + Distance( + p, + Type<Span<int8_t>>(L::Partial(1, 0, 0).Slice<int8_t>(p)).data())); EXPECT_EQ( 4, Distance( - p, Type<Span<int32_t>>(L::Partial(1, 0, 0).Slice<int32_t>(p)).data())); + p, + Type<Span<int32_t>>(L::Partial(1, 0, 0).Slice<int32_t>(p)).data())); EXPECT_EQ( 8, Distance( p, Type<Span<Int128>>(L::Partial(1, 0, 0).Slice<Int128>(p)).data())); EXPECT_EQ( - 0, Distance( - p, Type<Span<int8_t>>(L::Partial(5, 3, 1).Slice<int8_t>(p)).data())); + 0, + Distance( + p, + Type<Span<int8_t>>(L::Partial(5, 3, 1).Slice<int8_t>(p)).data())); EXPECT_EQ( 24, Distance( @@ -1150,14 +1212,16 @@ TEST(Layout, MutableSliceByTypeData) { EXPECT_EQ( 8, Distance( - p, Type<Span<int32_t>>(L::Partial(5, 3, 1).Slice<int32_t>(p)).data())); - EXPECT_EQ(0, - Distance(p, Type<Span<int8_t>>(L(5, 3, 1).Slice<int8_t>(p)).data())); + p, + Type<Span<int32_t>>(L::Partial(5, 3, 1).Slice<int32_t>(p)).data())); + EXPECT_EQ( + 0, Distance(p, Type<Span<int8_t>>(L(5, 3, 1).Slice<int8_t>(p)).data())); EXPECT_EQ( 24, Distance(p, Type<Span<Int128>>(L(5, 3, 1).Slice<Int128>(p)).data())); EXPECT_EQ( - 8, Distance(p, Type<Span<int32_t>>(L(5, 3, 1).Slice<int32_t>(p)).data())); + 8, + Distance(p, Type<Span<int32_t>>(L(5, 3, 1).Slice<int32_t>(p)).data())); } } @@ -1256,17 +1320,17 @@ TEST(Layout, MutableSlices) { } { const auto x = L::Partial(1, 2, 3); - EXPECT_THAT( - (Type<std::tuple<Span<int8_t>, Span<int8_t>, Span<Int128>>>(x.Slices(p))), - Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)), - IsSameSlice(x.Slice<2>(p)))); + EXPECT_THAT((Type<std::tuple<Span<int8_t>, Span<int8_t>, Span<Int128>>>( + x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)), + IsSameSlice(x.Slice<2>(p)))); } { const L x(1, 2, 3); - EXPECT_THAT( - (Type<std::tuple<Span<int8_t>, Span<int8_t>, Span<Int128>>>(x.Slices(p))), - Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)), - IsSameSlice(x.Slice<2>(p)))); + EXPECT_THAT((Type<std::tuple<Span<int8_t>, Span<int8_t>, Span<Int128>>>( + x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)), + IsSameSlice(x.Slice<2>(p)))); } } @@ -1398,7 +1462,8 @@ TEST(Layout, DebugString) { x.DebugString()); } { - constexpr auto x = Layout<int8_t, int32_t, int8_t, Int128>::Partial(1, 2, 3); + constexpr auto x = + Layout<int8_t, int32_t, int8_t, Int128>::Partial(1, 2, 3); EXPECT_EQ( "@0<signed char>(1)[1]; @4<int>(4)[2]; @12<signed char>(1)[3]; " "@16" + @@ -1406,7 +1471,8 @@ TEST(Layout, DebugString) { x.DebugString()); } { - constexpr auto x = Layout<int8_t, int32_t, int8_t, Int128>::Partial(1, 2, 3, 4); + constexpr auto x = + Layout<int8_t, int32_t, int8_t, Int128>::Partial(1, 2, 3, 4); EXPECT_EQ( "@0<signed char>(1)[1]; @4<int>(4)[2]; @12<signed char>(1)[3]; " "@16" + diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index 919ac074..bfef071f 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc @@ -27,7 +27,7 @@ constexpr size_t Group::kWidth; // Returns "random" seed. inline size_t RandomSeed() { -#if ABSL_HAVE_THREAD_LOCAL +#ifdef ABSL_HAVE_THREAD_LOCAL static thread_local size_t counter = 0; size_t value = ++counter; #else // ABSL_HAVE_THREAD_LOCAL @@ -43,6 +43,19 @@ bool ShouldInsertBackwards(size_t hash, ctrl_t* ctrl) { return (H1(hash, ctrl) ^ RandomSeed()) % 13 > 6; } +void ConvertDeletedToEmptyAndFullToDeleted( + ctrl_t* ctrl, size_t capacity) { + assert(ctrl[capacity] == kSentinel); + assert(IsValidCapacity(capacity)); + for (ctrl_t* pos = ctrl; pos != ctrl + capacity + 1; pos += Group::kWidth) { + Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos); + } + // Copy the cloned ctrl bytes. + std::memcpy(ctrl + capacity + 1, ctrl, Group::kWidth); + ctrl[capacity] = kSentinel; +} + + } // namespace container_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index ec13a2f7..8615de8b 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -102,7 +102,6 @@ #include <type_traits> #include <utility> -#include "absl/base/internal/bits.h" #include "absl/base/internal/endian.h" #include "absl/base/optimization.h" #include "absl/base/port.h" @@ -116,6 +115,7 @@ #include "absl/container/internal/layout.h" #include "absl/memory/memory.h" #include "absl/meta/type_traits.h" +#include "absl/numeric/bits.h" #include "absl/utility/utility.h" namespace absl { @@ -189,18 +189,9 @@ constexpr bool IsNoThrowSwappable(std::false_type /* is_swappable */) { } template <typename T> -int TrailingZeros(T x) { - return sizeof(T) == 8 ? base_internal::CountTrailingZerosNonZero64( - static_cast<uint64_t>(x)) - : base_internal::CountTrailingZerosNonZero32( - static_cast<uint32_t>(x)); -} - -template <typename T> -int LeadingZeros(T x) { - return sizeof(T) == 8 - ? base_internal::CountLeadingZeros64(static_cast<uint64_t>(x)) - : base_internal::CountLeadingZeros32(static_cast<uint32_t>(x)); +uint32_t TrailingZeros(T x) { + ABSL_INTERNAL_ASSUME(x != 0); + return countr_zero(x); } // An abstraction over a bitmask. It provides an easy way to iterate through the @@ -230,26 +221,24 @@ class BitMask { } explicit operator bool() const { return mask_ != 0; } int operator*() const { return LowestBitSet(); } - int LowestBitSet() const { + uint32_t LowestBitSet() const { return container_internal::TrailingZeros(mask_) >> Shift; } - int HighestBitSet() const { - return (sizeof(T) * CHAR_BIT - container_internal::LeadingZeros(mask_) - - 1) >> - Shift; + uint32_t HighestBitSet() const { + return static_cast<uint32_t>((bit_width(mask_) - 1) >> Shift); } BitMask begin() const { return *this; } BitMask end() const { return BitMask(0); } - int TrailingZeros() const { + uint32_t TrailingZeros() const { return container_internal::TrailingZeros(mask_) >> Shift; } - int LeadingZeros() const { + uint32_t LeadingZeros() const { constexpr int total_significant_bits = SignificantBits << Shift; constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; - return container_internal::LeadingZeros(mask_ << extra_bits) >> Shift; + return countl_zero(mask_ << extra_bits) >> Shift; } private: @@ -380,8 +369,8 @@ struct GroupSse2Impl { // Returns the number of trailing empty or deleted elements in the group. uint32_t CountLeadingEmptyOrDeleted() const { auto special = _mm_set1_epi8(kSentinel); - return TrailingZeros( - _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1); + return TrailingZeros(static_cast<uint32_t>( + _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1)); } void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { @@ -472,25 +461,23 @@ inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } // DELETED -> EMPTY // EMPTY -> EMPTY // FULL -> DELETED -inline void ConvertDeletedToEmptyAndFullToDeleted( - ctrl_t* ctrl, size_t capacity) { - assert(ctrl[capacity] == kSentinel); - assert(IsValidCapacity(capacity)); - for (ctrl_t* pos = ctrl; pos != ctrl + capacity + 1; pos += Group::kWidth) { - Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos); - } - // Copy the cloned ctrl bytes. - std::memcpy(ctrl + capacity + 1, ctrl, Group::kWidth); - ctrl[capacity] = kSentinel; -} +void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity); // Rounds up the capacity to the next power of 2 minus 1, with a minimum of 1. inline size_t NormalizeCapacity(size_t n) { - return n ? ~size_t{} >> LeadingZeros(n) : 1; + return n ? ~size_t{} >> countl_zero(n) : 1; } -// We use 7/8th as maximum load factor. -// For 16-wide groups, that gives an average of two empty slots per group. +// General notes on capacity/growth methods below: +// - We use 7/8th as maximum load factor. For 16-wide groups, that gives an +// average of two empty slots per group. +// - For (capacity+1) >= Group::kWidth, growth is 7/8*capacity. +// - For (capacity+1) < Group::kWidth, growth == capacity. In this case, we +// never need to probe (the whole table fits in one group) so we don't need a +// load factor less than 1. + +// Given `capacity` of the table, returns the size (i.e. number of full slots) +// at which we should grow the capacity. inline size_t CapacityToGrowth(size_t capacity) { assert(IsValidCapacity(capacity)); // `capacity*7/8` @@ -501,7 +488,7 @@ inline size_t CapacityToGrowth(size_t capacity) { return capacity - capacity / 8; } // From desired "growth" to a lowerbound of the necessary capacity. -// Might not be a valid one and required NormalizeCapacity(). +// Might not be a valid one and requires NormalizeCapacity(). inline size_t GrowthToLowerboundCapacity(size_t growth) { // `growth*8/7` if (Group::kWidth == 8 && growth == 7) { @@ -523,6 +510,64 @@ inline void AssertIsValid(ctrl_t* ctrl) { "been erased, or the table might have rehashed."); } +struct FindInfo { + size_t offset; + size_t probe_length; +}; + +// The representation of the object has two modes: +// - small: For capacities < kWidth-1 +// - large: For the rest. +// +// Differences: +// - In small mode we are able to use the whole capacity. The extra control +// bytes give us at least one "empty" control byte to stop the iteration. +// This is important to make 1 a valid capacity. +// +// - In small mode only the first `capacity()` control bytes after the +// sentinel are valid. The rest contain dummy kEmpty values that do not +// represent a real slot. This is important to take into account on +// find_first_non_full(), where we never try ShouldInsertBackwards() for +// small tables. +inline bool is_small(size_t capacity) { return capacity < Group::kWidth - 1; } + +inline probe_seq<Group::kWidth> probe(ctrl_t* ctrl, size_t hash, + size_t capacity) { + return probe_seq<Group::kWidth>(H1(hash, ctrl), capacity); +} + +// Probes the raw_hash_set with the probe sequence for hash and returns the +// pointer to the first empty or deleted slot. +// NOTE: this function must work with tables having both kEmpty and kDelete +// in one group. Such tables appears during drop_deletes_without_resize. +// +// This function is very useful when insertions happen and: +// - the input is already a set +// - there are enough slots +// - the element with the hash is not in the table +inline FindInfo find_first_non_full(ctrl_t* ctrl, size_t hash, + size_t capacity) { + auto seq = probe(ctrl, hash, capacity); + while (true) { + Group g{ctrl + seq.offset()}; + auto mask = g.MatchEmptyOrDeleted(); + if (mask) { +#if !defined(NDEBUG) + // We want to add entropy even when ASLR is not enabled. + // In debug build we will randomly insert in either the front or back of + // the group. + // TODO(kfm,sbenza): revisit after we do unconditional mixing + if (!is_small(capacity) && ShouldInsertBackwards(hash, ctrl)) { + return {seq.offset(mask.HighestBitSet()), seq.index()}; + } +#endif + return {seq.offset(mask.LowestBitSet()), seq.index()}; + } + seq.next(); + assert(seq.index() < capacity && "full table!"); + } +} + // Policy: a policy defines how to perform different operations on // the slots of the hashtable (see hash_policy_traits.h for the full interface // of policy). @@ -747,10 +792,10 @@ class raw_hash_set { explicit raw_hash_set(size_t bucket_count, const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : ctrl_(EmptyGroup()), settings_(0, hash, eq, alloc) { + : ctrl_(EmptyGroup()), + settings_(0, HashtablezInfoHandle(), hash, eq, alloc) { if (bucket_count) { capacity_ = NormalizeCapacity(bucket_count); - reset_growth_left(); initialize_slots(); } } @@ -856,10 +901,10 @@ class raw_hash_set { // than a full `insert`. for (const auto& v : that) { const size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, v); - auto target = find_first_non_full(hash); + auto target = find_first_non_full(ctrl_, hash, capacity_); set_ctrl(target.offset, H2(hash)); emplace_at(target.offset, v); - infoz_.RecordInsert(hash, target.probe_length); + infoz().RecordInsert(hash, target.probe_length); } size_ = that.size(); growth_left() -= that.size(); @@ -873,28 +918,27 @@ class raw_hash_set { slots_(absl::exchange(that.slots_, nullptr)), size_(absl::exchange(that.size_, 0)), capacity_(absl::exchange(that.capacity_, 0)), - infoz_(absl::exchange(that.infoz_, HashtablezInfoHandle())), // Hash, equality and allocator are copied instead of moved because // `that` must be left valid. If Hash is std::function<Key>, moving it // would create a nullptr functor that cannot be called. - settings_(that.settings_) { - // growth_left was copied above, reset the one from `that`. - that.growth_left() = 0; - } + settings_(absl::exchange(that.growth_left(), 0), + absl::exchange(that.infoz(), HashtablezInfoHandle()), + that.hash_ref(), that.eq_ref(), that.alloc_ref()) {} raw_hash_set(raw_hash_set&& that, const allocator_type& a) : ctrl_(EmptyGroup()), slots_(nullptr), size_(0), capacity_(0), - settings_(0, that.hash_ref(), that.eq_ref(), a) { + settings_(0, HashtablezInfoHandle(), that.hash_ref(), that.eq_ref(), + a) { if (a == that.alloc_ref()) { std::swap(ctrl_, that.ctrl_); std::swap(slots_, that.slots_); std::swap(size_, that.size_); std::swap(capacity_, that.capacity_); std::swap(growth_left(), that.growth_left()); - std::swap(infoz_, that.infoz_); + std::swap(infoz(), that.infoz()); } else { reserve(that.size()); // Note: this will copy elements of dense_set and unordered_set instead of @@ -965,7 +1009,7 @@ class raw_hash_set { reset_growth_left(); } assert(empty()); - infoz_.RecordStorageChanged(0, capacity_); + infoz().RecordStorageChanged(0, capacity_); } // This overload kicks in when the argument is an rvalue of insertable and @@ -1038,7 +1082,7 @@ class raw_hash_set { template <class InputIt> void insert(InputIt first, InputIt last) { - for (; first != last; ++first) insert(*first); + for (; first != last; ++first) emplace(*first); } template <class T, RequiresNotInit<T> = 0, RequiresInsertable<const T&> = 0> @@ -1065,7 +1109,9 @@ class raw_hash_set { } iterator insert(const_iterator, node_type&& node) { - return insert(std::move(node)).first; + auto res = insert(std::move(node)); + node = std::move(res.node); + return res.position; } // This overload kicks in if we can deduce the key from args. This enables us @@ -1255,7 +1301,7 @@ class raw_hash_set { swap(growth_left(), that.growth_left()); swap(hash_ref(), that.hash_ref()); swap(eq_ref(), that.eq_ref()); - swap(infoz_, that.infoz_); + swap(infoz(), that.infoz()); SwapAlloc(alloc_ref(), that.alloc_ref(), typename AllocTraits::propagate_on_container_swap{}); } @@ -1264,7 +1310,7 @@ class raw_hash_set { if (n == 0 && capacity_ == 0) return; if (n == 0 && size_ == 0) { destroy_slots(); - infoz_.RecordStorageChanged(0, 0); + infoz().RecordStorageChanged(0, 0); return; } // bitor is a faster way of doing `max` here. We will round up to the next @@ -1276,7 +1322,12 @@ class raw_hash_set { } } - void reserve(size_t n) { rehash(GrowthToLowerboundCapacity(n)); } + void reserve(size_t n) { + size_t m = GrowthToLowerboundCapacity(n); + if (m > capacity_) { + resize(NormalizeCapacity(m)); + } + } // Extension API: support for heterogeneous keys. // @@ -1301,7 +1352,7 @@ class raw_hash_set { void prefetch(const key_arg<K>& key) const { (void)key; #if defined(__GNUC__) - auto seq = probe(hash_ref()(key)); + auto seq = probe(ctrl_, hash_ref()(key), capacity_); __builtin_prefetch(static_cast<const void*>(ctrl_ + seq.offset())); __builtin_prefetch(static_cast<const void*>(slots_ + seq.offset())); #endif // __GNUC__ @@ -1316,7 +1367,7 @@ class raw_hash_set { // called heterogeneous key support. template <class K = key_type> iterator find(const key_arg<K>& key, size_t hash) { - auto seq = probe(hash); + auto seq = probe(ctrl_, hash, capacity_); while (true) { Group g{ctrl_ + seq.offset()}; for (int i : g.Match(H2(hash))) { @@ -1477,7 +1528,7 @@ class raw_hash_set { set_ctrl(index, was_never_full ? kEmpty : kDeleted); growth_left() += was_never_full; - infoz_.RecordErase(); + infoz().RecordErase(); } void initialize_slots() { @@ -1494,7 +1545,7 @@ class raw_hash_set { // bound more carefully. if (std::is_same<SlotAlloc, std::allocator<slot_type>>::value && slots_ == nullptr) { - infoz_ = Sample(); + infoz() = Sample(); } auto layout = MakeLayout(capacity_); @@ -1504,7 +1555,7 @@ class raw_hash_set { slots_ = layout.template Pointer<1>(mem); reset_ctrl(); reset_growth_left(); - infoz_.RecordStorageChanged(size_, capacity_); + infoz().RecordStorageChanged(size_, capacity_); } void destroy_slots() { @@ -1538,7 +1589,7 @@ class raw_hash_set { if (IsFull(old_ctrl[i])) { size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, PolicyTraits::element(old_slots + i)); - auto target = find_first_non_full(hash); + auto target = find_first_non_full(ctrl_, hash, capacity_); size_t new_i = target.offset; total_probe_length += target.probe_length; set_ctrl(new_i, H2(hash)); @@ -1552,12 +1603,12 @@ class raw_hash_set { Deallocate<Layout::Alignment()>(&alloc_ref(), old_ctrl, layout.AllocSize()); } - infoz_.RecordRehash(total_probe_length); + infoz().RecordRehash(total_probe_length); } void drop_deletes_without_resize() ABSL_ATTRIBUTE_NOINLINE { assert(IsValidCapacity(capacity_)); - assert(!is_small()); + assert(!is_small(capacity_)); // Algorithm: // - mark all DELETED slots as EMPTY // - mark all FULL slots as DELETED @@ -1582,7 +1633,7 @@ class raw_hash_set { if (!IsDeleted(ctrl_[i])) continue; size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, PolicyTraits::element(slots_ + i)); - auto target = find_first_non_full(hash); + auto target = find_first_non_full(ctrl_, hash, capacity_); size_t new_i = target.offset; total_probe_length += target.probe_length; @@ -1590,7 +1641,8 @@ class raw_hash_set { // If they do, we don't need to move the object as it falls already in the // best probe we can. const auto probe_index = [&](size_t pos) { - return ((pos - probe(hash).offset()) & capacity_) / Group::kWidth; + return ((pos - probe(ctrl_, hash, capacity_).offset()) & capacity_) / + Group::kWidth; }; // Element doesn't move. @@ -1617,7 +1669,7 @@ class raw_hash_set { } } reset_growth_left(); - infoz_.RecordRehash(total_probe_length); + infoz().RecordRehash(total_probe_length); } void rehash_and_grow_if_necessary() { @@ -1634,7 +1686,7 @@ class raw_hash_set { bool has_element(const value_type& elem) const { size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, elem); - auto seq = probe(hash); + auto seq = probe(ctrl_, hash, capacity_); while (true) { Group g{ctrl_ + seq.offset()}; for (int i : g.Match(H2(hash))) { @@ -1649,41 +1701,6 @@ class raw_hash_set { return false; } - // Probes the raw_hash_set with the probe sequence for hash and returns the - // pointer to the first empty or deleted slot. - // NOTE: this function must work with tables having both kEmpty and kDelete - // in one group. Such tables appears during drop_deletes_without_resize. - // - // This function is very useful when insertions happen and: - // - the input is already a set - // - there are enough slots - // - the element with the hash is not in the table - struct FindInfo { - size_t offset; - size_t probe_length; - }; - FindInfo find_first_non_full(size_t hash) { - auto seq = probe(hash); - while (true) { - Group g{ctrl_ + seq.offset()}; - auto mask = g.MatchEmptyOrDeleted(); - if (mask) { -#if !defined(NDEBUG) - // We want to add entropy even when ASLR is not enabled. - // In debug build we will randomly insert in either the front or back of - // the group. - // TODO(kfm,sbenza): revisit after we do unconditional mixing - if (!is_small() && ShouldInsertBackwards(hash, ctrl_)) { - return {seq.offset(mask.HighestBitSet()), seq.index()}; - } -#endif - return {seq.offset(mask.LowestBitSet()), seq.index()}; - } - seq.next(); - assert(seq.index() < capacity_ && "full table!"); - } - } - // TODO(alkis): Optimize this assuming *this and that don't overlap. raw_hash_set& move_assign(raw_hash_set&& that, std::true_type) { raw_hash_set tmp(std::move(that)); @@ -1700,7 +1717,7 @@ class raw_hash_set { template <class K> std::pair<size_t, bool> find_or_prepare_insert(const K& key) { auto hash = hash_ref()(key); - auto seq = probe(hash); + auto seq = probe(ctrl_, hash, capacity_); while (true) { Group g{ctrl_ + seq.offset()}; for (int i : g.Match(H2(hash))) { @@ -1717,16 +1734,16 @@ class raw_hash_set { } size_t prepare_insert(size_t hash) ABSL_ATTRIBUTE_NOINLINE { - auto target = find_first_non_full(hash); + auto target = find_first_non_full(ctrl_, hash, capacity_); if (ABSL_PREDICT_FALSE(growth_left() == 0 && !IsDeleted(ctrl_[target.offset]))) { rehash_and_grow_if_necessary(); - target = find_first_non_full(hash); + target = find_first_non_full(ctrl_, hash, capacity_); } ++size_; growth_left() -= IsEmpty(ctrl_[target.offset]); set_ctrl(target.offset, H2(hash)); - infoz_.RecordInsert(hash, target.probe_length); + infoz().RecordInsert(hash, target.probe_length); return target.offset; } @@ -1754,10 +1771,6 @@ class raw_hash_set { private: friend struct RawHashSetTestOnlyAccess; - probe_seq<Group::kWidth> probe(size_t hash) const { - return probe_seq<Group::kWidth>(H1(hash, ctrl_), capacity_); - } - // Reset all ctrl bytes back to kEmpty, except the sentinel. void reset_ctrl() { std::memset(ctrl_, kEmpty, capacity_ + Group::kWidth); @@ -1787,29 +1800,15 @@ class raw_hash_set { size_t& growth_left() { return settings_.template get<0>(); } - // The representation of the object has two modes: - // - small: For capacities < kWidth-1 - // - large: For the rest. - // - // Differences: - // - In small mode we are able to use the whole capacity. The extra control - // bytes give us at least one "empty" control byte to stop the iteration. - // This is important to make 1 a valid capacity. - // - // - In small mode only the first `capacity()` control bytes after the - // sentinel are valid. The rest contain dummy kEmpty values that do not - // represent a real slot. This is important to take into account on - // find_first_non_full(), where we never try ShouldInsertBackwards() for - // small tables. - bool is_small() const { return capacity_ < Group::kWidth - 1; } - - hasher& hash_ref() { return settings_.template get<1>(); } - const hasher& hash_ref() const { return settings_.template get<1>(); } - key_equal& eq_ref() { return settings_.template get<2>(); } - const key_equal& eq_ref() const { return settings_.template get<2>(); } - allocator_type& alloc_ref() { return settings_.template get<3>(); } + HashtablezInfoHandle& infoz() { return settings_.template get<1>(); } + + hasher& hash_ref() { return settings_.template get<2>(); } + const hasher& hash_ref() const { return settings_.template get<2>(); } + key_equal& eq_ref() { return settings_.template get<3>(); } + const key_equal& eq_ref() const { return settings_.template get<3>(); } + allocator_type& alloc_ref() { return settings_.template get<4>(); } const allocator_type& alloc_ref() const { - return settings_.template get<3>(); + return settings_.template get<4>(); } // TODO(alkis): Investigate removing some of these fields: @@ -1819,10 +1818,11 @@ class raw_hash_set { slot_type* slots_ = nullptr; // [capacity * slot_type] size_t size_ = 0; // number of full slots size_t capacity_ = 0; // total number of slots - HashtablezInfoHandle infoz_; - absl::container_internal::CompressedTuple<size_t /* growth_left */, hasher, + absl::container_internal::CompressedTuple<size_t /* growth_left */, + HashtablezInfoHandle, hasher, key_equal, allocator_type> - settings_{0, hasher{}, key_equal{}, allocator_type{}}; + settings_{0, HashtablezInfoHandle{}, hasher{}, key_equal{}, + allocator_type{}}; }; // Erases all elements that satisfy the predicate `pred` from the container `c`. @@ -1846,7 +1846,7 @@ struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> { const typename Set::key_type& key) { size_t num_probes = 0; size_t hash = set.hash_ref()(key); - auto seq = set.probe(hash); + auto seq = probe(set.ctrl_, hash, set.capacity_); while (true) { container_internal::Group g{set.ctrl_ + seq.offset()}; for (int i : g.Match(container_internal::H2(hash))) { diff --git a/absl/container/internal/raw_hash_set_allocator_test.cc b/absl/container/internal/raw_hash_set_allocator_test.cc index 1a036085..e73f53fd 100644 --- a/absl/container/internal/raw_hash_set_allocator_test.cc +++ b/absl/container/internal/raw_hash_set_allocator_test.cc @@ -466,6 +466,9 @@ class PAlloc { size_t id_ = std::numeric_limits<size_t>::max(); }; +// This doesn't compile with GCC 5.4 and 5.5 due to a bug in noexcept handing. +#if !defined(__GNUC__) || __GNUC__ != 5 || (__GNUC_MINOR__ != 4 && \ + __GNUC_MINOR__ != 5) TEST(NoPropagateOn, Swap) { using PA = PAlloc<char>; using Table = raw_hash_set<Policy, Identity, std::equal_to<int32_t>, PA>; @@ -475,6 +478,7 @@ TEST(NoPropagateOn, Swap) { EXPECT_EQ(t1.get_allocator(), PA(1)); EXPECT_EQ(t2.get_allocator(), PA(2)); } +#endif TEST(NoPropagateOn, CopyConstruct) { using PA = PAlloc<char>; diff --git a/absl/container/internal/raw_hash_set_benchmark.cc b/absl/container/internal/raw_hash_set_benchmark.cc new file mode 100644 index 00000000..f9be2c5a --- /dev/null +++ b/absl/container/internal/raw_hash_set_benchmark.cc @@ -0,0 +1,396 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/raw_hash_set.h" + +#include <numeric> +#include <random> + +#include "absl/base/internal/raw_logging.h" +#include "absl/container/internal/hash_function_defaults.h" +#include "absl/strings/str_format.h" +#include "benchmark/benchmark.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +struct RawHashSetTestOnlyAccess { + template <typename C> + static auto GetSlots(const C& c) -> decltype(c.slots_) { + return c.slots_; + } +}; + +namespace { + +struct IntPolicy { + using slot_type = int64_t; + using key_type = int64_t; + using init_type = int64_t; + + static void construct(void*, int64_t* slot, int64_t v) { *slot = v; } + static void destroy(void*, int64_t*) {} + static void transfer(void*, int64_t* new_slot, int64_t* old_slot) { + *new_slot = *old_slot; + } + + static int64_t& element(slot_type* slot) { return *slot; } + + template <class F> + static auto apply(F&& f, int64_t x) -> decltype(std::forward<F>(f)(x, x)) { + return std::forward<F>(f)(x, x); + } +}; + +class StringPolicy { + template <class F, class K, class V, + class = typename std::enable_if< + std::is_convertible<const K&, absl::string_view>::value>::type> + decltype(std::declval<F>()( + std::declval<const absl::string_view&>(), std::piecewise_construct, + std::declval<std::tuple<K>>(), + std::declval<V>())) static apply_impl(F&& f, + std::pair<std::tuple<K>, V> p) { + const absl::string_view& key = std::get<0>(p.first); + return std::forward<F>(f)(key, std::piecewise_construct, std::move(p.first), + std::move(p.second)); + } + + public: + struct slot_type { + struct ctor {}; + + template <class... Ts> + slot_type(ctor, Ts&&... ts) : pair(std::forward<Ts>(ts)...) {} + + std::pair<std::string, std::string> pair; + }; + + using key_type = std::string; + using init_type = std::pair<std::string, std::string>; + + template <class allocator_type, class... Args> + static void construct(allocator_type* alloc, slot_type* slot, Args... args) { + std::allocator_traits<allocator_type>::construct( + *alloc, slot, typename slot_type::ctor(), std::forward<Args>(args)...); + } + + template <class allocator_type> + static void destroy(allocator_type* alloc, slot_type* slot) { + std::allocator_traits<allocator_type>::destroy(*alloc, slot); + } + + template <class allocator_type> + static void transfer(allocator_type* alloc, slot_type* new_slot, + slot_type* old_slot) { + construct(alloc, new_slot, std::move(old_slot->pair)); + destroy(alloc, old_slot); + } + + static std::pair<std::string, std::string>& element(slot_type* slot) { + return slot->pair; + } + + template <class F, class... Args> + static auto apply(F&& f, Args&&... args) + -> decltype(apply_impl(std::forward<F>(f), + PairArgs(std::forward<Args>(args)...))) { + return apply_impl(std::forward<F>(f), + PairArgs(std::forward<Args>(args)...)); + } +}; + +struct StringHash : container_internal::hash_default_hash<absl::string_view> { + using is_transparent = void; +}; +struct StringEq : std::equal_to<absl::string_view> { + using is_transparent = void; +}; + +struct StringTable + : raw_hash_set<StringPolicy, StringHash, StringEq, std::allocator<int>> { + using Base = typename StringTable::raw_hash_set; + StringTable() {} + using Base::Base; +}; + +struct IntTable + : raw_hash_set<IntPolicy, container_internal::hash_default_hash<int64_t>, + std::equal_to<int64_t>, std::allocator<int64_t>> { + using Base = typename IntTable::raw_hash_set; + IntTable() {} + using Base::Base; +}; + +struct string_generator { + template <class RNG> + std::string operator()(RNG& rng) const { + std::string res; + res.resize(12); + std::uniform_int_distribution<uint32_t> printable_ascii(0x20, 0x7E); + std::generate(res.begin(), res.end(), [&] { return printable_ascii(rng); }); + return res; + } + + size_t size; +}; + +// Model a cache in steady state. +// +// On a table of size N, keep deleting the LRU entry and add a random one. +void BM_CacheInSteadyState(benchmark::State& state) { + std::random_device rd; + std::mt19937 rng(rd()); + string_generator gen{12}; + StringTable t; + std::deque<std::string> keys; + while (t.size() < state.range(0)) { + auto x = t.emplace(gen(rng), gen(rng)); + if (x.second) keys.push_back(x.first->first); + } + ABSL_RAW_CHECK(state.range(0) >= 10, ""); + while (state.KeepRunning()) { + // Some cache hits. + std::deque<std::string>::const_iterator it; + for (int i = 0; i != 90; ++i) { + if (i % 10 == 0) it = keys.end(); + ::benchmark::DoNotOptimize(t.find(*--it)); + } + // Some cache misses. + for (int i = 0; i != 10; ++i) ::benchmark::DoNotOptimize(t.find(gen(rng))); + ABSL_RAW_CHECK(t.erase(keys.front()), keys.front().c_str()); + keys.pop_front(); + while (true) { + auto x = t.emplace(gen(rng), gen(rng)); + if (x.second) { + keys.push_back(x.first->first); + break; + } + } + } + state.SetItemsProcessed(state.iterations()); + state.SetLabel(absl::StrFormat("load_factor=%.2f", t.load_factor())); +} + +template <typename Benchmark> +void CacheInSteadyStateArgs(Benchmark* bm) { + // The default. + const float max_load_factor = 0.875; + // When the cache is at the steady state, the probe sequence will equal + // capacity if there is no reclamation of deleted slots. Pick a number large + // enough to make the benchmark slow for that case. + const size_t capacity = 1 << 10; + + // Check N data points to cover load factors in [0.4, 0.8). + const size_t kNumPoints = 10; + for (size_t i = 0; i != kNumPoints; ++i) + bm->Arg(std::ceil( + capacity * (max_load_factor + i * max_load_factor / kNumPoints) / 2)); +} +BENCHMARK(BM_CacheInSteadyState)->Apply(CacheInSteadyStateArgs); + +void BM_EndComparison(benchmark::State& state) { + std::random_device rd; + std::mt19937 rng(rd()); + string_generator gen{12}; + StringTable t; + while (t.size() < state.range(0)) { + t.emplace(gen(rng), gen(rng)); + } + + for (auto _ : state) { + for (auto it = t.begin(); it != t.end(); ++it) { + benchmark::DoNotOptimize(it); + benchmark::DoNotOptimize(t); + benchmark::DoNotOptimize(it != t.end()); + } + } +} +BENCHMARK(BM_EndComparison)->Arg(400); + +void BM_CopyCtor(benchmark::State& state) { + std::random_device rd; + std::mt19937 rng(rd()); + IntTable t; + std::uniform_int_distribution<uint64_t> dist(0, ~uint64_t{}); + + while (t.size() < state.range(0)) { + t.emplace(dist(rng)); + } + + for (auto _ : state) { + IntTable t2 = t; + benchmark::DoNotOptimize(t2); + } +} +BENCHMARK(BM_CopyCtor)->Range(128, 4096); + +void BM_CopyAssign(benchmark::State& state) { + std::random_device rd; + std::mt19937 rng(rd()); + IntTable t; + std::uniform_int_distribution<uint64_t> dist(0, ~uint64_t{}); + while (t.size() < state.range(0)) { + t.emplace(dist(rng)); + } + + IntTable t2; + for (auto _ : state) { + t2 = t; + benchmark::DoNotOptimize(t2); + } +} +BENCHMARK(BM_CopyAssign)->Range(128, 4096); + +void BM_NoOpReserveIntTable(benchmark::State& state) { + IntTable t; + t.reserve(100000); + for (auto _ : state) { + benchmark::DoNotOptimize(t); + t.reserve(100000); + } +} +BENCHMARK(BM_NoOpReserveIntTable); + +void BM_NoOpReserveStringTable(benchmark::State& state) { + StringTable t; + t.reserve(100000); + for (auto _ : state) { + benchmark::DoNotOptimize(t); + t.reserve(100000); + } +} +BENCHMARK(BM_NoOpReserveStringTable); + +void BM_ReserveIntTable(benchmark::State& state) { + int reserve_size = state.range(0); + for (auto _ : state) { + state.PauseTiming(); + IntTable t; + state.ResumeTiming(); + benchmark::DoNotOptimize(t); + t.reserve(reserve_size); + } +} +BENCHMARK(BM_ReserveIntTable)->Range(128, 4096); + +void BM_ReserveStringTable(benchmark::State& state) { + int reserve_size = state.range(0); + for (auto _ : state) { + state.PauseTiming(); + StringTable t; + state.ResumeTiming(); + benchmark::DoNotOptimize(t); + t.reserve(reserve_size); + } +} +BENCHMARK(BM_ReserveStringTable)->Range(128, 4096); + +void BM_Group_Match(benchmark::State& state) { + std::array<ctrl_t, Group::kWidth> group; + std::iota(group.begin(), group.end(), -4); + Group g{group.data()}; + h2_t h = 1; + for (auto _ : state) { + ::benchmark::DoNotOptimize(h); + ::benchmark::DoNotOptimize(g.Match(h)); + } +} +BENCHMARK(BM_Group_Match); + +void BM_Group_MatchEmpty(benchmark::State& state) { + std::array<ctrl_t, Group::kWidth> group; + std::iota(group.begin(), group.end(), -4); + Group g{group.data()}; + for (auto _ : state) ::benchmark::DoNotOptimize(g.MatchEmpty()); +} +BENCHMARK(BM_Group_MatchEmpty); + +void BM_Group_MatchEmptyOrDeleted(benchmark::State& state) { + std::array<ctrl_t, Group::kWidth> group; + std::iota(group.begin(), group.end(), -4); + Group g{group.data()}; + for (auto _ : state) ::benchmark::DoNotOptimize(g.MatchEmptyOrDeleted()); +} +BENCHMARK(BM_Group_MatchEmptyOrDeleted); + +void BM_Group_CountLeadingEmptyOrDeleted(benchmark::State& state) { + std::array<ctrl_t, Group::kWidth> group; + std::iota(group.begin(), group.end(), -2); + Group g{group.data()}; + for (auto _ : state) + ::benchmark::DoNotOptimize(g.CountLeadingEmptyOrDeleted()); +} +BENCHMARK(BM_Group_CountLeadingEmptyOrDeleted); + +void BM_Group_MatchFirstEmptyOrDeleted(benchmark::State& state) { + std::array<ctrl_t, Group::kWidth> group; + std::iota(group.begin(), group.end(), -2); + Group g{group.data()}; + for (auto _ : state) ::benchmark::DoNotOptimize(*g.MatchEmptyOrDeleted()); +} +BENCHMARK(BM_Group_MatchFirstEmptyOrDeleted); + +void BM_DropDeletes(benchmark::State& state) { + constexpr size_t capacity = (1 << 20) - 1; + std::vector<ctrl_t> ctrl(capacity + 1 + Group::kWidth); + ctrl[capacity] = kSentinel; + std::vector<ctrl_t> pattern = {kEmpty, 2, kDeleted, 2, kEmpty, 1, kDeleted}; + for (size_t i = 0; i != capacity; ++i) { + ctrl[i] = pattern[i % pattern.size()]; + } + while (state.KeepRunning()) { + state.PauseTiming(); + std::vector<ctrl_t> ctrl_copy = ctrl; + state.ResumeTiming(); + ConvertDeletedToEmptyAndFullToDeleted(ctrl_copy.data(), capacity); + ::benchmark::DoNotOptimize(ctrl_copy[capacity]); + } +} +BENCHMARK(BM_DropDeletes); + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +// These methods are here to make it easy to examine the assembly for targeted +// parts of the API. +auto CodegenAbslRawHashSetInt64Find(absl::container_internal::IntTable* table, + int64_t key) -> decltype(table->find(key)) { + return table->find(key); +} + +bool CodegenAbslRawHashSetInt64FindNeEnd( + absl::container_internal::IntTable* table, int64_t key) { + return table->find(key) != table->end(); +} + +bool CodegenAbslRawHashSetInt64Contains( + absl::container_internal::IntTable* table, int64_t key) { + return table->contains(key); +} + +void CodegenAbslRawHashSetInt64Iterate( + absl::container_internal::IntTable* table) { + for (auto x : *table) benchmark::DoNotOptimize(x); +} + +int odr = + (::benchmark::DoNotOptimize(std::make_tuple( + &CodegenAbslRawHashSetInt64Find, &CodegenAbslRawHashSetInt64FindNeEnd, + &CodegenAbslRawHashSetInt64Contains, + &CodegenAbslRawHashSetInt64Iterate)), + 1); diff --git a/absl/container/internal/raw_hash_set_probe_benchmark.cc b/absl/container/internal/raw_hash_set_probe_benchmark.cc new file mode 100644 index 00000000..7169a2e2 --- /dev/null +++ b/absl/container/internal/raw_hash_set_probe_benchmark.cc @@ -0,0 +1,590 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Generates probe length statistics for many combinations of key types and key +// distributions, all using the default hash function for swisstable. + +#include <memory> +#include <regex> // NOLINT +#include <vector> + +#include "absl/container/flat_hash_map.h" +#include "absl/container/internal/hash_function_defaults.h" +#include "absl/container/internal/hashtable_debug.h" +#include "absl/container/internal/raw_hash_set.h" +#include "absl/random/distributions.h" +#include "absl/random/random.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" + +namespace { + +enum class OutputStyle { kRegular, kBenchmark }; + +// The --benchmark command line flag. +// This is populated from main(). +// When run in "benchmark" mode, we have different output. This allows +// A/B comparisons with tools like `benchy`. +absl::string_view benchmarks; + +OutputStyle output() { + return !benchmarks.empty() ? OutputStyle::kBenchmark : OutputStyle::kRegular; +} + +template <class T> +struct Policy { + using slot_type = T; + using key_type = T; + using init_type = T; + + template <class allocator_type, class Arg> + static void construct(allocator_type* alloc, slot_type* slot, + const Arg& arg) { + std::allocator_traits<allocator_type>::construct(*alloc, slot, arg); + } + + template <class allocator_type> + static void destroy(allocator_type* alloc, slot_type* slot) { + std::allocator_traits<allocator_type>::destroy(*alloc, slot); + } + + static slot_type& element(slot_type* slot) { return *slot; } + + template <class F, class... Args> + static auto apply(F&& f, const slot_type& arg) + -> decltype(std::forward<F>(f)(arg, arg)) { + return std::forward<F>(f)(arg, arg); + } +}; + +absl::BitGen& GlobalBitGen() { + static auto* value = new absl::BitGen; + return *value; +} + +// Keeps a pool of allocations and randomly gives one out. +// This introduces more randomization to the addresses given to swisstable and +// should help smooth out this factor from probe length calculation. +template <class T> +class RandomizedAllocator { + public: + using value_type = T; + + RandomizedAllocator() = default; + template <typename U> + RandomizedAllocator(RandomizedAllocator<U>) {} // NOLINT + + static T* allocate(size_t n) { + auto& pointers = GetPointers(n); + // Fill the pool + while (pointers.size() < kRandomPool) { + pointers.push_back(std::allocator<T>{}.allocate(n)); + } + + // Choose a random one. + size_t i = absl::Uniform<size_t>(GlobalBitGen(), 0, pointers.size()); + T* result = pointers[i]; + pointers[i] = pointers.back(); + pointers.pop_back(); + return result; + } + + static void deallocate(T* p, size_t n) { + // Just put it back on the pool. No need to release the memory. + GetPointers(n).push_back(p); + } + + private: + // We keep at least kRandomPool allocations for each size. + static constexpr size_t kRandomPool = 20; + + static std::vector<T*>& GetPointers(size_t n) { + static auto* m = new absl::flat_hash_map<size_t, std::vector<T*>>(); + return (*m)[n]; + } +}; + +template <class T> +struct DefaultHash { + using type = absl::container_internal::hash_default_hash<T>; +}; + +template <class T> +using DefaultHashT = typename DefaultHash<T>::type; + +template <class T> +struct Table : absl::container_internal::raw_hash_set< + Policy<T>, DefaultHashT<T>, + absl::container_internal::hash_default_eq<T>, + RandomizedAllocator<T>> {}; + +struct LoadSizes { + size_t min_load; + size_t max_load; +}; + +LoadSizes GetMinMaxLoadSizes() { + static const auto sizes = [] { + Table<int> t; + + // First, fill enough to have a good distribution. + constexpr size_t kMinSize = 10000; + while (t.size() < kMinSize) t.insert(t.size()); + + const auto reach_min_load_factor = [&] { + const double lf = t.load_factor(); + while (lf <= t.load_factor()) t.insert(t.size()); + }; + + // Then, insert until we reach min load factor. + reach_min_load_factor(); + const size_t min_load_size = t.size(); + + // Keep going until we hit min load factor again, then go back one. + t.insert(t.size()); + reach_min_load_factor(); + + return LoadSizes{min_load_size, t.size() - 1}; + }(); + return sizes; +} + +struct Ratios { + double min_load; + double avg_load; + double max_load; +}; + +// See absl/container/internal/hashtable_debug.h for details on +// probe length calculation. +template <class ElemFn> +Ratios CollectMeanProbeLengths() { + const auto min_max_sizes = GetMinMaxLoadSizes(); + + ElemFn elem; + using Key = decltype(elem()); + Table<Key> t; + + Ratios result; + while (t.size() < min_max_sizes.min_load) t.insert(elem()); + result.min_load = + absl::container_internal::GetHashtableDebugProbeSummary(t).mean; + + while (t.size() < (min_max_sizes.min_load + min_max_sizes.max_load) / 2) + t.insert(elem()); + result.avg_load = + absl::container_internal::GetHashtableDebugProbeSummary(t).mean; + + while (t.size() < min_max_sizes.max_load) t.insert(elem()); + result.max_load = + absl::container_internal::GetHashtableDebugProbeSummary(t).mean; + + return result; +} + +template <int Align> +uintptr_t PointerForAlignment() { + alignas(Align) static constexpr uintptr_t kInitPointer = 0; + return reinterpret_cast<uintptr_t>(&kInitPointer); +} + +// This incomplete type is used for testing hash of pointers of different +// alignments. +// NOTE: We are generating invalid pointer values on the fly with +// reinterpret_cast. There are not "safely derived" pointers so using them is +// technically UB. It is unlikely to be a problem, though. +template <int Align> +struct Ptr; + +template <int Align> +Ptr<Align>* MakePtr(uintptr_t v) { + if (sizeof(v) == 8) { + constexpr int kCopyBits = 16; + // Ensure high bits are all the same. + v = static_cast<uintptr_t>(static_cast<intptr_t>(v << kCopyBits) >> + kCopyBits); + } + return reinterpret_cast<Ptr<Align>*>(v); +} + +struct IntIdentity { + uint64_t i; + friend bool operator==(IntIdentity a, IntIdentity b) { return a.i == b.i; } + IntIdentity operator++(int) { return IntIdentity{i++}; } +}; + +template <int Align> +struct PtrIdentity { + explicit PtrIdentity(uintptr_t val = PointerForAlignment<Align>()) : i(val) {} + uintptr_t i; + friend bool operator==(PtrIdentity a, PtrIdentity b) { return a.i == b.i; } + PtrIdentity operator++(int) { + PtrIdentity p(i); + i += Align; + return p; + } +}; + +constexpr char kStringFormat[] = "/path/to/file/name-%07d-of-9999999.txt"; + +template <bool small> +struct String { + std::string value; + static std::string Make(uint32_t v) { + return {small ? absl::StrCat(v) : absl::StrFormat(kStringFormat, v)}; + } +}; + +template <> +struct DefaultHash<IntIdentity> { + struct type { + size_t operator()(IntIdentity t) const { return t.i; } + }; +}; + +template <int Align> +struct DefaultHash<PtrIdentity<Align>> { + struct type { + size_t operator()(PtrIdentity<Align> t) const { return t.i; } + }; +}; + +template <class T> +struct Sequential { + T operator()() const { return current++; } + mutable T current{}; +}; + +template <int Align> +struct Sequential<Ptr<Align>*> { + Ptr<Align>* operator()() const { + auto* result = MakePtr<Align>(current); + current += Align; + return result; + } + mutable uintptr_t current = PointerForAlignment<Align>(); +}; + + +template <bool small> +struct Sequential<String<small>> { + std::string operator()() const { return String<small>::Make(current++); } + mutable uint32_t current = 0; +}; + +template <class T, class U> +struct Sequential<std::pair<T, U>> { + mutable Sequential<T> tseq; + mutable Sequential<U> useq; + + using RealT = decltype(tseq()); + using RealU = decltype(useq()); + + mutable std::vector<RealT> ts; + mutable std::vector<RealU> us; + mutable size_t ti = 0, ui = 0; + + std::pair<RealT, RealU> operator()() const { + std::pair<RealT, RealU> value{get_t(), get_u()}; + if (ti == 0) { + ti = ui + 1; + ui = 0; + } else { + --ti; + ++ui; + } + return value; + } + + RealT get_t() const { + while (ti >= ts.size()) ts.push_back(tseq()); + return ts[ti]; + } + + RealU get_u() const { + while (ui >= us.size()) us.push_back(useq()); + return us[ui]; + } +}; + +template <class T, int percent_skip> +struct AlmostSequential { + mutable Sequential<T> current; + + auto operator()() const -> decltype(current()) { + while (absl::Uniform(GlobalBitGen(), 0.0, 1.0) <= percent_skip / 100.) + current(); + return current(); + } +}; + +struct Uniform { + template <typename T> + T operator()(T) const { + return absl::Uniform<T>(absl::IntervalClosed, GlobalBitGen(), T{0}, ~T{0}); + } +}; + +struct Gaussian { + template <typename T> + T operator()(T) const { + double d; + do { + d = absl::Gaussian<double>(GlobalBitGen(), 1e6, 1e4); + } while (d <= 0 || d > std::numeric_limits<T>::max() / 2); + return static_cast<T>(d); + } +}; + +struct Zipf { + template <typename T> + T operator()(T) const { + return absl::Zipf<T>(GlobalBitGen(), std::numeric_limits<T>::max(), 1.6); + } +}; + +template <class T, class Dist> +struct Random { + T operator()() const { return Dist{}(T{}); } +}; + +template <class Dist, int Align> +struct Random<Ptr<Align>*, Dist> { + Ptr<Align>* operator()() const { + return MakePtr<Align>(Random<uintptr_t, Dist>{}() * Align); + } +}; + +template <class Dist> +struct Random<IntIdentity, Dist> { + IntIdentity operator()() const { + return IntIdentity{Random<uint64_t, Dist>{}()}; + } +}; + +template <class Dist, int Align> +struct Random<PtrIdentity<Align>, Dist> { + PtrIdentity<Align> operator()() const { + return PtrIdentity<Align>{Random<uintptr_t, Dist>{}() * Align}; + } +}; + +template <class Dist, bool small> +struct Random<String<small>, Dist> { + std::string operator()() const { + return String<small>::Make(Random<uint32_t, Dist>{}()); + } +}; + +template <class T, class U, class Dist> +struct Random<std::pair<T, U>, Dist> { + auto operator()() const + -> decltype(std::make_pair(Random<T, Dist>{}(), Random<U, Dist>{}())) { + return std::make_pair(Random<T, Dist>{}(), Random<U, Dist>{}()); + } +}; + +template <typename> +std::string Name(); + +std::string Name(uint32_t*) { return "u32"; } +std::string Name(uint64_t*) { return "u64"; } +std::string Name(IntIdentity*) { return "IntIdentity"; } + +template <int Align> +std::string Name(Ptr<Align>**) { + return absl::StrCat("Ptr", Align); +} + +template <int Align> +std::string Name(PtrIdentity<Align>*) { + return absl::StrCat("PtrIdentity", Align); +} + +template <bool small> +std::string Name(String<small>*) { + return small ? "StrS" : "StrL"; +} + +template <class T, class U> +std::string Name(std::pair<T, U>*) { + if (output() == OutputStyle::kBenchmark) + return absl::StrCat("P_", Name<T>(), "_", Name<U>()); + return absl::StrCat("P<", Name<T>(), ",", Name<U>(), ">"); +} + +template <class T> +std::string Name(Sequential<T>*) { + return "Sequential"; +} + +template <class T, int P> +std::string Name(AlmostSequential<T, P>*) { + return absl::StrCat("AlmostSeq_", P); +} + +template <class T> +std::string Name(Random<T, Uniform>*) { + return "UnifRand"; +} + +template <class T> +std::string Name(Random<T, Gaussian>*) { + return "GausRand"; +} + +template <class T> +std::string Name(Random<T, Zipf>*) { + return "ZipfRand"; +} + +template <typename T> +std::string Name() { + return Name(static_cast<T*>(nullptr)); +} + +constexpr int kNameWidth = 15; +constexpr int kDistWidth = 16; + +bool CanRunBenchmark(absl::string_view name) { + static std::regex* const filter = []() -> std::regex* { + return benchmarks.empty() || benchmarks == "all" + ? nullptr + : new std::regex(std::string(benchmarks)); + }(); + return filter == nullptr || std::regex_search(std::string(name), *filter); +} + +struct Result { + std::string name; + std::string dist_name; + Ratios ratios; +}; + +template <typename T, typename Dist> +void RunForTypeAndDistribution(std::vector<Result>& results) { + std::string name = absl::StrCat(Name<T>(), "/", Name<Dist>()); + // We have to check against all three names (min/avg/max) before we run it. + // If any of them is enabled, we run it. + if (!CanRunBenchmark(absl::StrCat(name, "/min")) && + !CanRunBenchmark(absl::StrCat(name, "/avg")) && + !CanRunBenchmark(absl::StrCat(name, "/max"))) { + return; + } + results.push_back({Name<T>(), Name<Dist>(), CollectMeanProbeLengths<Dist>()}); +} + +template <class T> +void RunForType(std::vector<Result>& results) { + RunForTypeAndDistribution<T, Sequential<T>>(results); + RunForTypeAndDistribution<T, AlmostSequential<T, 20>>(results); + RunForTypeAndDistribution<T, AlmostSequential<T, 50>>(results); + RunForTypeAndDistribution<T, Random<T, Uniform>>(results); +#ifdef NDEBUG + // Disable these in non-opt mode because they take too long. + RunForTypeAndDistribution<T, Random<T, Gaussian>>(results); + RunForTypeAndDistribution<T, Random<T, Zipf>>(results); +#endif // NDEBUG +} + +} // namespace + +int main(int argc, char** argv) { + // Parse the benchmark flags. Ignore all of them except the regex pattern. + for (int i = 1; i < argc; ++i) { + absl::string_view arg = argv[i]; + const auto next = [&] { return argv[std::min(i + 1, argc - 1)]; }; + + if (absl::ConsumePrefix(&arg, "--benchmark_filter")) { + if (arg == "") { + // --benchmark_filter X + benchmarks = next(); + } else if (absl::ConsumePrefix(&arg, "=")) { + // --benchmark_filter=X + benchmarks = arg; + } + } + + // Any --benchmark flag turns on the mode. + if (absl::ConsumePrefix(&arg, "--benchmark")) { + if (benchmarks.empty()) benchmarks="all"; + } + } + + std::vector<Result> results; + RunForType<uint64_t>(results); + RunForType<IntIdentity>(results); + RunForType<Ptr<8>*>(results); + RunForType<Ptr<16>*>(results); + RunForType<Ptr<32>*>(results); + RunForType<Ptr<64>*>(results); + RunForType<PtrIdentity<8>>(results); + RunForType<PtrIdentity<16>>(results); + RunForType<PtrIdentity<32>>(results); + RunForType<PtrIdentity<64>>(results); + RunForType<std::pair<uint32_t, uint32_t>>(results); + RunForType<String<true>>(results); + RunForType<String<false>>(results); + RunForType<std::pair<uint64_t, String<true>>>(results); + RunForType<std::pair<String<true>, uint64_t>>(results); + RunForType<std::pair<uint64_t, String<false>>>(results); + RunForType<std::pair<String<false>, uint64_t>>(results); + + switch (output()) { + case OutputStyle::kRegular: + absl::PrintF("%-*s%-*s Min Avg Max\n%s\n", kNameWidth, + "Type", kDistWidth, "Distribution", + std::string(kNameWidth + kDistWidth + 10 * 3, '-')); + for (const auto& result : results) { + absl::PrintF("%-*s%-*s %8.4f %8.4f %8.4f\n", kNameWidth, result.name, + kDistWidth, result.dist_name, result.ratios.min_load, + result.ratios.avg_load, result.ratios.max_load); + } + break; + case OutputStyle::kBenchmark: { + absl::PrintF("{\n"); + absl::PrintF(" \"benchmarks\": [\n"); + absl::string_view comma; + for (const auto& result : results) { + auto print = [&](absl::string_view stat, double Ratios::*val) { + std::string name = + absl::StrCat(result.name, "/", result.dist_name, "/", stat); + // Check the regex again. We might had have enabled only one of the + // stats for the benchmark. + if (!CanRunBenchmark(name)) return; + absl::PrintF(" %s{\n", comma); + absl::PrintF(" \"cpu_time\": %f,\n", 1e9 * result.ratios.*val); + absl::PrintF(" \"real_time\": %f,\n", 1e9 * result.ratios.*val); + absl::PrintF(" \"iterations\": 1,\n"); + absl::PrintF(" \"name\": \"%s\",\n", name); + absl::PrintF(" \"time_unit\": \"ns\"\n"); + absl::PrintF(" }\n"); + comma = ","; + }; + print("min", &Ratios::min_load); + print("avg", &Ratios::avg_load); + print("max", &Ratios::max_load); + } + absl::PrintF(" ],\n"); + absl::PrintF(" \"context\": {\n"); + absl::PrintF(" }\n"); + absl::PrintF("}\n"); + break; + } + } + + return 0; +} diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index f5ae83c4..7dac65a0 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -14,6 +14,7 @@ #include "absl/container/internal/raw_hash_set.h" +#include <atomic> #include <cmath> #include <cstdint> #include <deque> @@ -22,6 +23,8 @@ #include <numeric> #include <random> #include <string> +#include <unordered_map> +#include <unordered_set> #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -48,11 +51,10 @@ struct RawHashSetTestOnlyAccess { namespace { -using ::testing::DoubleNear; using ::testing::ElementsAre; +using ::testing::Eq; using ::testing::Ge; using ::testing::Lt; -using ::testing::Optional; using ::testing::Pair; using ::testing::UnorderedElementsAre; @@ -75,8 +77,14 @@ TEST(Util, GrowthAndCapacity) { for (size_t growth = 0; growth < 10000; ++growth) { SCOPED_TRACE(growth); size_t capacity = NormalizeCapacity(GrowthToLowerboundCapacity(growth)); - // The capacity is large enough for `growth` + // The capacity is large enough for `growth`. EXPECT_THAT(CapacityToGrowth(capacity), Ge(growth)); + // For (capacity+1) < kWidth, growth should equal capacity. + if (capacity + 1 < Group::kWidth) { + EXPECT_THAT(CapacityToGrowth(capacity), Eq(capacity)); + } else { + EXPECT_THAT(CapacityToGrowth(capacity), Lt(capacity)); + } if (growth != 0 && capacity > 1) { // There is no smaller capacity that works. EXPECT_THAT(CapacityToGrowth(capacity / 2), Lt(growth)); @@ -250,25 +258,43 @@ TEST(Group, CountLeadingEmptyOrDeleted) { } } -struct IntPolicy { - using slot_type = int64_t; - using key_type = int64_t; - using init_type = int64_t; +template <class T> +struct ValuePolicy { + using slot_type = T; + using key_type = T; + using init_type = T; - static void construct(void*, int64_t* slot, int64_t v) { *slot = v; } - static void destroy(void*, int64_t*) {} - static void transfer(void*, int64_t* new_slot, int64_t* old_slot) { - *new_slot = *old_slot; + template <class Allocator, class... Args> + static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { + absl::allocator_traits<Allocator>::construct(*alloc, slot, + std::forward<Args>(args)...); } - static int64_t& element(slot_type* slot) { return *slot; } + template <class Allocator> + static void destroy(Allocator* alloc, slot_type* slot) { + absl::allocator_traits<Allocator>::destroy(*alloc, slot); + } - template <class F> - static auto apply(F&& f, int64_t x) -> decltype(std::forward<F>(f)(x, x)) { - return std::forward<F>(f)(x, x); + template <class Allocator> + static void transfer(Allocator* alloc, slot_type* new_slot, + slot_type* old_slot) { + construct(alloc, new_slot, std::move(*old_slot)); + destroy(alloc, old_slot); + } + + static T& element(slot_type* slot) { return *slot; } + + template <class F, class... Args> + static decltype(absl::container_internal::DecomposeValue( + std::declval<F>(), std::declval<Args>()...)) + apply(F&& f, Args&&... args) { + return absl::container_internal::DecomposeValue( + std::forward<F>(f), std::forward<Args>(args)...); } }; +using IntPolicy = ValuePolicy<int64_t>; + class StringPolicy { template <class F, class K, class V, class = typename std::enable_if< @@ -393,6 +419,13 @@ TEST(Table, EmptyFunctorOptimization) { size_t growth_left; void* infoz; }; + struct MockTableInfozDisabled { + void* ctrl; + void* slots; + size_t size; + size_t capacity; + size_t growth_left; + }; struct StatelessHash { size_t operator()(absl::string_view) const { return 0; } }; @@ -400,17 +433,27 @@ TEST(Table, EmptyFunctorOptimization) { size_t dummy; }; - EXPECT_EQ( - sizeof(MockTable), - sizeof( - raw_hash_set<StringPolicy, StatelessHash, - std::equal_to<absl::string_view>, std::allocator<int>>)); + if (std::is_empty<HashtablezInfoHandle>::value) { + EXPECT_EQ(sizeof(MockTableInfozDisabled), + sizeof(raw_hash_set<StringPolicy, StatelessHash, + std::equal_to<absl::string_view>, + std::allocator<int>>)); - EXPECT_EQ( - sizeof(MockTable) + sizeof(StatefulHash), - sizeof( - raw_hash_set<StringPolicy, StatefulHash, - std::equal_to<absl::string_view>, std::allocator<int>>)); + EXPECT_EQ(sizeof(MockTableInfozDisabled) + sizeof(StatefulHash), + sizeof(raw_hash_set<StringPolicy, StatefulHash, + std::equal_to<absl::string_view>, + std::allocator<int>>)); + } else { + EXPECT_EQ(sizeof(MockTable), + sizeof(raw_hash_set<StringPolicy, StatelessHash, + std::equal_to<absl::string_view>, + std::allocator<int>>)); + + EXPECT_EQ(sizeof(MockTable) + sizeof(StatefulHash), + sizeof(raw_hash_set<StringPolicy, StatefulHash, + std::equal_to<absl::string_view>, + std::allocator<int>>)); + } } TEST(Table, Empty) { @@ -847,7 +890,8 @@ TEST(Table, EraseMaintainsValidIterator) { std::vector<int64_t> CollectBadMergeKeys(size_t N) { static constexpr int kGroupSize = Group::kWidth - 1; - auto topk_range = [](size_t b, size_t e, IntTable* t) -> std::vector<int64_t> { + auto topk_range = [](size_t b, size_t e, + IntTable* t) -> std::vector<int64_t> { for (size_t i = b; i != e; ++i) { t->emplace(i); } @@ -1001,8 +1045,8 @@ using ProbeStatsPerSize = std::map<size_t, ProbeStats>; // 1. Create new table and reserve it to keys.size() * 2 // 2. Insert all keys xored with seed // 3. Collect ProbeStats from final table. -ProbeStats CollectProbeStatsOnKeysXoredWithSeed(const std::vector<int64_t>& keys, - size_t num_iters) { +ProbeStats CollectProbeStatsOnKeysXoredWithSeed( + const std::vector<int64_t>& keys, size_t num_iters) { const size_t reserve_size = keys.size() * 2; ProbeStats stats; @@ -1656,6 +1700,38 @@ TEST(Table, Merge) { EXPECT_THAT(t2, UnorderedElementsAre(Pair("0", "~0"))); } +TEST(Table, IteratorEmplaceConstructibleRequirement) { + struct Value { + explicit Value(absl::string_view view) : value(view) {} + std::string value; + + bool operator==(const Value& other) const { return value == other.value; } + }; + struct H { + size_t operator()(const Value& v) const { + return absl::Hash<std::string>{}(v.value); + } + }; + + struct Table : raw_hash_set<ValuePolicy<Value>, H, std::equal_to<Value>, + std::allocator<Value>> { + using Base = typename Table::raw_hash_set; + using Base::Base; + }; + + std::string input[3]{"A", "B", "C"}; + + Table t(std::begin(input), std::end(input)); + EXPECT_THAT(t, UnorderedElementsAre(Value{"A"}, Value{"B"}, Value{"C"})); + + input[0] = "D"; + input[1] = "E"; + input[2] = "F"; + t.insert(std::begin(input), std::end(input)); + EXPECT_THAT(t, UnorderedElementsAre(Value{"A"}, Value{"B"}, Value{"C"}, + Value{"D"}, Value{"E"}, Value{"F"})); +} + TEST(Nodes, EmptyNodeType) { using node_type = StringTable::node_type; node_type n; @@ -1710,6 +1786,26 @@ TEST(Nodes, ExtractInsert) { EXPECT_FALSE(node); } +TEST(Nodes, HintInsert) { + IntTable t = {1, 2, 3}; + auto node = t.extract(1); + EXPECT_THAT(t, UnorderedElementsAre(2, 3)); + auto it = t.insert(t.begin(), std::move(node)); + EXPECT_THAT(t, UnorderedElementsAre(1, 2, 3)); + EXPECT_EQ(*it, 1); + EXPECT_FALSE(node); + + node = t.extract(2); + EXPECT_THAT(t, UnorderedElementsAre(1, 3)); + // reinsert 2 to make the next insert fail. + t.insert(2); + EXPECT_THAT(t, UnorderedElementsAre(1, 2, 3)); + it = t.insert(t.begin(), std::move(node)); + EXPECT_EQ(*it, 2); + // The node was not emptied by the insert call. + EXPECT_TRUE(node); +} + IntTable MakeSimpleTable(size_t size) { IntTable t; while (t.size() < size) t.insert(t.size()); @@ -1804,18 +1900,34 @@ TEST(RawHashSamplerTest, Sample) { auto& sampler = HashtablezSampler::Global(); size_t start_size = 0; - start_size += sampler.Iterate([&](const HashtablezInfo&) { ++start_size; }); + std::unordered_set<const HashtablezInfo*> preexisting_info; + start_size += sampler.Iterate([&](const HashtablezInfo& info) { + preexisting_info.insert(&info); + ++start_size; + }); std::vector<IntTable> tables; for (int i = 0; i < 1000000; ++i) { tables.emplace_back(); tables.back().insert(1); + tables.back().insert(i % 5); } size_t end_size = 0; - end_size += sampler.Iterate([&](const HashtablezInfo&) { ++end_size; }); + std::unordered_map<size_t, int> observed_checksums; + end_size += sampler.Iterate([&](const HashtablezInfo& info) { + if (preexisting_info.count(&info) == 0) { + observed_checksums[info.hashes_bitwise_xor.load( + std::memory_order_relaxed)]++; + } + ++end_size; + }); EXPECT_NEAR((end_size - start_size) / static_cast<double>(tables.size()), 0.01, 0.005); + EXPECT_EQ(observed_checksums.size(), 5); + for (const auto& [_, count] : observed_checksums) { + EXPECT_NEAR((100 * count) / static_cast<double>(tables.size()), 0.2, 0.05); + } } #endif // ABSL_INTERNAL_HASHTABLEZ_SAMPLE diff --git a/absl/container/internal/unordered_map_constructor_test.h b/absl/container/internal/unordered_map_constructor_test.h index 76ee95e6..3f90ad7c 100644 --- a/absl/container/internal/unordered_map_constructor_test.h +++ b/absl/container/internal/unordered_map_constructor_test.h @@ -16,6 +16,7 @@ #define ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_CONSTRUCTOR_TEST_H_ #include <algorithm> +#include <unordered_map> #include <vector> #include "gmock/gmock.h" |