From e80c0b3536e1bdee68a874d529a9ba951faffe8b Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Fri, 27 Nov 2020 09:40:56 -0800 Subject: Export of internal Abseil changes -- d85f04af95a6fdafb102f7dc393d78d4431b47e5 by Abseil Team : Internal change for cord ring PiperOrigin-RevId: 344541196 -- 1ff57908e31a09ec0c98d8316da1263092cc3a1c by Abseil Team : Fix typo in comment. PiperOrigin-RevId: 344214280 GitOrigin-RevId: d85f04af95a6fdafb102f7dc393d78d4431b47e5 Change-Id: I58b3c28f62a5d10dd665b17d58a121f371e1260a --- absl/strings/internal/cord_rep_flat.h | 129 ++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 absl/strings/internal/cord_rep_flat.h (limited to 'absl/strings/internal/cord_rep_flat.h') diff --git a/absl/strings/internal/cord_rep_flat.h b/absl/strings/internal/cord_rep_flat.h new file mode 100644 index 00000000..3e2cd33c --- /dev/null +++ b/absl/strings/internal/cord_rep_flat.h @@ -0,0 +1,129 @@ +// Copyright 2020 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORD_REP_FLAT_H_ +#define ABSL_STRINGS_INTERNAL_CORD_REP_FLAT_H_ + +#include +#include +#include +#include + +#include "absl/strings/internal/cord_internal.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// Note: all constants below are never ODR used and internal to cord, we define +// these as static constexpr to avoid 'in struct' definition and usage clutter. + +// Largest and smallest flat node lengths we are willing to allocate +// Flat allocation size is stored in tag, which currently can encode sizes up +// to 4K, encoded as multiple of either 8 or 32 bytes. +// If we allow for larger sizes, we need to change this to 8/64, 16/128, etc. +// kMinFlatSize is bounded by tag needing to be at least FLAT * 8 bytes, and +// ideally a 'nice' size aligning with allocation and cacheline sizes like 32. +// kMaxFlatSize is bounded by the size resulting in a computed tag no greater +// than MAX_FLAT_TAG. MAX_FLAT_TAG provides for additional 'high' tag values. +static constexpr size_t kFlatOverhead = offsetof(CordRep, data); +static constexpr size_t kMinFlatSize = 32; +static constexpr size_t kMaxFlatSize = 4096; +static constexpr size_t kMaxFlatLength = kMaxFlatSize - kFlatOverhead; +static constexpr size_t kMinFlatLength = kMinFlatSize - kFlatOverhead; + +static constexpr size_t AllocatedSizeToTagUnchecked(size_t size) { + return (size <= 1024) ? size / 8 : 128 + size / 32 - 1024 / 32; +} + +static_assert(kMinFlatSize / 8 >= FLAT, ""); +static_assert(AllocatedSizeToTagUnchecked(kMaxFlatSize) <= MAX_FLAT_TAG, ""); + +// Helper functions for rounded div, and rounding to exact sizes. +static size_t DivUp(size_t n, size_t m) { return (n + m - 1) / m; } +static size_t RoundUp(size_t n, size_t m) { return DivUp(n, m) * m; } + +// Returns the size to the nearest equal or larger value that can be +// expressed exactly as a tag value. +static size_t RoundUpForTag(size_t size) { + return RoundUp(size, (size <= 1024) ? 8 : 32); +} + +// Converts the allocated size to a tag, rounding down if the size +// does not exactly match a 'tag expressible' size value. The result is +// undefined if the size exceeds the maximum size that can be encoded in +// a tag, i.e., if size is larger than TagToAllocatedSize(). +static uint8_t AllocatedSizeToTag(size_t size) { + const size_t tag = AllocatedSizeToTagUnchecked(size); + assert(tag <= MAX_FLAT_TAG); + return tag; +} + +// Converts the provided tag to the corresponding allocated size +static constexpr size_t TagToAllocatedSize(uint8_t tag) { + return (tag <= 128) ? (tag * 8) : (1024 + (tag - 128) * 32); +} + +// Converts the provided tag to the corresponding available data length +static constexpr size_t TagToLength(uint8_t tag) { + return TagToAllocatedSize(tag) - kFlatOverhead; +} + +// Enforce that kMaxFlatSize maps to a well-known exact tag value. +static_assert(TagToAllocatedSize(224) == kMaxFlatSize, "Bad tag logic"); + +struct CordRepFlat : public CordRep { + // Creates a new flat node. + static CordRepFlat* New(size_t len) { + if (len <= kMinFlatLength) { + len = kMinFlatLength; + } else if (len > kMaxFlatLength) { + len = kMaxFlatLength; + } + + // Round size up so it matches a size we can exactly express in a tag. + const size_t size = RoundUpForTag(len + kFlatOverhead); + void* const raw_rep = ::operator new(size); + CordRepFlat* rep = new (raw_rep) CordRepFlat(); + rep->tag = AllocatedSizeToTag(size); + return rep; + } + + // Deletes a CordRepFlat instance created previously through a call to New(). + // Flat CordReps are allocated and constructed with raw ::operator new and + // placement new, and must be destructed and deallocated accordingly. + static void Delete(CordRep*rep) { + assert(rep->tag >= FLAT); +#if defined(__cpp_sized_deallocation) + size_t size = TagToAllocatedSize(rep->tag); + rep->~CordRep(); + ::operator delete(rep, size); +#else + rep->~CordRep(); + ::operator delete(rep); +#endif + } + + // Returns the maximum capacity (payload size) of this instance. + size_t Capacity() const { return TagToLength(tag); } + + // Returns the allocated size (payload + overhead) of this instance. + size_t AllocatedSize() const { return TagToAllocatedSize(tag); } +}; + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORD_REP_FLAT_H_ -- cgit v1.2.3 From fbdff6f3ae0ba977a69f172e85ecaede535e70f6 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Mon, 7 Dec 2020 09:56:14 -0800 Subject: Export of internal Abseil changes -- ff793052bd01e1e4fcf639f94d7c30c4855a9372 by Evan Brown : Roll forward of btree_iterator refactoring. PiperOrigin-RevId: 346116047 -- 17984679f16e3e2139b0f14fa76f4a6ca16a3ef9 by Chris Kennelly : Extend absl::StrContains to accept single character needles. Single characters are more efficient to search for. Extending this API allows the abseil-string-find-str-contains Clang Tidy to include this pattern. The C++ committee has adopted http://wg21.link/P1679 for inclusion in C++23. PiperOrigin-RevId: 346095060 -- ef20b31c501b1dcaa25e244fd8f8aa43dec09bd6 by Jorg Brown : Internal change for cord ring PiperOrigin-RevId: 346087545 -- b70f2c1cb77fc9e733a126e790967d45c5fd1dc7 by Derek Mauro : Release layout_benchmark PiperOrigin-RevId: 345968909 -- 3a0eda337ee43622f92cfe14c2aa06f72dc71ee5 by Derek Mauro : Release raw_hash_set_probe_benchmark PiperOrigin-RevId: 345965969 -- abffdb4bb241a2264cb4e73a6262b660bb10447d by Derek Mauro : Internal change PiperOrigin-RevId: 345733599 -- 7c9e24a71188df945be17fe98f700bdb51f81b16 by Derek Mauro : Release hash_benchmark PiperOrigin-RevId: 345721635 -- d68f33f17f9a8cd3f6da8eee3870bdb46402cdc8 by Derek Mauro : Release raw_hash_set_benchmark PiperOrigin-RevId: 345708384 -- 6e6c547d4d1327b226c0ffe8ff34d0aa103ce24b by Abseil Team : Updates the implementation of InlinedVector to accurately express the value-initialization semantics of the default constructor PiperOrigin-RevId: 345548260 -- 1532424deda97d468444c217cc0fa4614099c7c1 by Evan Brown : Rollback btree_iterator refactoring. PiperOrigin-RevId: 345543900 GitOrigin-RevId: ff793052bd01e1e4fcf639f94d7c30c4855a9372 Change-Id: I719831981fd056de41939f9addfee3d85e3b49b2 --- WORKSPACE | 6 +- absl/container/BUILD.bazel | 55 ++ absl/container/internal/inlined_vector.h | 5 +- absl/container/internal/layout_benchmark.cc | 122 +++++ absl/container/internal/raw_hash_set_benchmark.cc | 396 ++++++++++++++ .../internal/raw_hash_set_probe_benchmark.cc | 590 +++++++++++++++++++++ absl/hash/BUILD.bazel | 18 + absl/hash/hash_benchmark.cc | 249 +++++++++ absl/strings/cord.cc | 22 +- absl/strings/internal/cord_internal.h | 2 + absl/strings/internal/cord_rep_flat.h | 14 +- absl/strings/match.h | 4 + absl/strings/match_test.cc | 17 + 13 files changed, 1481 insertions(+), 19 deletions(-) create mode 100644 absl/container/internal/layout_benchmark.cc create mode 100644 absl/container/internal/raw_hash_set_benchmark.cc create mode 100644 absl/container/internal/raw_hash_set_probe_benchmark.cc create mode 100644 absl/hash/hash_benchmark.cc (limited to 'absl/strings/internal/cord_rep_flat.h') diff --git a/WORKSPACE b/WORKSPACE index ed90d2ba..258d23b5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -29,9 +29,9 @@ http_archive( # Google benchmark. http_archive( name = "com_github_google_benchmark", - urls = ["https://github.com/google/benchmark/archive/16703ff83c1ae6d53e5155df3bb3ab0bc96083be.zip"], - strip_prefix = "benchmark-16703ff83c1ae6d53e5155df3bb3ab0bc96083be", - sha256 = "59f918c8ccd4d74b6ac43484467b500f1d64b40cc1010daa055375b322a43ba3", + urls = ["https://github.com/google/benchmark/archive/bf585a2789e30585b4e3ce6baf11ef2750b54677.zip"], # 2020-11-26T11:14:03Z + strip_prefix = "benchmark-bf585a2789e30585b4e3ce6baf11ef2750b54677", + sha256 = "2a778d821997df7d8646c9c59b8edb9a573a6e04c534c01892a40aa524a7b68c", ) # C++ rules for Bazel. diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 8e72ad03..70977143 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -630,6 +630,45 @@ cc_test( ], ) +cc_binary( + name = "raw_hash_set_benchmark", + testonly = 1, + srcs = ["internal/raw_hash_set_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":hash_function_defaults", + ":raw_hash_set", + "//absl/base:raw_logging_internal", + "//absl/strings:str_format", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_binary( + name = "raw_hash_set_probe_benchmark", + testonly = 1, + srcs = ["internal/raw_hash_set_probe_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = select({ + "//conditions:default": [], + }) + ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":flat_hash_map", + ":hash_function_defaults", + ":hashtable_debug", + ":raw_hash_set", + "//absl/random", + "//absl/random:distributions", + "//absl/strings", + "//absl/strings:str_format", + ], +) + cc_test( name = "raw_hash_set_allocator_test", size = "small", @@ -677,6 +716,22 @@ cc_test( ], ) +cc_binary( + name = "layout_benchmark", + testonly = 1, + srcs = ["internal/layout_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":layout", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "@com_github_google_benchmark//:benchmark_main", + ], +) + cc_library( name = "tracked", testonly = 1, diff --git a/absl/container/internal/inlined_vector.h b/absl/container/internal/inlined_vector.h index c98c25c4..502ea3dd 100644 --- a/absl/container/internal/inlined_vector.h +++ b/absl/container/internal/inlined_vector.h @@ -298,9 +298,10 @@ class Storage { // Storage Constructors and Destructor // --------------------------------------------------------------------------- - Storage() : metadata_() {} + Storage() : metadata_(allocator_type(), /* size and is_allocated */ 0) {} - explicit Storage(const allocator_type& alloc) : metadata_(alloc, {}) {} + explicit Storage(const allocator_type& alloc) + : metadata_(alloc, /* size and is_allocated */ 0) {} ~Storage() { pointer data = GetIsAllocated() ? GetAllocatedData() : GetInlinedData(); diff --git a/absl/container/internal/layout_benchmark.cc b/absl/container/internal/layout_benchmark.cc new file mode 100644 index 00000000..d8636e8d --- /dev/null +++ b/absl/container/internal/layout_benchmark.cc @@ -0,0 +1,122 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Every benchmark should have the same performance as the corresponding +// headroom benchmark. + +#include "absl/base/internal/raw_logging.h" +#include "absl/container/internal/layout.h" +#include "benchmark/benchmark.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +using ::benchmark::DoNotOptimize; + +using Int128 = int64_t[2]; + +// This benchmark provides the upper bound on performance for BM_OffsetConstant. +template +void BM_OffsetConstantHeadroom(benchmark::State& state) { + for (auto _ : state) { + DoNotOptimize(Offset); + } +} + +template +void BM_OffsetConstant(benchmark::State& state) { + using L = Layout; + ABSL_RAW_CHECK(L::Partial(3, 5, 7).template Offset<3>() == Offset, + "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(L::Partial(3, 5, 7).template Offset<3>()); + } +} + +template +size_t VariableOffset(size_t n, size_t m, size_t k); + +template <> +size_t VariableOffset(size_t n, size_t m, + size_t k) { + auto Align = [](size_t n, size_t m) { return (n + m - 1) & ~(m - 1); }; + return Align(Align(Align(n * 1, 2) + m * 2, 4) + k * 4, 8); +} + +template <> +size_t VariableOffset(size_t n, size_t m, + size_t k) { + // No alignment is necessary. + return n * 16 + m * 4 + k * 2; +} + +// This benchmark provides the upper bound on performance for BM_OffsetVariable. +template +void BM_OffsetVariableHeadroom(benchmark::State& state) { + size_t n = 3; + size_t m = 5; + size_t k = 7; + ABSL_RAW_CHECK(VariableOffset(n, m, k) == Offset, "Invalid offset"); + for (auto _ : state) { + DoNotOptimize(n); + DoNotOptimize(m); + DoNotOptimize(k); + DoNotOptimize(VariableOffset(n, m, k)); + } +} + +template +void BM_OffsetVariable(benchmark::State& state) { + using L = Layout; + size_t n = 3; + size_t m = 5; + size_t k = 7; + ABSL_RAW_CHECK(L::Partial(n, m, k).template Offset<3>() == Offset, + "Inavlid offset"); + for (auto _ : state) { + DoNotOptimize(n); + DoNotOptimize(m); + DoNotOptimize(k); + DoNotOptimize(L::Partial(n, m, k).template Offset<3>()); + } +} + +// Run all benchmarks in two modes: +// +// Layout with padding: int8_t[3], int16_t[5], int32_t[7], Int128[?]. +// Layout without padding: Int128[3], int32_t[5], int16_t[7], int8_t[?]. + +#define OFFSET_BENCHMARK(NAME, OFFSET, T1, T2, T3, T4) \ + auto& NAME##_##OFFSET##_##T1##_##T2##_##T3##_##T4 = \ + NAME; \ + BENCHMARK(NAME##_##OFFSET##_##T1##_##T2##_##T3##_##T4) + +OFFSET_BENCHMARK(BM_OffsetConstantHeadroom, 48, int8_t, int16_t, int32_t, + Int128); +OFFSET_BENCHMARK(BM_OffsetConstant, 48, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_OffsetConstantHeadroom, 82, Int128, int32_t, int16_t, + int8_t); +OFFSET_BENCHMARK(BM_OffsetConstant, 82, Int128, int32_t, int16_t, int8_t); +OFFSET_BENCHMARK(BM_OffsetVariableHeadroom, 48, int8_t, int16_t, int32_t, + Int128); +OFFSET_BENCHMARK(BM_OffsetVariable, 48, int8_t, int16_t, int32_t, Int128); +OFFSET_BENCHMARK(BM_OffsetVariableHeadroom, 82, Int128, int32_t, int16_t, + int8_t); +OFFSET_BENCHMARK(BM_OffsetVariable, 82, Int128, int32_t, int16_t, int8_t); +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/container/internal/raw_hash_set_benchmark.cc b/absl/container/internal/raw_hash_set_benchmark.cc new file mode 100644 index 00000000..f9be2c5a --- /dev/null +++ b/absl/container/internal/raw_hash_set_benchmark.cc @@ -0,0 +1,396 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/raw_hash_set.h" + +#include +#include + +#include "absl/base/internal/raw_logging.h" +#include "absl/container/internal/hash_function_defaults.h" +#include "absl/strings/str_format.h" +#include "benchmark/benchmark.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +struct RawHashSetTestOnlyAccess { + template + static auto GetSlots(const C& c) -> decltype(c.slots_) { + return c.slots_; + } +}; + +namespace { + +struct IntPolicy { + using slot_type = int64_t; + using key_type = int64_t; + using init_type = int64_t; + + static void construct(void*, int64_t* slot, int64_t v) { *slot = v; } + static void destroy(void*, int64_t*) {} + static void transfer(void*, int64_t* new_slot, int64_t* old_slot) { + *new_slot = *old_slot; + } + + static int64_t& element(slot_type* slot) { return *slot; } + + template + static auto apply(F&& f, int64_t x) -> decltype(std::forward(f)(x, x)) { + return std::forward(f)(x, x); + } +}; + +class StringPolicy { + template ::value>::type> + decltype(std::declval()( + std::declval(), std::piecewise_construct, + std::declval>(), + std::declval())) static apply_impl(F&& f, + std::pair, V> p) { + const absl::string_view& key = std::get<0>(p.first); + return std::forward(f)(key, std::piecewise_construct, std::move(p.first), + std::move(p.second)); + } + + public: + struct slot_type { + struct ctor {}; + + template + slot_type(ctor, Ts&&... ts) : pair(std::forward(ts)...) {} + + std::pair pair; + }; + + using key_type = std::string; + using init_type = std::pair; + + template + static void construct(allocator_type* alloc, slot_type* slot, Args... args) { + std::allocator_traits::construct( + *alloc, slot, typename slot_type::ctor(), std::forward(args)...); + } + + template + static void destroy(allocator_type* alloc, slot_type* slot) { + std::allocator_traits::destroy(*alloc, slot); + } + + template + static void transfer(allocator_type* alloc, slot_type* new_slot, + slot_type* old_slot) { + construct(alloc, new_slot, std::move(old_slot->pair)); + destroy(alloc, old_slot); + } + + static std::pair& element(slot_type* slot) { + return slot->pair; + } + + template + static auto apply(F&& f, Args&&... args) + -> decltype(apply_impl(std::forward(f), + PairArgs(std::forward(args)...))) { + return apply_impl(std::forward(f), + PairArgs(std::forward(args)...)); + } +}; + +struct StringHash : container_internal::hash_default_hash { + using is_transparent = void; +}; +struct StringEq : std::equal_to { + using is_transparent = void; +}; + +struct StringTable + : raw_hash_set> { + using Base = typename StringTable::raw_hash_set; + StringTable() {} + using Base::Base; +}; + +struct IntTable + : raw_hash_set, + std::equal_to, std::allocator> { + using Base = typename IntTable::raw_hash_set; + IntTable() {} + using Base::Base; +}; + +struct string_generator { + template + std::string operator()(RNG& rng) const { + std::string res; + res.resize(12); + std::uniform_int_distribution printable_ascii(0x20, 0x7E); + std::generate(res.begin(), res.end(), [&] { return printable_ascii(rng); }); + return res; + } + + size_t size; +}; + +// Model a cache in steady state. +// +// On a table of size N, keep deleting the LRU entry and add a random one. +void BM_CacheInSteadyState(benchmark::State& state) { + std::random_device rd; + std::mt19937 rng(rd()); + string_generator gen{12}; + StringTable t; + std::deque keys; + while (t.size() < state.range(0)) { + auto x = t.emplace(gen(rng), gen(rng)); + if (x.second) keys.push_back(x.first->first); + } + ABSL_RAW_CHECK(state.range(0) >= 10, ""); + while (state.KeepRunning()) { + // Some cache hits. + std::deque::const_iterator it; + for (int i = 0; i != 90; ++i) { + if (i % 10 == 0) it = keys.end(); + ::benchmark::DoNotOptimize(t.find(*--it)); + } + // Some cache misses. + for (int i = 0; i != 10; ++i) ::benchmark::DoNotOptimize(t.find(gen(rng))); + ABSL_RAW_CHECK(t.erase(keys.front()), keys.front().c_str()); + keys.pop_front(); + while (true) { + auto x = t.emplace(gen(rng), gen(rng)); + if (x.second) { + keys.push_back(x.first->first); + break; + } + } + } + state.SetItemsProcessed(state.iterations()); + state.SetLabel(absl::StrFormat("load_factor=%.2f", t.load_factor())); +} + +template +void CacheInSteadyStateArgs(Benchmark* bm) { + // The default. + const float max_load_factor = 0.875; + // When the cache is at the steady state, the probe sequence will equal + // capacity if there is no reclamation of deleted slots. Pick a number large + // enough to make the benchmark slow for that case. + const size_t capacity = 1 << 10; + + // Check N data points to cover load factors in [0.4, 0.8). + const size_t kNumPoints = 10; + for (size_t i = 0; i != kNumPoints; ++i) + bm->Arg(std::ceil( + capacity * (max_load_factor + i * max_load_factor / kNumPoints) / 2)); +} +BENCHMARK(BM_CacheInSteadyState)->Apply(CacheInSteadyStateArgs); + +void BM_EndComparison(benchmark::State& state) { + std::random_device rd; + std::mt19937 rng(rd()); + string_generator gen{12}; + StringTable t; + while (t.size() < state.range(0)) { + t.emplace(gen(rng), gen(rng)); + } + + for (auto _ : state) { + for (auto it = t.begin(); it != t.end(); ++it) { + benchmark::DoNotOptimize(it); + benchmark::DoNotOptimize(t); + benchmark::DoNotOptimize(it != t.end()); + } + } +} +BENCHMARK(BM_EndComparison)->Arg(400); + +void BM_CopyCtor(benchmark::State& state) { + std::random_device rd; + std::mt19937 rng(rd()); + IntTable t; + std::uniform_int_distribution dist(0, ~uint64_t{}); + + while (t.size() < state.range(0)) { + t.emplace(dist(rng)); + } + + for (auto _ : state) { + IntTable t2 = t; + benchmark::DoNotOptimize(t2); + } +} +BENCHMARK(BM_CopyCtor)->Range(128, 4096); + +void BM_CopyAssign(benchmark::State& state) { + std::random_device rd; + std::mt19937 rng(rd()); + IntTable t; + std::uniform_int_distribution dist(0, ~uint64_t{}); + while (t.size() < state.range(0)) { + t.emplace(dist(rng)); + } + + IntTable t2; + for (auto _ : state) { + t2 = t; + benchmark::DoNotOptimize(t2); + } +} +BENCHMARK(BM_CopyAssign)->Range(128, 4096); + +void BM_NoOpReserveIntTable(benchmark::State& state) { + IntTable t; + t.reserve(100000); + for (auto _ : state) { + benchmark::DoNotOptimize(t); + t.reserve(100000); + } +} +BENCHMARK(BM_NoOpReserveIntTable); + +void BM_NoOpReserveStringTable(benchmark::State& state) { + StringTable t; + t.reserve(100000); + for (auto _ : state) { + benchmark::DoNotOptimize(t); + t.reserve(100000); + } +} +BENCHMARK(BM_NoOpReserveStringTable); + +void BM_ReserveIntTable(benchmark::State& state) { + int reserve_size = state.range(0); + for (auto _ : state) { + state.PauseTiming(); + IntTable t; + state.ResumeTiming(); + benchmark::DoNotOptimize(t); + t.reserve(reserve_size); + } +} +BENCHMARK(BM_ReserveIntTable)->Range(128, 4096); + +void BM_ReserveStringTable(benchmark::State& state) { + int reserve_size = state.range(0); + for (auto _ : state) { + state.PauseTiming(); + StringTable t; + state.ResumeTiming(); + benchmark::DoNotOptimize(t); + t.reserve(reserve_size); + } +} +BENCHMARK(BM_ReserveStringTable)->Range(128, 4096); + +void BM_Group_Match(benchmark::State& state) { + std::array group; + std::iota(group.begin(), group.end(), -4); + Group g{group.data()}; + h2_t h = 1; + for (auto _ : state) { + ::benchmark::DoNotOptimize(h); + ::benchmark::DoNotOptimize(g.Match(h)); + } +} +BENCHMARK(BM_Group_Match); + +void BM_Group_MatchEmpty(benchmark::State& state) { + std::array group; + std::iota(group.begin(), group.end(), -4); + Group g{group.data()}; + for (auto _ : state) ::benchmark::DoNotOptimize(g.MatchEmpty()); +} +BENCHMARK(BM_Group_MatchEmpty); + +void BM_Group_MatchEmptyOrDeleted(benchmark::State& state) { + std::array group; + std::iota(group.begin(), group.end(), -4); + Group g{group.data()}; + for (auto _ : state) ::benchmark::DoNotOptimize(g.MatchEmptyOrDeleted()); +} +BENCHMARK(BM_Group_MatchEmptyOrDeleted); + +void BM_Group_CountLeadingEmptyOrDeleted(benchmark::State& state) { + std::array group; + std::iota(group.begin(), group.end(), -2); + Group g{group.data()}; + for (auto _ : state) + ::benchmark::DoNotOptimize(g.CountLeadingEmptyOrDeleted()); +} +BENCHMARK(BM_Group_CountLeadingEmptyOrDeleted); + +void BM_Group_MatchFirstEmptyOrDeleted(benchmark::State& state) { + std::array group; + std::iota(group.begin(), group.end(), -2); + Group g{group.data()}; + for (auto _ : state) ::benchmark::DoNotOptimize(*g.MatchEmptyOrDeleted()); +} +BENCHMARK(BM_Group_MatchFirstEmptyOrDeleted); + +void BM_DropDeletes(benchmark::State& state) { + constexpr size_t capacity = (1 << 20) - 1; + std::vector ctrl(capacity + 1 + Group::kWidth); + ctrl[capacity] = kSentinel; + std::vector pattern = {kEmpty, 2, kDeleted, 2, kEmpty, 1, kDeleted}; + for (size_t i = 0; i != capacity; ++i) { + ctrl[i] = pattern[i % pattern.size()]; + } + while (state.KeepRunning()) { + state.PauseTiming(); + std::vector ctrl_copy = ctrl; + state.ResumeTiming(); + ConvertDeletedToEmptyAndFullToDeleted(ctrl_copy.data(), capacity); + ::benchmark::DoNotOptimize(ctrl_copy[capacity]); + } +} +BENCHMARK(BM_DropDeletes); + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +// These methods are here to make it easy to examine the assembly for targeted +// parts of the API. +auto CodegenAbslRawHashSetInt64Find(absl::container_internal::IntTable* table, + int64_t key) -> decltype(table->find(key)) { + return table->find(key); +} + +bool CodegenAbslRawHashSetInt64FindNeEnd( + absl::container_internal::IntTable* table, int64_t key) { + return table->find(key) != table->end(); +} + +bool CodegenAbslRawHashSetInt64Contains( + absl::container_internal::IntTable* table, int64_t key) { + return table->contains(key); +} + +void CodegenAbslRawHashSetInt64Iterate( + absl::container_internal::IntTable* table) { + for (auto x : *table) benchmark::DoNotOptimize(x); +} + +int odr = + (::benchmark::DoNotOptimize(std::make_tuple( + &CodegenAbslRawHashSetInt64Find, &CodegenAbslRawHashSetInt64FindNeEnd, + &CodegenAbslRawHashSetInt64Contains, + &CodegenAbslRawHashSetInt64Iterate)), + 1); diff --git a/absl/container/internal/raw_hash_set_probe_benchmark.cc b/absl/container/internal/raw_hash_set_probe_benchmark.cc new file mode 100644 index 00000000..7169a2e2 --- /dev/null +++ b/absl/container/internal/raw_hash_set_probe_benchmark.cc @@ -0,0 +1,590 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Generates probe length statistics for many combinations of key types and key +// distributions, all using the default hash function for swisstable. + +#include +#include // NOLINT +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/container/internal/hash_function_defaults.h" +#include "absl/container/internal/hashtable_debug.h" +#include "absl/container/internal/raw_hash_set.h" +#include "absl/random/distributions.h" +#include "absl/random/random.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" + +namespace { + +enum class OutputStyle { kRegular, kBenchmark }; + +// The --benchmark command line flag. +// This is populated from main(). +// When run in "benchmark" mode, we have different output. This allows +// A/B comparisons with tools like `benchy`. +absl::string_view benchmarks; + +OutputStyle output() { + return !benchmarks.empty() ? OutputStyle::kBenchmark : OutputStyle::kRegular; +} + +template +struct Policy { + using slot_type = T; + using key_type = T; + using init_type = T; + + template + static void construct(allocator_type* alloc, slot_type* slot, + const Arg& arg) { + std::allocator_traits::construct(*alloc, slot, arg); + } + + template + static void destroy(allocator_type* alloc, slot_type* slot) { + std::allocator_traits::destroy(*alloc, slot); + } + + static slot_type& element(slot_type* slot) { return *slot; } + + template + static auto apply(F&& f, const slot_type& arg) + -> decltype(std::forward(f)(arg, arg)) { + return std::forward(f)(arg, arg); + } +}; + +absl::BitGen& GlobalBitGen() { + static auto* value = new absl::BitGen; + return *value; +} + +// Keeps a pool of allocations and randomly gives one out. +// This introduces more randomization to the addresses given to swisstable and +// should help smooth out this factor from probe length calculation. +template +class RandomizedAllocator { + public: + using value_type = T; + + RandomizedAllocator() = default; + template + RandomizedAllocator(RandomizedAllocator) {} // NOLINT + + static T* allocate(size_t n) { + auto& pointers = GetPointers(n); + // Fill the pool + while (pointers.size() < kRandomPool) { + pointers.push_back(std::allocator{}.allocate(n)); + } + + // Choose a random one. + size_t i = absl::Uniform(GlobalBitGen(), 0, pointers.size()); + T* result = pointers[i]; + pointers[i] = pointers.back(); + pointers.pop_back(); + return result; + } + + static void deallocate(T* p, size_t n) { + // Just put it back on the pool. No need to release the memory. + GetPointers(n).push_back(p); + } + + private: + // We keep at least kRandomPool allocations for each size. + static constexpr size_t kRandomPool = 20; + + static std::vector& GetPointers(size_t n) { + static auto* m = new absl::flat_hash_map>(); + return (*m)[n]; + } +}; + +template +struct DefaultHash { + using type = absl::container_internal::hash_default_hash; +}; + +template +using DefaultHashT = typename DefaultHash::type; + +template +struct Table : absl::container_internal::raw_hash_set< + Policy, DefaultHashT, + absl::container_internal::hash_default_eq, + RandomizedAllocator> {}; + +struct LoadSizes { + size_t min_load; + size_t max_load; +}; + +LoadSizes GetMinMaxLoadSizes() { + static const auto sizes = [] { + Table t; + + // First, fill enough to have a good distribution. + constexpr size_t kMinSize = 10000; + while (t.size() < kMinSize) t.insert(t.size()); + + const auto reach_min_load_factor = [&] { + const double lf = t.load_factor(); + while (lf <= t.load_factor()) t.insert(t.size()); + }; + + // Then, insert until we reach min load factor. + reach_min_load_factor(); + const size_t min_load_size = t.size(); + + // Keep going until we hit min load factor again, then go back one. + t.insert(t.size()); + reach_min_load_factor(); + + return LoadSizes{min_load_size, t.size() - 1}; + }(); + return sizes; +} + +struct Ratios { + double min_load; + double avg_load; + double max_load; +}; + +// See absl/container/internal/hashtable_debug.h for details on +// probe length calculation. +template +Ratios CollectMeanProbeLengths() { + const auto min_max_sizes = GetMinMaxLoadSizes(); + + ElemFn elem; + using Key = decltype(elem()); + Table t; + + Ratios result; + while (t.size() < min_max_sizes.min_load) t.insert(elem()); + result.min_load = + absl::container_internal::GetHashtableDebugProbeSummary(t).mean; + + while (t.size() < (min_max_sizes.min_load + min_max_sizes.max_load) / 2) + t.insert(elem()); + result.avg_load = + absl::container_internal::GetHashtableDebugProbeSummary(t).mean; + + while (t.size() < min_max_sizes.max_load) t.insert(elem()); + result.max_load = + absl::container_internal::GetHashtableDebugProbeSummary(t).mean; + + return result; +} + +template +uintptr_t PointerForAlignment() { + alignas(Align) static constexpr uintptr_t kInitPointer = 0; + return reinterpret_cast(&kInitPointer); +} + +// This incomplete type is used for testing hash of pointers of different +// alignments. +// NOTE: We are generating invalid pointer values on the fly with +// reinterpret_cast. There are not "safely derived" pointers so using them is +// technically UB. It is unlikely to be a problem, though. +template +struct Ptr; + +template +Ptr* MakePtr(uintptr_t v) { + if (sizeof(v) == 8) { + constexpr int kCopyBits = 16; + // Ensure high bits are all the same. + v = static_cast(static_cast(v << kCopyBits) >> + kCopyBits); + } + return reinterpret_cast*>(v); +} + +struct IntIdentity { + uint64_t i; + friend bool operator==(IntIdentity a, IntIdentity b) { return a.i == b.i; } + IntIdentity operator++(int) { return IntIdentity{i++}; } +}; + +template +struct PtrIdentity { + explicit PtrIdentity(uintptr_t val = PointerForAlignment()) : i(val) {} + uintptr_t i; + friend bool operator==(PtrIdentity a, PtrIdentity b) { return a.i == b.i; } + PtrIdentity operator++(int) { + PtrIdentity p(i); + i += Align; + return p; + } +}; + +constexpr char kStringFormat[] = "/path/to/file/name-%07d-of-9999999.txt"; + +template +struct String { + std::string value; + static std::string Make(uint32_t v) { + return {small ? absl::StrCat(v) : absl::StrFormat(kStringFormat, v)}; + } +}; + +template <> +struct DefaultHash { + struct type { + size_t operator()(IntIdentity t) const { return t.i; } + }; +}; + +template +struct DefaultHash> { + struct type { + size_t operator()(PtrIdentity t) const { return t.i; } + }; +}; + +template +struct Sequential { + T operator()() const { return current++; } + mutable T current{}; +}; + +template +struct Sequential*> { + Ptr* operator()() const { + auto* result = MakePtr(current); + current += Align; + return result; + } + mutable uintptr_t current = PointerForAlignment(); +}; + + +template +struct Sequential> { + std::string operator()() const { return String::Make(current++); } + mutable uint32_t current = 0; +}; + +template +struct Sequential> { + mutable Sequential tseq; + mutable Sequential useq; + + using RealT = decltype(tseq()); + using RealU = decltype(useq()); + + mutable std::vector ts; + mutable std::vector us; + mutable size_t ti = 0, ui = 0; + + std::pair operator()() const { + std::pair value{get_t(), get_u()}; + if (ti == 0) { + ti = ui + 1; + ui = 0; + } else { + --ti; + ++ui; + } + return value; + } + + RealT get_t() const { + while (ti >= ts.size()) ts.push_back(tseq()); + return ts[ti]; + } + + RealU get_u() const { + while (ui >= us.size()) us.push_back(useq()); + return us[ui]; + } +}; + +template +struct AlmostSequential { + mutable Sequential current; + + auto operator()() const -> decltype(current()) { + while (absl::Uniform(GlobalBitGen(), 0.0, 1.0) <= percent_skip / 100.) + current(); + return current(); + } +}; + +struct Uniform { + template + T operator()(T) const { + return absl::Uniform(absl::IntervalClosed, GlobalBitGen(), T{0}, ~T{0}); + } +}; + +struct Gaussian { + template + T operator()(T) const { + double d; + do { + d = absl::Gaussian(GlobalBitGen(), 1e6, 1e4); + } while (d <= 0 || d > std::numeric_limits::max() / 2); + return static_cast(d); + } +}; + +struct Zipf { + template + T operator()(T) const { + return absl::Zipf(GlobalBitGen(), std::numeric_limits::max(), 1.6); + } +}; + +template +struct Random { + T operator()() const { return Dist{}(T{}); } +}; + +template +struct Random*, Dist> { + Ptr* operator()() const { + return MakePtr(Random{}() * Align); + } +}; + +template +struct Random { + IntIdentity operator()() const { + return IntIdentity{Random{}()}; + } +}; + +template +struct Random, Dist> { + PtrIdentity operator()() const { + return PtrIdentity{Random{}() * Align}; + } +}; + +template +struct Random, Dist> { + std::string operator()() const { + return String::Make(Random{}()); + } +}; + +template +struct Random, Dist> { + auto operator()() const + -> decltype(std::make_pair(Random{}(), Random{}())) { + return std::make_pair(Random{}(), Random{}()); + } +}; + +template +std::string Name(); + +std::string Name(uint32_t*) { return "u32"; } +std::string Name(uint64_t*) { return "u64"; } +std::string Name(IntIdentity*) { return "IntIdentity"; } + +template +std::string Name(Ptr**) { + return absl::StrCat("Ptr", Align); +} + +template +std::string Name(PtrIdentity*) { + return absl::StrCat("PtrIdentity", Align); +} + +template +std::string Name(String*) { + return small ? "StrS" : "StrL"; +} + +template +std::string Name(std::pair*) { + if (output() == OutputStyle::kBenchmark) + return absl::StrCat("P_", Name(), "_", Name()); + return absl::StrCat("P<", Name(), ",", Name(), ">"); +} + +template +std::string Name(Sequential*) { + return "Sequential"; +} + +template +std::string Name(AlmostSequential*) { + return absl::StrCat("AlmostSeq_", P); +} + +template +std::string Name(Random*) { + return "UnifRand"; +} + +template +std::string Name(Random*) { + return "GausRand"; +} + +template +std::string Name(Random*) { + return "ZipfRand"; +} + +template +std::string Name() { + return Name(static_cast(nullptr)); +} + +constexpr int kNameWidth = 15; +constexpr int kDistWidth = 16; + +bool CanRunBenchmark(absl::string_view name) { + static std::regex* const filter = []() -> std::regex* { + return benchmarks.empty() || benchmarks == "all" + ? nullptr + : new std::regex(std::string(benchmarks)); + }(); + return filter == nullptr || std::regex_search(std::string(name), *filter); +} + +struct Result { + std::string name; + std::string dist_name; + Ratios ratios; +}; + +template +void RunForTypeAndDistribution(std::vector& results) { + std::string name = absl::StrCat(Name(), "/", Name()); + // We have to check against all three names (min/avg/max) before we run it. + // If any of them is enabled, we run it. + if (!CanRunBenchmark(absl::StrCat(name, "/min")) && + !CanRunBenchmark(absl::StrCat(name, "/avg")) && + !CanRunBenchmark(absl::StrCat(name, "/max"))) { + return; + } + results.push_back({Name(), Name(), CollectMeanProbeLengths()}); +} + +template +void RunForType(std::vector& results) { + RunForTypeAndDistribution>(results); + RunForTypeAndDistribution>(results); + RunForTypeAndDistribution>(results); + RunForTypeAndDistribution>(results); +#ifdef NDEBUG + // Disable these in non-opt mode because they take too long. + RunForTypeAndDistribution>(results); + RunForTypeAndDistribution>(results); +#endif // NDEBUG +} + +} // namespace + +int main(int argc, char** argv) { + // Parse the benchmark flags. Ignore all of them except the regex pattern. + for (int i = 1; i < argc; ++i) { + absl::string_view arg = argv[i]; + const auto next = [&] { return argv[std::min(i + 1, argc - 1)]; }; + + if (absl::ConsumePrefix(&arg, "--benchmark_filter")) { + if (arg == "") { + // --benchmark_filter X + benchmarks = next(); + } else if (absl::ConsumePrefix(&arg, "=")) { + // --benchmark_filter=X + benchmarks = arg; + } + } + + // Any --benchmark flag turns on the mode. + if (absl::ConsumePrefix(&arg, "--benchmark")) { + if (benchmarks.empty()) benchmarks="all"; + } + } + + std::vector results; + RunForType(results); + RunForType(results); + RunForType*>(results); + RunForType*>(results); + RunForType*>(results); + RunForType*>(results); + RunForType>(results); + RunForType>(results); + RunForType>(results); + RunForType>(results); + RunForType>(results); + RunForType>(results); + RunForType>(results); + RunForType>>(results); + RunForType, uint64_t>>(results); + RunForType>>(results); + RunForType, uint64_t>>(results); + + switch (output()) { + case OutputStyle::kRegular: + absl::PrintF("%-*s%-*s Min Avg Max\n%s\n", kNameWidth, + "Type", kDistWidth, "Distribution", + std::string(kNameWidth + kDistWidth + 10 * 3, '-')); + for (const auto& result : results) { + absl::PrintF("%-*s%-*s %8.4f %8.4f %8.4f\n", kNameWidth, result.name, + kDistWidth, result.dist_name, result.ratios.min_load, + result.ratios.avg_load, result.ratios.max_load); + } + break; + case OutputStyle::kBenchmark: { + absl::PrintF("{\n"); + absl::PrintF(" \"benchmarks\": [\n"); + absl::string_view comma; + for (const auto& result : results) { + auto print = [&](absl::string_view stat, double Ratios::*val) { + std::string name = + absl::StrCat(result.name, "/", result.dist_name, "/", stat); + // Check the regex again. We might had have enabled only one of the + // stats for the benchmark. + if (!CanRunBenchmark(name)) return; + absl::PrintF(" %s{\n", comma); + absl::PrintF(" \"cpu_time\": %f,\n", 1e9 * result.ratios.*val); + absl::PrintF(" \"real_time\": %f,\n", 1e9 * result.ratios.*val); + absl::PrintF(" \"iterations\": 1,\n"); + absl::PrintF(" \"name\": \"%s\",\n", name); + absl::PrintF(" \"time_unit\": \"ns\"\n"); + absl::PrintF(" }\n"); + comma = ","; + }; + print("min", &Ratios::min_load); + print("avg", &Ratios::avg_load); + print("max", &Ratios::max_load); + } + absl::PrintF(" ],\n"); + absl::PrintF(" \"context\": {\n"); + absl::PrintF(" }\n"); + absl::PrintF("}\n"); + break; + } + } + + return 0; +} diff --git a/absl/hash/BUILD.bazel b/absl/hash/BUILD.bazel index 40c8f207..acd490ba 100644 --- a/absl/hash/BUILD.bazel +++ b/absl/hash/BUILD.bazel @@ -82,6 +82,24 @@ cc_test( ], ) +cc_binary( + name = "hash_benchmark", + testonly = 1, + srcs = ["hash_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":hash", + "//absl/base:core_headers", + "//absl/random", + "//absl/strings:cord", + "//absl/strings:cord_test_helpers", + "@com_github_google_benchmark//:benchmark_main", + ], +) + cc_library( name = "spy_hash_state", testonly = 1, diff --git a/absl/hash/hash_benchmark.cc b/absl/hash/hash_benchmark.cc new file mode 100644 index 00000000..27e52719 --- /dev/null +++ b/absl/hash/hash_benchmark.cc @@ -0,0 +1,249 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "absl/base/attributes.h" +#include "absl/hash/hash.h" +#include "absl/random/random.h" +#include "absl/strings/cord.h" +#include "absl/strings/cord_test_helpers.h" +#include "benchmark/benchmark.h" + +namespace { + +using absl::Hash; + +template