diff options
Diffstat (limited to 'absl/base/internal')
51 files changed, 1472 insertions, 198 deletions
diff --git a/absl/base/internal/atomic_hook.h b/absl/base/internal/atomic_hook.h index 6757e75b..ae21cd7f 100644 --- a/absl/base/internal/atomic_hook.h +++ b/absl/base/internal/atomic_hook.h @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// #ifndef ABSL_BASE_INTERNAL_ATOMIC_HOOK_H_ #define ABSL_BASE_INTERNAL_ATOMIC_HOOK_H_ @@ -21,29 +20,51 @@ #include <cstdint> #include <utility> -#ifdef _MSC_FULL_VER +#include "absl/base/attributes.h" +#include "absl/base/config.h" + +#if defined(_MSC_VER) && !defined(__clang__) +#define ABSL_HAVE_WORKING_CONSTEXPR_STATIC_INIT 0 +#else +#define ABSL_HAVE_WORKING_CONSTEXPR_STATIC_INIT 1 +#endif + +#if defined(_MSC_VER) #define ABSL_HAVE_WORKING_ATOMIC_POINTER 0 #else #define ABSL_HAVE_WORKING_ATOMIC_POINTER 1 #endif namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { template <typename T> class AtomicHook; -// AtomicHook is a helper class, templatized on a raw function pointer type, for -// implementing Abseil customization hooks. It is a callable object that -// dispatches to the registered hook. +// To workaround AtomicHook not being constant-initializable on some platforms, +// prefer to annotate instances with `ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES` +// instead of `ABSL_CONST_INIT`. +#if ABSL_HAVE_WORKING_CONSTEXPR_STATIC_INIT +#define ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_CONST_INIT +#else +#define ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES +#endif + +// `AtomicHook` is a helper class, templatized on a raw function pointer type, +// for implementing Abseil customization hooks. It is a callable object that +// dispatches to the registered hook. Objects of type `AtomicHook` must have +// static or thread storage duration. // // A default constructed object performs a no-op (and returns a default // constructed object) if no hook has been registered. // -// Hooks can be pre-registered via constant initialization, for example, -// ABSL_CONST_INIT static AtomicHook<void(*)()> my_hook(DefaultAction); -// and then changed at runtime via a call to Store(). +// Hooks can be pre-registered via constant initialization, for example: +// +// ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static AtomicHook<void(*)()> +// my_hook(DefaultAction); +// +// and then changed at runtime via a call to `Store()`. // // Reads and writes guarantee memory_order_acquire/memory_order_release // semantics. @@ -58,12 +79,23 @@ class AtomicHook<ReturnType (*)(Args...)> { // Constructs an object that by default dispatches to/returns the // pre-registered default_fn when no hook has been registered at runtime. -#if ABSL_HAVE_WORKING_ATOMIC_POINTER +#if ABSL_HAVE_WORKING_ATOMIC_POINTER && ABSL_HAVE_WORKING_CONSTEXPR_STATIC_INIT explicit constexpr AtomicHook(FnPtr default_fn) : hook_(default_fn), default_fn_(default_fn) {} -#else +#elif ABSL_HAVE_WORKING_CONSTEXPR_STATIC_INIT explicit constexpr AtomicHook(FnPtr default_fn) : hook_(kUninitialized), default_fn_(default_fn) {} +#else + // As of January 2020, on all known versions of MSVC this constructor runs in + // the global constructor sequence. If `Store()` is called by a dynamic + // initializer, we want to preserve the value, even if this constructor runs + // after the call to `Store()`. If not, `hook_` will be + // zero-initialized by the linker and we have no need to set it. + // https://developercommunity.visualstudio.com/content/problem/336946/class-with-constexpr-constructor-not-using-static.html + explicit constexpr AtomicHook(FnPtr default_fn) + : /* hook_(deliberately omitted), */ default_fn_(default_fn) { + static_assert(kUninitialized == 0, "here we rely on zero-initialization"); + } #endif // Stores the provided function pointer as the value for this hook. @@ -159,9 +191,10 @@ class AtomicHook<ReturnType (*)(Args...)> { }; #undef ABSL_HAVE_WORKING_ATOMIC_POINTER +#undef ABSL_HAVE_WORKING_CONSTEXPR_STATIC_INIT } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_ATOMIC_HOOK_H_ diff --git a/absl/base/internal/atomic_hook_test.cc b/absl/base/internal/atomic_hook_test.cc index ecc80406..e577a8fd 100644 --- a/absl/base/internal/atomic_hook_test.cc +++ b/absl/base/internal/atomic_hook_test.cc @@ -14,16 +14,22 @@ #include "absl/base/internal/atomic_hook.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/base/attributes.h" +#include "absl/base/internal/atomic_hook_test_helper.h" namespace { +using ::testing::Eq; + int value = 0; void TestHook(int x) { value = x; } TEST(AtomicHookTest, NoDefaultFunction) { - ABSL_CONST_INIT static absl::base_internal::AtomicHook<void(*)(int)> hook; + ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook< + void (*)(int)> + hook; value = 0; // Test the default DummyFunction. @@ -49,8 +55,9 @@ TEST(AtomicHookTest, NoDefaultFunction) { TEST(AtomicHookTest, WithDefaultFunction) { // Set the default value to TestHook at compile-time. - ABSL_CONST_INIT static absl::base_internal::AtomicHook<void (*)(int)> hook( - TestHook); + ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook< + void (*)(int)> + hook(TestHook); value = 0; // Test the default value is TestHook. @@ -67,4 +74,24 @@ TEST(AtomicHookTest, WithDefaultFunction) { EXPECT_EQ(value, 2); } +ABSL_CONST_INIT int override_func_calls = 0; +void OverrideFunc() { override_func_calls++; } +static struct OverrideInstaller { + OverrideInstaller() { absl::atomic_hook_internal::func.Store(OverrideFunc); } +} override_installer; + +TEST(AtomicHookTest, DynamicInitFromAnotherTU) { + // MSVC 14.2 doesn't do constexpr static init correctly; in particular it + // tends to sequence static init (i.e. defaults) of `AtomicHook` objects + // after their dynamic init (i.e. overrides), overwriting whatever value was + // written during dynamic init. This regression test validates the fix. + // https://developercommunity.visualstudio.com/content/problem/336946/class-with-constexpr-constructor-not-using-static.html + EXPECT_THAT(absl::atomic_hook_internal::default_func_calls, Eq(0)); + EXPECT_THAT(override_func_calls, Eq(0)); + absl::atomic_hook_internal::func(); + EXPECT_THAT(absl::atomic_hook_internal::default_func_calls, Eq(0)); + EXPECT_THAT(override_func_calls, Eq(1)); + EXPECT_THAT(absl::atomic_hook_internal::func.Load(), Eq(OverrideFunc)); +} + } // namespace diff --git a/absl/base/internal/atomic_hook_test_helper.cc b/absl/base/internal/atomic_hook_test_helper.cc new file mode 100644 index 00000000..537d47cd --- /dev/null +++ b/absl/base/internal/atomic_hook_test_helper.cc @@ -0,0 +1,32 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/atomic_hook_test_helper.h" + +#include "absl/base/attributes.h" +#include "absl/base/internal/atomic_hook.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace atomic_hook_internal { + +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES absl::base_internal::AtomicHook<VoidF> + func(DefaultFunc); +ABSL_CONST_INIT int default_func_calls = 0; +void DefaultFunc() { default_func_calls++; } +void RegisterFunc(VoidF f) { func.Store(f); } + +} // namespace atomic_hook_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/base/internal/atomic_hook_test_helper.h b/absl/base/internal/atomic_hook_test_helper.h new file mode 100644 index 00000000..3e72b497 --- /dev/null +++ b/absl/base/internal/atomic_hook_test_helper.h @@ -0,0 +1,34 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_ATOMIC_HOOK_TEST_HELPER_H_ +#define ABSL_BASE_ATOMIC_HOOK_TEST_HELPER_H_ + +#include "absl/base/internal/atomic_hook.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace atomic_hook_internal { + +using VoidF = void (*)(); +extern absl::base_internal::AtomicHook<VoidF> func; +extern int default_func_calls; +void DefaultFunc(); +void RegisterFunc(VoidF func); + +} // namespace atomic_hook_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_ATOMIC_HOOK_TEST_HELPER_H_ diff --git a/absl/base/internal/bits.h b/absl/base/internal/bits.h index ef978e9b..8b03453c 100644 --- a/absl/base/internal/bits.h +++ b/absl/base/internal/bits.h @@ -20,6 +20,8 @@ #include <cstdint> +#include "absl/base/config.h" + // Clang on Windows has __builtin_clzll; otherwise we need to use the // windows intrinsic functions. #if defined(_MSC_VER) @@ -46,15 +48,27 @@ namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64Slow(uint64_t n) { int zeroes = 60; - if (n >> 32) zeroes -= 32, n >>= 32; - if (n >> 16) zeroes -= 16, n >>= 16; - if (n >> 8) zeroes -= 8, n >>= 8; - if (n >> 4) zeroes -= 4, n >>= 4; + if (n >> 32) { + zeroes -= 32; + n >>= 32; + } + if (n >> 16) { + zeroes -= 16; + n >>= 16; + } + if (n >> 8) { + zeroes -= 8; + n >>= 8; + } + if (n >> 4) { + zeroes -= 4; + n >>= 4; + } return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; } @@ -96,9 +110,18 @@ ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64(uint64_t n) { ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32Slow(uint64_t n) { int zeroes = 28; - if (n >> 16) zeroes -= 16, n >>= 16; - if (n >> 8) zeroes -= 8, n >>= 8; - if (n >> 4) zeroes -= 4, n >>= 4; + if (n >> 16) { + zeroes -= 16; + n >>= 16; + } + if (n >> 8) { + zeroes -= 8; + n >>= 8; + } + if (n >> 4) { + zeroes -= 4; + n >>= 4; + } return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; } @@ -189,7 +212,7 @@ ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32(uint32_t n) { #undef ABSL_BASE_INTERNAL_FORCEINLINE } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_BITS_H_ diff --git a/absl/base/internal/cycleclock.cc b/absl/base/internal/cycleclock.cc index 9868e549..0e65005b 100644 --- a/absl/base/internal/cycleclock.cc +++ b/absl/base/internal/cycleclock.cc @@ -28,7 +28,7 @@ #include "absl/base/internal/unscaledcycleclock.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { #if ABSL_USE_UNSCALED_CYCLECLOCK @@ -103,5 +103,5 @@ double CycleClock::Frequency() { #endif } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/cycleclock.h b/absl/base/internal/cycleclock.h index 39bce06a..a18b5844 100644 --- a/absl/base/internal/cycleclock.h +++ b/absl/base/internal/cycleclock.h @@ -44,8 +44,10 @@ #include <cstdint> +#include "absl/base/config.h" + namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { // ----------------------------------------------------------------------------- @@ -86,7 +88,7 @@ class CycleClockSource { }; } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_CYCLECLOCK_H_ diff --git a/absl/base/internal/direct_mmap.h b/absl/base/internal/direct_mmap.h index 2e5e422c..5618867b 100644 --- a/absl/base/internal/direct_mmap.h +++ b/absl/base/internal/direct_mmap.h @@ -62,7 +62,7 @@ extern "C" void* __mmap2(void*, size_t, int, int, int, size_t); #endif // __BIONIC__ namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { // Platform specific logic extracted from @@ -129,7 +129,7 @@ inline int DirectMunmap(void* start, size_t length) { } } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #else // !__linux__ @@ -138,7 +138,7 @@ inline int DirectMunmap(void* start, size_t length) { // actual mmap()/munmap() methods. namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { inline void* DirectMmap(void* start, size_t length, int prot, int flags, int fd, @@ -151,7 +151,7 @@ inline int DirectMunmap(void* start, size_t length) { } } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // __linux__ diff --git a/absl/base/internal/endian.h b/absl/base/internal/endian.h index 8643bffa..9677530e 100644 --- a/absl/base/internal/endian.h +++ b/absl/base/internal/endian.h @@ -19,9 +19,6 @@ // The following guarantees declaration of the byte swap functions #ifdef _MSC_VER #include <stdlib.h> // NOLINT(build/include) -#elif defined(__APPLE__) -// macOS / Darwin features -#include <libkern/OSByteOrder.h> #elif defined(__FreeBSD__) #include <sys/endian.h> #elif defined(__GLIBC__) @@ -34,7 +31,7 @@ #include "absl/base/port.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN // Use compiler byte-swapping intrinsics if they are available. 32-bit // and 64-bit versions are available in Clang and GCC as of GCC 4.3.0. @@ -64,11 +61,6 @@ inline uint16_t gbswap_16(uint16_t host_int) { return _byteswap_ushort(host_int); } -#elif defined(__APPLE__) -inline uint64_t gbswap_64(uint64_t host_int) { return OSSwapInt16(host_int); } -inline uint32_t gbswap_32(uint32_t host_int) { return OSSwapInt32(host_int); } -inline uint16_t gbswap_16(uint16_t host_int) { return OSSwapInt64(host_int); } - #else inline uint64_t gbswap_64(uint64_t host_int) { #if defined(__GNUC__) && defined(__x86_64__) && !defined(__APPLE__) @@ -114,7 +106,7 @@ inline uint16_t gbswap_16(uint16_t host_int) { #endif } -#endif // intrinics available +#endif // intrinsics available #ifdef ABSL_IS_LITTLE_ENDIAN @@ -268,7 +260,7 @@ inline void Store64(void *p, uint64_t v) { } // namespace big_endian -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_ENDIAN_H_ diff --git a/absl/base/internal/endian_test.cc b/absl/base/internal/endian_test.cc index 0c683286..aa6b8496 100644 --- a/absl/base/internal/endian_test.cc +++ b/absl/base/internal/endian_test.cc @@ -24,7 +24,7 @@ #include "absl/base/config.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace { const uint64_t kInitialNumber{0x0123456789abcdef}; @@ -261,5 +261,5 @@ TEST(EndianessTest, big_endian) { } } // namespace -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/errno_saver.h b/absl/base/internal/errno_saver.h new file mode 100644 index 00000000..251de510 --- /dev/null +++ b/absl/base/internal/errno_saver.h @@ -0,0 +1,43 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_ERRNO_SAVER_H_ +#define ABSL_BASE_INTERNAL_ERRNO_SAVER_H_ + +#include <cerrno> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// `ErrnoSaver` captures the value of `errno` upon construction and restores it +// upon deletion. It is used in low-level code and must be super fast. Do not +// add instrumentation, even in debug modes. +class ErrnoSaver { + public: + ErrnoSaver() : saved_errno_(errno) {} + ~ErrnoSaver() { errno = saved_errno_; } + int operator()() const { return saved_errno_; } + + private: + const int saved_errno_; +}; + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_ERRNO_SAVER_H_ diff --git a/absl/base/internal/errno_saver_test.cc b/absl/base/internal/errno_saver_test.cc new file mode 100644 index 00000000..b845e2dd --- /dev/null +++ b/absl/base/internal/errno_saver_test.cc @@ -0,0 +1,44 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/errno_saver.h" + +#include <cerrno> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace { +using ::testing::Eq; + +struct ErrnoPrinter { + int no; +}; +std::ostream &operator<<(std::ostream &os, ErrnoPrinter ep) { + return os << strerror(ep.no) << " [" << ep.no << "]"; +} +bool operator==(ErrnoPrinter one, ErrnoPrinter two) { return one.no == two.no; } + +TEST(ErrnoSaverTest, Works) { + errno = EDOM; + { + absl::base_internal::ErrnoSaver errno_saver; + EXPECT_THAT(ErrnoPrinter{errno}, Eq(ErrnoPrinter{EDOM})); + errno = ERANGE; + EXPECT_THAT(ErrnoPrinter{errno}, Eq(ErrnoPrinter{ERANGE})); + EXPECT_THAT(ErrnoPrinter{errno_saver()}, Eq(ErrnoPrinter{EDOM})); + } + EXPECT_THAT(ErrnoPrinter{errno}, Eq(ErrnoPrinter{EDOM})); +} +} // namespace diff --git a/absl/base/internal/exception_safety_testing.cc b/absl/base/internal/exception_safety_testing.cc index 6ef4325c..6ccac418 100644 --- a/absl/base/internal/exception_safety_testing.cc +++ b/absl/base/internal/exception_safety_testing.cc @@ -14,6 +14,8 @@ #include "absl/base/internal/exception_safety_testing.h" +#ifdef ABSL_HAVE_EXCEPTIONS + #include "gtest/gtest.h" #include "absl/meta/type_traits.h" @@ -73,3 +75,5 @@ std::string GetSpecString(AllocSpec spec) { } // namespace exceptions_internal } // namespace testing + +#endif // ABSL_HAVE_EXCEPTIONS diff --git a/absl/base/internal/exception_safety_testing.h b/absl/base/internal/exception_safety_testing.h index be38ba54..6ba89d05 100644 --- a/absl/base/internal/exception_safety_testing.h +++ b/absl/base/internal/exception_safety_testing.h @@ -17,6 +17,10 @@ #ifndef ABSL_BASE_INTERNAL_EXCEPTION_SAFETY_TESTING_H_ #define ABSL_BASE_INTERNAL_EXCEPTION_SAFETY_TESTING_H_ +#include "absl/base/config.h" + +#ifdef ABSL_HAVE_EXCEPTIONS + #include <cstddef> #include <cstdint> #include <functional> @@ -27,7 +31,6 @@ #include <unordered_map> #include "gtest/gtest.h" -#include "absl/base/config.h" #include "absl/base/internal/pretty_function.h" #include "absl/memory/memory.h" #include "absl/meta/type_traits.h" @@ -1093,4 +1096,6 @@ class ExceptionSafetyTestBuilder { } // namespace testing +#endif // ABSL_HAVE_EXCEPTIONS + #endif // ABSL_BASE_INTERNAL_EXCEPTION_SAFETY_TESTING_H_ diff --git a/absl/base/internal/exponential_biased.cc b/absl/base/internal/exponential_biased.cc new file mode 100644 index 00000000..1b30c061 --- /dev/null +++ b/absl/base/internal/exponential_biased.cc @@ -0,0 +1,93 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/exponential_biased.h" + +#include <stdint.h> + +#include <algorithm> +#include <atomic> +#include <cmath> +#include <limits> + +#include "absl/base/attributes.h" +#include "absl/base/optimization.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// The algorithm generates a random number between 0 and 1 and applies the +// inverse cumulative distribution function for an exponential. Specifically: +// Let m be the inverse of the sample period, then the probability +// distribution function is m*exp(-mx) so the CDF is +// p = 1 - exp(-mx), so +// q = 1 - p = exp(-mx) +// log_e(q) = -mx +// -log_e(q)/m = x +// log_2(q) * (-log_e(2) * 1/m) = x +// In the code, q is actually in the range 1 to 2**26, hence the -26 below +int64_t ExponentialBiased::GetSkipCount(int64_t mean) { + if (ABSL_PREDICT_FALSE(!initialized_)) { + Initialize(); + } + + uint64_t rng = NextRandom(rng_); + rng_ = rng; + + // Take the top 26 bits as the random number + // (This plus the 1<<58 sampling bound give a max possible step of + // 5194297183973780480 bytes.) + // The uint32_t cast is to prevent a (hard-to-reproduce) NAN + // under piii debug for some binaries. + double q = static_cast<uint32_t>(rng >> (kPrngNumBits - 26)) + 1.0; + // Put the computed p-value through the CDF of a geometric. + double interval = bias_ + (std::log2(q) - 26) * (-std::log(2.0) * mean); + // Very large values of interval overflow int64_t. To avoid that, we will + // cheat and clamp any huge values to (int64_t max)/2. This is a potential + // source of bias, but the mean would need to be such a large value that it's + // not likely to come up. For example, with a mean of 1e18, the probability of + // hitting this condition is about 1/1000. For a mean of 1e17, standard + // calculators claim that this event won't happen. + if (interval > static_cast<double>(std::numeric_limits<int64_t>::max() / 2)) { + // Assume huge values are bias neutral, retain bias for next call. + return std::numeric_limits<int64_t>::max() / 2; + } + double value = std::round(interval); + bias_ = interval - value; + return value; +} + +int64_t ExponentialBiased::GetStride(int64_t mean) { + return GetSkipCount(mean - 1) + 1; +} + +void ExponentialBiased::Initialize() { + // We don't get well distributed numbers from `this` so we call NextRandom() a + // bunch to mush the bits around. We use a global_rand to handle the case + // where the same thread (by memory address) gets created and destroyed + // repeatedly. + ABSL_CONST_INIT static std::atomic<uint32_t> global_rand(0); + uint64_t r = reinterpret_cast<uint64_t>(this) + + global_rand.fetch_add(1, std::memory_order_relaxed); + for (int i = 0; i < 20; ++i) { + r = NextRandom(r); + } + rng_ = r; + initialized_ = true; +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/base/internal/exponential_biased.h b/absl/base/internal/exponential_biased.h new file mode 100644 index 00000000..94f79a33 --- /dev/null +++ b/absl/base/internal/exponential_biased.h @@ -0,0 +1,130 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_ +#define ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_ + +#include <stdint.h> + +#include "absl/base/config.h" +#include "absl/base/macros.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// ExponentialBiased provides a small and fast random number generator for a +// rounded exponential distribution. This generator manages very little state, +// and imposes no synchronization overhead. This makes it useful in specialized +// scenarios requiring minimum overhead, such as stride based periodic sampling. +// +// ExponentialBiased provides two closely related functions, GetSkipCount() and +// GetStride(), both returning a rounded integer defining a number of events +// required before some event with a given mean probability occurs. +// +// The distribution is useful to generate a random wait time or some periodic +// event with a given mean probability. For example, if an action is supposed to +// happen on average once every 'N' events, then we can get a random 'stride' +// counting down how long before the event to happen. For example, if we'd want +// to sample one in every 1000 'Frobber' calls, our code could look like this: +// +// Frobber::Frobber() { +// stride_ = exponential_biased_.GetStride(1000); +// } +// +// void Frobber::Frob(int arg) { +// if (--stride == 0) { +// SampleFrob(arg); +// stride_ = exponential_biased_.GetStride(1000); +// } +// ... +// } +// +// The rounding of the return value creates a bias, especially for smaller means +// where the distribution of the fraction is not evenly distributed. We correct +// this bias by tracking the fraction we rounded up or down on each iteration, +// effectively tracking the distance between the cumulative value, and the +// rounded cumulative value. For example, given a mean of 2: +// +// raw = 1.63076, cumulative = 1.63076, rounded = 2, bias = -0.36923 +// raw = 0.14624, cumulative = 1.77701, rounded = 2, bias = 0.14624 +// raw = 4.93194, cumulative = 6.70895, rounded = 7, bias = -0.06805 +// raw = 0.24206, cumulative = 6.95101, rounded = 7, bias = 0.24206 +// etc... +// +// Adjusting with rounding bias is relatively trivial: +// +// double value = bias_ + exponential_distribution(mean)(); +// double rounded_value = std::round(value); +// bias_ = value - rounded_value; +// return rounded_value; +// +// This class is thread-compatible. +class ExponentialBiased { + public: + // The number of bits set by NextRandom. + static constexpr int kPrngNumBits = 48; + + // `GetSkipCount()` returns the number of events to skip before some chosen + // event happens. For example, randomly tossing a coin, we will on average + // throw heads once before we get tails. We can simulate random coin tosses + // using GetSkipCount() as: + // + // ExponentialBiased eb; + // for (...) { + // int number_of_heads_before_tail = eb.GetSkipCount(1); + // for (int flips = 0; flips < number_of_heads_before_tail; ++flips) { + // printf("head..."); + // } + // printf("tail\n"); + // } + // + int64_t GetSkipCount(int64_t mean); + + // GetStride() returns the number of events required for a specific event to + // happen. See the class comments for a usage example. `GetStride()` is + // equivalent to `GetSkipCount(mean - 1) + 1`. When to use `GetStride()` or + // `GetSkipCount()` depends mostly on what best fits the use case. + int64_t GetStride(int64_t mean); + + // Computes a random number in the range [0, 1<<(kPrngNumBits+1) - 1] + // + // This is public to enable testing. + static uint64_t NextRandom(uint64_t rnd); + + private: + void Initialize(); + + uint64_t rng_{0}; + double bias_{0}; + bool initialized_{false}; +}; + +// Returns the next prng value. +// pRNG is: aX+b mod c with a = 0x5DEECE66D, b = 0xB, c = 1<<48 +// This is the lrand64 generator. +inline uint64_t ExponentialBiased::NextRandom(uint64_t rnd) { + const uint64_t prng_mult = uint64_t{0x5DEECE66D}; + const uint64_t prng_add = 0xB; + const uint64_t prng_mod_power = 48; + const uint64_t prng_mod_mask = + ~((~static_cast<uint64_t>(0)) << prng_mod_power); + return (prng_mult * rnd + prng_add) & prng_mod_mask; +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_ diff --git a/absl/base/internal/exponential_biased_test.cc b/absl/base/internal/exponential_biased_test.cc new file mode 100644 index 00000000..90a482d2 --- /dev/null +++ b/absl/base/internal/exponential_biased_test.cc @@ -0,0 +1,199 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/exponential_biased.h" + +#include <stddef.h> + +#include <cmath> +#include <cstdint> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/strings/str_cat.h" + +using ::testing::Ge; + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +MATCHER_P2(IsBetween, a, b, + absl::StrCat(std::string(negation ? "isn't" : "is"), " between ", a, + " and ", b)) { + return a <= arg && arg <= b; +} + +// Tests of the quality of the random numbers generated +// This uses the Anderson Darling test for uniformity. +// See "Evaluating the Anderson-Darling Distribution" by Marsaglia +// for details. + +// Short cut version of ADinf(z), z>0 (from Marsaglia) +// This returns the p-value for Anderson Darling statistic in +// the limit as n-> infinity. For finite n, apply the error fix below. +double AndersonDarlingInf(double z) { + if (z < 2) { + return exp(-1.2337141 / z) / sqrt(z) * + (2.00012 + + (0.247105 - + (0.0649821 - (0.0347962 - (0.011672 - 0.00168691 * z) * z) * z) * + z) * + z); + } + return exp( + -exp(1.0776 - + (2.30695 - + (0.43424 - (0.082433 - (0.008056 - 0.0003146 * z) * z) * z) * z) * + z)); +} + +// Corrects the approximation error in AndersonDarlingInf for small values of n +// Add this to AndersonDarlingInf to get a better approximation +// (from Marsaglia) +double AndersonDarlingErrFix(int n, double x) { + if (x > 0.8) { + return (-130.2137 + + (745.2337 - + (1705.091 - (1950.646 - (1116.360 - 255.7844 * x) * x) * x) * x) * + x) / + n; + } + double cutoff = 0.01265 + 0.1757 / n; + if (x < cutoff) { + double t = x / cutoff; + t = sqrt(t) * (1 - t) * (49 * t - 102); + return t * (0.0037 / (n * n) + 0.00078 / n + 0.00006) / n; + } else { + double t = (x - cutoff) / (0.8 - cutoff); + t = -0.00022633 + + (6.54034 - (14.6538 - (14.458 - (8.259 - 1.91864 * t) * t) * t) * t) * + t; + return t * (0.04213 + 0.01365 / n) / n; + } +} + +// Returns the AndersonDarling p-value given n and the value of the statistic +double AndersonDarlingPValue(int n, double z) { + double ad = AndersonDarlingInf(z); + double errfix = AndersonDarlingErrFix(n, ad); + return ad + errfix; +} + +double AndersonDarlingStatistic(const std::vector<double>& random_sample) { + int n = random_sample.size(); + double ad_sum = 0; + for (int i = 0; i < n; i++) { + ad_sum += (2 * i + 1) * + std::log(random_sample[i] * (1 - random_sample[n - 1 - i])); + } + double ad_statistic = -n - 1 / static_cast<double>(n) * ad_sum; + return ad_statistic; +} + +// Tests if the array of doubles is uniformly distributed. +// Returns the p-value of the Anderson Darling Statistic +// for the given set of sorted random doubles +// See "Evaluating the Anderson-Darling Distribution" by +// Marsaglia and Marsaglia for details. +double AndersonDarlingTest(const std::vector<double>& random_sample) { + double ad_statistic = AndersonDarlingStatistic(random_sample); + double p = AndersonDarlingPValue(random_sample.size(), ad_statistic); + return p; +} + +TEST(ExponentialBiasedTest, CoinTossDemoWithGetSkipCount) { + ExponentialBiased eb; + for (int runs = 0; runs < 10; ++runs) { + for (int flips = eb.GetSkipCount(1); flips > 0; --flips) { + printf("head..."); + } + printf("tail\n"); + } + int heads = 0; + for (int i = 0; i < 10000000; i += 1 + eb.GetSkipCount(1)) { + ++heads; + } + printf("Heads = %d (%f%%)\n", heads, 100.0 * heads / 10000000); +} + +TEST(ExponentialBiasedTest, SampleDemoWithStride) { + ExponentialBiased eb; + int stride = eb.GetStride(10); + int samples = 0; + for (int i = 0; i < 10000000; ++i) { + if (--stride == 0) { + ++samples; + stride = eb.GetStride(10); + } + } + printf("Samples = %d (%f%%)\n", samples, 100.0 * samples / 10000000); +} + + +// Testing that NextRandom generates uniform random numbers. Applies the +// Anderson-Darling test for uniformity +TEST(ExponentialBiasedTest, TestNextRandom) { + for (auto n : std::vector<int>({ + 10, // Check short-range correlation + 100, 1000, + 10000 // Make sure there's no systemic error + })) { + uint64_t x = 1; + // This assumes that the prng returns 48 bit numbers + uint64_t max_prng_value = static_cast<uint64_t>(1) << 48; + // Initialize. + for (int i = 1; i <= 20; i++) { + x = ExponentialBiased::NextRandom(x); + } + std::vector<uint64_t> int_random_sample(n); + // Collect samples + for (int i = 0; i < n; i++) { + int_random_sample[i] = x; + x = ExponentialBiased::NextRandom(x); + } + // First sort them... + std::sort(int_random_sample.begin(), int_random_sample.end()); + std::vector<double> random_sample(n); + // Convert them to uniform randoms (in the range [0,1]) + for (int i = 0; i < n; i++) { + random_sample[i] = + static_cast<double>(int_random_sample[i]) / max_prng_value; + } + // Now compute the Anderson-Darling statistic + double ad_pvalue = AndersonDarlingTest(random_sample); + EXPECT_GT(std::min(ad_pvalue, 1 - ad_pvalue), 0.0001) + << "prng is not uniform: n = " << n << " p = " << ad_pvalue; + } +} + +// The generator needs to be available as a thread_local and as a static +// variable. +TEST(ExponentialBiasedTest, InitializationModes) { + ABSL_CONST_INIT static ExponentialBiased eb_static; + EXPECT_THAT(eb_static.GetSkipCount(2), Ge(0)); + +#if ABSL_HAVE_THREAD_LOCAL + thread_local ExponentialBiased eb_thread; + EXPECT_THAT(eb_thread.GetSkipCount(2), Ge(0)); +#endif + + ExponentialBiased eb_stack; + EXPECT_THAT(eb_stack.GetSkipCount(2), Ge(0)); +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/base/internal/hide_ptr.h b/absl/base/internal/hide_ptr.h index 3720db18..1dba8090 100644 --- a/absl/base/internal/hide_ptr.h +++ b/absl/base/internal/hide_ptr.h @@ -17,8 +17,10 @@ #include <cstdint> +#include "absl/base/config.h" + namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { // Arbitrary value with high bits set. Xor'ing with it is unlikely @@ -43,7 +45,7 @@ inline T* UnhidePtr(uintptr_t hidden) { } } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_HIDE_PTR_H_ diff --git a/absl/base/internal/identity.h b/absl/base/internal/identity.h index b9d0f621..a3154ed7 100644 --- a/absl/base/internal/identity.h +++ b/absl/base/internal/identity.h @@ -16,8 +16,10 @@ #ifndef ABSL_BASE_INTERNAL_IDENTITY_H_ #define ABSL_BASE_INTERNAL_IDENTITY_H_ +#include "absl/base/config.h" + namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace internal { template <typename T> @@ -29,7 +31,7 @@ template <typename T> using identity_t = typename identity<T>::type; } // namespace internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_IDENTITY_H_ diff --git a/absl/base/internal/inline_variable_testing.h b/absl/base/internal/inline_variable_testing.h index 0ebdb9d8..3856b9f8 100644 --- a/absl/base/internal/inline_variable_testing.h +++ b/absl/base/internal/inline_variable_testing.h @@ -18,7 +18,7 @@ #include "absl/base/internal/inline_variable.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace inline_variable_testing_internal { struct Foo { @@ -40,7 +40,7 @@ const int& get_int_a(); const int& get_int_b(); } // namespace inline_variable_testing_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INLINE_VARIABLE_TESTING_H_ diff --git a/absl/base/internal/invoke.h b/absl/base/internal/invoke.h index 030b98df..c4eceebd 100644 --- a/absl/base/internal/invoke.h +++ b/absl/base/internal/invoke.h @@ -45,7 +45,7 @@ // top of this file for the API documentation. namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { // The five classes below each implement one of the clauses from the definition @@ -181,7 +181,7 @@ InvokeT<F, Args...> Invoke(F&& f, Args&&... args) { std::forward<Args>(args)...); } } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_INVOKE_H_ diff --git a/absl/base/internal/low_level_alloc.cc b/absl/base/internal/low_level_alloc.cc index 419c0e45..1bf94438 100644 --- a/absl/base/internal/low_level_alloc.cc +++ b/absl/base/internal/low_level_alloc.cc @@ -63,7 +63,7 @@ #endif // __APPLE__ namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { // A first-fit allocator with amortized logarithmic free() time. @@ -204,32 +204,33 @@ struct LowLevelAlloc::Arena { base_internal::SpinLock mu; // Head of free list, sorted by address - AllocList freelist GUARDED_BY(mu); + AllocList freelist ABSL_GUARDED_BY(mu); // Count of allocated blocks - int32_t allocation_count GUARDED_BY(mu); + int32_t allocation_count ABSL_GUARDED_BY(mu); // flags passed to NewArena const uint32_t flags; // Result of sysconf(_SC_PAGESIZE) const size_t pagesize; // Lowest power of two >= max(16, sizeof(AllocList)) - const size_t roundup; + const size_t round_up; // Smallest allocation block size const size_t min_size; // PRNG state - uint32_t random GUARDED_BY(mu); + uint32_t random ABSL_GUARDED_BY(mu); }; namespace { -using ArenaStorage = std::aligned_storage<sizeof(LowLevelAlloc::Arena), - alignof(LowLevelAlloc::Arena)>::type; - // Static storage space for the lazily-constructed, default global arena // instances. We require this space because the whole point of LowLevelAlloc // is to avoid relying on malloc/new. -ArenaStorage default_arena_storage; -ArenaStorage unhooked_arena_storage; +alignas(LowLevelAlloc::Arena) unsigned char default_arena_storage[sizeof( + LowLevelAlloc::Arena)]; +alignas(LowLevelAlloc::Arena) unsigned char unhooked_arena_storage[sizeof( + LowLevelAlloc::Arena)]; #ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING -ArenaStorage unhooked_async_sig_safe_arena_storage; +alignas( + LowLevelAlloc::Arena) unsigned char unhooked_async_sig_safe_arena_storage + [sizeof(LowLevelAlloc::Arena)]; #endif // We must use LowLevelCallOnce here to construct the global arenas, rather than @@ -276,10 +277,10 @@ static const uintptr_t kMagicAllocated = 0x4c833e95U; static const uintptr_t kMagicUnallocated = ~kMagicAllocated; namespace { -class SCOPED_LOCKABLE ArenaLock { +class ABSL_SCOPED_LOCKABLE ArenaLock { public: explicit ArenaLock(LowLevelAlloc::Arena *arena) - EXCLUSIVE_LOCK_FUNCTION(arena->mu) + ABSL_EXCLUSIVE_LOCK_FUNCTION(arena->mu) : arena_(arena) { #ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING if ((arena->flags & LowLevelAlloc::kAsyncSignalSafe) != 0) { @@ -291,7 +292,7 @@ class SCOPED_LOCKABLE ArenaLock { arena_->mu.Lock(); } ~ArenaLock() { ABSL_RAW_CHECK(left_, "haven't left Arena region"); } - void Leave() UNLOCK_FUNCTION() { + void Leave() ABSL_UNLOCK_FUNCTION() { arena_->mu.Unlock(); #ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING if (mask_valid_) { @@ -337,11 +338,11 @@ size_t GetPageSize() { size_t RoundedUpBlockSize() { // Round up block sizes to a power of two close to the header size. - size_t roundup = 16; - while (roundup < sizeof(AllocList::Header)) { - roundup += roundup; + size_t round_up = 16; + while (round_up < sizeof(AllocList::Header)) { + round_up += round_up; } - return roundup; + return round_up; } } // namespace @@ -351,8 +352,8 @@ LowLevelAlloc::Arena::Arena(uint32_t flags_value) allocation_count(0), flags(flags_value), pagesize(GetPageSize()), - roundup(RoundedUpBlockSize()), - min_size(2 * roundup), + round_up(RoundedUpBlockSize()), + min_size(2 * round_up), random(0) { freelist.header.size = 0; freelist.header.magic = @@ -448,7 +449,7 @@ static inline uintptr_t RoundUp(uintptr_t addr, uintptr_t align) { // that the freelist is in the correct order, that it // consists of regions marked "unallocated", and that no two regions // are adjacent in memory (they should have been coalesced). -// L < arena->mu +// L >= arena->mu static AllocList *Next(int i, AllocList *prev, LowLevelAlloc::Arena *arena) { ABSL_RAW_CHECK(i < prev->levels, "too few levels in Next()"); AllocList *next = prev->next[i]; @@ -509,8 +510,6 @@ void LowLevelAlloc::Free(void *v) { if (v != nullptr) { AllocList *f = reinterpret_cast<AllocList *>( reinterpret_cast<char *>(v) - sizeof (f->header)); - ABSL_RAW_CHECK(f->header.magic == Magic(kMagicAllocated, &f->header), - "bad magic number in Free()"); LowLevelAlloc::Arena *arena = f->header.arena; ArenaLock section(arena); AddToFreelist(v, arena); @@ -529,7 +528,7 @@ static void *DoAllocWithArena(size_t request, LowLevelAlloc::Arena *arena) { ArenaLock section(arena); // round up with header size_t req_rnd = RoundUp(CheckedAdd(request, sizeof (s->header)), - arena->roundup); + arena->round_up); for (;;) { // loop until we find a suitable region // find the minimum levels that a block of this size must have int i = LLA_SkiplistLevels(req_rnd, arena->min_size, nullptr) - 1; @@ -615,7 +614,7 @@ void *LowLevelAlloc::AllocWithArena(size_t request, Arena *arena) { } } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_LOW_LEVEL_ALLOC_MISSING diff --git a/absl/base/internal/low_level_alloc.h b/absl/base/internal/low_level_alloc.h index 32b6ec17..db91951c 100644 --- a/absl/base/internal/low_level_alloc.h +++ b/absl/base/internal/low_level_alloc.h @@ -55,7 +55,7 @@ #include "absl/base/port.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { class LowLevelAlloc { @@ -120,7 +120,7 @@ class LowLevelAlloc { }; } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_LOW_LEVEL_ALLOC_H_ diff --git a/absl/base/internal/low_level_alloc_test.cc b/absl/base/internal/low_level_alloc_test.cc index b6eb8b30..7abbbf9c 100644 --- a/absl/base/internal/low_level_alloc_test.cc +++ b/absl/base/internal/low_level_alloc_test.cc @@ -22,7 +22,7 @@ #include <utility> namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { namespace { @@ -150,7 +150,7 @@ static struct BeforeMain { } // namespace } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl int main(int argc, char *argv[]) { diff --git a/absl/base/internal/low_level_scheduling.h b/absl/base/internal/low_level_scheduling.h index b762279d..961cc981 100644 --- a/absl/base/internal/low_level_scheduling.h +++ b/absl/base/internal/low_level_scheduling.h @@ -28,7 +28,7 @@ extern "C" bool __google_disable_rescheduling(void); extern "C" void __google_enable_rescheduling(bool disable_result); namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { class SchedulingHelper; // To allow use of SchedulingGuard. @@ -101,7 +101,7 @@ inline void SchedulingGuard::EnableRescheduling(bool /* disable_result */) { } } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_LOW_LEVEL_SCHEDULING_H_ diff --git a/absl/base/internal/periodic_sampler.cc b/absl/base/internal/periodic_sampler.cc new file mode 100644 index 00000000..520dabba --- /dev/null +++ b/absl/base/internal/periodic_sampler.cc @@ -0,0 +1,53 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/periodic_sampler.h" + +#include <atomic> + +#include "absl/base/internal/exponential_biased.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +int64_t PeriodicSamplerBase::GetExponentialBiased(int period) noexcept { + return rng_.GetStride(period); +} + +bool PeriodicSamplerBase::SubtleConfirmSample() noexcept { + int current_period = period(); + + // Deal with period case 0 (always off) and 1 (always on) + if (ABSL_PREDICT_FALSE(current_period < 2)) { + stride_ = 0; + return current_period == 1; + } + + // Check if this is the first call to Sample() + if (ABSL_PREDICT_FALSE(stride_ == 1)) { + stride_ = static_cast<uint64_t>(-GetExponentialBiased(current_period)); + if (static_cast<int64_t>(stride_) < -1) { + ++stride_; + return false; + } + } + + stride_ = static_cast<uint64_t>(-GetExponentialBiased(current_period)); + return true; +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/base/internal/periodic_sampler.h b/absl/base/internal/periodic_sampler.h new file mode 100644 index 00000000..f8a86796 --- /dev/null +++ b/absl/base/internal/periodic_sampler.h @@ -0,0 +1,211 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_PERIODIC_SAMPLER_H_ +#define ABSL_BASE_INTERNAL_PERIODIC_SAMPLER_H_ + +#include <stdint.h> + +#include <atomic> + +#include "absl/base/internal/exponential_biased.h" +#include "absl/base/optimization.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// PeriodicSamplerBase provides the basic period sampler implementation. +// +// This is the base class for the templated PeriodicSampler class, which holds +// a global std::atomic value identified by a user defined tag, such that +// each specific PeriodSampler implementation holds its own global period. +// +// PeriodicSamplerBase is thread-compatible except where stated otherwise. +class PeriodicSamplerBase { + public: + // PeriodicSamplerBase is trivial / copyable / movable / destructible. + PeriodicSamplerBase() = default; + PeriodicSamplerBase(PeriodicSamplerBase&&) = default; + PeriodicSamplerBase(const PeriodicSamplerBase&) = default; + + // Returns true roughly once every `period` calls. This is established by a + // randomly picked `stride` that is counted down on each call to `Sample`. + // This stride is picked such that the probability of `Sample()` returning + // true is 1 in `period`. + inline bool Sample() noexcept; + + // The below methods are intended for optimized use cases where the + // size of the inlined fast path code is highly important. Applications + // should use the `Sample()` method unless they have proof that their + // specific use case requires the optimizations offered by these methods. + // + // An example of such a use case is SwissTable sampling. All sampling checks + // are in inlined SwissTable methods, and the number of call sites is huge. + // In this case, the inlined code size added to each translation unit calling + // SwissTable methods is non-trivial. + // + // The `SubtleMaybeSample()` function spuriously returns true even if the + // function should not be sampled, applications MUST match each call to + // 'SubtleMaybeSample()' returning true with a `SubtleConfirmSample()` call, + // and use the result of the latter as the sampling decision. + // In other words: the code should logically be equivalent to: + // + // if (SubtleMaybeSample() && SubtleConfirmSample()) { + // // Sample this call + // } + // + // In the 'inline-size' optimized case, the `SubtleConfirmSample()` call can + // be placed out of line, for example, the typical use case looks as follows: + // + // // --- frobber.h ----------- + // void FrobberSampled(); + // + // inline void FrobberImpl() { + // // ... + // } + // + // inline void Frobber() { + // if (ABSL_PREDICT_FALSE(sampler.SubtleMaybeSample())) { + // FrobberSampled(); + // } else { + // FrobberImpl(); + // } + // } + // + // // --- frobber.cc ----------- + // void FrobberSampled() { + // if (!sampler.SubtleConfirmSample())) { + // // Spurious false positive + // FrobberImpl(); + // return; + // } + // + // // Sampled execution + // // ... + // } + inline bool SubtleMaybeSample() noexcept; + bool SubtleConfirmSample() noexcept; + + protected: + // We explicitly don't use a virtual destructor as this class is never + // virtually destroyed, and it keeps the class trivial, which avoids TLS + // prologue and epilogue code for our TLS instances. + ~PeriodicSamplerBase() = default; + + // Returns the next stride for our sampler. + // This function is virtual for testing purposes only. + virtual int64_t GetExponentialBiased(int period) noexcept; + + private: + // Returns the current period of this sampler. Thread-safe. + virtual int period() const noexcept = 0; + + // Keep and decrement stride_ as an unsigned integer, but compare the value + // to zero casted as a signed int. clang and msvc do not create optimum code + // if we use signed for the combined decrement and sign comparison. + // + // Below 3 alternative options, all compiles generate the best code + // using the unsigned increment <---> signed int comparison option. + // + // Option 1: + // int64_t stride_; + // if (ABSL_PREDICT_TRUE(++stride_ < 0)) { ... } + // + // GCC x64 (OK) : https://gcc.godbolt.org/z/R5MzzA + // GCC ppc (OK) : https://gcc.godbolt.org/z/z7NZAt + // Clang x64 (BAD): https://gcc.godbolt.org/z/t4gPsd + // ICC x64 (OK) : https://gcc.godbolt.org/z/rE6s8W + // MSVC x64 (OK) : https://gcc.godbolt.org/z/ARMXqS + // + // Option 2: + // int64_t stride_ = 0; + // if (ABSL_PREDICT_TRUE(--stride_ >= 0)) { ... } + // + // GCC x64 (OK) : https://gcc.godbolt.org/z/jSQxYK + // GCC ppc (OK) : https://gcc.godbolt.org/z/VJdYaA + // Clang x64 (BAD): https://gcc.godbolt.org/z/Xm4NjX + // ICC x64 (OK) : https://gcc.godbolt.org/z/4snaFd + // MSVC x64 (BAD): https://gcc.godbolt.org/z/BgnEKE + // + // Option 3: + // uint64_t stride_; + // if (ABSL_PREDICT_TRUE(static_cast<int64_t>(++stride_) < 0)) { ... } + // + // GCC x64 (OK) : https://gcc.godbolt.org/z/bFbfPy + // GCC ppc (OK) : https://gcc.godbolt.org/z/S9KkUE + // Clang x64 (OK) : https://gcc.godbolt.org/z/UYzRb4 + // ICC x64 (OK) : https://gcc.godbolt.org/z/ptTNfD + // MSVC x64 (OK) : https://gcc.godbolt.org/z/76j4-5 + uint64_t stride_ = 0; + ExponentialBiased rng_; +}; + +inline bool PeriodicSamplerBase::SubtleMaybeSample() noexcept { + // See comments on `stride_` for the unsigned increment / signed compare. + if (ABSL_PREDICT_TRUE(static_cast<int64_t>(++stride_) < 0)) { + return false; + } + return true; +} + +inline bool PeriodicSamplerBase::Sample() noexcept { + return ABSL_PREDICT_FALSE(SubtleMaybeSample()) ? SubtleConfirmSample() + : false; +} + +// PeriodicSampler is a concreted periodic sampler implementation. +// The user provided Tag identifies the implementation, and is required to +// isolate the global state of this instance from other instances. +// +// Typical use case: +// +// struct HashTablezTag {}; +// thread_local PeriodicSampler sampler; +// +// void HashTableSamplingLogic(...) { +// if (sampler.Sample()) { +// HashTableSlowSamplePath(...); +// } +// } +// +template <typename Tag, int default_period = 0> +class PeriodicSampler final : public PeriodicSamplerBase { + public: + ~PeriodicSampler() = default; + + int period() const noexcept final { + return period_.load(std::memory_order_relaxed); + } + + // Sets the global period for this sampler. Thread-safe. + // Setting a period of 0 disables the sampler, i.e., every call to Sample() + // will return false. Setting a period of 1 puts the sampler in 'always on' + // mode, i.e., every call to Sample() returns true. + static void SetGlobalPeriod(int period) { + period_.store(period, std::memory_order_relaxed); + } + + private: + static std::atomic<int> period_; +}; + +template <typename Tag, int default_period> +std::atomic<int> PeriodicSampler<Tag, default_period>::period_(default_period); + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_PERIODIC_SAMPLER_H_ diff --git a/absl/base/internal/periodic_sampler_benchmark.cc b/absl/base/internal/periodic_sampler_benchmark.cc new file mode 100644 index 00000000..5ad469ce --- /dev/null +++ b/absl/base/internal/periodic_sampler_benchmark.cc @@ -0,0 +1,79 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" +#include "absl/base/internal/periodic_sampler.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { +namespace { + +template <typename Sampler> +void BM_Sample(Sampler* sampler, benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(sampler); + benchmark::DoNotOptimize(sampler->Sample()); + } +} + +template <typename Sampler> +void BM_SampleMinunumInlined(Sampler* sampler, benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(sampler); + if (ABSL_PREDICT_FALSE(sampler->SubtleMaybeSample())) { + benchmark::DoNotOptimize(sampler->SubtleConfirmSample()); + } + } +} + +void BM_PeriodicSampler_TinySample(benchmark::State& state) { + struct Tag {}; + PeriodicSampler<Tag, 10> sampler; + BM_Sample(&sampler, state); +} +BENCHMARK(BM_PeriodicSampler_TinySample); + +void BM_PeriodicSampler_ShortSample(benchmark::State& state) { + struct Tag {}; + PeriodicSampler<Tag, 1024> sampler; + BM_Sample(&sampler, state); +} +BENCHMARK(BM_PeriodicSampler_ShortSample); + +void BM_PeriodicSampler_LongSample(benchmark::State& state) { + struct Tag {}; + PeriodicSampler<Tag, 1024 * 1024> sampler; + BM_Sample(&sampler, state); +} +BENCHMARK(BM_PeriodicSampler_LongSample); + +void BM_PeriodicSampler_LongSampleMinunumInlined(benchmark::State& state) { + struct Tag {}; + PeriodicSampler<Tag, 1024 * 1024> sampler; + BM_SampleMinunumInlined(&sampler, state); +} +BENCHMARK(BM_PeriodicSampler_LongSampleMinunumInlined); + +void BM_PeriodicSampler_Disabled(benchmark::State& state) { + struct Tag {}; + PeriodicSampler<Tag, 0> sampler; + BM_Sample(&sampler, state); +} +BENCHMARK(BM_PeriodicSampler_Disabled); + +} // namespace +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/base/internal/periodic_sampler_test.cc b/absl/base/internal/periodic_sampler_test.cc new file mode 100644 index 00000000..3b301e37 --- /dev/null +++ b/absl/base/internal/periodic_sampler_test.cc @@ -0,0 +1,177 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/periodic_sampler.h" + +#include <thread> // NOLINT(build/c++11) + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/base/macros.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { +namespace { + +using testing::Eq; +using testing::Return; +using testing::StrictMock; + +class MockPeriodicSampler : public PeriodicSamplerBase { + public: + virtual ~MockPeriodicSampler() = default; + + MOCK_METHOD(int, period, (), (const, noexcept)); + MOCK_METHOD(int64_t, GetExponentialBiased, (int), (noexcept)); +}; + +TEST(PeriodicSamplerBaseTest, Sample) { + StrictMock<MockPeriodicSampler> sampler; + + EXPECT_CALL(sampler, period()).Times(3).WillRepeatedly(Return(16)); + EXPECT_CALL(sampler, GetExponentialBiased(16)) + .WillOnce(Return(2)) + .WillOnce(Return(3)) + .WillOnce(Return(4)); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_TRUE(sampler.Sample()); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); + EXPECT_TRUE(sampler.Sample()); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); +} + +TEST(PeriodicSamplerBaseTest, ImmediatelySample) { + StrictMock<MockPeriodicSampler> sampler; + + EXPECT_CALL(sampler, period()).Times(2).WillRepeatedly(Return(16)); + EXPECT_CALL(sampler, GetExponentialBiased(16)) + .WillOnce(Return(1)) + .WillOnce(Return(2)) + .WillOnce(Return(3)); + + EXPECT_TRUE(sampler.Sample()); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_TRUE(sampler.Sample()); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); +} + +TEST(PeriodicSamplerBaseTest, Disabled) { + StrictMock<MockPeriodicSampler> sampler; + + EXPECT_CALL(sampler, period()).Times(3).WillRepeatedly(Return(0)); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); +} + +TEST(PeriodicSamplerBaseTest, AlwaysOn) { + StrictMock<MockPeriodicSampler> sampler; + + EXPECT_CALL(sampler, period()).Times(3).WillRepeatedly(Return(1)); + + EXPECT_TRUE(sampler.Sample()); + EXPECT_TRUE(sampler.Sample()); + EXPECT_TRUE(sampler.Sample()); +} + +TEST(PeriodicSamplerBaseTest, Disable) { + StrictMock<MockPeriodicSampler> sampler; + + EXPECT_CALL(sampler, period()).WillOnce(Return(16)); + EXPECT_CALL(sampler, GetExponentialBiased(16)).WillOnce(Return(3)); + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); + + EXPECT_CALL(sampler, period()).Times(2).WillRepeatedly(Return(0)); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); +} + +TEST(PeriodicSamplerBaseTest, Enable) { + StrictMock<MockPeriodicSampler> sampler; + + EXPECT_CALL(sampler, period()).WillOnce(Return(0)); + EXPECT_FALSE(sampler.Sample()); + + EXPECT_CALL(sampler, period()).Times(2).WillRepeatedly(Return(16)); + EXPECT_CALL(sampler, GetExponentialBiased(16)) + .Times(2) + .WillRepeatedly(Return(3)); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); + EXPECT_TRUE(sampler.Sample()); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); +} + +TEST(PeriodicSamplerTest, ConstructConstInit) { + struct Tag {}; + ABSL_CONST_INIT static PeriodicSampler<Tag> sampler; + (void)sampler; +} + +TEST(PeriodicSamplerTest, DefaultPeriod0) { + struct Tag {}; + PeriodicSampler<Tag> sampler; + EXPECT_THAT(sampler.period(), Eq(0)); +} + +TEST(PeriodicSamplerTest, DefaultPeriod) { + struct Tag {}; + PeriodicSampler<Tag, 100> sampler; + EXPECT_THAT(sampler.period(), Eq(100)); +} + +TEST(PeriodicSamplerTest, SetGlobalPeriod) { + struct Tag1 {}; + struct Tag2 {}; + PeriodicSampler<Tag1, 25> sampler1; + PeriodicSampler<Tag2, 50> sampler2; + + EXPECT_THAT(sampler1.period(), Eq(25)); + EXPECT_THAT(sampler2.period(), Eq(50)); + + std::thread thread([] { + PeriodicSampler<Tag1, 25> sampler1; + PeriodicSampler<Tag2, 50> sampler2; + EXPECT_THAT(sampler1.period(), Eq(25)); + EXPECT_THAT(sampler2.period(), Eq(50)); + sampler1.SetGlobalPeriod(10); + sampler2.SetGlobalPeriod(20); + }); + thread.join(); + + EXPECT_THAT(sampler1.period(), Eq(10)); + EXPECT_THAT(sampler2.period(), Eq(20)); +} + +} // namespace +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/base/internal/raw_logging.cc b/absl/base/internal/raw_logging.cc index 1a36d5b6..40cea550 100644 --- a/absl/base/internal/raw_logging.cc +++ b/absl/base/internal/raw_logging.cc @@ -37,9 +37,9 @@ // this, consider moving both to config.h instead. #if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ defined(__Fuchsia__) || defined(__native_client__) || \ - defined(__EMSCRIPTEN__) -#include <unistd.h> + defined(__EMSCRIPTEN__) || defined(__ASYLO__) +#include <unistd.h> #define ABSL_HAVE_POSIX_WRITE 1 #define ABSL_LOW_LEVEL_WRITE_SUPPORTED 1 @@ -71,10 +71,12 @@ // Explicitly #error out when not ABSL_LOW_LEVEL_WRITE_SUPPORTED, except for a // whitelisted set of platforms for which we expect not to be able to raw log. -ABSL_CONST_INIT static absl::base_internal::AtomicHook< - absl::raw_logging_internal::LogPrefixHook> log_prefix_hook; -ABSL_CONST_INIT static absl::base_internal::AtomicHook< - absl::raw_logging_internal::AbortHook> abort_hook; +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook< + absl::raw_logging_internal::LogPrefixHook> + log_prefix_hook; +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook< + absl::raw_logging_internal::AbortHook> + abort_hook; #ifdef ABSL_LOW_LEVEL_WRITE_SUPPORTED static const char kTruncated[] = " ... (message truncated)\n"; @@ -182,7 +184,7 @@ void RawLogVA(absl::LogSeverity severity, const char* file, int line, } // namespace namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace raw_logging_internal { void SafeWriteToStderr(const char *s, size_t len) { #if defined(ABSL_HAVE_SYSCALL_WRITE) @@ -225,13 +227,14 @@ bool RawLoggingFullySupported() { #endif // !ABSL_LOW_LEVEL_WRITE_SUPPORTED } -ABSL_CONST_INIT absl::base_internal::AtomicHook<InternalLogFunction> - internal_log_function(DefaultInternalLog); +ABSL_DLL ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES + absl::base_internal::AtomicHook<InternalLogFunction> + internal_log_function(DefaultInternalLog); void RegisterInternalLogFunction(InternalLogFunction func) { internal_log_function.Store(func); } } // namespace raw_logging_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/raw_logging.h b/absl/base/internal/raw_logging.h index 8e5a34ef..418d6c85 100644 --- a/absl/base/internal/raw_logging.h +++ b/absl/base/internal/raw_logging.h @@ -22,9 +22,11 @@ #include <string> #include "absl/base/attributes.h" +#include "absl/base/config.h" #include "absl/base/internal/atomic_hook.h" #include "absl/base/log_severity.h" #include "absl/base/macros.h" +#include "absl/base/optimization.h" #include "absl/base/port.h" // This is similar to LOG(severity) << format..., but @@ -70,14 +72,10 @@ // // The API is a subset of the above: each macro only takes two arguments. Use // StrCat if you need to build a richer message. -#define ABSL_INTERNAL_LOG(severity, message) \ - do { \ - constexpr const char* absl_raw_logging_internal_basename = \ - ::absl::raw_logging_internal::Basename(__FILE__, \ - sizeof(__FILE__) - 1); \ - ::absl::raw_logging_internal::internal_log_function( \ - ABSL_RAW_LOGGING_INTERNAL_##severity, \ - absl_raw_logging_internal_basename, __LINE__, message); \ +#define ABSL_INTERNAL_LOG(severity, message) \ + do { \ + ::absl::raw_logging_internal::internal_log_function( \ + ABSL_RAW_LOGGING_INTERNAL_##severity, __FILE__, __LINE__, message); \ } while (0) #define ABSL_INTERNAL_CHECK(condition, message) \ @@ -97,7 +95,7 @@ ::absl::NormalizeLogSeverity(severity) namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace raw_logging_internal { // Helper function to implement ABSL_RAW_LOG @@ -158,7 +156,7 @@ using LogPrefixHook = bool (*)(absl::LogSeverity severity, const char* file, // // 'file' and 'line' are the file and line number where the ABSL_RAW_LOG macro // was located. -// The null-terminated logged message lives in the buffer between 'buf_start' +// The NUL-terminated logged message lives in the buffer between 'buf_start' // and 'buf_end'. 'prefix_end' points to the first non-prefix character of the // buffer (as written by the LogPrefixHook.) using AbortHook = void (*)(const char* file, int line, const char* buf_start, @@ -172,12 +170,14 @@ using InternalLogFunction = void (*)(absl::LogSeverity severity, const char* file, int line, const std::string& message); -extern base_internal::AtomicHook<InternalLogFunction> internal_log_function; +ABSL_DLL ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES extern base_internal::AtomicHook< + InternalLogFunction> + internal_log_function; void RegisterInternalLogFunction(InternalLogFunction func); } // namespace raw_logging_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_RAW_LOGGING_H_ diff --git a/absl/base/internal/scheduling_mode.h b/absl/base/internal/scheduling_mode.h index 48767920..8be5ab6d 100644 --- a/absl/base/internal/scheduling_mode.h +++ b/absl/base/internal/scheduling_mode.h @@ -18,8 +18,10 @@ #ifndef ABSL_BASE_INTERNAL_SCHEDULING_MODE_H_ #define ABSL_BASE_INTERNAL_SCHEDULING_MODE_H_ +#include "absl/base/config.h" + namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { // Used to describe how a thread may be scheduled. Typically associated with @@ -50,7 +52,7 @@ enum SchedulingMode { }; } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_SCHEDULING_MODE_H_ diff --git a/absl/base/internal/scoped_set_env.cc b/absl/base/internal/scoped_set_env.cc index eb7a0116..8a934cb5 100644 --- a/absl/base/internal/scoped_set_env.cc +++ b/absl/base/internal/scoped_set_env.cc @@ -23,7 +23,7 @@ #include "absl/base/internal/raw_logging.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { namespace { @@ -77,5 +77,5 @@ ScopedSetEnv::~ScopedSetEnv() { } } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/scoped_set_env.h b/absl/base/internal/scoped_set_env.h index 709843bc..19ec7b5d 100644 --- a/absl/base/internal/scoped_set_env.h +++ b/absl/base/internal/scoped_set_env.h @@ -19,8 +19,10 @@ #include <string> +#include "absl/base/config.h" + namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { class ScopedSetEnv { @@ -37,7 +39,7 @@ class ScopedSetEnv { }; } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_SCOPED_SET_ENV_H_ diff --git a/absl/base/internal/spinlock.cc b/absl/base/internal/spinlock.cc index 5f443bfa..830d4729 100644 --- a/absl/base/internal/spinlock.cc +++ b/absl/base/internal/spinlock.cc @@ -54,11 +54,11 @@ // holder to acquire the lock. There may be outstanding waiter(s). namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { -ABSL_CONST_INIT static base_internal::AtomicHook<void (*)(const void *lock, - int64_t wait_cycles)> +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static base_internal::AtomicHook<void (*)( + const void *lock, int64_t wait_cycles)> submit_profile_data; void RegisterSpinLockProfiler(void (*fn)(const void *contendedlock, @@ -229,5 +229,5 @@ uint64_t SpinLock::DecodeWaitCycles(uint32_t lock_value) { } } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/spinlock.h b/absl/base/internal/spinlock.h index df947661..24e2e9a6 100644 --- a/absl/base/internal/spinlock.h +++ b/absl/base/internal/spinlock.h @@ -46,10 +46,10 @@ #include "absl/base/thread_annotations.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { -class LOCKABLE SpinLock { +class ABSL_LOCKABLE SpinLock { public: SpinLock() : lockword_(kSpinLockCooperative) { ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static); @@ -80,7 +80,7 @@ class LOCKABLE SpinLock { ~SpinLock() { ABSL_TSAN_MUTEX_DESTROY(this, __tsan_mutex_not_static); } // Acquire this SpinLock. - inline void Lock() EXCLUSIVE_LOCK_FUNCTION() { + inline void Lock() ABSL_EXCLUSIVE_LOCK_FUNCTION() { ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); if (!TryLockImpl()) { SlowLock(); @@ -92,7 +92,7 @@ class LOCKABLE SpinLock { // acquisition was successful. If the lock was not acquired, false is // returned. If this SpinLock is free at the time of the call, TryLock // will return true with high probability. - inline bool TryLock() EXCLUSIVE_TRYLOCK_FUNCTION(true) { + inline bool TryLock() ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(true) { ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_try_lock); bool res = TryLockImpl(); ABSL_TSAN_MUTEX_POST_LOCK( @@ -102,7 +102,7 @@ class LOCKABLE SpinLock { } // Release this SpinLock, which must be held by the calling thread. - inline void Unlock() UNLOCK_FUNCTION() { + inline void Unlock() ABSL_UNLOCK_FUNCTION() { ABSL_TSAN_MUTEX_PRE_UNLOCK(this, 0); uint32_t lock_value = lockword_.load(std::memory_order_relaxed); lock_value = lockword_.exchange(lock_value & kSpinLockCooperative, @@ -180,13 +180,13 @@ class LOCKABLE SpinLock { // Corresponding locker object that arranges to acquire a spinlock for // the duration of a C++ scope. -class SCOPED_LOCKABLE SpinLockHolder { +class ABSL_SCOPED_LOCKABLE SpinLockHolder { public: - inline explicit SpinLockHolder(SpinLock* l) EXCLUSIVE_LOCK_FUNCTION(l) + inline explicit SpinLockHolder(SpinLock* l) ABSL_EXCLUSIVE_LOCK_FUNCTION(l) : lock_(l) { l->Lock(); } - inline ~SpinLockHolder() UNLOCK_FUNCTION() { lock_->Unlock(); } + inline ~SpinLockHolder() ABSL_UNLOCK_FUNCTION() { lock_->Unlock(); } SpinLockHolder(const SpinLockHolder&) = delete; SpinLockHolder& operator=(const SpinLockHolder&) = delete; @@ -226,11 +226,10 @@ inline uint32_t SpinLock::TryLockInternal(uint32_t lock_value, } } - if (lockword_.compare_exchange_strong( + if (!lockword_.compare_exchange_strong( lock_value, kSpinLockHeld | lock_value | wait_cycles | sched_disabled_bit, std::memory_order_acquire, std::memory_order_relaxed)) { - } else { base_internal::SchedulingGuard::EnableRescheduling(sched_disabled_bit != 0); } @@ -238,7 +237,7 @@ inline uint32_t SpinLock::TryLockInternal(uint32_t lock_value, } } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_SPINLOCK_H_ diff --git a/absl/base/internal/spinlock_linux.inc b/absl/base/internal/spinlock_linux.inc index 28e29d19..323edd62 100644 --- a/absl/base/internal/spinlock_linux.inc +++ b/absl/base/internal/spinlock_linux.inc @@ -19,12 +19,12 @@ #include <unistd.h> #include <atomic> -#include <cerrno> #include <climits> #include <cstdint> #include <ctime> #include "absl/base/attributes.h" +#include "absl/base/internal/errno_saver.h" // The SpinLock lockword is `std::atomic<uint32_t>`. Here we assert that // `std::atomic<uint32_t>` is bitwise equivalent of the `int` expected @@ -51,12 +51,11 @@ extern "C" { ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay( std::atomic<uint32_t> *w, uint32_t value, int loop, absl::base_internal::SchedulingMode) { - int save_errno = errno; + absl::base_internal::ErrnoSaver errno_saver; struct timespec tm; tm.tv_sec = 0; tm.tv_nsec = absl::base_internal::SpinLockSuggestedDelayNS(loop); syscall(SYS_futex, w, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, value, &tm); - errno = save_errno; } ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockWake(std::atomic<uint32_t> *w, diff --git a/absl/base/internal/spinlock_posix.inc b/absl/base/internal/spinlock_posix.inc index f025b5f8..fcd21b15 100644 --- a/absl/base/internal/spinlock_posix.inc +++ b/absl/base/internal/spinlock_posix.inc @@ -15,10 +15,11 @@ // This file is a Posix-specific part of spinlock_wait.cc #include <sched.h> + #include <atomic> #include <ctime> -#include <cerrno> +#include "absl/base/internal/errno_saver.h" #include "absl/base/internal/scheduling_mode.h" #include "absl/base/port.h" @@ -27,7 +28,7 @@ extern "C" { ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay( std::atomic<uint32_t>* /* lock_word */, uint32_t /* value */, int loop, absl::base_internal::SchedulingMode /* mode */) { - int save_errno = errno; + absl::base_internal::ErrnoSaver errno_saver; if (loop == 0) { } else if (loop == 1) { sched_yield(); @@ -37,7 +38,6 @@ ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay( tm.tv_nsec = absl::base_internal::SpinLockSuggestedDelayNS(loop); nanosleep(&tm, nullptr); } - errno = save_errno; } ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockWake( diff --git a/absl/base/internal/spinlock_wait.cc b/absl/base/internal/spinlock_wait.cc index 60a85196..fa824be1 100644 --- a/absl/base/internal/spinlock_wait.cc +++ b/absl/base/internal/spinlock_wait.cc @@ -32,7 +32,7 @@ #endif namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { // See spinlock_wait.h for spec. @@ -77,5 +77,5 @@ int SpinLockSuggestedDelayNS(int loop) { } } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/spinlock_wait.h b/absl/base/internal/spinlock_wait.h index 7e139de0..169bc749 100644 --- a/absl/base/internal/spinlock_wait.h +++ b/absl/base/internal/spinlock_wait.h @@ -24,7 +24,7 @@ #include "absl/base/internal/scheduling_mode.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { // SpinLockWait() waits until it can perform one of several transitions from @@ -63,7 +63,7 @@ void SpinLockDelay(std::atomic<uint32_t> *w, uint32_t value, int loop, int SpinLockSuggestedDelayNS(int loop); } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl // In some build configurations we pass --detect-odr-violations to the diff --git a/absl/base/internal/sysinfo.cc b/absl/base/internal/sysinfo.cc index 0d5cf821..a0930e97 100644 --- a/absl/base/internal/sysinfo.cc +++ b/absl/base/internal/sysinfo.cc @@ -17,7 +17,6 @@ #include "absl/base/attributes.h" #ifdef _WIN32 -#include <shlwapi.h> #include <windows.h> #else #include <fcntl.h> @@ -56,13 +55,9 @@ #include "absl/base/internal/unscaledcycleclock.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { -static once_flag init_system_info_once; -static int num_cpus = 0; -static double nominal_cpu_frequency = 1.0; // 0.0 might be dangerous. - static int GetNumCPUs() { #if defined(__myriad2__) return 1; @@ -77,14 +72,23 @@ static int GetNumCPUs() { #if defined(_WIN32) static double GetNominalCPUFrequency() { - DWORD data; - DWORD data_size = sizeof(data); - #pragma comment(lib, "shlwapi.lib") // For SHGetValue(). - if (SUCCEEDED( - SHGetValueA(HKEY_LOCAL_MACHINE, - "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", - "~MHz", nullptr, &data, &data_size))) { - return data * 1e6; // Value is MHz. +#pragma comment(lib, "advapi32.lib") // For Reg* functions. + HKEY key; + // Use the Reg* functions rather than the SH functions because shlwapi.dll + // pulls in gdi32.dll which makes process destruction much more costly. + if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, + "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", 0, + KEY_READ, &key) == ERROR_SUCCESS) { + DWORD type = 0; + DWORD data = 0; + DWORD data_size = sizeof(data); + auto result = RegQueryValueExA(key, "~MHz", 0, &type, + reinterpret_cast<LPBYTE>(&data), &data_size); + RegCloseKey(key); + if (result == ERROR_SUCCESS && type == REG_DWORD && + data_size == sizeof(data)) { + return data * 1e6; // Value is MHz. + } } return 1.0; } @@ -257,28 +261,34 @@ static double GetNominalCPUFrequency() { #endif -// InitializeSystemInfo() may be called before main() and before -// malloc is properly initialized, therefore this must not allocate -// memory. -static void InitializeSystemInfo() { - num_cpus = GetNumCPUs(); - nominal_cpu_frequency = GetNominalCPUFrequency(); -} +ABSL_CONST_INIT static once_flag init_num_cpus_once; +ABSL_CONST_INIT static int num_cpus = 0; +// NumCPUs() may be called before main() and before malloc is properly +// initialized, therefore this must not allocate memory. int NumCPUs() { - base_internal::LowLevelCallOnce(&init_system_info_once, InitializeSystemInfo); + base_internal::LowLevelCallOnce( + &init_num_cpus_once, []() { num_cpus = GetNumCPUs(); }); return num_cpus; } +// A default frequency of 0.0 might be dangerous if it is used in division. +ABSL_CONST_INIT static once_flag init_nominal_cpu_frequency_once; +ABSL_CONST_INIT static double nominal_cpu_frequency = 1.0; + +// NominalCPUFrequency() may be called before main() and before malloc is +// properly initialized, therefore this must not allocate memory. double NominalCPUFrequency() { - base_internal::LowLevelCallOnce(&init_system_info_once, InitializeSystemInfo); + base_internal::LowLevelCallOnce( + &init_nominal_cpu_frequency_once, + []() { nominal_cpu_frequency = GetNominalCPUFrequency(); }); return nominal_cpu_frequency; } #if defined(_WIN32) pid_t GetTID() { - return GetCurrentThreadId(); + return pid_t{GetCurrentThreadId()}; } #elif defined(__linux__) @@ -402,5 +412,5 @@ pid_t GetTID() { #endif } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/sysinfo.h b/absl/base/internal/sysinfo.h index 22739aeb..7246d5dd 100644 --- a/absl/base/internal/sysinfo.h +++ b/absl/base/internal/sysinfo.h @@ -26,14 +26,14 @@ #ifndef _WIN32 #include <sys/types.h> -#else -#include <intsafe.h> #endif +#include <cstdint> + #include "absl/base/port.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { // Nominal core processor cycles per second of each processor. This is _not_ @@ -52,14 +52,15 @@ int NumCPUs(); // On Linux, you may send a signal to the resulting ID with kill(). However, // it is recommended for portability that you use pthread_kill() instead. #ifdef _WIN32 -// On Windows, process id and thread id are of the same type according to -// the return types of GetProcessId() and GetThreadId() are both DWORD. -using pid_t = DWORD; +// On Windows, process id and thread id are of the same type according to the +// return types of GetProcessId() and GetThreadId() are both DWORD, an unsigned +// 32-bit type. +using pid_t = uint32_t; #endif pid_t GetTID(); } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_SYSINFO_H_ diff --git a/absl/base/internal/sysinfo_test.cc b/absl/base/internal/sysinfo_test.cc index c8323e52..fa8b88b1 100644 --- a/absl/base/internal/sysinfo_test.cc +++ b/absl/base/internal/sysinfo_test.cc @@ -28,7 +28,7 @@ #include "absl/synchronization/mutex.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { namespace { @@ -59,8 +59,8 @@ TEST(SysinfoTest, GetTID) { #endif // Test that TIDs are unique to each thread. // Uses a few loops to exercise implementations that reallocate IDs. - for (int i = 0; i < 32; ++i) { - constexpr int kNumThreads = 64; + for (int i = 0; i < 10; ++i) { + constexpr int kNumThreads = 10; Barrier all_threads_done(kNumThreads); std::vector<std::thread> threads; @@ -96,5 +96,5 @@ TEST(SysinfoTest, LinuxGetTID) { } // namespace } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/thread_identity.cc b/absl/base/internal/thread_identity.cc index dbec326a..d63a04ae 100644 --- a/absl/base/internal/thread_identity.cc +++ b/absl/base/internal/thread_identity.cc @@ -28,7 +28,7 @@ #include "absl/base/internal/spinlock.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { #if ABSL_THREAD_IDENTITY_MODE != ABSL_THREAD_IDENTITY_MODE_USE_CPP11 @@ -56,7 +56,12 @@ void AllocateThreadIdentityKey(ThreadIdentityReclaimerFunction reclaimer) { #ifdef __GNUC__ __attribute__((visibility("protected"))) #endif // __GNUC__ - ABSL_PER_THREAD_TLS_KEYWORD ThreadIdentity* thread_identity_ptr; +#if ABSL_PER_THREAD_TLS +// Prefer __thread to thread_local as benchmarks indicate it is a bit faster. +ABSL_PER_THREAD_TLS_KEYWORD ThreadIdentity* thread_identity_ptr = nullptr; +#elif defined(ABSL_HAVE_THREAD_LOCAL) +thread_local ThreadIdentity* thread_identity_ptr = nullptr; +#endif // ABSL_PER_THREAD_TLS #endif // TLS or CPP11 void SetCurrentThreadIdentity( @@ -70,8 +75,8 @@ void SetCurrentThreadIdentity( absl::call_once(init_thread_identity_key_once, AllocateThreadIdentityKey, reclaimer); -#ifdef __EMSCRIPTEN__ - // Emscripten PThread implementation does not support signals. +#if defined(__EMSCRIPTEN__) || defined(__MINGW32__) + // Emscripten and MinGW pthread implementations does not support signals. // See https://kripken.github.io/emscripten-site/docs/porting/pthreads.html // for more information. pthread_setspecific(thread_identity_pthread_key, @@ -90,7 +95,7 @@ void SetCurrentThreadIdentity( pthread_setspecific(thread_identity_pthread_key, reinterpret_cast<void*>(identity)); pthread_sigmask(SIG_SETMASK, &curr_signals, nullptr); -#endif // !__EMSCRIPTEN__ +#endif // !__EMSCRIPTEN__ && !__MINGW32__ #elif ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_TLS // NOTE: Not async-safe. But can be open-coded. @@ -108,6 +113,18 @@ void SetCurrentThreadIdentity( #endif } +#if ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_TLS || \ + ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_CPP11 + +// Please see the comment on `CurrentThreadIdentityIfPresent` in +// thread_identity.h. Because DLLs cannot expose thread_local variables in +// headers, we opt for the correct-but-slower option of placing the definition +// of this function only in a translation unit inside DLL. +#if defined(ABSL_BUILD_DLL) || defined(ABSL_CONSUME_DLL) +ThreadIdentity* CurrentThreadIdentityIfPresent() { return thread_identity_ptr; } +#endif +#endif + void ClearCurrentThreadIdentity() { #if ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_TLS || \ ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_CPP11 @@ -131,5 +148,5 @@ ThreadIdentity* CurrentThreadIdentityIfPresent() { #endif } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/thread_identity.h b/absl/base/internal/thread_identity.h index bb5aa074..ceb109b4 100644 --- a/absl/base/internal/thread_identity.h +++ b/absl/base/internal/thread_identity.h @@ -30,10 +30,11 @@ #include <atomic> #include <cstdint> +#include "absl/base/config.h" #include "absl/base/internal/per_thread_tls.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN struct SynchLocksHeld; struct SynchWaitParams; @@ -209,7 +210,7 @@ void ClearCurrentThreadIdentity(); #error ABSL_THREAD_IDENTITY_MODE cannot be direcly set #elif defined(ABSL_FORCE_THREAD_IDENTITY_MODE) #define ABSL_THREAD_IDENTITY_MODE ABSL_FORCE_THREAD_IDENTITY_MODE -#elif defined(_WIN32) +#elif defined(_WIN32) && !defined(__MINGW32__) #define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_CPP11 #elif ABSL_PER_THREAD_TLS && defined(__GOOGLE_GRTE_VERSION__) && \ (__GOOGLE_GRTE_VERSION__ >= 20140228L) @@ -225,11 +226,26 @@ void ClearCurrentThreadIdentity(); #if ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_TLS || \ ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_CPP11 -extern ABSL_PER_THREAD_TLS_KEYWORD ThreadIdentity* thread_identity_ptr; +#if ABSL_PER_THREAD_TLS +ABSL_CONST_INIT extern ABSL_PER_THREAD_TLS_KEYWORD ThreadIdentity* + thread_identity_ptr; +#elif defined(ABSL_HAVE_THREAD_LOCAL) +ABSL_CONST_INIT extern thread_local ThreadIdentity* thread_identity_ptr; +#else +#error Thread-local storage not detected on this platform +#endif +// thread_local variables cannot be in headers exposed by DLLs. However, it is +// important for performance reasons in general that +// `CurrentThreadIdentityIfPresent` be inlined. This is not possible across a +// DLL boundary so, with DLLs, we opt to have the function not be inlined. Note +// that `CurrentThreadIdentityIfPresent` is declared above so we can exclude +// this entire inline definition when compiling as a DLL. +#if !defined(ABSL_BUILD_DLL) && !defined(ABSL_CONSUME_DLL) inline ThreadIdentity* CurrentThreadIdentityIfPresent() { return thread_identity_ptr; } +#endif #elif ABSL_THREAD_IDENTITY_MODE != \ ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC @@ -237,7 +253,7 @@ inline ThreadIdentity* CurrentThreadIdentityIfPresent() { #endif } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_THREAD_IDENTITY_H_ diff --git a/absl/base/internal/thread_identity_test.cc b/absl/base/internal/thread_identity_test.cc index 8de6d9eb..3685779c 100644 --- a/absl/base/internal/thread_identity_test.cc +++ b/absl/base/internal/thread_identity_test.cc @@ -25,7 +25,7 @@ #include "absl/synchronization/mutex.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { namespace { @@ -124,5 +124,5 @@ TEST(ThreadIdentityTest, ReusedThreadIdentityMutexTest) { } // namespace } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/throw_delegate.cc b/absl/base/internal/throw_delegate.cc index bee404d4..c055f75d 100644 --- a/absl/base/internal/throw_delegate.cc +++ b/absl/base/internal/throw_delegate.cc @@ -22,7 +22,7 @@ #include "absl/base/internal/raw_logging.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { namespace { @@ -104,5 +104,5 @@ void ThrowStdBadFunctionCall() { Throw(std::bad_function_call()); } void ThrowStdBadAlloc() { Throw(std::bad_alloc()); } } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/throw_delegate.h b/absl/base/internal/throw_delegate.h index 60ed4f32..075f5272 100644 --- a/absl/base/internal/throw_delegate.h +++ b/absl/base/internal/throw_delegate.h @@ -19,8 +19,10 @@ #include <string> +#include "absl/base/config.h" + namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { // Helper functions that allow throwing exceptions consistently from anywhere. @@ -67,7 +69,7 @@ namespace base_internal { // [[noreturn]] void ThrowStdBadArrayNewLength(); } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_BASE_INTERNAL_THROW_DELEGATE_H_ diff --git a/absl/base/internal/unaligned_access.h b/absl/base/internal/unaligned_access.h index a709a446..6be56c86 100644 --- a/absl/base/internal/unaligned_access.h +++ b/absl/base/internal/unaligned_access.h @@ -18,9 +18,11 @@ #define ABSL_BASE_INTERNAL_UNALIGNED_ACCESS_H_ #include <string.h> + #include <cstdint> #include "absl/base/attributes.h" +#include "absl/base/config.h" // unaligned APIs @@ -56,7 +58,7 @@ void __sanitizer_unaligned_store64(void *p, uint64_t v); } // extern "C" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { inline uint16_t UnalignedLoad16(const void *p) { @@ -84,7 +86,7 @@ inline void UnalignedStore64(void *p, uint64_t v) { } } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #define ABSL_INTERNAL_UNALIGNED_LOAD16(_p) \ @@ -104,7 +106,7 @@ inline void UnalignedStore64(void *p, uint64_t v) { #else namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { inline uint16_t UnalignedLoad16(const void *p) { @@ -132,7 +134,7 @@ inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); } inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); } } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #define ABSL_INTERNAL_UNALIGNED_LOAD16(_p) \ diff --git a/absl/base/internal/unscaledcycleclock.cc b/absl/base/internal/unscaledcycleclock.cc index 29a927d3..f1e7bbef 100644 --- a/absl/base/internal/unscaledcycleclock.cc +++ b/absl/base/internal/unscaledcycleclock.cc @@ -21,13 +21,18 @@ #endif #if defined(__powerpc__) || defined(__ppc__) +#ifdef __GLIBC__ #include <sys/platform/ppc.h> +#elif defined(__FreeBSD__) +#include <sys/sysctl.h> +#include <sys/types.h> +#endif #endif #include "absl/base/internal/sysinfo.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace base_internal { #if defined(__i386__) @@ -57,11 +62,43 @@ double UnscaledCycleClock::Frequency() { #elif defined(__powerpc__) || defined(__ppc__) int64_t UnscaledCycleClock::Now() { +#ifdef __GLIBC__ return __ppc_get_timebase(); +#else +#ifdef __powerpc64__ + int64_t tbr; + asm volatile("mfspr %0, 268" : "=r"(tbr)); + return tbr; +#else + int32_t tbu, tbl, tmp; + asm volatile( + "0:\n" + "mftbu %[hi32]\n" + "mftb %[lo32]\n" + "mftbu %[tmp]\n" + "cmpw %[tmp],%[hi32]\n" + "bne 0b\n" + : [ hi32 ] "=r"(tbu), [ lo32 ] "=r"(tbl), [ tmp ] "=r"(tmp)); + return (static_cast<int64_t>(tbu) << 32) | tbl; +#endif +#endif } double UnscaledCycleClock::Frequency() { +#ifdef __GLIBC__ return __ppc_get_timebase_freq(); +#elif defined(__FreeBSD__) + static once_flag init_timebase_frequency_once; + static double timebase_frequency = 0.0; + base_internal::LowLevelCallOnce(&init_timebase_frequency_once, [&]() { + size_t length = sizeof(timebase_frequency); + sysctlbyname("kern.timecounter.tc.timebase.frequency", &timebase_frequency, + &length, nullptr, 0); + }); + return timebase_frequency; +#else +#error Must implement UnscaledCycleClock::Frequency() +#endif } #elif defined(__aarch64__) @@ -97,7 +134,7 @@ double UnscaledCycleClock::Frequency() { #endif } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_USE_UNSCALED_CYCLECLOCK diff --git a/absl/base/internal/unscaledcycleclock.h b/absl/base/internal/unscaledcycleclock.h index e6fc9103..cdce9bf8 100644 --- a/absl/base/internal/unscaledcycleclock.h +++ b/absl/base/internal/unscaledcycleclock.h @@ -86,7 +86,7 @@ #endif namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace time_internal { class UnscaledCycleClockWrapperForGetCurrentTime; } // namespace time_internal @@ -116,7 +116,7 @@ class UnscaledCycleClock { }; } // namespace base_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_USE_UNSCALED_CYCLECLOCK |