diff options
Diffstat (limited to 'absl/base/internal')
44 files changed, 969 insertions, 1702 deletions
diff --git a/absl/base/internal/bits.h b/absl/base/internal/bits.h deleted file mode 100644 index 14c51d8b..00000000 --- a/absl/base/internal/bits.h +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef ABSL_BASE_INTERNAL_BITS_H_ -#define ABSL_BASE_INTERNAL_BITS_H_ - -// This file contains bitwise ops which are implementation details of various -// absl libraries. - -#include <cstdint> - -#include "absl/base/config.h" - -// Clang on Windows has __builtin_clzll; otherwise we need to use the -// windows intrinsic functions. -#if defined(_MSC_VER) && !defined(__clang__) -#include <intrin.h> -#if defined(_M_X64) -#pragma intrinsic(_BitScanReverse64) -#pragma intrinsic(_BitScanForward64) -#endif -#pragma intrinsic(_BitScanReverse) -#pragma intrinsic(_BitScanForward) -#endif - -#include "absl/base/attributes.h" - -#if defined(_MSC_VER) && !defined(__clang__) -// We can achieve something similar to attribute((always_inline)) with MSVC by -// using the __forceinline keyword, however this is not perfect. MSVC is -// much less aggressive about inlining, and even with the __forceinline keyword. -#define ABSL_BASE_INTERNAL_FORCEINLINE __forceinline -#else -// Use default attribute inline. -#define ABSL_BASE_INTERNAL_FORCEINLINE inline ABSL_ATTRIBUTE_ALWAYS_INLINE -#endif - - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace base_internal { - -ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64Slow(uint64_t n) { - int zeroes = 60; - if (n >> 32) { - zeroes -= 32; - n >>= 32; - } - if (n >> 16) { - zeroes -= 16; - n >>= 16; - } - if (n >> 8) { - zeroes -= 8; - n >>= 8; - } - if (n >> 4) { - zeroes -= 4; - n >>= 4; - } - return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; -} - -ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64(uint64_t n) { -#if defined(_MSC_VER) && !defined(__clang__) && defined(_M_X64) - // MSVC does not have __buitin_clzll. Use _BitScanReverse64. - unsigned long result = 0; // NOLINT(runtime/int) - if (_BitScanReverse64(&result, n)) { - return 63 - result; - } - return 64; -#elif defined(_MSC_VER) && !defined(__clang__) - // MSVC does not have __buitin_clzll. Compose two calls to _BitScanReverse - unsigned long result = 0; // NOLINT(runtime/int) - if ((n >> 32) && _BitScanReverse(&result, n >> 32)) { - return 31 - result; - } - if (_BitScanReverse(&result, n)) { - return 63 - result; - } - return 64; -#elif defined(__GNUC__) || defined(__clang__) - // Use __builtin_clzll, which uses the following instructions: - // x86: bsr - // ARM64: clz - // PPC: cntlzd - static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int) - "__builtin_clzll does not take 64-bit arg"); - - // Handle 0 as a special case because __builtin_clzll(0) is undefined. - if (n == 0) { - return 64; - } - return __builtin_clzll(n); -#else - return CountLeadingZeros64Slow(n); -#endif -} - -ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32Slow(uint64_t n) { - int zeroes = 28; - if (n >> 16) { - zeroes -= 16; - n >>= 16; - } - if (n >> 8) { - zeroes -= 8; - n >>= 8; - } - if (n >> 4) { - zeroes -= 4; - n >>= 4; - } - return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; -} - -ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32(uint32_t n) { -#if defined(_MSC_VER) && !defined(__clang__) - unsigned long result = 0; // NOLINT(runtime/int) - if (_BitScanReverse(&result, n)) { - return 31 - result; - } - return 32; -#elif defined(__GNUC__) || defined(__clang__) - // Use __builtin_clz, which uses the following instructions: - // x86: bsr - // ARM64: clz - // PPC: cntlzd - static_assert(sizeof(int) == sizeof(n), - "__builtin_clz does not take 32-bit arg"); - - // Handle 0 as a special case because __builtin_clz(0) is undefined. - if (n == 0) { - return 32; - } - return __builtin_clz(n); -#else - return CountLeadingZeros32Slow(n); -#endif -} - -ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64Slow(uint64_t n) { - int c = 63; - n &= ~n + 1; - if (n & 0x00000000FFFFFFFF) c -= 32; - if (n & 0x0000FFFF0000FFFF) c -= 16; - if (n & 0x00FF00FF00FF00FF) c -= 8; - if (n & 0x0F0F0F0F0F0F0F0F) c -= 4; - if (n & 0x3333333333333333) c -= 2; - if (n & 0x5555555555555555) c -= 1; - return c; -} - -ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64(uint64_t n) { -#if defined(_MSC_VER) && !defined(__clang__) && defined(_M_X64) - unsigned long result = 0; // NOLINT(runtime/int) - _BitScanForward64(&result, n); - return result; -#elif defined(_MSC_VER) && !defined(__clang__) - unsigned long result = 0; // NOLINT(runtime/int) - if (static_cast<uint32_t>(n) == 0) { - _BitScanForward(&result, n >> 32); - return result + 32; - } - _BitScanForward(&result, n); - return result; -#elif defined(__GNUC__) || defined(__clang__) - static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int) - "__builtin_ctzll does not take 64-bit arg"); - return __builtin_ctzll(n); -#else - return CountTrailingZerosNonZero64Slow(n); -#endif -} - -ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32Slow(uint32_t n) { - int c = 31; - n &= ~n + 1; - if (n & 0x0000FFFF) c -= 16; - if (n & 0x00FF00FF) c -= 8; - if (n & 0x0F0F0F0F) c -= 4; - if (n & 0x33333333) c -= 2; - if (n & 0x55555555) c -= 1; - return c; -} - -ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32(uint32_t n) { -#if defined(_MSC_VER) && !defined(__clang__) - unsigned long result = 0; // NOLINT(runtime/int) - _BitScanForward(&result, n); - return result; -#elif defined(__GNUC__) || defined(__clang__) - static_assert(sizeof(int) == sizeof(n), - "__builtin_ctz does not take 32-bit arg"); - return __builtin_ctz(n); -#else - return CountTrailingZerosNonZero32Slow(n); -#endif -} - -#undef ABSL_BASE_INTERNAL_FORCEINLINE - -} // namespace base_internal -ABSL_NAMESPACE_END -} // namespace absl - -#endif // ABSL_BASE_INTERNAL_BITS_H_ diff --git a/absl/base/internal/bits_test.cc b/absl/base/internal/bits_test.cc deleted file mode 100644 index 7855fa62..00000000 --- a/absl/base/internal/bits_test.cc +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/base/internal/bits.h" - -#include "gtest/gtest.h" - -namespace { - -int CLZ64(uint64_t n) { - int fast = absl::base_internal::CountLeadingZeros64(n); - int slow = absl::base_internal::CountLeadingZeros64Slow(n); - EXPECT_EQ(fast, slow) << n; - return fast; -} - -TEST(BitsTest, CountLeadingZeros64) { - EXPECT_EQ(64, CLZ64(uint64_t{})); - EXPECT_EQ(0, CLZ64(~uint64_t{})); - - for (int index = 0; index < 64; index++) { - uint64_t x = static_cast<uint64_t>(1) << index; - const auto cnt = 63 - index; - ASSERT_EQ(cnt, CLZ64(x)) << index; - ASSERT_EQ(cnt, CLZ64(x + x - 1)) << index; - } -} - -int CLZ32(uint32_t n) { - int fast = absl::base_internal::CountLeadingZeros32(n); - int slow = absl::base_internal::CountLeadingZeros32Slow(n); - EXPECT_EQ(fast, slow) << n; - return fast; -} - -TEST(BitsTest, CountLeadingZeros32) { - EXPECT_EQ(32, CLZ32(uint32_t{})); - EXPECT_EQ(0, CLZ32(~uint32_t{})); - - for (int index = 0; index < 32; index++) { - uint32_t x = static_cast<uint32_t>(1) << index; - const auto cnt = 31 - index; - ASSERT_EQ(cnt, CLZ32(x)) << index; - ASSERT_EQ(cnt, CLZ32(x + x - 1)) << index; - ASSERT_EQ(CLZ64(x), CLZ32(x) + 32); - } -} - -int CTZ64(uint64_t n) { - int fast = absl::base_internal::CountTrailingZerosNonZero64(n); - int slow = absl::base_internal::CountTrailingZerosNonZero64Slow(n); - EXPECT_EQ(fast, slow) << n; - return fast; -} - -TEST(BitsTest, CountTrailingZerosNonZero64) { - EXPECT_EQ(0, CTZ64(~uint64_t{})); - - for (int index = 0; index < 64; index++) { - uint64_t x = static_cast<uint64_t>(1) << index; - const auto cnt = index; - ASSERT_EQ(cnt, CTZ64(x)) << index; - ASSERT_EQ(cnt, CTZ64(~(x - 1))) << index; - } -} - -int CTZ32(uint32_t n) { - int fast = absl::base_internal::CountTrailingZerosNonZero32(n); - int slow = absl::base_internal::CountTrailingZerosNonZero32Slow(n); - EXPECT_EQ(fast, slow) << n; - return fast; -} - -TEST(BitsTest, CountTrailingZerosNonZero32) { - EXPECT_EQ(0, CTZ32(~uint32_t{})); - - for (int index = 0; index < 32; index++) { - uint32_t x = static_cast<uint32_t>(1) << index; - const auto cnt = index; - ASSERT_EQ(cnt, CTZ32(x)) << index; - ASSERT_EQ(cnt, CTZ32(~(x - 1))) << index; - } -} - - -} // namespace diff --git a/absl/base/internal/cycleclock.cc b/absl/base/internal/cycleclock.cc index 0e65005b..f6e64242 100644 --- a/absl/base/internal/cycleclock.cc +++ b/absl/base/internal/cycleclock.cc @@ -25,6 +25,7 @@ #include <atomic> #include <chrono> // NOLINT(build/c++11) +#include "absl/base/attributes.h" #include "absl/base/internal/unscaledcycleclock.h" namespace absl { @@ -33,44 +34,18 @@ namespace base_internal { #if ABSL_USE_UNSCALED_CYCLECLOCK -namespace { +constexpr int32_t CycleClock::kShift; +constexpr double CycleClock::kFrequencyScale; -#ifdef NDEBUG -#ifdef ABSL_INTERNAL_UNSCALED_CYCLECLOCK_FREQUENCY_IS_CPU_FREQUENCY -// Not debug mode and the UnscaledCycleClock frequency is the CPU -// frequency. Scale the CycleClock to prevent overflow if someone -// tries to represent the time as cycles since the Unix epoch. -static constexpr int32_t kShift = 1; -#else -// Not debug mode and the UnscaledCycleClock isn't operating at the -// raw CPU frequency. There is no need to do any scaling, so don't -// needlessly sacrifice precision. -static constexpr int32_t kShift = 0; -#endif -#else -// In debug mode use a different shift to discourage depending on a -// particular shift value. -static constexpr int32_t kShift = 2; -#endif - -static constexpr double kFrequencyScale = 1.0 / (1 << kShift); -static std::atomic<CycleClockSourceFunc> cycle_clock_source; +ABSL_CONST_INIT std::atomic<CycleClockSourceFunc> + CycleClock::cycle_clock_source_{nullptr}; -CycleClockSourceFunc LoadCycleClockSource() { - // Optimize for the common case (no callback) by first doing a relaxed load; - // this is significantly faster on non-x86 platforms. - if (cycle_clock_source.load(std::memory_order_relaxed) == nullptr) { - return nullptr; - } - // This corresponds to the store(std::memory_order_release) in - // CycleClockSource::Register, and makes sure that any updates made prior to - // registering the callback are visible to this thread before the callback is - // invoked. - return cycle_clock_source.load(std::memory_order_acquire); +void CycleClockSource::Register(CycleClockSourceFunc source) { + // Corresponds to the load(std::memory_order_acquire) in LoadCycleClockSource. + CycleClock::cycle_clock_source_.store(source, std::memory_order_release); } -} // namespace - +#ifdef _WIN32 int64_t CycleClock::Now() { auto fn = LoadCycleClockSource(); if (fn == nullptr) { @@ -78,15 +53,7 @@ int64_t CycleClock::Now() { } return fn() >> kShift; } - -double CycleClock::Frequency() { - return kFrequencyScale * base_internal::UnscaledCycleClock::Frequency(); -} - -void CycleClockSource::Register(CycleClockSourceFunc source) { - // Corresponds to the load(std::memory_order_acquire) in LoadCycleClockSource. - cycle_clock_source.store(source, std::memory_order_release); -} +#endif #else diff --git a/absl/base/internal/cycleclock.h b/absl/base/internal/cycleclock.h index a18b5844..9704e388 100644 --- a/absl/base/internal/cycleclock.h +++ b/absl/base/internal/cycleclock.h @@ -42,14 +42,19 @@ #ifndef ABSL_BASE_INTERNAL_CYCLECLOCK_H_ #define ABSL_BASE_INTERNAL_CYCLECLOCK_H_ +#include <atomic> #include <cstdint> +#include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/internal/unscaledcycleclock.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace base_internal { +using CycleClockSourceFunc = int64_t (*)(); + // ----------------------------------------------------------------------------- // CycleClock // ----------------------------------------------------------------------------- @@ -68,12 +73,37 @@ class CycleClock { static double Frequency(); private: +#if ABSL_USE_UNSCALED_CYCLECLOCK + static CycleClockSourceFunc LoadCycleClockSource(); + +#ifdef NDEBUG +#ifdef ABSL_INTERNAL_UNSCALED_CYCLECLOCK_FREQUENCY_IS_CPU_FREQUENCY + // Not debug mode and the UnscaledCycleClock frequency is the CPU + // frequency. Scale the CycleClock to prevent overflow if someone + // tries to represent the time as cycles since the Unix epoch. + static constexpr int32_t kShift = 1; +#else + // Not debug mode and the UnscaledCycleClock isn't operating at the + // raw CPU frequency. There is no need to do any scaling, so don't + // needlessly sacrifice precision. + static constexpr int32_t kShift = 0; +#endif +#else // NDEBUG + // In debug mode use a different shift to discourage depending on a + // particular shift value. + static constexpr int32_t kShift = 2; +#endif // NDEBUG + + static constexpr double kFrequencyScale = 1.0 / (1 << kShift); + ABSL_CONST_INIT static std::atomic<CycleClockSourceFunc> cycle_clock_source_; +#endif // ABSL_USE_UNSCALED_CYCLECLOC + CycleClock() = delete; // no instances CycleClock(const CycleClock&) = delete; CycleClock& operator=(const CycleClock&) = delete; -}; -using CycleClockSourceFunc = int64_t (*)(); + friend class CycleClockSource; +}; class CycleClockSource { private: @@ -87,6 +117,41 @@ class CycleClockSource { static void Register(CycleClockSourceFunc source); }; +#if ABSL_USE_UNSCALED_CYCLECLOCK + +inline CycleClockSourceFunc CycleClock::LoadCycleClockSource() { +#if !defined(__x86_64__) + // Optimize for the common case (no callback) by first doing a relaxed load; + // this is significantly faster on non-x86 platforms. + if (cycle_clock_source_.load(std::memory_order_relaxed) == nullptr) { + return nullptr; + } +#endif // !defined(__x86_64__) + + // This corresponds to the store(std::memory_order_release) in + // CycleClockSource::Register, and makes sure that any updates made prior to + // registering the callback are visible to this thread before the callback + // is invoked. + return cycle_clock_source_.load(std::memory_order_acquire); +} + +// Accessing globals in inlined code in Window DLLs is problematic. +#ifndef _WIN32 +inline int64_t CycleClock::Now() { + auto fn = LoadCycleClockSource(); + if (fn == nullptr) { + return base_internal::UnscaledCycleClock::Now() >> kShift; + } + return fn() >> kShift; +} +#endif + +inline double CycleClock::Frequency() { + return kFrequencyScale * base_internal::UnscaledCycleClock::Frequency(); +} + +#endif // ABSL_USE_UNSCALED_CYCLECLOCK + } // namespace base_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/direct_mmap.h b/absl/base/internal/direct_mmap.h index 16accf09..a01d6122 100644 --- a/absl/base/internal/direct_mmap.h +++ b/absl/base/internal/direct_mmap.h @@ -20,7 +20,7 @@ #include "absl/base/config.h" -#if ABSL_HAVE_MMAP +#ifdef ABSL_HAVE_MMAP #include <sys/mman.h> @@ -74,10 +74,13 @@ namespace base_internal { inline void* DirectMmap(void* start, size_t length, int prot, int flags, int fd, off64_t offset) noexcept { #if defined(__i386__) || defined(__ARM_ARCH_3__) || defined(__ARM_EABI__) || \ + defined(__m68k__) || defined(__sh__) || \ + (defined(__hppa__) && !defined(__LP64__)) || \ (defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI32) || \ (defined(__PPC__) && !defined(__PPC64__)) || \ (defined(__riscv) && __riscv_xlen == 32) || \ - (defined(__s390__) && !defined(__s390x__)) + (defined(__s390__) && !defined(__s390x__)) || \ + (defined(__sparc__) && !defined(__arch64__)) // On these architectures, implement mmap with mmap2. static int pagesize = 0; if (pagesize == 0) { diff --git a/absl/base/internal/dynamic_annotations.h b/absl/base/internal/dynamic_annotations.h index 7d80f41c..b23c5ec1 100644 --- a/absl/base/internal/dynamic_annotations.h +++ b/absl/base/internal/dynamic_annotations.h @@ -58,8 +58,6 @@ #if defined(__clang__) && !defined(SWIG) #define ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED 1 -#else -#define ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED 0 #endif #if DYNAMIC_ANNOTATIONS_ENABLED != 0 @@ -84,19 +82,16 @@ // ANNOTALYSIS_ENABLED == 1 when IGNORE_READ_ATTRIBUTE_ENABLED == 1 #define ABSL_INTERNAL_ANNOTALYSIS_ENABLED \ - ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED + defined(ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED) // Read/write annotations are enabled in Annotalysis mode; disabled otherwise. #define ABSL_INTERNAL_READS_WRITES_ANNOTATIONS_ENABLED \ ABSL_INTERNAL_ANNOTALYSIS_ENABLED #endif // Memory annotations are also made available to LLVM's Memory Sanitizer -#if defined(MEMORY_SANITIZER) && defined(__has_feature) && \ - !defined(__native_client__) -#if __has_feature(memory_sanitizer) +#if defined(ABSL_HAVE_MEMORY_SANITIZER) && !defined(__native_client__) #define ABSL_INTERNAL_MEMORY_ANNOTATIONS_ENABLED 1 #endif -#endif #ifndef ABSL_INTERNAL_MEMORY_ANNOTATIONS_ENABLED #define ABSL_INTERNAL_MEMORY_ANNOTATIONS_ENABLED 0 @@ -162,7 +157,7 @@ ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateRWLockCreate)(__FILE__, __LINE__, lock) // Report that a linker initialized lock has been created at address `lock`. -#ifdef THREAD_SANITIZER +#ifdef ABSL_HAVE_THREAD_SANITIZER #define ANNOTATE_RWLOCK_CREATE_STATIC(lock) \ ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateRWLockCreateStatic) \ (__FILE__, __LINE__, lock) @@ -250,19 +245,19 @@ // ------------------------------------------------------------------------- // Define IGNORE_READS_BEGIN/_END attributes. -#if ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED == 1 +#if defined(ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED) #define ABSL_INTERNAL_IGNORE_READS_BEGIN_ATTRIBUTE \ __attribute((exclusive_lock_function("*"))) #define ABSL_INTERNAL_IGNORE_READS_END_ATTRIBUTE \ __attribute((unlock_function("*"))) -#else // ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED == 0 +#else // !defined(ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED) #define ABSL_INTERNAL_IGNORE_READS_BEGIN_ATTRIBUTE // empty #define ABSL_INTERNAL_IGNORE_READS_END_ATTRIBUTE // empty -#endif // ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED +#endif // defined(ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED) // ------------------------------------------------------------------------- // Define IGNORE_READS_BEGIN/_END annotations. @@ -367,7 +362,7 @@ // ------------------------------------------------------------------------- // Address sanitizer annotations -#ifdef ADDRESS_SANITIZER +#ifdef ABSL_HAVE_ADDRESS_SANITIZER // Describe the current state of a contiguous container such as e.g. // std::vector or std::string. For more details see // sanitizer/common_interface_defs.h, which is provided by the compiler. @@ -385,7 +380,7 @@ #define ANNOTATE_CONTIGUOUS_CONTAINER(beg, end, old_mid, new_mid) #define ADDRESS_SANITIZER_REDZONE(name) static_assert(true, "") -#endif // ADDRESS_SANITIZER +#endif // ABSL_HAVE_ADDRESS_SANITIZER // ------------------------------------------------------------------------- // Undefine the macros intended only for this file. diff --git a/absl/base/internal/endian.h b/absl/base/internal/endian.h index 9677530e..50747d75 100644 --- a/absl/base/internal/endian.h +++ b/absl/base/internal/endian.h @@ -16,16 +16,10 @@ #ifndef ABSL_BASE_INTERNAL_ENDIAN_H_ #define ABSL_BASE_INTERNAL_ENDIAN_H_ -// The following guarantees declaration of the byte swap functions -#ifdef _MSC_VER -#include <stdlib.h> // NOLINT(build/include) -#elif defined(__FreeBSD__) -#include <sys/endian.h> -#elif defined(__GLIBC__) -#include <byteswap.h> // IWYU pragma: export -#endif - #include <cstdint> +#include <cstdlib> + +#include "absl/base/casts.h" #include "absl/base/config.h" #include "absl/base/internal/unaligned_access.h" #include "absl/base/port.h" @@ -33,47 +27,11 @@ namespace absl { ABSL_NAMESPACE_BEGIN -// Use compiler byte-swapping intrinsics if they are available. 32-bit -// and 64-bit versions are available in Clang and GCC as of GCC 4.3.0. -// The 16-bit version is available in Clang and GCC only as of GCC 4.8.0. -// For simplicity, we enable them all only for GCC 4.8.0 or later. -#if defined(__clang__) || \ - (defined(__GNUC__) && \ - ((__GNUC__ == 4 && __GNUC_MINOR__ >= 8) || __GNUC__ >= 5)) inline uint64_t gbswap_64(uint64_t host_int) { +#if ABSL_HAVE_BUILTIN(__builtin_bswap64) || defined(__GNUC__) return __builtin_bswap64(host_int); -} -inline uint32_t gbswap_32(uint32_t host_int) { - return __builtin_bswap32(host_int); -} -inline uint16_t gbswap_16(uint16_t host_int) { - return __builtin_bswap16(host_int); -} - #elif defined(_MSC_VER) -inline uint64_t gbswap_64(uint64_t host_int) { return _byteswap_uint64(host_int); -} -inline uint32_t gbswap_32(uint32_t host_int) { - return _byteswap_ulong(host_int); -} -inline uint16_t gbswap_16(uint16_t host_int) { - return _byteswap_ushort(host_int); -} - -#else -inline uint64_t gbswap_64(uint64_t host_int) { -#if defined(__GNUC__) && defined(__x86_64__) && !defined(__APPLE__) - // Adapted from /usr/include/byteswap.h. Not available on Mac. - if (__builtin_constant_p(host_int)) { - return __bswap_constant_64(host_int); - } else { - uint64_t result; - __asm__("bswap %0" : "=r"(result) : "0"(host_int)); - return result; - } -#elif defined(__GLIBC__) - return bswap_64(host_int); #else return (((host_int & uint64_t{0xFF}) << 56) | ((host_int & uint64_t{0xFF00}) << 40) | @@ -83,12 +41,14 @@ inline uint64_t gbswap_64(uint64_t host_int) { ((host_int & uint64_t{0xFF0000000000}) >> 24) | ((host_int & uint64_t{0xFF000000000000}) >> 40) | ((host_int & uint64_t{0xFF00000000000000}) >> 56)); -#endif // bswap_64 +#endif } inline uint32_t gbswap_32(uint32_t host_int) { -#if defined(__GLIBC__) - return bswap_32(host_int); +#if ABSL_HAVE_BUILTIN(__builtin_bswap32) || defined(__GNUC__) + return __builtin_bswap32(host_int); +#elif defined(_MSC_VER) + return _byteswap_ulong(host_int); #else return (((host_int & uint32_t{0xFF}) << 24) | ((host_int & uint32_t{0xFF00}) << 8) | @@ -98,33 +58,29 @@ inline uint32_t gbswap_32(uint32_t host_int) { } inline uint16_t gbswap_16(uint16_t host_int) { -#if defined(__GLIBC__) - return bswap_16(host_int); +#if ABSL_HAVE_BUILTIN(__builtin_bswap16) || defined(__GNUC__) + return __builtin_bswap16(host_int); +#elif defined(_MSC_VER) + return _byteswap_ushort(host_int); #else return (((host_int & uint16_t{0xFF}) << 8) | ((host_int & uint16_t{0xFF00}) >> 8)); #endif } -#endif // intrinsics available - #ifdef ABSL_IS_LITTLE_ENDIAN -// Definitions for ntohl etc. that don't require us to include -// netinet/in.h. We wrap gbswap_32 and gbswap_16 in functions rather -// than just #defining them because in debug mode, gcc doesn't -// correctly handle the (rather involved) definitions of bswap_32. -// gcc guarantees that inline functions are as fast as macros, so -// this isn't a performance hit. +// Portable definitions for htonl (host-to-network) and friends on little-endian +// architectures. inline uint16_t ghtons(uint16_t x) { return gbswap_16(x); } inline uint32_t ghtonl(uint32_t x) { return gbswap_32(x); } inline uint64_t ghtonll(uint64_t x) { return gbswap_64(x); } #elif defined ABSL_IS_BIG_ENDIAN -// These definitions are simpler on big-endian machines -// These are functions instead of macros to avoid self-assignment warnings -// on calls such as "i = ghtnol(i);". This also provides type checking. +// Portable definitions for htonl (host-to-network) etc on big-endian +// architectures. These definitions are simpler since the host byte order is the +// same as network byte order. inline uint16_t ghtons(uint16_t x) { return x; } inline uint32_t ghtonl(uint32_t x) { return x; } inline uint64_t ghtonll(uint64_t x) { return x; } @@ -173,6 +129,36 @@ inline constexpr bool IsLittleEndian() { return false; } #endif /* ENDIAN */ +inline uint8_t FromHost(uint8_t x) { return x; } +inline uint16_t FromHost(uint16_t x) { return FromHost16(x); } +inline uint32_t FromHost(uint32_t x) { return FromHost32(x); } +inline uint64_t FromHost(uint64_t x) { return FromHost64(x); } +inline uint8_t ToHost(uint8_t x) { return x; } +inline uint16_t ToHost(uint16_t x) { return ToHost16(x); } +inline uint32_t ToHost(uint32_t x) { return ToHost32(x); } +inline uint64_t ToHost(uint64_t x) { return ToHost64(x); } + +inline int8_t FromHost(int8_t x) { return x; } +inline int16_t FromHost(int16_t x) { + return bit_cast<int16_t>(FromHost16(bit_cast<uint16_t>(x))); +} +inline int32_t FromHost(int32_t x) { + return bit_cast<int32_t>(FromHost32(bit_cast<uint32_t>(x))); +} +inline int64_t FromHost(int64_t x) { + return bit_cast<int64_t>(FromHost64(bit_cast<uint64_t>(x))); +} +inline int8_t ToHost(int8_t x) { return x; } +inline int16_t ToHost(int16_t x) { + return bit_cast<int16_t>(ToHost16(bit_cast<uint16_t>(x))); +} +inline int32_t ToHost(int32_t x) { + return bit_cast<int32_t>(ToHost32(bit_cast<uint32_t>(x))); +} +inline int64_t ToHost(int64_t x) { + return bit_cast<int64_t>(ToHost64(bit_cast<uint64_t>(x))); +} + // Functions to do unaligned loads and stores in little-endian order. inline uint16_t Load16(const void *p) { return ToHost16(ABSL_INTERNAL_UNALIGNED_LOAD16(p)); @@ -233,6 +219,36 @@ inline constexpr bool IsLittleEndian() { return false; } #endif /* ENDIAN */ +inline uint8_t FromHost(uint8_t x) { return x; } +inline uint16_t FromHost(uint16_t x) { return FromHost16(x); } +inline uint32_t FromHost(uint32_t x) { return FromHost32(x); } +inline uint64_t FromHost(uint64_t x) { return FromHost64(x); } +inline uint8_t ToHost(uint8_t x) { return x; } +inline uint16_t ToHost(uint16_t x) { return ToHost16(x); } +inline uint32_t ToHost(uint32_t x) { return ToHost32(x); } +inline uint64_t ToHost(uint64_t x) { return ToHost64(x); } + +inline int8_t FromHost(int8_t x) { return x; } +inline int16_t FromHost(int16_t x) { + return bit_cast<int16_t>(FromHost16(bit_cast<uint16_t>(x))); +} +inline int32_t FromHost(int32_t x) { + return bit_cast<int32_t>(FromHost32(bit_cast<uint32_t>(x))); +} +inline int64_t FromHost(int64_t x) { + return bit_cast<int64_t>(FromHost64(bit_cast<uint64_t>(x))); +} +inline int8_t ToHost(int8_t x) { return x; } +inline int16_t ToHost(int16_t x) { + return bit_cast<int16_t>(ToHost16(bit_cast<uint16_t>(x))); +} +inline int32_t ToHost(int32_t x) { + return bit_cast<int32_t>(ToHost32(bit_cast<uint32_t>(x))); +} +inline int64_t ToHost(int64_t x) { + return bit_cast<int64_t>(ToHost64(bit_cast<uint64_t>(x))); +} + // Functions to do unaligned loads and stores in big-endian order. inline uint16_t Load16(const void *p) { return ToHost16(ABSL_INTERNAL_UNALIGNED_LOAD16(p)); diff --git a/absl/base/internal/exception_safety_testing.h b/absl/base/internal/exception_safety_testing.h index 6ba89d05..77a5aec6 100644 --- a/absl/base/internal/exception_safety_testing.h +++ b/absl/base/internal/exception_safety_testing.h @@ -536,7 +536,22 @@ class ThrowingValue : private exceptions_internal::TrackedObject { } // Memory management operators - // Args.. allows us to overload regular and placement new in one shot + static void* operator new(size_t s) noexcept( + IsSpecified(TypeSpec::kNoThrowNew)) { + if (!IsSpecified(TypeSpec::kNoThrowNew)) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION, true); + } + return ::operator new(s); + } + + static void* operator new[](size_t s) noexcept( + IsSpecified(TypeSpec::kNoThrowNew)) { + if (!IsSpecified(TypeSpec::kNoThrowNew)) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION, true); + } + return ::operator new[](s); + } + template <typename... Args> static void* operator new(size_t s, Args&&... args) noexcept( IsSpecified(TypeSpec::kNoThrowNew)) { @@ -557,12 +572,6 @@ class ThrowingValue : private exceptions_internal::TrackedObject { // Abseil doesn't support throwing overloaded operator delete. These are // provided so a throwing operator-new can clean up after itself. - // - // We provide both regular and templated operator delete because if only the - // templated version is provided as we did with operator new, the compiler has - // no way of knowing which overload of operator delete to call. See - // https://en.cppreference.com/w/cpp/memory/new/operator_delete and - // https://en.cppreference.com/w/cpp/language/delete for the gory details. void operator delete(void* p) noexcept { ::operator delete(p); } template <typename... Args> @@ -726,9 +735,8 @@ class ThrowingAllocator : private exceptions_internal::TrackedObject { ThrowingAllocator select_on_container_copy_construction() noexcept( IsSpecified(AllocSpec::kNoThrowAllocate)) { - auto& out = *this; ReadStateAndMaybeThrow(ABSL_PRETTY_FUNCTION); - return out; + return *this; } template <typename U> diff --git a/absl/base/internal/exponential_biased.cc b/absl/base/internal/exponential_biased.cc deleted file mode 100644 index 1b30c061..00000000 --- a/absl/base/internal/exponential_biased.cc +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2019 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/base/internal/exponential_biased.h" - -#include <stdint.h> - -#include <algorithm> -#include <atomic> -#include <cmath> -#include <limits> - -#include "absl/base/attributes.h" -#include "absl/base/optimization.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace base_internal { - -// The algorithm generates a random number between 0 and 1 and applies the -// inverse cumulative distribution function for an exponential. Specifically: -// Let m be the inverse of the sample period, then the probability -// distribution function is m*exp(-mx) so the CDF is -// p = 1 - exp(-mx), so -// q = 1 - p = exp(-mx) -// log_e(q) = -mx -// -log_e(q)/m = x -// log_2(q) * (-log_e(2) * 1/m) = x -// In the code, q is actually in the range 1 to 2**26, hence the -26 below -int64_t ExponentialBiased::GetSkipCount(int64_t mean) { - if (ABSL_PREDICT_FALSE(!initialized_)) { - Initialize(); - } - - uint64_t rng = NextRandom(rng_); - rng_ = rng; - - // Take the top 26 bits as the random number - // (This plus the 1<<58 sampling bound give a max possible step of - // 5194297183973780480 bytes.) - // The uint32_t cast is to prevent a (hard-to-reproduce) NAN - // under piii debug for some binaries. - double q = static_cast<uint32_t>(rng >> (kPrngNumBits - 26)) + 1.0; - // Put the computed p-value through the CDF of a geometric. - double interval = bias_ + (std::log2(q) - 26) * (-std::log(2.0) * mean); - // Very large values of interval overflow int64_t. To avoid that, we will - // cheat and clamp any huge values to (int64_t max)/2. This is a potential - // source of bias, but the mean would need to be such a large value that it's - // not likely to come up. For example, with a mean of 1e18, the probability of - // hitting this condition is about 1/1000. For a mean of 1e17, standard - // calculators claim that this event won't happen. - if (interval > static_cast<double>(std::numeric_limits<int64_t>::max() / 2)) { - // Assume huge values are bias neutral, retain bias for next call. - return std::numeric_limits<int64_t>::max() / 2; - } - double value = std::round(interval); - bias_ = interval - value; - return value; -} - -int64_t ExponentialBiased::GetStride(int64_t mean) { - return GetSkipCount(mean - 1) + 1; -} - -void ExponentialBiased::Initialize() { - // We don't get well distributed numbers from `this` so we call NextRandom() a - // bunch to mush the bits around. We use a global_rand to handle the case - // where the same thread (by memory address) gets created and destroyed - // repeatedly. - ABSL_CONST_INIT static std::atomic<uint32_t> global_rand(0); - uint64_t r = reinterpret_cast<uint64_t>(this) + - global_rand.fetch_add(1, std::memory_order_relaxed); - for (int i = 0; i < 20; ++i) { - r = NextRandom(r); - } - rng_ = r; - initialized_ = true; -} - -} // namespace base_internal -ABSL_NAMESPACE_END -} // namespace absl diff --git a/absl/base/internal/exponential_biased.h b/absl/base/internal/exponential_biased.h deleted file mode 100644 index 94f79a33..00000000 --- a/absl/base/internal/exponential_biased.h +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2019 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_ -#define ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_ - -#include <stdint.h> - -#include "absl/base/config.h" -#include "absl/base/macros.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace base_internal { - -// ExponentialBiased provides a small and fast random number generator for a -// rounded exponential distribution. This generator manages very little state, -// and imposes no synchronization overhead. This makes it useful in specialized -// scenarios requiring minimum overhead, such as stride based periodic sampling. -// -// ExponentialBiased provides two closely related functions, GetSkipCount() and -// GetStride(), both returning a rounded integer defining a number of events -// required before some event with a given mean probability occurs. -// -// The distribution is useful to generate a random wait time or some periodic -// event with a given mean probability. For example, if an action is supposed to -// happen on average once every 'N' events, then we can get a random 'stride' -// counting down how long before the event to happen. For example, if we'd want -// to sample one in every 1000 'Frobber' calls, our code could look like this: -// -// Frobber::Frobber() { -// stride_ = exponential_biased_.GetStride(1000); -// } -// -// void Frobber::Frob(int arg) { -// if (--stride == 0) { -// SampleFrob(arg); -// stride_ = exponential_biased_.GetStride(1000); -// } -// ... -// } -// -// The rounding of the return value creates a bias, especially for smaller means -// where the distribution of the fraction is not evenly distributed. We correct -// this bias by tracking the fraction we rounded up or down on each iteration, -// effectively tracking the distance between the cumulative value, and the -// rounded cumulative value. For example, given a mean of 2: -// -// raw = 1.63076, cumulative = 1.63076, rounded = 2, bias = -0.36923 -// raw = 0.14624, cumulative = 1.77701, rounded = 2, bias = 0.14624 -// raw = 4.93194, cumulative = 6.70895, rounded = 7, bias = -0.06805 -// raw = 0.24206, cumulative = 6.95101, rounded = 7, bias = 0.24206 -// etc... -// -// Adjusting with rounding bias is relatively trivial: -// -// double value = bias_ + exponential_distribution(mean)(); -// double rounded_value = std::round(value); -// bias_ = value - rounded_value; -// return rounded_value; -// -// This class is thread-compatible. -class ExponentialBiased { - public: - // The number of bits set by NextRandom. - static constexpr int kPrngNumBits = 48; - - // `GetSkipCount()` returns the number of events to skip before some chosen - // event happens. For example, randomly tossing a coin, we will on average - // throw heads once before we get tails. We can simulate random coin tosses - // using GetSkipCount() as: - // - // ExponentialBiased eb; - // for (...) { - // int number_of_heads_before_tail = eb.GetSkipCount(1); - // for (int flips = 0; flips < number_of_heads_before_tail; ++flips) { - // printf("head..."); - // } - // printf("tail\n"); - // } - // - int64_t GetSkipCount(int64_t mean); - - // GetStride() returns the number of events required for a specific event to - // happen. See the class comments for a usage example. `GetStride()` is - // equivalent to `GetSkipCount(mean - 1) + 1`. When to use `GetStride()` or - // `GetSkipCount()` depends mostly on what best fits the use case. - int64_t GetStride(int64_t mean); - - // Computes a random number in the range [0, 1<<(kPrngNumBits+1) - 1] - // - // This is public to enable testing. - static uint64_t NextRandom(uint64_t rnd); - - private: - void Initialize(); - - uint64_t rng_{0}; - double bias_{0}; - bool initialized_{false}; -}; - -// Returns the next prng value. -// pRNG is: aX+b mod c with a = 0x5DEECE66D, b = 0xB, c = 1<<48 -// This is the lrand64 generator. -inline uint64_t ExponentialBiased::NextRandom(uint64_t rnd) { - const uint64_t prng_mult = uint64_t{0x5DEECE66D}; - const uint64_t prng_add = 0xB; - const uint64_t prng_mod_power = 48; - const uint64_t prng_mod_mask = - ~((~static_cast<uint64_t>(0)) << prng_mod_power); - return (prng_mult * rnd + prng_add) & prng_mod_mask; -} - -} // namespace base_internal -ABSL_NAMESPACE_END -} // namespace absl - -#endif // ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_ diff --git a/absl/base/internal/exponential_biased_test.cc b/absl/base/internal/exponential_biased_test.cc deleted file mode 100644 index 90a482d2..00000000 --- a/absl/base/internal/exponential_biased_test.cc +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2019 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/base/internal/exponential_biased.h" - -#include <stddef.h> - -#include <cmath> -#include <cstdint> -#include <vector> - -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include "absl/strings/str_cat.h" - -using ::testing::Ge; - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace base_internal { - -MATCHER_P2(IsBetween, a, b, - absl::StrCat(std::string(negation ? "isn't" : "is"), " between ", a, - " and ", b)) { - return a <= arg && arg <= b; -} - -// Tests of the quality of the random numbers generated -// This uses the Anderson Darling test for uniformity. -// See "Evaluating the Anderson-Darling Distribution" by Marsaglia -// for details. - -// Short cut version of ADinf(z), z>0 (from Marsaglia) -// This returns the p-value for Anderson Darling statistic in -// the limit as n-> infinity. For finite n, apply the error fix below. -double AndersonDarlingInf(double z) { - if (z < 2) { - return exp(-1.2337141 / z) / sqrt(z) * - (2.00012 + - (0.247105 - - (0.0649821 - (0.0347962 - (0.011672 - 0.00168691 * z) * z) * z) * - z) * - z); - } - return exp( - -exp(1.0776 - - (2.30695 - - (0.43424 - (0.082433 - (0.008056 - 0.0003146 * z) * z) * z) * z) * - z)); -} - -// Corrects the approximation error in AndersonDarlingInf for small values of n -// Add this to AndersonDarlingInf to get a better approximation -// (from Marsaglia) -double AndersonDarlingErrFix(int n, double x) { - if (x > 0.8) { - return (-130.2137 + - (745.2337 - - (1705.091 - (1950.646 - (1116.360 - 255.7844 * x) * x) * x) * x) * - x) / - n; - } - double cutoff = 0.01265 + 0.1757 / n; - if (x < cutoff) { - double t = x / cutoff; - t = sqrt(t) * (1 - t) * (49 * t - 102); - return t * (0.0037 / (n * n) + 0.00078 / n + 0.00006) / n; - } else { - double t = (x - cutoff) / (0.8 - cutoff); - t = -0.00022633 + - (6.54034 - (14.6538 - (14.458 - (8.259 - 1.91864 * t) * t) * t) * t) * - t; - return t * (0.04213 + 0.01365 / n) / n; - } -} - -// Returns the AndersonDarling p-value given n and the value of the statistic -double AndersonDarlingPValue(int n, double z) { - double ad = AndersonDarlingInf(z); - double errfix = AndersonDarlingErrFix(n, ad); - return ad + errfix; -} - -double AndersonDarlingStatistic(const std::vector<double>& random_sample) { - int n = random_sample.size(); - double ad_sum = 0; - for (int i = 0; i < n; i++) { - ad_sum += (2 * i + 1) * - std::log(random_sample[i] * (1 - random_sample[n - 1 - i])); - } - double ad_statistic = -n - 1 / static_cast<double>(n) * ad_sum; - return ad_statistic; -} - -// Tests if the array of doubles is uniformly distributed. -// Returns the p-value of the Anderson Darling Statistic -// for the given set of sorted random doubles -// See "Evaluating the Anderson-Darling Distribution" by -// Marsaglia and Marsaglia for details. -double AndersonDarlingTest(const std::vector<double>& random_sample) { - double ad_statistic = AndersonDarlingStatistic(random_sample); - double p = AndersonDarlingPValue(random_sample.size(), ad_statistic); - return p; -} - -TEST(ExponentialBiasedTest, CoinTossDemoWithGetSkipCount) { - ExponentialBiased eb; - for (int runs = 0; runs < 10; ++runs) { - for (int flips = eb.GetSkipCount(1); flips > 0; --flips) { - printf("head..."); - } - printf("tail\n"); - } - int heads = 0; - for (int i = 0; i < 10000000; i += 1 + eb.GetSkipCount(1)) { - ++heads; - } - printf("Heads = %d (%f%%)\n", heads, 100.0 * heads / 10000000); -} - -TEST(ExponentialBiasedTest, SampleDemoWithStride) { - ExponentialBiased eb; - int stride = eb.GetStride(10); - int samples = 0; - for (int i = 0; i < 10000000; ++i) { - if (--stride == 0) { - ++samples; - stride = eb.GetStride(10); - } - } - printf("Samples = %d (%f%%)\n", samples, 100.0 * samples / 10000000); -} - - -// Testing that NextRandom generates uniform random numbers. Applies the -// Anderson-Darling test for uniformity -TEST(ExponentialBiasedTest, TestNextRandom) { - for (auto n : std::vector<int>({ - 10, // Check short-range correlation - 100, 1000, - 10000 // Make sure there's no systemic error - })) { - uint64_t x = 1; - // This assumes that the prng returns 48 bit numbers - uint64_t max_prng_value = static_cast<uint64_t>(1) << 48; - // Initialize. - for (int i = 1; i <= 20; i++) { - x = ExponentialBiased::NextRandom(x); - } - std::vector<uint64_t> int_random_sample(n); - // Collect samples - for (int i = 0; i < n; i++) { - int_random_sample[i] = x; - x = ExponentialBiased::NextRandom(x); - } - // First sort them... - std::sort(int_random_sample.begin(), int_random_sample.end()); - std::vector<double> random_sample(n); - // Convert them to uniform randoms (in the range [0,1]) - for (int i = 0; i < n; i++) { - random_sample[i] = - static_cast<double>(int_random_sample[i]) / max_prng_value; - } - // Now compute the Anderson-Darling statistic - double ad_pvalue = AndersonDarlingTest(random_sample); - EXPECT_GT(std::min(ad_pvalue, 1 - ad_pvalue), 0.0001) - << "prng is not uniform: n = " << n << " p = " << ad_pvalue; - } -} - -// The generator needs to be available as a thread_local and as a static -// variable. -TEST(ExponentialBiasedTest, InitializationModes) { - ABSL_CONST_INIT static ExponentialBiased eb_static; - EXPECT_THAT(eb_static.GetSkipCount(2), Ge(0)); - -#if ABSL_HAVE_THREAD_LOCAL - thread_local ExponentialBiased eb_thread; - EXPECT_THAT(eb_thread.GetSkipCount(2), Ge(0)); -#endif - - ExponentialBiased eb_stack; - EXPECT_THAT(eb_stack.GetSkipCount(2), Ge(0)); -} - -} // namespace base_internal -ABSL_NAMESPACE_END -} // namespace absl diff --git a/absl/base/internal/invoke.h b/absl/base/internal/invoke.h index c4eceebd..4a644c68 100644 --- a/absl/base/internal/invoke.h +++ b/absl/base/internal/invoke.h @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// absl::base_internal::Invoke(f, args...) is an implementation of +// absl::base_internal::invoke(f, args...) is an implementation of // INVOKE(f, args...) from section [func.require] of the C++ standard. +// When compiled as C++17 and later versions, it is implemented as an alias of +// std::invoke. // // [func.require] // Define INVOKE (f, t1, t2, ..., tN) as follows: @@ -29,12 +31,32 @@ // is not one of the types described in the previous item; // 5. f(t1, t2, ..., tN) in all other cases. // -// The implementation is SFINAE-friendly: substitution failure within Invoke() +// The implementation is SFINAE-friendly: substitution failure within invoke() // isn't an error. #ifndef ABSL_BASE_INTERNAL_INVOKE_H_ #define ABSL_BASE_INTERNAL_INVOKE_H_ +#include "absl/base/config.h" + +#if __cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) + +#include <functional> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +using std::invoke; +using std::invoke_result_t; +using std::is_invocable_r; + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#else // __cplusplus >= 201703L + #include <algorithm> #include <type_traits> #include <utility> @@ -170,18 +192,40 @@ struct Invoker { // The result type of Invoke<F, Args...>. template <typename F, typename... Args> -using InvokeT = decltype(Invoker<F, Args...>::type::Invoke( +using invoke_result_t = decltype(Invoker<F, Args...>::type::Invoke( std::declval<F>(), std::declval<Args>()...)); // Invoke(f, args...) is an implementation of INVOKE(f, args...) from section // [func.require] of the C++ standard. template <typename F, typename... Args> -InvokeT<F, Args...> Invoke(F&& f, Args&&... args) { +invoke_result_t<F, Args...> invoke(F&& f, Args&&... args) { return Invoker<F, Args...>::type::Invoke(std::forward<F>(f), std::forward<Args>(args)...); } + +template <typename AlwaysVoid, typename, typename, typename...> +struct IsInvocableRImpl : std::false_type {}; + +template <typename R, typename F, typename... Args> +struct IsInvocableRImpl< + absl::void_t<absl::base_internal::invoke_result_t<F, Args...> >, R, F, + Args...> + : std::integral_constant< + bool, + std::is_convertible<absl::base_internal::invoke_result_t<F, Args...>, + R>::value || + std::is_void<R>::value> {}; + +// Type trait whose member `value` is true if invoking `F` with `Args` is valid, +// and either the return type is convertible to `R`, or `R` is void. +// C++11-compatible version of `std::is_invocable_r`. +template <typename R, typename F, typename... Args> +using is_invocable_r = IsInvocableRImpl<void, R, F, Args...>; + } // namespace base_internal ABSL_NAMESPACE_END } // namespace absl +#endif // __cplusplus >= 201703L + #endif // ABSL_BASE_INTERNAL_INVOKE_H_ diff --git a/absl/base/internal/low_level_alloc.cc b/absl/base/internal/low_level_alloc.cc index 1bf94438..229ab916 100644 --- a/absl/base/internal/low_level_alloc.cc +++ b/absl/base/internal/low_level_alloc.cc @@ -598,7 +598,7 @@ static void *DoAllocWithArena(size_t request, LowLevelAlloc::Arena *arena) { section.Leave(); result = &s->levels; } - ANNOTATE_MEMORY_IS_UNINITIALIZED(result, request); + ABSL_ANNOTATE_MEMORY_IS_UNINITIALIZED(result, request); return result; } diff --git a/absl/base/internal/low_level_alloc_test.cc b/absl/base/internal/low_level_alloc_test.cc index 2f2eaffa..8fdec09e 100644 --- a/absl/base/internal/low_level_alloc_test.cc +++ b/absl/base/internal/low_level_alloc_test.cc @@ -21,6 +21,10 @@ #include <unordered_map> #include <utility> +#ifdef __EMSCRIPTEN__ +#include <emscripten.h> +#endif + #include "absl/container/node_hash_map.h" namespace absl { @@ -82,7 +86,7 @@ static void Test(bool use_new_arena, bool call_malloc_hook, int n) { AllocMap::iterator it; BlockDesc block_desc; int rnd; - LowLevelAlloc::Arena *arena = 0; + LowLevelAlloc::Arena *arena = nullptr; if (use_new_arena) { int32_t flags = call_malloc_hook ? LowLevelAlloc::kCallMallocHook : 0; arena = LowLevelAlloc::NewArena(flags); @@ -97,11 +101,10 @@ static void Test(bool use_new_arena, bool call_malloc_hook, int n) { case 0: // coin came up heads: add a block using_low_level_alloc = true; block_desc.len = rand() & 0x3fff; - block_desc.ptr = - reinterpret_cast<char *>( - arena == 0 - ? LowLevelAlloc::Alloc(block_desc.len) - : LowLevelAlloc::AllocWithArena(block_desc.len, arena)); + block_desc.ptr = reinterpret_cast<char *>( + arena == nullptr + ? LowLevelAlloc::Alloc(block_desc.len) + : LowLevelAlloc::AllocWithArena(block_desc.len, arena)); using_low_level_alloc = false; RandomizeBlockDesc(&block_desc); rnd = rand(); @@ -158,5 +161,20 @@ ABSL_NAMESPACE_END int main(int argc, char *argv[]) { // The actual test runs in the global constructor of `before_main`. printf("PASS\n"); +#ifdef __EMSCRIPTEN__ + // clang-format off +// This is JS here. Don't try to format it. + MAIN_THREAD_EM_ASM({ + if (ENVIRONMENT_IS_WEB) { + if (typeof TEST_FINISH === 'function') { + TEST_FINISH($0); + } else { + console.error('Attempted to exit with status ' + $0); + console.error('But TEST_FINSIHED is not a function.'); + } + } + }, 0); +// clang-format on +#endif return 0; } diff --git a/absl/base/internal/low_level_scheduling.h b/absl/base/internal/low_level_scheduling.h index 961cc981..9baccc06 100644 --- a/absl/base/internal/low_level_scheduling.h +++ b/absl/base/internal/low_level_scheduling.h @@ -18,6 +18,7 @@ #ifndef ABSL_BASE_INTERNAL_LOW_LEVEL_SCHEDULING_H_ #define ABSL_BASE_INTERNAL_LOW_LEVEL_SCHEDULING_H_ +#include "absl/base/internal/raw_logging.h" #include "absl/base/internal/scheduling_mode.h" #include "absl/base/macros.h" @@ -29,6 +30,13 @@ extern "C" void __google_enable_rescheduling(bool disable_result); namespace absl { ABSL_NAMESPACE_BEGIN +class CondVar; +class Mutex; + +namespace synchronization_internal { +int MutexDelay(int32_t c, int mode); +} // namespace synchronization_internal + namespace base_internal { class SchedulingHelper; // To allow use of SchedulingGuard. @@ -53,6 +61,8 @@ class SchedulingGuard { public: // Returns true iff the calling thread may be cooperatively rescheduled. static bool ReschedulingIsAllowed(); + SchedulingGuard(const SchedulingGuard&) = delete; + SchedulingGuard& operator=(const SchedulingGuard&) = delete; private: // Disable cooperative rescheduling of the calling thread. It may still @@ -76,12 +86,23 @@ class SchedulingGuard { bool disabled; }; - // Access to SchedulingGuard is explicitly white-listed. + // A scoped helper to enable rescheduling temporarily. + // REQUIRES: destructor must run in same thread as constructor. + class ScopedEnable { + public: + ScopedEnable(); + ~ScopedEnable(); + + private: + int scheduling_disabled_depth_; + }; + + // Access to SchedulingGuard is explicitly permitted. + friend class absl::CondVar; + friend class absl::Mutex; friend class SchedulingHelper; friend class SpinLock; - - SchedulingGuard(const SchedulingGuard&) = delete; - SchedulingGuard& operator=(const SchedulingGuard&) = delete; + friend int absl::synchronization_internal::MutexDelay(int32_t c, int mode); }; //------------------------------------------------------------------------------ @@ -100,6 +121,12 @@ inline void SchedulingGuard::EnableRescheduling(bool /* disable_result */) { return; } +inline SchedulingGuard::ScopedEnable::ScopedEnable() + : scheduling_disabled_depth_(0) {} +inline SchedulingGuard::ScopedEnable::~ScopedEnable() { + ABSL_RAW_CHECK(scheduling_disabled_depth_ == 0, "disable unused warning"); +} + } // namespace base_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/periodic_sampler.cc b/absl/base/internal/periodic_sampler.cc deleted file mode 100644 index 520dabba..00000000 --- a/absl/base/internal/periodic_sampler.cc +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2019 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/base/internal/periodic_sampler.h" - -#include <atomic> - -#include "absl/base/internal/exponential_biased.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace base_internal { - -int64_t PeriodicSamplerBase::GetExponentialBiased(int period) noexcept { - return rng_.GetStride(period); -} - -bool PeriodicSamplerBase::SubtleConfirmSample() noexcept { - int current_period = period(); - - // Deal with period case 0 (always off) and 1 (always on) - if (ABSL_PREDICT_FALSE(current_period < 2)) { - stride_ = 0; - return current_period == 1; - } - - // Check if this is the first call to Sample() - if (ABSL_PREDICT_FALSE(stride_ == 1)) { - stride_ = static_cast<uint64_t>(-GetExponentialBiased(current_period)); - if (static_cast<int64_t>(stride_) < -1) { - ++stride_; - return false; - } - } - - stride_ = static_cast<uint64_t>(-GetExponentialBiased(current_period)); - return true; -} - -} // namespace base_internal -ABSL_NAMESPACE_END -} // namespace absl diff --git a/absl/base/internal/periodic_sampler.h b/absl/base/internal/periodic_sampler.h deleted file mode 100644 index f8a86796..00000000 --- a/absl/base/internal/periodic_sampler.h +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2019 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef ABSL_BASE_INTERNAL_PERIODIC_SAMPLER_H_ -#define ABSL_BASE_INTERNAL_PERIODIC_SAMPLER_H_ - -#include <stdint.h> - -#include <atomic> - -#include "absl/base/internal/exponential_biased.h" -#include "absl/base/optimization.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace base_internal { - -// PeriodicSamplerBase provides the basic period sampler implementation. -// -// This is the base class for the templated PeriodicSampler class, which holds -// a global std::atomic value identified by a user defined tag, such that -// each specific PeriodSampler implementation holds its own global period. -// -// PeriodicSamplerBase is thread-compatible except where stated otherwise. -class PeriodicSamplerBase { - public: - // PeriodicSamplerBase is trivial / copyable / movable / destructible. - PeriodicSamplerBase() = default; - PeriodicSamplerBase(PeriodicSamplerBase&&) = default; - PeriodicSamplerBase(const PeriodicSamplerBase&) = default; - - // Returns true roughly once every `period` calls. This is established by a - // randomly picked `stride` that is counted down on each call to `Sample`. - // This stride is picked such that the probability of `Sample()` returning - // true is 1 in `period`. - inline bool Sample() noexcept; - - // The below methods are intended for optimized use cases where the - // size of the inlined fast path code is highly important. Applications - // should use the `Sample()` method unless they have proof that their - // specific use case requires the optimizations offered by these methods. - // - // An example of such a use case is SwissTable sampling. All sampling checks - // are in inlined SwissTable methods, and the number of call sites is huge. - // In this case, the inlined code size added to each translation unit calling - // SwissTable methods is non-trivial. - // - // The `SubtleMaybeSample()` function spuriously returns true even if the - // function should not be sampled, applications MUST match each call to - // 'SubtleMaybeSample()' returning true with a `SubtleConfirmSample()` call, - // and use the result of the latter as the sampling decision. - // In other words: the code should logically be equivalent to: - // - // if (SubtleMaybeSample() && SubtleConfirmSample()) { - // // Sample this call - // } - // - // In the 'inline-size' optimized case, the `SubtleConfirmSample()` call can - // be placed out of line, for example, the typical use case looks as follows: - // - // // --- frobber.h ----------- - // void FrobberSampled(); - // - // inline void FrobberImpl() { - // // ... - // } - // - // inline void Frobber() { - // if (ABSL_PREDICT_FALSE(sampler.SubtleMaybeSample())) { - // FrobberSampled(); - // } else { - // FrobberImpl(); - // } - // } - // - // // --- frobber.cc ----------- - // void FrobberSampled() { - // if (!sampler.SubtleConfirmSample())) { - // // Spurious false positive - // FrobberImpl(); - // return; - // } - // - // // Sampled execution - // // ... - // } - inline bool SubtleMaybeSample() noexcept; - bool SubtleConfirmSample() noexcept; - - protected: - // We explicitly don't use a virtual destructor as this class is never - // virtually destroyed, and it keeps the class trivial, which avoids TLS - // prologue and epilogue code for our TLS instances. - ~PeriodicSamplerBase() = default; - - // Returns the next stride for our sampler. - // This function is virtual for testing purposes only. - virtual int64_t GetExponentialBiased(int period) noexcept; - - private: - // Returns the current period of this sampler. Thread-safe. - virtual int period() const noexcept = 0; - - // Keep and decrement stride_ as an unsigned integer, but compare the value - // to zero casted as a signed int. clang and msvc do not create optimum code - // if we use signed for the combined decrement and sign comparison. - // - // Below 3 alternative options, all compiles generate the best code - // using the unsigned increment <---> signed int comparison option. - // - // Option 1: - // int64_t stride_; - // if (ABSL_PREDICT_TRUE(++stride_ < 0)) { ... } - // - // GCC x64 (OK) : https://gcc.godbolt.org/z/R5MzzA - // GCC ppc (OK) : https://gcc.godbolt.org/z/z7NZAt - // Clang x64 (BAD): https://gcc.godbolt.org/z/t4gPsd - // ICC x64 (OK) : https://gcc.godbolt.org/z/rE6s8W - // MSVC x64 (OK) : https://gcc.godbolt.org/z/ARMXqS - // - // Option 2: - // int64_t stride_ = 0; - // if (ABSL_PREDICT_TRUE(--stride_ >= 0)) { ... } - // - // GCC x64 (OK) : https://gcc.godbolt.org/z/jSQxYK - // GCC ppc (OK) : https://gcc.godbolt.org/z/VJdYaA - // Clang x64 (BAD): https://gcc.godbolt.org/z/Xm4NjX - // ICC x64 (OK) : https://gcc.godbolt.org/z/4snaFd - // MSVC x64 (BAD): https://gcc.godbolt.org/z/BgnEKE - // - // Option 3: - // uint64_t stride_; - // if (ABSL_PREDICT_TRUE(static_cast<int64_t>(++stride_) < 0)) { ... } - // - // GCC x64 (OK) : https://gcc.godbolt.org/z/bFbfPy - // GCC ppc (OK) : https://gcc.godbolt.org/z/S9KkUE - // Clang x64 (OK) : https://gcc.godbolt.org/z/UYzRb4 - // ICC x64 (OK) : https://gcc.godbolt.org/z/ptTNfD - // MSVC x64 (OK) : https://gcc.godbolt.org/z/76j4-5 - uint64_t stride_ = 0; - ExponentialBiased rng_; -}; - -inline bool PeriodicSamplerBase::SubtleMaybeSample() noexcept { - // See comments on `stride_` for the unsigned increment / signed compare. - if (ABSL_PREDICT_TRUE(static_cast<int64_t>(++stride_) < 0)) { - return false; - } - return true; -} - -inline bool PeriodicSamplerBase::Sample() noexcept { - return ABSL_PREDICT_FALSE(SubtleMaybeSample()) ? SubtleConfirmSample() - : false; -} - -// PeriodicSampler is a concreted periodic sampler implementation. -// The user provided Tag identifies the implementation, and is required to -// isolate the global state of this instance from other instances. -// -// Typical use case: -// -// struct HashTablezTag {}; -// thread_local PeriodicSampler sampler; -// -// void HashTableSamplingLogic(...) { -// if (sampler.Sample()) { -// HashTableSlowSamplePath(...); -// } -// } -// -template <typename Tag, int default_period = 0> -class PeriodicSampler final : public PeriodicSamplerBase { - public: - ~PeriodicSampler() = default; - - int period() const noexcept final { - return period_.load(std::memory_order_relaxed); - } - - // Sets the global period for this sampler. Thread-safe. - // Setting a period of 0 disables the sampler, i.e., every call to Sample() - // will return false. Setting a period of 1 puts the sampler in 'always on' - // mode, i.e., every call to Sample() returns true. - static void SetGlobalPeriod(int period) { - period_.store(period, std::memory_order_relaxed); - } - - private: - static std::atomic<int> period_; -}; - -template <typename Tag, int default_period> -std::atomic<int> PeriodicSampler<Tag, default_period>::period_(default_period); - -} // namespace base_internal -ABSL_NAMESPACE_END -} // namespace absl - -#endif // ABSL_BASE_INTERNAL_PERIODIC_SAMPLER_H_ diff --git a/absl/base/internal/periodic_sampler_benchmark.cc b/absl/base/internal/periodic_sampler_benchmark.cc deleted file mode 100644 index 5ad469ce..00000000 --- a/absl/base/internal/periodic_sampler_benchmark.cc +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2019 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "benchmark/benchmark.h" -#include "absl/base/internal/periodic_sampler.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace base_internal { -namespace { - -template <typename Sampler> -void BM_Sample(Sampler* sampler, benchmark::State& state) { - for (auto _ : state) { - benchmark::DoNotOptimize(sampler); - benchmark::DoNotOptimize(sampler->Sample()); - } -} - -template <typename Sampler> -void BM_SampleMinunumInlined(Sampler* sampler, benchmark::State& state) { - for (auto _ : state) { - benchmark::DoNotOptimize(sampler); - if (ABSL_PREDICT_FALSE(sampler->SubtleMaybeSample())) { - benchmark::DoNotOptimize(sampler->SubtleConfirmSample()); - } - } -} - -void BM_PeriodicSampler_TinySample(benchmark::State& state) { - struct Tag {}; - PeriodicSampler<Tag, 10> sampler; - BM_Sample(&sampler, state); -} -BENCHMARK(BM_PeriodicSampler_TinySample); - -void BM_PeriodicSampler_ShortSample(benchmark::State& state) { - struct Tag {}; - PeriodicSampler<Tag, 1024> sampler; - BM_Sample(&sampler, state); -} -BENCHMARK(BM_PeriodicSampler_ShortSample); - -void BM_PeriodicSampler_LongSample(benchmark::State& state) { - struct Tag {}; - PeriodicSampler<Tag, 1024 * 1024> sampler; - BM_Sample(&sampler, state); -} -BENCHMARK(BM_PeriodicSampler_LongSample); - -void BM_PeriodicSampler_LongSampleMinunumInlined(benchmark::State& state) { - struct Tag {}; - PeriodicSampler<Tag, 1024 * 1024> sampler; - BM_SampleMinunumInlined(&sampler, state); -} -BENCHMARK(BM_PeriodicSampler_LongSampleMinunumInlined); - -void BM_PeriodicSampler_Disabled(benchmark::State& state) { - struct Tag {}; - PeriodicSampler<Tag, 0> sampler; - BM_Sample(&sampler, state); -} -BENCHMARK(BM_PeriodicSampler_Disabled); - -} // namespace -} // namespace base_internal -ABSL_NAMESPACE_END -} // namespace absl diff --git a/absl/base/internal/periodic_sampler_test.cc b/absl/base/internal/periodic_sampler_test.cc deleted file mode 100644 index 3b301e37..00000000 --- a/absl/base/internal/periodic_sampler_test.cc +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2019 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/base/internal/periodic_sampler.h" - -#include <thread> // NOLINT(build/c++11) - -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include "absl/base/attributes.h" -#include "absl/base/macros.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace base_internal { -namespace { - -using testing::Eq; -using testing::Return; -using testing::StrictMock; - -class MockPeriodicSampler : public PeriodicSamplerBase { - public: - virtual ~MockPeriodicSampler() = default; - - MOCK_METHOD(int, period, (), (const, noexcept)); - MOCK_METHOD(int64_t, GetExponentialBiased, (int), (noexcept)); -}; - -TEST(PeriodicSamplerBaseTest, Sample) { - StrictMock<MockPeriodicSampler> sampler; - - EXPECT_CALL(sampler, period()).Times(3).WillRepeatedly(Return(16)); - EXPECT_CALL(sampler, GetExponentialBiased(16)) - .WillOnce(Return(2)) - .WillOnce(Return(3)) - .WillOnce(Return(4)); - - EXPECT_FALSE(sampler.Sample()); - EXPECT_TRUE(sampler.Sample()); - - EXPECT_FALSE(sampler.Sample()); - EXPECT_FALSE(sampler.Sample()); - EXPECT_TRUE(sampler.Sample()); - - EXPECT_FALSE(sampler.Sample()); - EXPECT_FALSE(sampler.Sample()); - EXPECT_FALSE(sampler.Sample()); -} - -TEST(PeriodicSamplerBaseTest, ImmediatelySample) { - StrictMock<MockPeriodicSampler> sampler; - - EXPECT_CALL(sampler, period()).Times(2).WillRepeatedly(Return(16)); - EXPECT_CALL(sampler, GetExponentialBiased(16)) - .WillOnce(Return(1)) - .WillOnce(Return(2)) - .WillOnce(Return(3)); - - EXPECT_TRUE(sampler.Sample()); - - EXPECT_FALSE(sampler.Sample()); - EXPECT_TRUE(sampler.Sample()); - - EXPECT_FALSE(sampler.Sample()); - EXPECT_FALSE(sampler.Sample()); -} - -TEST(PeriodicSamplerBaseTest, Disabled) { - StrictMock<MockPeriodicSampler> sampler; - - EXPECT_CALL(sampler, period()).Times(3).WillRepeatedly(Return(0)); - - EXPECT_FALSE(sampler.Sample()); - EXPECT_FALSE(sampler.Sample()); - EXPECT_FALSE(sampler.Sample()); -} - -TEST(PeriodicSamplerBaseTest, AlwaysOn) { - StrictMock<MockPeriodicSampler> sampler; - - EXPECT_CALL(sampler, period()).Times(3).WillRepeatedly(Return(1)); - - EXPECT_TRUE(sampler.Sample()); - EXPECT_TRUE(sampler.Sample()); - EXPECT_TRUE(sampler.Sample()); -} - -TEST(PeriodicSamplerBaseTest, Disable) { - StrictMock<MockPeriodicSampler> sampler; - - EXPECT_CALL(sampler, period()).WillOnce(Return(16)); - EXPECT_CALL(sampler, GetExponentialBiased(16)).WillOnce(Return(3)); - EXPECT_FALSE(sampler.Sample()); - EXPECT_FALSE(sampler.Sample()); - - EXPECT_CALL(sampler, period()).Times(2).WillRepeatedly(Return(0)); - - EXPECT_FALSE(sampler.Sample()); - EXPECT_FALSE(sampler.Sample()); -} - -TEST(PeriodicSamplerBaseTest, Enable) { - StrictMock<MockPeriodicSampler> sampler; - - EXPECT_CALL(sampler, period()).WillOnce(Return(0)); - EXPECT_FALSE(sampler.Sample()); - - EXPECT_CALL(sampler, period()).Times(2).WillRepeatedly(Return(16)); - EXPECT_CALL(sampler, GetExponentialBiased(16)) - .Times(2) - .WillRepeatedly(Return(3)); - - EXPECT_FALSE(sampler.Sample()); - EXPECT_FALSE(sampler.Sample()); - EXPECT_TRUE(sampler.Sample()); - - EXPECT_FALSE(sampler.Sample()); - EXPECT_FALSE(sampler.Sample()); -} - -TEST(PeriodicSamplerTest, ConstructConstInit) { - struct Tag {}; - ABSL_CONST_INIT static PeriodicSampler<Tag> sampler; - (void)sampler; -} - -TEST(PeriodicSamplerTest, DefaultPeriod0) { - struct Tag {}; - PeriodicSampler<Tag> sampler; - EXPECT_THAT(sampler.period(), Eq(0)); -} - -TEST(PeriodicSamplerTest, DefaultPeriod) { - struct Tag {}; - PeriodicSampler<Tag, 100> sampler; - EXPECT_THAT(sampler.period(), Eq(100)); -} - -TEST(PeriodicSamplerTest, SetGlobalPeriod) { - struct Tag1 {}; - struct Tag2 {}; - PeriodicSampler<Tag1, 25> sampler1; - PeriodicSampler<Tag2, 50> sampler2; - - EXPECT_THAT(sampler1.period(), Eq(25)); - EXPECT_THAT(sampler2.period(), Eq(50)); - - std::thread thread([] { - PeriodicSampler<Tag1, 25> sampler1; - PeriodicSampler<Tag2, 50> sampler2; - EXPECT_THAT(sampler1.period(), Eq(25)); - EXPECT_THAT(sampler2.period(), Eq(50)); - sampler1.SetGlobalPeriod(10); - sampler2.SetGlobalPeriod(20); - }); - thread.join(); - - EXPECT_THAT(sampler1.period(), Eq(10)); - EXPECT_THAT(sampler2.period(), Eq(20)); -} - -} // namespace -} // namespace base_internal -ABSL_NAMESPACE_END -} // namespace absl diff --git a/absl/base/internal/prefetch.h b/absl/base/internal/prefetch.h new file mode 100644 index 00000000..06419283 --- /dev/null +++ b/absl/base/internal/prefetch.h @@ -0,0 +1,138 @@ +// Copyright 2022 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_PREFETCH_H_ +#define ABSL_BASE_INTERNAL_PREFETCH_H_ + +#include "absl/base/config.h" + +#ifdef __SSE__ +#include <xmmintrin.h> +#endif + +#if defined(_MSC_VER) && defined(ABSL_INTERNAL_HAVE_SSE) +#include <intrin.h> +#pragma intrinsic(_mm_prefetch) +#endif + +// Compatibility wrappers around __builtin_prefetch, to prefetch data +// for read if supported by the toolchain. + +// Move data into the cache before it is read, or "prefetch" it. +// +// The value of `addr` is the address of the memory to prefetch. If +// the target and compiler support it, data prefetch instructions are +// generated. If the prefetch is done some time before the memory is +// read, it may be in the cache by the time the read occurs. +// +// The function names specify the temporal locality heuristic applied, +// using the names of Intel prefetch instructions: +// +// T0 - high degree of temporal locality; data should be left in as +// many levels of the cache possible +// T1 - moderate degree of temporal locality +// T2 - low degree of temporal locality +// Nta - no temporal locality, data need not be left in the cache +// after the read +// +// Incorrect or gratuitous use of these functions can degrade +// performance, so use them only when representative benchmarks show +// an improvement. +// +// Example usage: +// +// absl::base_internal::PrefetchT0(addr); +// +// Currently, the different prefetch calls behave on some Intel +// architectures as follows: +// +// SNB..SKL SKX +// PrefetchT0() L1/L2/L3 L1/L2 +// PrefetchT1() L2/L3 L2 +// PrefetchT2() L2/L3 L2 +// PrefetchNta() L1/--/L3 L1* +// +// * On SKX PrefetchNta() will bring the line into L1 but will evict +// from L3 cache. This might result in surprising behavior. +// +// SNB = Sandy Bridge, SKL = Skylake, SKX = Skylake Xeon. +// +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +void PrefetchT0(const void* addr); +void PrefetchT1(const void* addr); +void PrefetchT2(const void* addr); +void PrefetchNta(const void* addr); + +// Implementation details follow. + +#if ABSL_HAVE_BUILTIN(__builtin_prefetch) || defined(__GNUC__) + +#define ABSL_INTERNAL_HAVE_PREFETCH 1 + +// See __builtin_prefetch: +// https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html. +// +// These functions speculatively load for read only. This is +// safe for all currently supported platforms. However, prefetch for +// store may have problems depending on the target platform. +// +inline void PrefetchT0(const void* addr) { + // Note: this uses prefetcht0 on Intel. + __builtin_prefetch(addr, 0, 3); +} +inline void PrefetchT1(const void* addr) { + // Note: this uses prefetcht1 on Intel. + __builtin_prefetch(addr, 0, 2); +} +inline void PrefetchT2(const void* addr) { + // Note: this uses prefetcht2 on Intel. + __builtin_prefetch(addr, 0, 1); +} +inline void PrefetchNta(const void* addr) { + // Note: this uses prefetchtnta on Intel. + __builtin_prefetch(addr, 0, 0); +} + +#elif defined(ABSL_INTERNAL_HAVE_SSE) + +#define ABSL_INTERNAL_HAVE_PREFETCH 1 + +inline void PrefetchT0(const void* addr) { + _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T0); +} +inline void PrefetchT1(const void* addr) { + _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T1); +} +inline void PrefetchT2(const void* addr) { + _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T2); +} +inline void PrefetchNta(const void* addr) { + _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_NTA); +} + +#else +inline void PrefetchT0(const void*) {} +inline void PrefetchT1(const void*) {} +inline void PrefetchT2(const void*) {} +inline void PrefetchNta(const void*) {} +#endif + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_PREFETCH_H_ diff --git a/absl/base/internal/prefetch_test.cc b/absl/base/internal/prefetch_test.cc new file mode 100644 index 00000000..7c1dae46 --- /dev/null +++ b/absl/base/internal/prefetch_test.cc @@ -0,0 +1,43 @@ +// Copyright 2022 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/prefetch.h" + +#include "gtest/gtest.h" + +namespace { + +int number = 42; + +TEST(Prefetch, TemporalLocalityNone) { + absl::base_internal::PrefetchNta(&number); + EXPECT_EQ(number, 42); +} + +TEST(Prefetch, TemporalLocalityLow) { + absl::base_internal::PrefetchT2(&number); + EXPECT_EQ(number, 42); +} + +TEST(Prefetch, TemporalLocalityMedium) { + absl::base_internal::PrefetchT1(&number); + EXPECT_EQ(number, 42); +} + +TEST(Prefetch, TemporalLocalityHigh) { + absl::base_internal::PrefetchT0(&number); + EXPECT_EQ(number, 42); +} + +} // namespace diff --git a/absl/base/internal/raw_logging.cc b/absl/base/internal/raw_logging.cc index 40cea550..54e71a3f 100644 --- a/absl/base/internal/raw_logging.cc +++ b/absl/base/internal/raw_logging.cc @@ -14,15 +14,17 @@ #include "absl/base/internal/raw_logging.h" -#include <stddef.h> #include <cstdarg> +#include <cstddef> #include <cstdio> #include <cstdlib> #include <cstring> +#include <string> #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/atomic_hook.h" +#include "absl/base/internal/errno_saver.h" #include "absl/base/log_severity.h" // We know how to perform low-level writes to stderr in POSIX and Windows. For @@ -36,8 +38,8 @@ // This preprocessor token is also defined in raw_io.cc. If you need to copy // this, consider moving both to config.h instead. #if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ - defined(__Fuchsia__) || defined(__native_client__) || \ - defined(__EMSCRIPTEN__) || defined(__ASYLO__) + defined(__Fuchsia__) || defined(__native_client__) || \ + defined(__OpenBSD__) || defined(__EMSCRIPTEN__) || defined(__ASYLO__) #include <unistd.h> @@ -50,7 +52,8 @@ // ABSL_HAVE_SYSCALL_WRITE is defined when the platform provides the syscall // syscall(SYS_write, /*int*/ fd, /*char* */ buf, /*size_t*/ len); // for low level operations that want to avoid libc. -#if (defined(__linux__) || defined(__FreeBSD__)) && !defined(__ANDROID__) +#if (defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__)) && \ + !defined(__ANDROID__) #include <sys/syscall.h> #define ABSL_HAVE_SYSCALL_WRITE 1 #define ABSL_LOW_LEVEL_WRITE_SUPPORTED 1 @@ -67,28 +70,25 @@ #undef ABSL_HAVE_RAW_IO #endif -// TODO(gfalcon): We want raw-logging to work on as many platforms as possible. -// Explicitly #error out when not ABSL_LOW_LEVEL_WRITE_SUPPORTED, except for a -// whitelisted set of platforms for which we expect not to be able to raw log. +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace raw_logging_internal { +namespace { -ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook< - absl::raw_logging_internal::LogPrefixHook> - log_prefix_hook; -ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook< - absl::raw_logging_internal::AbortHook> - abort_hook; +// TODO(gfalcon): We want raw-logging to work on as many platforms as possible. +// Explicitly `#error` out when not `ABSL_LOW_LEVEL_WRITE_SUPPORTED`, except for +// a selected set of platforms for which we expect not to be able to raw log. #ifdef ABSL_LOW_LEVEL_WRITE_SUPPORTED -static const char kTruncated[] = " ... (message truncated)\n"; +constexpr char kTruncated[] = " ... (message truncated)\n"; // sprintf the format to the buffer, adjusting *buf and *size to reflect the // consumed bytes, and return whether the message fit without truncation. If // truncation occurred, if possible leave room in the buffer for the message // kTruncated[]. -inline static bool VADoRawLog(char** buf, int* size, const char* format, - va_list ap) ABSL_PRINTF_ATTRIBUTE(3, 0); -inline static bool VADoRawLog(char** buf, int* size, - const char* format, va_list ap) { +bool VADoRawLog(char** buf, int* size, const char* format, va_list ap) + ABSL_PRINTF_ATTRIBUTE(3, 0); +bool VADoRawLog(char** buf, int* size, const char* format, va_list ap) { int n = vsnprintf(*buf, *size, format, ap); bool result = true; if (n < 0 || n > *size) { @@ -96,7 +96,7 @@ inline static bool VADoRawLog(char** buf, int* size, if (static_cast<size_t>(*size) > sizeof(kTruncated)) { n = *size - sizeof(kTruncated); // room for truncation message } else { - n = 0; // no room for truncation message + n = 0; // no room for truncation message } } *size -= n; @@ -105,9 +105,7 @@ inline static bool VADoRawLog(char** buf, int* size, } #endif // ABSL_LOW_LEVEL_WRITE_SUPPORTED -static constexpr int kLogBufSize = 3000; - -namespace { +constexpr int kLogBufSize = 3000; // CAVEAT: vsnprintf called from *DoRawLog below has some (exotic) code paths // that invoke malloc() and getenv() that might acquire some locks. @@ -128,6 +126,18 @@ bool DoRawLog(char** buf, int* size, const char* format, ...) { return true; } +bool DefaultLogFilterAndPrefix(absl::LogSeverity, const char* file, int line, + char** buf, int* buf_size) { + DoRawLog(buf, buf_size, "[%s : %d] RAW: ", file, line); + return true; +} + +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES +absl::base_internal::AtomicHook<LogFilterAndPrefixHook> + log_filter_and_prefix_hook(DefaultLogFilterAndPrefix); +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES +absl::base_internal::AtomicHook<AbortHook> abort_hook; + void RawLogVA(absl::LogSeverity severity, const char* file, int line, const char* format, va_list ap) ABSL_PRINTF_ATTRIBUTE(4, 0); void RawLogVA(absl::LogSeverity severity, const char* file, int line, @@ -148,14 +158,7 @@ void RawLogVA(absl::LogSeverity severity, const char* file, int line, } #endif - auto log_prefix_hook_ptr = log_prefix_hook.Load(); - if (log_prefix_hook_ptr) { - enabled = log_prefix_hook_ptr(severity, file, line, &buf, &size); - } else { - if (enabled) { - DoRawLog(&buf, &size, "[%s : %d] RAW: ", file, line); - } - } + enabled = log_filter_and_prefix_hook(severity, file, line, &buf, &size); const char* const prefix_end = buf; #ifdef ABSL_LOW_LEVEL_WRITE_SUPPORTED @@ -166,11 +169,12 @@ void RawLogVA(absl::LogSeverity severity, const char* file, int line, } else { DoRawLog(&buf, &size, "%s", kTruncated); } - absl::raw_logging_internal::SafeWriteToStderr(buffer, strlen(buffer)); + AsyncSignalSafeWriteToStderr(buffer, strlen(buffer)); } #else static_cast<void>(format); static_cast<void>(ap); + static_cast<void>(enabled); #endif // Abort the process after logging a FATAL message, even if the output itself @@ -181,13 +185,23 @@ void RawLogVA(absl::LogSeverity severity, const char* file, int line, } } +// Non-formatting version of RawLog(). +// +// TODO(gfalcon): When string_view no longer depends on base, change this +// interface to take its message as a string_view instead. +void DefaultInternalLog(absl::LogSeverity severity, const char* file, int line, + const std::string& message) { + RawLog(severity, file, line, "%.*s", static_cast<int>(message.size()), + message.data()); +} + } // namespace -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace raw_logging_internal { -void SafeWriteToStderr(const char *s, size_t len) { +void AsyncSignalSafeWriteToStderr(const char* s, size_t len) { + absl::base_internal::ErrnoSaver errno_saver; #if defined(ABSL_HAVE_SYSCALL_WRITE) + // We prefer calling write via `syscall` to minimize the risk of libc doing + // something "helpful". syscall(SYS_write, STDERR_FILENO, s, len); #elif defined(ABSL_HAVE_POSIX_WRITE) write(STDERR_FILENO, s, len); @@ -201,8 +215,6 @@ void SafeWriteToStderr(const char *s, size_t len) { } void RawLog(absl::LogSeverity severity, const char* file, int line, - const char* format, ...) ABSL_PRINTF_ATTRIBUTE(4, 5); -void RawLog(absl::LogSeverity severity, const char* file, int line, const char* format, ...) { va_list ap; va_start(ap, format); @@ -210,15 +222,6 @@ void RawLog(absl::LogSeverity severity, const char* file, int line, va_end(ap); } -// Non-formatting version of RawLog(). -// -// TODO(gfalcon): When string_view no longer depends on base, change this -// interface to take its message as a string_view instead. -static void DefaultInternalLog(absl::LogSeverity severity, const char* file, - int line, const std::string& message) { - RawLog(severity, file, line, "%s", message.c_str()); -} - bool RawLoggingFullySupported() { #ifdef ABSL_LOW_LEVEL_WRITE_SUPPORTED return true; @@ -227,10 +230,16 @@ bool RawLoggingFullySupported() { #endif // !ABSL_LOW_LEVEL_WRITE_SUPPORTED } -ABSL_DLL ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_DLL absl::base_internal::AtomicHook<InternalLogFunction> internal_log_function(DefaultInternalLog); +void RegisterLogFilterAndPrefixHook(LogFilterAndPrefixHook func) { + log_filter_and_prefix_hook.Store(func); +} + +void RegisterAbortHook(AbortHook func) { abort_hook.Store(func); } + void RegisterInternalLogFunction(InternalLogFunction func) { internal_log_function.Store(func); } diff --git a/absl/base/internal/raw_logging.h b/absl/base/internal/raw_logging.h index 418d6c85..0747c9df 100644 --- a/absl/base/internal/raw_logging.h +++ b/absl/base/internal/raw_logging.h @@ -72,10 +72,14 @@ // // The API is a subset of the above: each macro only takes two arguments. Use // StrCat if you need to build a richer message. -#define ABSL_INTERNAL_LOG(severity, message) \ - do { \ - ::absl::raw_logging_internal::internal_log_function( \ - ABSL_RAW_LOGGING_INTERNAL_##severity, __FILE__, __LINE__, message); \ +#define ABSL_INTERNAL_LOG(severity, message) \ + do { \ + constexpr const char* absl_raw_logging_internal_filename = __FILE__; \ + ::absl::raw_logging_internal::internal_log_function( \ + ABSL_RAW_LOGGING_INTERNAL_##severity, \ + absl_raw_logging_internal_filename, __LINE__, message); \ + if (ABSL_RAW_LOGGING_INTERNAL_##severity == ::absl::LogSeverity::kFatal) \ + ABSL_INTERNAL_UNREACHABLE; \ } while (0) #define ABSL_INTERNAL_CHECK(condition, message) \ @@ -105,12 +109,9 @@ namespace raw_logging_internal { void RawLog(absl::LogSeverity severity, const char* file, int line, const char* format, ...) ABSL_PRINTF_ATTRIBUTE(4, 5); -// Writes the provided buffer directly to stderr, in a safe, low-level manner. -// -// In POSIX this means calling write(), which is async-signal safe and does -// not malloc. If the platform supports the SYS_write syscall, we invoke that -// directly to side-step any libc interception. -void SafeWriteToStderr(const char *s, size_t len); +// Writes the provided buffer directly to stderr, in a signal-safe, low-level +// manner. +void AsyncSignalSafeWriteToStderr(const char* s, size_t len); // compile-time function to get the "base" filename, that is, the part of // a filename after the last "/" or "\" path separator. The search starts at @@ -144,11 +145,12 @@ bool RawLoggingFullySupported(); // 'severity' is the severity level of the message being written. // 'file' and 'line' are the file and line number where the ABSL_RAW_LOG macro // was located. -// 'buffer' and 'buf_size' are pointers to the buffer and buffer size. If the -// hook writes a prefix, it must increment *buffer and decrement *buf_size +// 'buf' and 'buf_size' are pointers to the buffer and buffer size. If the +// hook writes a prefix, it must increment *buf and decrement *buf_size // accordingly. -using LogPrefixHook = bool (*)(absl::LogSeverity severity, const char* file, - int line, char** buffer, int* buf_size); +using LogFilterAndPrefixHook = bool (*)(absl::LogSeverity severity, + const char* file, int line, char** buf, + int* buf_size); // Function type for a raw_logging customization hook called to abort a process // when a FATAL message is logged. If the provided AbortHook() returns, the @@ -158,7 +160,10 @@ using LogPrefixHook = bool (*)(absl::LogSeverity severity, const char* file, // was located. // The NUL-terminated logged message lives in the buffer between 'buf_start' // and 'buf_end'. 'prefix_end' points to the first non-prefix character of the -// buffer (as written by the LogPrefixHook.) +// buffer (as written by the LogFilterAndPrefixHook.) +// +// The lifetime of the filename and message buffers will not end while the +// process remains alive. using AbortHook = void (*)(const char* file, int line, const char* buf_start, const char* prefix_end, const char* buf_end); @@ -170,10 +175,18 @@ using InternalLogFunction = void (*)(absl::LogSeverity severity, const char* file, int line, const std::string& message); -ABSL_DLL ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES extern base_internal::AtomicHook< +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_DLL extern base_internal::AtomicHook< InternalLogFunction> internal_log_function; +// Registers hooks of the above types. Only a single hook of each type may be +// registered. It is an error to call these functions multiple times with +// different input arguments. +// +// These functions are safe to call at any point during initialization; they do +// not block or malloc, and are async-signal safe. +void RegisterLogFilterAndPrefixHook(LogFilterAndPrefixHook func); +void RegisterAbortHook(AbortHook func); void RegisterInternalLogFunction(InternalLogFunction func); } // namespace raw_logging_internal diff --git a/absl/base/internal/spinlock.cc b/absl/base/internal/spinlock.cc index a7d44f3e..35c0696a 100644 --- a/absl/base/internal/spinlock.cc +++ b/absl/base/internal/spinlock.cc @@ -125,8 +125,9 @@ void SpinLock::SlowLock() { // it as having a sleeper. if ((lock_value & kWaitTimeMask) == 0) { // Here, just "mark" that the thread is going to sleep. Don't store the - // lock wait time in the lock as that will cause the current lock - // owner to think it experienced contention. + // lock wait time in the lock -- the lock word stores the amount of time + // that the current holder waited before acquiring the lock, not the wait + // time of any thread currently waiting to acquire it. if (lockword_.compare_exchange_strong( lock_value, lock_value | kSpinLockSleeper, std::memory_order_relaxed, std::memory_order_relaxed)) { @@ -140,6 +141,14 @@ void SpinLock::SlowLock() { // this thread obtains the lock. lock_value = TryLockInternal(lock_value, wait_cycles); continue; // Skip the delay at the end of the loop. + } else if ((lock_value & kWaitTimeMask) == 0) { + // The lock is still held, without a waiter being marked, but something + // else about the lock word changed, causing our CAS to fail. For + // example, a new lock holder may have acquired the lock with + // kSpinLockDisabledScheduling set, whereas the previous holder had not + // set that flag. In this case, attempt again to mark ourselves as a + // waiter. + continue; } } diff --git a/absl/base/internal/spinlock.h b/absl/base/internal/spinlock.h index 2222398b..6d8d8ddd 100644 --- a/absl/base/internal/spinlock.h +++ b/absl/base/internal/spinlock.h @@ -15,17 +15,16 @@ // // Most users requiring mutual exclusion should use Mutex. -// SpinLock is provided for use in three situations: -// - for use in code that Mutex itself depends on -// - to get a faster fast-path release under low contention (without an -// atomic read-modify-write) In return, SpinLock has worse behaviour under -// contention, which is why Mutex is preferred in most situations. +// SpinLock is provided for use in two situations: +// - for use by Abseil internal code that Mutex itself depends on // - for async signal safety (see below) // SpinLock is async signal safe. If a spinlock is used within a signal // handler, all code that acquires the lock must ensure that the signal cannot // arrive while they are holding the lock. Typically, this is done by blocking // the signal. +// +// Threads waiting on a SpinLock may be woken in an arbitrary order. #ifndef ABSL_BASE_INTERNAL_SPINLOCK_H_ #define ABSL_BASE_INTERNAL_SPINLOCK_H_ @@ -64,7 +63,14 @@ class ABSL_LOCKABLE SpinLock { constexpr SpinLock(absl::ConstInitType, base_internal::SchedulingMode mode) : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) {} + // For global SpinLock instances prefer trivial destructor when possible. + // Default but non-trivial destructor in some build configurations causes an + // extra static initializer. +#ifdef ABSL_INTERNAL_HAVE_TSAN_INTERFACE ~SpinLock() { ABSL_TSAN_MUTEX_DESTROY(this, __tsan_mutex_not_static); } +#else + ~SpinLock() = default; +#endif // Acquire this SpinLock. inline void Lock() ABSL_EXCLUSIVE_LOCK_FUNCTION() { @@ -114,6 +120,14 @@ class ABSL_LOCKABLE SpinLock { return (lockword_.load(std::memory_order_relaxed) & kSpinLockHeld) != 0; } + // Return immediately if this thread holds the SpinLock exclusively. + // Otherwise, report an error by crashing with a diagnostic. + inline void AssertHeld() const ABSL_ASSERT_EXCLUSIVE_LOCK() { + if (!IsHeld()) { + ABSL_RAW_LOG(FATAL, "thread should hold the lock on SpinLock"); + } + } + protected: // These should not be exported except for testing. @@ -133,8 +147,20 @@ class ABSL_LOCKABLE SpinLock { // // bit[0] encodes whether a lock is being held. // bit[1] encodes whether a lock uses cooperative scheduling. - // bit[2] encodes whether a lock disables scheduling. + // bit[2] encodes whether the current lock holder disabled scheduling when + // acquiring the lock. Only set when kSpinLockHeld is also set. // bit[3:31] encodes time a lock spent on waiting as a 29-bit unsigned int. + // This is set by the lock holder to indicate how long it waited on + // the lock before eventually acquiring it. The number of cycles is + // encoded as a 29-bit unsigned int, or in the case that the current + // holder did not wait but another waiter is queued, the LSB + // (kSpinLockSleeper) is set. The implementation does not explicitly + // track the number of queued waiters beyond this. It must always be + // assumed that waiters may exist if the current holder was required to + // queue. + // + // Invariant: if the lock is not held, the value is either 0 or + // kSpinLockCooperative. static constexpr uint32_t kSpinLockHeld = 1; static constexpr uint32_t kSpinLockCooperative = 2; static constexpr uint32_t kSpinLockDisabledScheduling = 4; diff --git a/absl/base/internal/spinlock_akaros.inc b/absl/base/internal/spinlock_akaros.inc index bc468940..7b0cada4 100644 --- a/absl/base/internal/spinlock_akaros.inc +++ b/absl/base/internal/spinlock_akaros.inc @@ -20,7 +20,7 @@ extern "C" { -ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay( +ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockDelay)( std::atomic<uint32_t>* /* lock_word */, uint32_t /* value */, int /* loop */, absl::base_internal::SchedulingMode /* mode */) { // In Akaros, one must take care not to call anything that could cause a @@ -29,7 +29,7 @@ ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay( // arbitrary code. } -ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockWake( +ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockWake)( std::atomic<uint32_t>* /* lock_word */, bool /* all */) {} } // extern "C" diff --git a/absl/base/internal/spinlock_linux.inc b/absl/base/internal/spinlock_linux.inc index e31c6ed4..fe8ba674 100644 --- a/absl/base/internal/spinlock_linux.inc +++ b/absl/base/internal/spinlock_linux.inc @@ -56,18 +56,15 @@ static_assert(sizeof(std::atomic<uint32_t>) == sizeof(int), extern "C" { -ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay( - std::atomic<uint32_t> *w, uint32_t value, int loop, +ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockDelay)( + std::atomic<uint32_t> *w, uint32_t value, int, absl::base_internal::SchedulingMode) { absl::base_internal::ErrnoSaver errno_saver; - struct timespec tm; - tm.tv_sec = 0; - tm.tv_nsec = absl::base_internal::SpinLockSuggestedDelayNS(loop); - syscall(SYS_futex, w, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, value, &tm); + syscall(SYS_futex, w, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, value, nullptr); } -ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockWake(std::atomic<uint32_t> *w, - bool all) { +ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockWake)( + std::atomic<uint32_t> *w, bool all) { syscall(SYS_futex, w, FUTEX_WAKE | FUTEX_PRIVATE_FLAG, all ? INT_MAX : 1, 0); } diff --git a/absl/base/internal/spinlock_posix.inc b/absl/base/internal/spinlock_posix.inc index fcd21b15..4f6f887d 100644 --- a/absl/base/internal/spinlock_posix.inc +++ b/absl/base/internal/spinlock_posix.inc @@ -25,7 +25,7 @@ extern "C" { -ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay( +ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockDelay)( std::atomic<uint32_t>* /* lock_word */, uint32_t /* value */, int loop, absl::base_internal::SchedulingMode /* mode */) { absl::base_internal::ErrnoSaver errno_saver; @@ -40,7 +40,7 @@ ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay( } } -ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockWake( +ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockWake)( std::atomic<uint32_t>* /* lock_word */, bool /* all */) {} } // extern "C" diff --git a/absl/base/internal/spinlock_wait.h b/absl/base/internal/spinlock_wait.h index 169bc749..9a1adcda 100644 --- a/absl/base/internal/spinlock_wait.h +++ b/absl/base/internal/spinlock_wait.h @@ -39,22 +39,22 @@ struct SpinLockWaitTransition { // satisfying 0<=i<n && trans[i].done, atomically make the transition, // then return the old value of *w. Make any other atomic transitions // where !trans[i].done, but continue waiting. +// +// Wakeups for threads blocked on SpinLockWait do not respect priorities. uint32_t SpinLockWait(std::atomic<uint32_t> *w, int n, const SpinLockWaitTransition trans[], SchedulingMode scheduling_mode); -// If possible, wake some thread that has called SpinLockDelay(w, ...). If -// "all" is true, wake all such threads. This call is a hint, and on some -// systems it may be a no-op; threads calling SpinLockDelay() will always wake -// eventually even if SpinLockWake() is never called. +// If possible, wake some thread that has called SpinLockDelay(w, ...). If `all` +// is true, wake all such threads. On some systems, this may be a no-op; on +// those systems, threads calling SpinLockDelay() will always wake eventually +// even if SpinLockWake() is never called. void SpinLockWake(std::atomic<uint32_t> *w, bool all); // Wait for an appropriate spin delay on iteration "loop" of a // spin loop on location *w, whose previously observed value was "value". // SpinLockDelay() may do nothing, may yield the CPU, may sleep a clock tick, -// or may wait for a delay that can be truncated by a call to SpinLockWake(w). -// In all cases, it must return in bounded time even if SpinLockWake() is not -// called. +// or may wait for a call to SpinLockWake(w). void SpinLockDelay(std::atomic<uint32_t> *w, uint32_t value, int loop, base_internal::SchedulingMode scheduling_mode); @@ -73,21 +73,23 @@ ABSL_NAMESPACE_END // By changing our extension points to be extern "C", we dodge this // check. extern "C" { -void AbslInternalSpinLockWake(std::atomic<uint32_t> *w, bool all); -void AbslInternalSpinLockDelay( +void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockWake)(std::atomic<uint32_t> *w, + bool all); +void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockDelay)( std::atomic<uint32_t> *w, uint32_t value, int loop, absl::base_internal::SchedulingMode scheduling_mode); } inline void absl::base_internal::SpinLockWake(std::atomic<uint32_t> *w, bool all) { - AbslInternalSpinLockWake(w, all); + ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockWake)(w, all); } inline void absl::base_internal::SpinLockDelay( std::atomic<uint32_t> *w, uint32_t value, int loop, absl::base_internal::SchedulingMode scheduling_mode) { - AbslInternalSpinLockDelay(w, value, loop, scheduling_mode); + ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockDelay) + (w, value, loop, scheduling_mode); } #endif // ABSL_BASE_INTERNAL_SPINLOCK_WAIT_H_ diff --git a/absl/base/internal/spinlock_win32.inc b/absl/base/internal/spinlock_win32.inc index 78654b5b..9d224813 100644 --- a/absl/base/internal/spinlock_win32.inc +++ b/absl/base/internal/spinlock_win32.inc @@ -20,9 +20,9 @@ extern "C" { -void AbslInternalSpinLockDelay(std::atomic<uint32_t>* /* lock_word */, - uint32_t /* value */, int loop, - absl::base_internal::SchedulingMode /* mode */) { +void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockDelay)( + std::atomic<uint32_t>* /* lock_word */, uint32_t /* value */, int loop, + absl::base_internal::SchedulingMode /* mode */) { if (loop == 0) { } else if (loop == 1) { Sleep(0); @@ -31,7 +31,7 @@ void AbslInternalSpinLockDelay(std::atomic<uint32_t>* /* lock_word */, } } -void AbslInternalSpinLockWake(std::atomic<uint32_t>* /* lock_word */, - bool /* all */) {} +void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockWake)( + std::atomic<uint32_t>* /* lock_word */, bool /* all */) {} } // extern "C" diff --git a/absl/base/internal/strerror.cc b/absl/base/internal/strerror.cc index af181513..0d6226fd 100644 --- a/absl/base/internal/strerror.cc +++ b/absl/base/internal/strerror.cc @@ -14,6 +14,7 @@ #include "absl/base/internal/strerror.h" +#include <array> #include <cerrno> #include <cstddef> #include <cstdio> @@ -21,13 +22,13 @@ #include <string> #include <type_traits> -#include "absl/base/attributes.h" #include "absl/base/internal/errno_saver.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace base_internal { namespace { + const char* StrErrorAdaptor(int errnum, char* buf, size_t buflen) { #if defined(_WIN32) int rc = strerror_s(buf, buflen, errnum); @@ -35,15 +36,6 @@ const char* StrErrorAdaptor(int errnum, char* buf, size_t buflen) { if (rc == 0 && strncmp(buf, "Unknown error", buflen) == 0) *buf = '\0'; return buf; #else -#if defined(__GLIBC__) || defined(__APPLE__) - // Use the BSD sys_errlist API provided by GNU glibc and others to - // avoid any need to copy the message into the local buffer first. - if (0 <= errnum && errnum < sys_nerr) { - if (const char* p = sys_errlist[errnum]) { - return p; - } - } -#endif // The type of `ret` is platform-specific; both of these branches must compile // either way but only one will execute on any given platform: auto ret = strerror_r(errnum, buf, buflen); @@ -57,10 +49,8 @@ const char* StrErrorAdaptor(int errnum, char* buf, size_t buflen) { } #endif } -} // namespace -std::string StrError(int errnum) { - absl::base_internal::ErrnoSaver errno_saver; +std::string StrErrorInternal(int errnum) { char buf[100]; const char* str = StrErrorAdaptor(errnum, buf, sizeof buf); if (*str == '\0') { @@ -70,6 +60,29 @@ std::string StrError(int errnum) { return str; } +// kSysNerr is the number of errors from a recent glibc. `StrError()` falls back +// to `StrErrorAdaptor()` if the value is larger than this. +constexpr int kSysNerr = 135; + +std::array<std::string, kSysNerr>* NewStrErrorTable() { + auto* table = new std::array<std::string, kSysNerr>; + for (int i = 0; i < static_cast<int>(table->size()); ++i) { + (*table)[i] = StrErrorInternal(i); + } + return table; +} + +} // namespace + +std::string StrError(int errnum) { + absl::base_internal::ErrnoSaver errno_saver; + static const auto* table = NewStrErrorTable(); + if (errnum >= 0 && errnum < static_cast<int>(table->size())) { + return (*table)[errnum]; + } + return StrErrorInternal(errnum); +} + } // namespace base_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/strerror_benchmark.cc b/absl/base/internal/strerror_benchmark.cc index d8ca86b9..c9ab14a8 100644 --- a/absl/base/internal/strerror_benchmark.cc +++ b/absl/base/internal/strerror_benchmark.cc @@ -20,15 +20,6 @@ #include "benchmark/benchmark.h" namespace { -#if defined(__GLIBC__) || defined(__APPLE__) -void BM_SysErrList(benchmark::State& state) { - for (auto _ : state) { - benchmark::DoNotOptimize(std::string(sys_errlist[ERANGE])); - } -} -BENCHMARK(BM_SysErrList); -#endif - void BM_AbslStrError(benchmark::State& state) { for (auto _ : state) { benchmark::DoNotOptimize(absl::base_internal::StrError(ERANGE)); diff --git a/absl/base/internal/strerror_test.cc b/absl/base/internal/strerror_test.cc index a53da97f..e32d5b5c 100644 --- a/absl/base/internal/strerror_test.cc +++ b/absl/base/internal/strerror_test.cc @@ -62,12 +62,14 @@ TEST(StrErrorTest, MultipleThreads) { ++counter; errno = ERANGE; const std::string value = absl::base_internal::StrError(i); + // EXPECT_* could change errno. Stash it first. + int check_err = errno; + EXPECT_THAT(check_err, Eq(ERANGE)); // Only the GNU implementation is guaranteed to provide the // string "Unknown error nnn". POSIX doesn't say anything. if (!absl::StartsWith(value, "Unknown error ")) { - EXPECT_THAT(absl::base_internal::StrError(i), Eq(expected_strings[i])); + EXPECT_THAT(value, Eq(expected_strings[i])); } - EXPECT_THAT(errno, Eq(ERANGE)); } }; diff --git a/absl/base/internal/sysinfo.cc b/absl/base/internal/sysinfo.cc index 6c69683f..c8366df1 100644 --- a/absl/base/internal/sysinfo.cc +++ b/absl/base/internal/sysinfo.cc @@ -39,6 +39,7 @@ #endif #include <string.h> + #include <cassert> #include <cstdint> #include <cstdio> @@ -50,6 +51,7 @@ #include <vector> #include "absl/base/call_once.h" +#include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/internal/spinlock.h" #include "absl/base/internal/unscaledcycleclock.h" @@ -59,9 +61,77 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace base_internal { +namespace { + +#if defined(_WIN32) + +// Returns number of bits set in `bitMask` +DWORD Win32CountSetBits(ULONG_PTR bitMask) { + for (DWORD bitSetCount = 0; ; ++bitSetCount) { + if (bitMask == 0) return bitSetCount; + bitMask &= bitMask - 1; + } +} + +// Returns the number of logical CPUs using GetLogicalProcessorInformation(), or +// 0 if the number of processors is not available or can not be computed. +// https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getlogicalprocessorinformation +int Win32NumCPUs() { +#pragma comment(lib, "kernel32.lib") + using Info = SYSTEM_LOGICAL_PROCESSOR_INFORMATION; + + DWORD info_size = sizeof(Info); + Info* info(static_cast<Info*>(malloc(info_size))); + if (info == nullptr) return 0; + + bool success = GetLogicalProcessorInformation(info, &info_size); + if (!success && GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + free(info); + info = static_cast<Info*>(malloc(info_size)); + if (info == nullptr) return 0; + success = GetLogicalProcessorInformation(info, &info_size); + } + + DWORD logicalProcessorCount = 0; + if (success) { + Info* ptr = info; + DWORD byteOffset = 0; + while (byteOffset + sizeof(Info) <= info_size) { + switch (ptr->Relationship) { + case RelationProcessorCore: + logicalProcessorCount += Win32CountSetBits(ptr->ProcessorMask); + break; + + case RelationNumaNode: + case RelationCache: + case RelationProcessorPackage: + // Ignore other entries + break; + + default: + // Ignore unknown entries + break; + } + byteOffset += sizeof(Info); + ptr++; + } + } + free(info); + return logicalProcessorCount; +} + +#endif + +} // namespace + static int GetNumCPUs() { #if defined(__myriad2__) return 1; +#elif defined(_WIN32) + const unsigned hardware_concurrency = Win32NumCPUs(); + return hardware_concurrency ? hardware_concurrency : 1; +#elif defined(_AIX) + return sysconf(_SC_NPROCESSORS_ONLN); #else // Other possibilities: // - Read /sys/devices/system/cpu/online and use cpumask_parse() @@ -420,6 +490,18 @@ pid_t GetTID() { #endif +// GetCachedTID() caches the thread ID in thread-local storage (which is a +// userspace construct) to avoid unnecessary system calls. Without this caching, +// it can take roughly 98ns, while it takes roughly 1ns with this caching. +pid_t GetCachedTID() { +#ifdef ABSL_HAVE_THREAD_LOCAL + static thread_local pid_t thread_id = GetTID(); + return thread_id; +#else + return GetTID(); +#endif // ABSL_HAVE_THREAD_LOCAL +} + } // namespace base_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/sysinfo.h b/absl/base/internal/sysinfo.h index 7246d5dd..119cf1f0 100644 --- a/absl/base/internal/sysinfo.h +++ b/absl/base/internal/sysinfo.h @@ -30,6 +30,7 @@ #include <cstdint> +#include "absl/base/config.h" #include "absl/base/port.h" namespace absl { @@ -59,6 +60,13 @@ using pid_t = uint32_t; #endif pid_t GetTID(); +// Like GetTID(), but caches the result in thread-local storage in order +// to avoid unnecessary system calls. Note that there are some cases where +// one must call through to GetTID directly, which is why this exists as a +// separate function. For example, GetCachedTID() is not safe to call in +// an asynchronous signal-handling context nor right after a call to fork(). +pid_t GetCachedTID(); + } // namespace base_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/sysinfo_test.cc b/absl/base/internal/sysinfo_test.cc index fa8b88b1..f305b6c5 100644 --- a/absl/base/internal/sysinfo_test.cc +++ b/absl/base/internal/sysinfo_test.cc @@ -37,18 +37,6 @@ TEST(SysinfoTest, NumCPUs) { << "NumCPUs() should not have the default value of 0"; } -TEST(SysinfoTest, NominalCPUFrequency) { -#if !(defined(__aarch64__) && defined(__linux__)) && !defined(__EMSCRIPTEN__) - EXPECT_GE(NominalCPUFrequency(), 1000.0) - << "NominalCPUFrequency() did not return a reasonable value"; -#else - // Aarch64 cannot read the CPU frequency from sysfs, so we get back 1.0. - // Emscripten does not have a sysfs to read from at all. - EXPECT_EQ(NominalCPUFrequency(), 1.0) - << "CPU frequency detection was fixed! Please update unittest."; -#endif -} - TEST(SysinfoTest, GetTID) { EXPECT_EQ(GetTID(), GetTID()); // Basic compile and equality test. #ifdef __native_client__ diff --git a/absl/base/internal/thread_identity.cc b/absl/base/internal/thread_identity.cc index d63a04ae..79853f09 100644 --- a/absl/base/internal/thread_identity.cc +++ b/absl/base/internal/thread_identity.cc @@ -14,7 +14,7 @@ #include "absl/base/internal/thread_identity.h" -#ifndef _WIN32 +#if !defined(_WIN32) || defined(__MINGW32__) #include <pthread.h> #include <signal.h> #endif @@ -23,6 +23,7 @@ #include <cassert> #include <memory> +#include "absl/base/attributes.h" #include "absl/base/call_once.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/internal/spinlock.h" @@ -53,9 +54,12 @@ void AllocateThreadIdentityKey(ThreadIdentityReclaimerFunction reclaimer) { // exist within a process (via dlopen() or similar), references to // thread_identity_ptr from each instance of the code will refer to // *different* instances of this ptr. -#ifdef __GNUC__ +// Apple platforms have the visibility attribute, but issue a compile warning +// that protected visibility is unsupported. +ABSL_CONST_INIT // Must come before __attribute__((visibility("protected"))) +#if ABSL_HAVE_ATTRIBUTE(visibility) && !defined(__APPLE__) __attribute__((visibility("protected"))) -#endif // __GNUC__ +#endif // ABSL_HAVE_ATTRIBUTE(visibility) && !defined(__APPLE__) #if ABSL_PER_THREAD_TLS // Prefer __thread to thread_local as benchmarks indicate it is a bit faster. ABSL_PER_THREAD_TLS_KEYWORD ThreadIdentity* thread_identity_ptr = nullptr; @@ -117,10 +121,10 @@ void SetCurrentThreadIdentity( ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_CPP11 // Please see the comment on `CurrentThreadIdentityIfPresent` in -// thread_identity.h. Because DLLs cannot expose thread_local variables in -// headers, we opt for the correct-but-slower option of placing the definition -// of this function only in a translation unit inside DLL. -#if defined(ABSL_BUILD_DLL) || defined(ABSL_CONSUME_DLL) +// thread_identity.h. When we cannot expose thread_local variables in +// headers, we opt for the correct-but-slower option of not inlining this +// function. +#ifndef ABSL_INTERNAL_INLINE_CURRENT_THREAD_IDENTITY_IF_PRESENT ThreadIdentity* CurrentThreadIdentityIfPresent() { return thread_identity_ptr; } #endif #endif diff --git a/absl/base/internal/thread_identity.h b/absl/base/internal/thread_identity.h index ceb109b4..659694b3 100644 --- a/absl/base/internal/thread_identity.h +++ b/absl/base/internal/thread_identity.h @@ -32,6 +32,7 @@ #include "absl/base/config.h" #include "absl/base/internal/per_thread_tls.h" +#include "absl/base/optimization.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -69,30 +70,28 @@ struct PerThreadSynch { // is using this PerThreadSynch as a terminator. Its // skip field must not be filled in because the loop // might then skip over the terminator. - - // The wait parameters of the current wait. waitp is null if the - // thread is not waiting. Transitions from null to non-null must - // occur before the enqueue commit point (state = kQueued in - // Enqueue() and CondVarEnqueue()). Transitions from non-null to - // null must occur after the wait is finished (state = kAvailable in - // Mutex::Block() and CondVar::WaitCommon()). This field may be - // changed only by the thread that describes this PerThreadSynch. A - // special case is Fer(), which calls Enqueue() on another thread, - // but with an identical SynchWaitParams pointer, thus leaving the - // pointer unchanged. - SynchWaitParams *waitp; - - bool suppress_fatal_errors; // If true, try to proceed even in the face of - // broken invariants. This is used within fatal - // signal handlers to improve the chances of - // debug logging information being output - // successfully. - - intptr_t readers; // Number of readers in mutex. - int priority; // Priority of thread (updated every so often). - - // When priority will next be read (cycles). - int64_t next_priority_read_cycles; + bool wake; // This thread is to be woken from a Mutex. + // If "x" is on a waiter list for a mutex, "x->cond_waiter" is true iff the + // waiter is waiting on the mutex as part of a CV Wait or Mutex Await. + // + // The value of "x->cond_waiter" is meaningless if "x" is not on a + // Mutex waiter list. + bool cond_waiter; + bool maybe_unlocking; // Valid at head of Mutex waiter queue; + // true if UnlockSlow could be searching + // for a waiter to wake. Used for an optimization + // in Enqueue(). true is always a valid value. + // Can be reset to false when the unlocker or any + // writer releases the lock, or a reader fully + // releases the lock. It may not be set to false + // by a reader that decrements the count to + // non-zero. protected by mutex spinlock + bool suppress_fatal_errors; // If true, try to proceed even in the face + // of broken invariants. This is used within + // fatal signal handlers to improve the + // chances of debug logging information being + // output successfully. + int priority; // Priority of thread (updated every so often). // State values: // kAvailable: This PerThreadSynch is available. @@ -111,30 +110,30 @@ struct PerThreadSynch { }; std::atomic<State> state; - bool maybe_unlocking; // Valid at head of Mutex waiter queue; - // true if UnlockSlow could be searching - // for a waiter to wake. Used for an optimization - // in Enqueue(). true is always a valid value. - // Can be reset to false when the unlocker or any - // writer releases the lock, or a reader fully releases - // the lock. It may not be set to false by a reader - // that decrements the count to non-zero. - // protected by mutex spinlock + // The wait parameters of the current wait. waitp is null if the + // thread is not waiting. Transitions from null to non-null must + // occur before the enqueue commit point (state = kQueued in + // Enqueue() and CondVarEnqueue()). Transitions from non-null to + // null must occur after the wait is finished (state = kAvailable in + // Mutex::Block() and CondVar::WaitCommon()). This field may be + // changed only by the thread that describes this PerThreadSynch. A + // special case is Fer(), which calls Enqueue() on another thread, + // but with an identical SynchWaitParams pointer, thus leaving the + // pointer unchanged. + SynchWaitParams* waitp; - bool wake; // This thread is to be woken from a Mutex. + intptr_t readers; // Number of readers in mutex. - // If "x" is on a waiter list for a mutex, "x->cond_waiter" is true iff the - // waiter is waiting on the mutex as part of a CV Wait or Mutex Await. - // - // The value of "x->cond_waiter" is meaningless if "x" is not on a - // Mutex waiter list. - bool cond_waiter; + // When priority will next be read (cycles). + int64_t next_priority_read_cycles; // Locks held; used during deadlock detection. // Allocated in Synch_GetAllLocks() and freed in ReclaimThreadIdentity(). SynchLocksHeld *all_locks; }; +// The instances of this class are allocated in NewThreadIdentity() with an +// alignment of PerThreadSynch::kAlignment. struct ThreadIdentity { // Must be the first member. The Mutex implementation requires that // the PerThreadSynch object associated with each thread is @@ -144,7 +143,7 @@ struct ThreadIdentity { // Private: Reserved for absl::synchronization_internal::Waiter. struct WaiterState { - char data[128]; + alignas(void*) char data[128]; } waiter_state; // Used by PerThreadSem::{Get,Set}ThreadBlockedCounter(). @@ -189,30 +188,32 @@ void ClearCurrentThreadIdentity(); // May be chosen at compile time via: -DABSL_FORCE_THREAD_IDENTITY_MODE=<mode // index> #ifdef ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC -#error ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC cannot be direcly set +#error ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC cannot be directly set #else #define ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC 0 #endif #ifdef ABSL_THREAD_IDENTITY_MODE_USE_TLS -#error ABSL_THREAD_IDENTITY_MODE_USE_TLS cannot be direcly set +#error ABSL_THREAD_IDENTITY_MODE_USE_TLS cannot be directly set #else #define ABSL_THREAD_IDENTITY_MODE_USE_TLS 1 #endif #ifdef ABSL_THREAD_IDENTITY_MODE_USE_CPP11 -#error ABSL_THREAD_IDENTITY_MODE_USE_CPP11 cannot be direcly set +#error ABSL_THREAD_IDENTITY_MODE_USE_CPP11 cannot be directly set #else #define ABSL_THREAD_IDENTITY_MODE_USE_CPP11 2 #endif #ifdef ABSL_THREAD_IDENTITY_MODE -#error ABSL_THREAD_IDENTITY_MODE cannot be direcly set +#error ABSL_THREAD_IDENTITY_MODE cannot be directly set #elif defined(ABSL_FORCE_THREAD_IDENTITY_MODE) #define ABSL_THREAD_IDENTITY_MODE ABSL_FORCE_THREAD_IDENTITY_MODE #elif defined(_WIN32) && !defined(__MINGW32__) #define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_CPP11 -#elif ABSL_PER_THREAD_TLS && defined(__GOOGLE_GRTE_VERSION__) && \ +#elif defined(__APPLE__) && defined(ABSL_HAVE_THREAD_LOCAL) +#define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_CPP11 +#elif ABSL_PER_THREAD_TLS && defined(__GOOGLE_GRTE_VERSION__) && \ (__GOOGLE_GRTE_VERSION__ >= 20140228L) // Support for async-safe TLS was specifically added in GRTEv4. It's not // present in the upstream eglibc. @@ -235,13 +236,18 @@ ABSL_CONST_INIT extern thread_local ThreadIdentity* thread_identity_ptr; #error Thread-local storage not detected on this platform #endif -// thread_local variables cannot be in headers exposed by DLLs. However, it is -// important for performance reasons in general that -// `CurrentThreadIdentityIfPresent` be inlined. This is not possible across a -// DLL boundary so, with DLLs, we opt to have the function not be inlined. Note +// thread_local variables cannot be in headers exposed by DLLs or in certain +// build configurations on Apple platforms. However, it is important for +// performance reasons in general that `CurrentThreadIdentityIfPresent` be +// inlined. In the other cases we opt to have the function not be inlined. Note // that `CurrentThreadIdentityIfPresent` is declared above so we can exclude -// this entire inline definition when compiling as a DLL. -#if !defined(ABSL_BUILD_DLL) && !defined(ABSL_CONSUME_DLL) +// this entire inline definition. +#if !defined(__APPLE__) && !defined(ABSL_BUILD_DLL) && \ + !defined(ABSL_CONSUME_DLL) +#define ABSL_INTERNAL_INLINE_CURRENT_THREAD_IDENTITY_IF_PRESENT 1 +#endif + +#ifdef ABSL_INTERNAL_INLINE_CURRENT_THREAD_IDENTITY_IF_PRESENT inline ThreadIdentity* CurrentThreadIdentityIfPresent() { return thread_identity_ptr; } diff --git a/absl/base/internal/thread_identity_test.cc b/absl/base/internal/thread_identity_test.cc index 624d5b96..46a6f743 100644 --- a/absl/base/internal/thread_identity_test.cc +++ b/absl/base/internal/thread_identity_test.cc @@ -75,7 +75,7 @@ TEST(ThreadIdentityTest, BasicIdentityWorksThreaded) { // - If a thread implementation chooses to recycle threads, that // correct re-initialization occurs. static const int kNumLoops = 3; - static const int kNumThreads = 400; + static const int kNumThreads = 32; for (int iter = 0; iter < kNumLoops; iter++) { std::vector<std::thread> threads; for (int i = 0; i < kNumThreads; ++i) { diff --git a/absl/base/internal/throw_delegate.cc b/absl/base/internal/throw_delegate.cc index c055f75d..c260ff1e 100644 --- a/absl/base/internal/throw_delegate.cc +++ b/absl/base/internal/throw_delegate.cc @@ -18,6 +18,7 @@ #include <functional> #include <new> #include <stdexcept> + #include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" @@ -25,83 +26,186 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace base_internal { +// NOTE: The various STL exception throwing functions are placed within the +// #ifdef blocks so the symbols aren't exposed on platforms that don't support +// them, such as the Android NDK. For example, ANGLE fails to link when building +// within AOSP without them, since the STL functions don't exist. namespace { +#ifdef ABSL_HAVE_EXCEPTIONS template <typename T> [[noreturn]] void Throw(const T& error) { -#ifdef ABSL_HAVE_EXCEPTIONS throw error; -#else - ABSL_RAW_LOG(FATAL, "%s", error.what()); - std::abort(); -#endif } +#endif } // namespace void ThrowStdLogicError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::logic_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif } void ThrowStdLogicError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::logic_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif } void ThrowStdInvalidArgument(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::invalid_argument(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif } void ThrowStdInvalidArgument(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::invalid_argument(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif } void ThrowStdDomainError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::domain_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif } void ThrowStdDomainError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::domain_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif } void ThrowStdLengthError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::length_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif } void ThrowStdLengthError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::length_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif } void ThrowStdOutOfRange(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::out_of_range(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif } void ThrowStdOutOfRange(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::out_of_range(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif } void ThrowStdRuntimeError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::runtime_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif } void ThrowStdRuntimeError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::runtime_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif } void ThrowStdRangeError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::range_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif } void ThrowStdRangeError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::range_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif } void ThrowStdOverflowError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::overflow_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif } void ThrowStdOverflowError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::overflow_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif } void ThrowStdUnderflowError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::underflow_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif } void ThrowStdUnderflowError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS Throw(std::underflow_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif } -void ThrowStdBadFunctionCall() { Throw(std::bad_function_call()); } +void ThrowStdBadFunctionCall() { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::bad_function_call()); +#else + std::abort(); +#endif +} -void ThrowStdBadAlloc() { Throw(std::bad_alloc()); } +void ThrowStdBadAlloc() { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::bad_alloc()); +#else + std::abort(); +#endif +} } // namespace base_internal ABSL_NAMESPACE_END diff --git a/absl/base/internal/tsan_mutex_interface.h b/absl/base/internal/tsan_mutex_interface.h index 2a510603..39207d8a 100644 --- a/absl/base/internal/tsan_mutex_interface.h +++ b/absl/base/internal/tsan_mutex_interface.h @@ -19,6 +19,8 @@ #ifndef ABSL_BASE_INTERNAL_TSAN_MUTEX_INTERFACE_H_ #define ABSL_BASE_INTERNAL_TSAN_MUTEX_INTERFACE_H_ +#include "absl/base/config.h" + // ABSL_INTERNAL_HAVE_TSAN_INTERFACE // Macro intended only for internal use. // @@ -28,7 +30,7 @@ #error "ABSL_INTERNAL_HAVE_TSAN_INTERFACE cannot be directly set." #endif -#if defined(THREAD_SANITIZER) && defined(__has_include) +#if defined(ABSL_HAVE_THREAD_SANITIZER) && defined(__has_include) #if __has_include(<sanitizer/tsan_interface.h>) #define ABSL_INTERNAL_HAVE_TSAN_INTERFACE 1 #endif diff --git a/absl/base/internal/unaligned_access.h b/absl/base/internal/unaligned_access.h index 6be56c86..093dd9b4 100644 --- a/absl/base/internal/unaligned_access.h +++ b/absl/base/internal/unaligned_access.h @@ -31,80 +31,6 @@ // The unaligned API is C++ only. The declarations use C++ features // (namespaces, inline) which are absent or incompatible in C. #if defined(__cplusplus) - -#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) ||\ - defined(MEMORY_SANITIZER) -// Consider we have an unaligned load/store of 4 bytes from address 0x...05. -// AddressSanitizer will treat it as a 3-byte access to the range 05:07 and -// will miss a bug if 08 is the first unaddressable byte. -// ThreadSanitizer will also treat this as a 3-byte access to 05:07 and will -// miss a race between this access and some other accesses to 08. -// MemorySanitizer will correctly propagate the shadow on unaligned stores -// and correctly report bugs on unaligned loads, but it may not properly -// update and report the origin of the uninitialized memory. -// For all three tools, replacing an unaligned access with a tool-specific -// callback solves the problem. - -// Make sure uint16_t/uint32_t/uint64_t are defined. -#include <stdint.h> - -extern "C" { -uint16_t __sanitizer_unaligned_load16(const void *p); -uint32_t __sanitizer_unaligned_load32(const void *p); -uint64_t __sanitizer_unaligned_load64(const void *p); -void __sanitizer_unaligned_store16(void *p, uint16_t v); -void __sanitizer_unaligned_store32(void *p, uint32_t v); -void __sanitizer_unaligned_store64(void *p, uint64_t v); -} // extern "C" - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace base_internal { - -inline uint16_t UnalignedLoad16(const void *p) { - return __sanitizer_unaligned_load16(p); -} - -inline uint32_t UnalignedLoad32(const void *p) { - return __sanitizer_unaligned_load32(p); -} - -inline uint64_t UnalignedLoad64(const void *p) { - return __sanitizer_unaligned_load64(p); -} - -inline void UnalignedStore16(void *p, uint16_t v) { - __sanitizer_unaligned_store16(p, v); -} - -inline void UnalignedStore32(void *p, uint32_t v) { - __sanitizer_unaligned_store32(p, v); -} - -inline void UnalignedStore64(void *p, uint64_t v) { - __sanitizer_unaligned_store64(p, v); -} - -} // namespace base_internal -ABSL_NAMESPACE_END -} // namespace absl - -#define ABSL_INTERNAL_UNALIGNED_LOAD16(_p) \ - (absl::base_internal::UnalignedLoad16(_p)) -#define ABSL_INTERNAL_UNALIGNED_LOAD32(_p) \ - (absl::base_internal::UnalignedLoad32(_p)) -#define ABSL_INTERNAL_UNALIGNED_LOAD64(_p) \ - (absl::base_internal::UnalignedLoad64(_p)) - -#define ABSL_INTERNAL_UNALIGNED_STORE16(_p, _val) \ - (absl::base_internal::UnalignedStore16(_p, _val)) -#define ABSL_INTERNAL_UNALIGNED_STORE32(_p, _val) \ - (absl::base_internal::UnalignedStore32(_p, _val)) -#define ABSL_INTERNAL_UNALIGNED_STORE64(_p, _val) \ - (absl::base_internal::UnalignedStore64(_p, _val)) - -#else - namespace absl { ABSL_NAMESPACE_BEGIN namespace base_internal { @@ -151,8 +77,6 @@ ABSL_NAMESPACE_END #define ABSL_INTERNAL_UNALIGNED_STORE64(_p, _val) \ (absl::base_internal::UnalignedStore64(_p, _val)) -#endif - #endif // defined(__cplusplus), end of unaligned API #endif // ABSL_BASE_INTERNAL_UNALIGNED_ACCESS_H_ diff --git a/absl/base/internal/unscaledcycleclock.cc b/absl/base/internal/unscaledcycleclock.cc index 83e2134e..b1c396c6 100644 --- a/absl/base/internal/unscaledcycleclock.cc +++ b/absl/base/internal/unscaledcycleclock.cc @@ -24,10 +24,13 @@ #ifdef __GLIBC__ #include <sys/platform/ppc.h> #elif defined(__FreeBSD__) -#include "absl/base/call_once.h" +// clang-format off +// This order does actually matter =(. #include <sys/types.h> #include <sys/sysctl.h> -#include <threads.h> +// clang-format on + +#include "absl/base/call_once.h" #endif #endif @@ -51,12 +54,6 @@ double UnscaledCycleClock::Frequency() { #elif defined(__x86_64__) -int64_t UnscaledCycleClock::Now() { - uint64_t low, high; - __asm__ volatile("rdtsc" : "=a"(low), "=d"(high)); - return (high << 32) | low; -} - double UnscaledCycleClock::Frequency() { return base_internal::NominalCPUFrequency(); } @@ -89,6 +86,10 @@ int64_t UnscaledCycleClock::Now() { double UnscaledCycleClock::Frequency() { #ifdef __GLIBC__ return __ppc_get_timebase_freq(); +#elif defined(_AIX) + // This is the same constant value as returned by + // __ppc_get_timebase_freq(). + return static_cast<double>(512000000); #elif defined(__FreeBSD__) static once_flag init_timebase_frequency_once; static double timebase_frequency = 0.0; @@ -121,13 +122,23 @@ double UnscaledCycleClock::Frequency() { return aarch64_timer_frequency; } +#elif defined(__riscv) + +int64_t UnscaledCycleClock::Now() { + int64_t virtual_timer_value; + asm volatile("rdcycle %0" : "=r"(virtual_timer_value)); + return virtual_timer_value; +} + +double UnscaledCycleClock::Frequency() { + return base_internal::NominalCPUFrequency(); +} + #elif defined(_M_IX86) || defined(_M_X64) #pragma intrinsic(__rdtsc) -int64_t UnscaledCycleClock::Now() { - return __rdtsc(); -} +int64_t UnscaledCycleClock::Now() { return __rdtsc(); } double UnscaledCycleClock::Frequency() { return base_internal::NominalCPUFrequency(); diff --git a/absl/base/internal/unscaledcycleclock.h b/absl/base/internal/unscaledcycleclock.h index cdce9bf8..2cbeae31 100644 --- a/absl/base/internal/unscaledcycleclock.h +++ b/absl/base/internal/unscaledcycleclock.h @@ -15,8 +15,8 @@ // UnscaledCycleClock // An UnscaledCycleClock yields the value and frequency of a cycle counter // that increments at a rate that is approximately constant. -// This class is for internal / whitelisted use only, you should consider -// using CycleClock instead. +// This class is for internal use only, you should consider using CycleClock +// instead. // // Notes: // The cycle counter frequency is not necessarily the core clock frequency. @@ -46,8 +46,8 @@ // The following platforms have an implementation of a hardware counter. #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) || \ - defined(__powerpc__) || defined(__ppc__) || \ - defined(_M_IX86) || defined(_M_X64) + defined(__powerpc__) || defined(__ppc__) || defined(__riscv) || \ + defined(_M_IX86) || (defined(_M_X64) && !defined(_M_ARM64EC)) #define ABSL_HAVE_UNSCALED_CYCLECLOCK_IMPLEMENTATION 1 #else #define ABSL_HAVE_UNSCALED_CYCLECLOCK_IMPLEMENTATION 0 @@ -59,8 +59,7 @@ // CycleClock that runs at atleast 1 MHz. We've found some Android // ARM64 devices where this is not the case, so we disable it by // default on Android ARM64. -#if defined(__native_client__) || \ - (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) || \ +#if defined(__native_client__) || (defined(__APPLE__)) || \ (defined(__ANDROID__) && defined(__aarch64__)) #define ABSL_USE_UNSCALED_CYCLECLOCK_DEFAULT 0 #else @@ -80,8 +79,8 @@ // This macro can be used to test if UnscaledCycleClock::Frequency() // is NominalCPUFrequency() on a particular platform. -#if (defined(__i386__) || defined(__x86_64__) || \ - defined(_M_IX86) || defined(_M_X64)) +#if (defined(__i386__) || defined(__x86_64__) || defined(__riscv) || \ + defined(_M_IX86) || defined(_M_X64)) #define ABSL_INTERNAL_UNSCALED_CYCLECLOCK_FREQUENCY_IS_CPU_FREQUENCY #endif @@ -109,12 +108,22 @@ class UnscaledCycleClock { // value. static double Frequency(); - // Whitelisted friends. + // Allowed users friend class base_internal::CycleClock; friend class time_internal::UnscaledCycleClockWrapperForGetCurrentTime; friend class base_internal::UnscaledCycleClockWrapperForInitializeFrequency; }; +#if defined(__x86_64__) + +inline int64_t UnscaledCycleClock::Now() { + uint64_t low, high; + __asm__ volatile("rdtsc" : "=a"(low), "=d"(high)); + return (high << 32) | low; +} + +#endif + } // namespace base_internal ABSL_NAMESPACE_END } // namespace absl |