From b3aaac8a37c467a1125c794196caa90d0957bdc3 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Thu, 9 Jan 2020 09:58:48 -0800 Subject: Export of internal Abseil changes -- 9beb68204986a015c9cb065b9fae4f9a8879a788 by Abseil Team : Move Base64EscapeInternal and CalculateBase64EscapedLenInternal to an internal header. PiperOrigin-RevId: 288917378 -- 90acfbe03b3f9f6de3ffa49c39343dfaa2c5d38c by Greg Falcon : Update macos CI script to support the ALTERNATE_OPTIONS environment variable. PiperOrigin-RevId: 288913564 -- f1572e870678cdcda6b48cb39780d1ad984e4c1b by Derek Mauro : Makes absl::NullSafeStringView constexpr Fixes https://github.com/abseil/abseil-cpp/issues/583 PiperOrigin-RevId: 288906940 -- d28a8471e32c10caa64bfffe6d6d4d0a8d144013 by Abseil Team : absl::GetFlag is lock free for small trivially copyable types. PiperOrigin-RevId: 288768172 -- 2643b8ed1a1dc836b38ab9e46538a1af129ffd67 by Gennadiy Rozental : Eliminate call to callback from flag initialization. We do not need to have this invocation inside FlagImpl::Init since SetCallback performs invocation anyways. Calling InitCallback from inside of Init complicates separation of value initialization from data guard initialization, which is about to happen. PiperOrigin-RevId: 288732526 -- 22caa880b7a4cb6da34e16a2e064a473c99e880b by Abseil Team : Fix the documentation on how to create a null string_view. PiperOrigin-RevId: 288727968 -- 10727c5cadc561837141176f4c9b9717cec9233a by Greg Falcon : Change CI scripts for gcc to use the ALTERNATE_OPTIONS file as well. PiperOrigin-RevId: 288718855 -- 5d1e2dd6c7fb12af8aa4337a0f61872f5f0c5992 by Greg Falcon : Add an option for using inline namespaces in Abseil. PiperOrigin-RevId: 288614491 GitOrigin-RevId: 9beb68204986a015c9cb065b9fae4f9a8879a788 Change-Id: If9acd46301e3df8cb231b4c16f7ed651bf4fb3c3 --- absl/flags/config.h | 11 ++++ absl/flags/flag.cc | 22 -------- absl/flags/flag.h | 45 ++++++++++++----- absl/flags/internal/commandlineflag.h | 32 ++++++------ absl/flags/internal/flag.cc | 15 ++++-- absl/flags/internal/flag.h | 95 ++++++++++++++++++++++++++++++++--- absl/flags/parse.cc | 4 +- 7 files changed, 160 insertions(+), 64 deletions(-) (limited to 'absl/flags') diff --git a/absl/flags/config.h b/absl/flags/config.h index a9fd97a..fbe3496 100644 --- a/absl/flags/config.h +++ b/absl/flags/config.h @@ -45,4 +45,15 @@ #define ABSL_FLAGS_STRIP_HELP ABSL_FLAGS_STRIP_NAMES #endif +// ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD macro is used for using atomics with +// double words, e.g. absl::Duration. +// For reasons in bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80878, modern +// versions of GCC do not support cmpxchg16b instruction in standard atomics. +#ifdef ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD +#error "ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD should not be defined." +#elif defined(__clang__) && defined(__x86_64__) && \ + defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) +#define ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD 1 +#endif + #endif // ABSL_FLAGS_CONFIG_H_ diff --git a/absl/flags/flag.cc b/absl/flags/flag.cc index 7faa7ad..491a66b 100644 --- a/absl/flags/flag.cc +++ b/absl/flags/flag.cc @@ -20,28 +20,6 @@ namespace absl { ABSL_NAMESPACE_BEGIN -// We want to validate the type mismatch between type definition and -// declaration. The lock-free implementation does not allow us to do it, -// so in debug builds we always use the slower implementation, which always -// validates the type. -#ifndef NDEBUG -#define ABSL_FLAGS_ATOMIC_GET(T) \ - T GetFlag(const absl::Flag& flag) { return flag.Get(); } -#else -#define ABSL_FLAGS_ATOMIC_GET(T) \ - T GetFlag(const absl::Flag& flag) { \ - T result; \ - if (flag.AtomicGet(&result)) { \ - return result; \ - } \ - return flag.Get(); \ - } -#endif - -ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(ABSL_FLAGS_ATOMIC_GET) - -#undef ABSL_FLAGS_ATOMIC_GET - // This global nutex protects on-demand construction of flag objects in MSVC // builds. #if defined(_MSC_VER) && !defined(__clang__) diff --git a/absl/flags/flag.h b/absl/flags/flag.h index 326fb8e..62e73f8 100644 --- a/absl/flags/flag.h +++ b/absl/flags/flag.h @@ -29,6 +29,8 @@ #ifndef ABSL_FLAGS_FLAG_H_ #define ABSL_FLAGS_FLAG_H_ +#include + #include "absl/base/attributes.h" #include "absl/base/casts.h" #include "absl/flags/config.h" @@ -181,23 +183,42 @@ class Flag { // // // FLAGS_firstname is a Flag of type `std::string` // std::string first_name = absl::GetFlag(FLAGS_firstname); -template +template ::value, int>::type = 0> ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag& flag) { -#define ABSL_FLAGS_INTERNAL_LOCK_FREE_VALIDATE(BIT) \ - static_assert( \ - !std::is_same::value, \ - "Do not specify explicit template parameters to absl::GetFlag"); - ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(ABSL_FLAGS_INTERNAL_LOCK_FREE_VALIDATE) -#undef ABSL_FLAGS_INTERNAL_LOCK_FREE_VALIDATE - return flag.Get(); } +// We want to validate the type mismatch between type definition and +// declaration. The lock-free implementation does not allow us to do it, +// so in debug builds we always use the slower implementation, which always +// validates the type. +#ifndef NDEBUG +template ::value, int>::type = 0> +ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag& flag) { + return flag.Get(); +} +#else // Overload for `GetFlag()` for types that support lock-free reads. -#define ABSL_FLAGS_INTERNAL_LOCK_FREE_EXPORT(T) \ - ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag& flag); -ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(ABSL_FLAGS_INTERNAL_LOCK_FREE_EXPORT) -#undef ABSL_FLAGS_INTERNAL_LOCK_FREE_EXPORT +template ::value, int>::type = 0> +ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag& flag) { + // T might not be default constructible. + union U { + T value; + U() {} + }; + U result; + if (flag.AtomicGet(&result.value)) { + return result.value; + } + return flag.Get(); +} +#endif // SetFlag() // diff --git a/absl/flags/internal/commandlineflag.h b/absl/flags/internal/commandlineflag.h index a0c18e8..1862306 100644 --- a/absl/flags/internal/commandlineflag.h +++ b/absl/flags/internal/commandlineflag.h @@ -259,22 +259,22 @@ class CommandLineFlag { virtual void Read(void* dst) const = 0; }; -// This macro is the "source of truth" for the list of supported flag types we -// expect to perform lock free operations on. Specifically it generates code, -// a one argument macro operating on a type, supplied as a macro argument, for -// each type in the list. -#define ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(A) \ - A(bool) \ - A(short) \ - A(unsigned short) \ - A(int) \ - A(unsigned int) \ - A(long) \ - A(unsigned long) \ - A(long long) \ - A(unsigned long long) \ - A(double) \ - A(float) +// This macro is the "source of truth" for the list of supported flag built-in +// types. +#define ABSL_FLAGS_INTERNAL_BUILTIN_TYPES(A) \ + A(bool) \ + A(short) \ + A(unsigned short) \ + A(int) \ + A(unsigned int) \ + A(long) \ + A(unsigned long) \ + A(long long) \ + A(unsigned long long) \ + A(double) \ + A(float) \ + A(std::string) \ + A(std::vector) } // namespace flags_internal ABSL_NAMESPACE_END diff --git a/absl/flags/internal/flag.cc b/absl/flags/internal/flag.cc index bb9a98f..6979dc4 100644 --- a/absl/flags/internal/flag.cc +++ b/absl/flags/internal/flag.cc @@ -16,6 +16,7 @@ #include "absl/flags/internal/flag.h" #include "absl/base/optimization.h" +#include "absl/flags/config.h" #include "absl/flags/usage_config.h" #include "absl/synchronization/mutex.h" @@ -35,9 +36,7 @@ namespace { bool ShouldValidateFlagValue(FlagOpFn flag_type_id) { #define DONT_VALIDATE(T) \ if (flag_type_id == &flags_internal::FlagOps) return false; - ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(DONT_VALIDATE) - DONT_VALIDATE(std::string) - DONT_VALIDATE(std::vector) + ABSL_FLAGS_INTERNAL_BUILTIN_TYPES(DONT_VALIDATE) #undef DONT_VALIDATE return true; @@ -85,7 +84,6 @@ void FlagImpl::Init() { cur_ = MakeInitValue().release(); StoreAtomic(); inited_.store(true, std::memory_order_release); - InvokeCallback(); } } @@ -264,8 +262,15 @@ void FlagImpl::StoreAtomic() { if (data_size <= sizeof(int64_t)) { int64_t t = 0; std::memcpy(&t, cur_, data_size); - atomic_.store(t, std::memory_order_release); + atomics_.small_atomic.store(t, std::memory_order_release); } +#if defined(ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD) + else if (data_size <= sizeof(FlagsInternalTwoWordsType)) { + FlagsInternalTwoWordsType t{0, 0}; + std::memcpy(&t, cur_, data_size); + atomics_.big_atomic.store(t, std::memory_order_release); + } +#endif } void FlagImpl::Write(const void* src, const flags_internal::FlagOpFn src_op) { diff --git a/absl/flags/internal/flag.h b/absl/flags/internal/flag.h index 7d5271c..a5edfd1 100644 --- a/absl/flags/internal/flag.h +++ b/absl/flags/internal/flag.h @@ -20,6 +20,7 @@ #include #include "absl/base/thread_annotations.h" +#include "absl/flags/config.h" #include "absl/flags/internal/commandlineflag.h" #include "absl/flags/internal/registry.h" #include "absl/memory/memory.h" @@ -30,7 +31,61 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace flags_internal { -constexpr int64_t AtomicInit() { return 0xababababababababll; } +// The minimum atomic size we believe to generate lock free code, i.e. all +// trivially copyable types not bigger this size generate lock free code. +static constexpr int kMinLockFreeAtomicSize = 8; + +// The same as kMinLockFreeAtomicSize but maximum atomic size. As double words +// might use two registers, we want to dispatch the logic for them. +#if defined(ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD) +static constexpr int kMaxLockFreeAtomicSize = 16; +#else +static constexpr int kMaxLockFreeAtomicSize = 8; +#endif + +// We can use atomic in cases when it fits in the register, trivially copyable +// in order to make memcpy operations. +template +struct IsAtomicFlagTypeTrait { + static constexpr bool value = + (sizeof(T) <= kMaxLockFreeAtomicSize && + type_traits_internal::is_trivially_copyable::value); +}; + +// Clang does not always produce cmpxchg16b instruction when alignment of a 16 +// bytes type is not 16. +struct alignas(16) FlagsInternalTwoWordsType { + int64_t first; + int64_t second; +}; + +constexpr bool operator==(const FlagsInternalTwoWordsType& that, + const FlagsInternalTwoWordsType& other) { + return that.first == other.first && that.second == other.second; +} +constexpr bool operator!=(const FlagsInternalTwoWordsType& that, + const FlagsInternalTwoWordsType& other) { + return !(that == other); +} + +constexpr int64_t SmallAtomicInit() { return 0xababababababababll; } + +template +struct BestAtomicType { + using type = int64_t; + static constexpr int64_t AtomicInit() { return SmallAtomicInit(); } +}; + +template +struct BestAtomicType< + T, typename std::enable_if<(kMinLockFreeAtomicSize < sizeof(T) && + sizeof(T) <= kMaxLockFreeAtomicSize), + void>::type> { + using type = FlagsInternalTwoWordsType; + static constexpr FlagsInternalTwoWordsType AtomicInit() { + return {SmallAtomicInit(), SmallAtomicInit()}; + } +}; template class Flag; @@ -182,14 +237,15 @@ class FlagImpl { // it replaces `dst` with the new value. bool TryParse(void** dst, absl::string_view value, std::string* err) const ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); + template bool AtomicGet(T* v) const { - const int64_t r = atomic_.load(std::memory_order_acquire); - if (r != flags_internal::AtomicInit()) { - std::memcpy(v, &r, sizeof(T)); + using U = flags_internal::BestAtomicType; + const typename U::type r = atomics_.template load(); + if (r != U::AtomicInit()) { + std::memcpy(static_cast(v), &r, sizeof(T)); return true; } - return false; } @@ -271,7 +327,34 @@ class FlagImpl { int64_t counter_ ABSL_GUARDED_BY(*DataGuard()) = 0; // For some types, a copy of the current value is kept in an atomically // accessible field. - std::atomic atomic_{flags_internal::AtomicInit()}; + union Atomics { + // Using small atomic for small types. + std::atomic small_atomic; + template ::type> + int64_t load() const { + return small_atomic.load(std::memory_order_acquire); + } + +#if defined(ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD) + // Using big atomics for big types. + std::atomic big_atomic; + template ::type> + FlagsInternalTwoWordsType load() const { + return big_atomic.load(std::memory_order_acquire); + } + constexpr Atomics() + : big_atomic{FlagsInternalTwoWordsType{SmallAtomicInit(), + SmallAtomicInit()}} {} +#else + constexpr Atomics() : small_atomic{SmallAtomicInit()} {} +#endif + }; + Atomics atomics_{}; struct CallbackData { FlagCallback func; diff --git a/absl/flags/parse.cc b/absl/flags/parse.cc index 2f07725..a288ace 100644 --- a/absl/flags/parse.cc +++ b/absl/flags/parse.cc @@ -280,9 +280,7 @@ void CheckDefaultValuesParsingRoundtrip() { #define IGNORE_TYPE(T) \ if (flag->IsOfType()) return; - ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(IGNORE_TYPE) - IGNORE_TYPE(std::string) - IGNORE_TYPE(std::vector) + ABSL_FLAGS_INTERNAL_BUILTIN_TYPES(IGNORE_TYPE) #undef IGNORE_TYPE flag->CheckDefaultValueParsingRoundtrip(); -- cgit v1.2.3