From b3aaac8a37c467a1125c794196caa90d0957bdc3 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Thu, 9 Jan 2020 09:58:48 -0800 Subject: Export of internal Abseil changes -- 9beb68204986a015c9cb065b9fae4f9a8879a788 by Abseil Team : Move Base64EscapeInternal and CalculateBase64EscapedLenInternal to an internal header. PiperOrigin-RevId: 288917378 -- 90acfbe03b3f9f6de3ffa49c39343dfaa2c5d38c by Greg Falcon : Update macos CI script to support the ALTERNATE_OPTIONS environment variable. PiperOrigin-RevId: 288913564 -- f1572e870678cdcda6b48cb39780d1ad984e4c1b by Derek Mauro : Makes absl::NullSafeStringView constexpr Fixes https://github.com/abseil/abseil-cpp/issues/583 PiperOrigin-RevId: 288906940 -- d28a8471e32c10caa64bfffe6d6d4d0a8d144013 by Abseil Team : absl::GetFlag is lock free for small trivially copyable types. PiperOrigin-RevId: 288768172 -- 2643b8ed1a1dc836b38ab9e46538a1af129ffd67 by Gennadiy Rozental : Eliminate call to callback from flag initialization. We do not need to have this invocation inside FlagImpl::Init since SetCallback performs invocation anyways. Calling InitCallback from inside of Init complicates separation of value initialization from data guard initialization, which is about to happen. PiperOrigin-RevId: 288732526 -- 22caa880b7a4cb6da34e16a2e064a473c99e880b by Abseil Team : Fix the documentation on how to create a null string_view. PiperOrigin-RevId: 288727968 -- 10727c5cadc561837141176f4c9b9717cec9233a by Greg Falcon : Change CI scripts for gcc to use the ALTERNATE_OPTIONS file as well. PiperOrigin-RevId: 288718855 -- 5d1e2dd6c7fb12af8aa4337a0f61872f5f0c5992 by Greg Falcon : Add an option for using inline namespaces in Abseil. PiperOrigin-RevId: 288614491 GitOrigin-RevId: 9beb68204986a015c9cb065b9fae4f9a8879a788 Change-Id: If9acd46301e3df8cb231b4c16f7ed651bf4fb3c3 --- absl/flags/internal/commandlineflag.h | 32 ++++++------ absl/flags/internal/flag.cc | 15 ++++-- absl/flags/internal/flag.h | 95 ++++++++++++++++++++++++++++++++--- 3 files changed, 115 insertions(+), 27 deletions(-) (limited to 'absl/flags/internal') diff --git a/absl/flags/internal/commandlineflag.h b/absl/flags/internal/commandlineflag.h index a0c18e80..1862306d 100644 --- a/absl/flags/internal/commandlineflag.h +++ b/absl/flags/internal/commandlineflag.h @@ -259,22 +259,22 @@ class CommandLineFlag { virtual void Read(void* dst) const = 0; }; -// This macro is the "source of truth" for the list of supported flag types we -// expect to perform lock free operations on. Specifically it generates code, -// a one argument macro operating on a type, supplied as a macro argument, for -// each type in the list. -#define ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(A) \ - A(bool) \ - A(short) \ - A(unsigned short) \ - A(int) \ - A(unsigned int) \ - A(long) \ - A(unsigned long) \ - A(long long) \ - A(unsigned long long) \ - A(double) \ - A(float) +// This macro is the "source of truth" for the list of supported flag built-in +// types. +#define ABSL_FLAGS_INTERNAL_BUILTIN_TYPES(A) \ + A(bool) \ + A(short) \ + A(unsigned short) \ + A(int) \ + A(unsigned int) \ + A(long) \ + A(unsigned long) \ + A(long long) \ + A(unsigned long long) \ + A(double) \ + A(float) \ + A(std::string) \ + A(std::vector) } // namespace flags_internal ABSL_NAMESPACE_END diff --git a/absl/flags/internal/flag.cc b/absl/flags/internal/flag.cc index bb9a98f3..6979dc46 100644 --- a/absl/flags/internal/flag.cc +++ b/absl/flags/internal/flag.cc @@ -16,6 +16,7 @@ #include "absl/flags/internal/flag.h" #include "absl/base/optimization.h" +#include "absl/flags/config.h" #include "absl/flags/usage_config.h" #include "absl/synchronization/mutex.h" @@ -35,9 +36,7 @@ namespace { bool ShouldValidateFlagValue(FlagOpFn flag_type_id) { #define DONT_VALIDATE(T) \ if (flag_type_id == &flags_internal::FlagOps) return false; - ABSL_FLAGS_INTERNAL_FOR_EACH_LOCK_FREE(DONT_VALIDATE) - DONT_VALIDATE(std::string) - DONT_VALIDATE(std::vector) + ABSL_FLAGS_INTERNAL_BUILTIN_TYPES(DONT_VALIDATE) #undef DONT_VALIDATE return true; @@ -85,7 +84,6 @@ void FlagImpl::Init() { cur_ = MakeInitValue().release(); StoreAtomic(); inited_.store(true, std::memory_order_release); - InvokeCallback(); } } @@ -264,8 +262,15 @@ void FlagImpl::StoreAtomic() { if (data_size <= sizeof(int64_t)) { int64_t t = 0; std::memcpy(&t, cur_, data_size); - atomic_.store(t, std::memory_order_release); + atomics_.small_atomic.store(t, std::memory_order_release); } +#if defined(ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD) + else if (data_size <= sizeof(FlagsInternalTwoWordsType)) { + FlagsInternalTwoWordsType t{0, 0}; + std::memcpy(&t, cur_, data_size); + atomics_.big_atomic.store(t, std::memory_order_release); + } +#endif } void FlagImpl::Write(const void* src, const flags_internal::FlagOpFn src_op) { diff --git a/absl/flags/internal/flag.h b/absl/flags/internal/flag.h index 7d5271c4..a5edfd17 100644 --- a/absl/flags/internal/flag.h +++ b/absl/flags/internal/flag.h @@ -20,6 +20,7 @@ #include #include "absl/base/thread_annotations.h" +#include "absl/flags/config.h" #include "absl/flags/internal/commandlineflag.h" #include "absl/flags/internal/registry.h" #include "absl/memory/memory.h" @@ -30,7 +31,61 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace flags_internal { -constexpr int64_t AtomicInit() { return 0xababababababababll; } +// The minimum atomic size we believe to generate lock free code, i.e. all +// trivially copyable types not bigger this size generate lock free code. +static constexpr int kMinLockFreeAtomicSize = 8; + +// The same as kMinLockFreeAtomicSize but maximum atomic size. As double words +// might use two registers, we want to dispatch the logic for them. +#if defined(ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD) +static constexpr int kMaxLockFreeAtomicSize = 16; +#else +static constexpr int kMaxLockFreeAtomicSize = 8; +#endif + +// We can use atomic in cases when it fits in the register, trivially copyable +// in order to make memcpy operations. +template +struct IsAtomicFlagTypeTrait { + static constexpr bool value = + (sizeof(T) <= kMaxLockFreeAtomicSize && + type_traits_internal::is_trivially_copyable::value); +}; + +// Clang does not always produce cmpxchg16b instruction when alignment of a 16 +// bytes type is not 16. +struct alignas(16) FlagsInternalTwoWordsType { + int64_t first; + int64_t second; +}; + +constexpr bool operator==(const FlagsInternalTwoWordsType& that, + const FlagsInternalTwoWordsType& other) { + return that.first == other.first && that.second == other.second; +} +constexpr bool operator!=(const FlagsInternalTwoWordsType& that, + const FlagsInternalTwoWordsType& other) { + return !(that == other); +} + +constexpr int64_t SmallAtomicInit() { return 0xababababababababll; } + +template +struct BestAtomicType { + using type = int64_t; + static constexpr int64_t AtomicInit() { return SmallAtomicInit(); } +}; + +template +struct BestAtomicType< + T, typename std::enable_if<(kMinLockFreeAtomicSize < sizeof(T) && + sizeof(T) <= kMaxLockFreeAtomicSize), + void>::type> { + using type = FlagsInternalTwoWordsType; + static constexpr FlagsInternalTwoWordsType AtomicInit() { + return {SmallAtomicInit(), SmallAtomicInit()}; + } +}; template class Flag; @@ -182,14 +237,15 @@ class FlagImpl { // it replaces `dst` with the new value. bool TryParse(void** dst, absl::string_view value, std::string* err) const ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); + template bool AtomicGet(T* v) const { - const int64_t r = atomic_.load(std::memory_order_acquire); - if (r != flags_internal::AtomicInit()) { - std::memcpy(v, &r, sizeof(T)); + using U = flags_internal::BestAtomicType; + const typename U::type r = atomics_.template load(); + if (r != U::AtomicInit()) { + std::memcpy(static_cast(v), &r, sizeof(T)); return true; } - return false; } @@ -271,7 +327,34 @@ class FlagImpl { int64_t counter_ ABSL_GUARDED_BY(*DataGuard()) = 0; // For some types, a copy of the current value is kept in an atomically // accessible field. - std::atomic atomic_{flags_internal::AtomicInit()}; + union Atomics { + // Using small atomic for small types. + std::atomic small_atomic; + template ::type> + int64_t load() const { + return small_atomic.load(std::memory_order_acquire); + } + +#if defined(ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD) + // Using big atomics for big types. + std::atomic big_atomic; + template ::type> + FlagsInternalTwoWordsType load() const { + return big_atomic.load(std::memory_order_acquire); + } + constexpr Atomics() + : big_atomic{FlagsInternalTwoWordsType{SmallAtomicInit(), + SmallAtomicInit()}} {} +#else + constexpr Atomics() : small_atomic{SmallAtomicInit()} {} +#endif + }; + Atomics atomics_{}; struct CallbackData { FlagCallback func; -- cgit v1.2.3