From 79e0dc11514df035a8d07a356f9ee1800fb2160c Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Thu, 26 Mar 2020 08:48:01 -0700 Subject: Export of internal Abseil changes -- 990253454819ce26ff1dda9ab4bbc145b61d01e4 by Xiaoyi Zhang : Import github PR https://github.com/abseil/abseil-cpp/pull/645 PiperOrigin-RevId: 303119797 -- 5ac845cb7929b7d1eaf59a309afd811db5001175 by Abseil Team : Fix internal exception spec compatibility error PiperOrigin-RevId: 303104081 -- 3290595dd866eecab3c7044e2e3ca0adb74f1bf5 by Gennadiy Rozental : Use FlagValue to represent the value of a flag. Place it directly after FlagImpl and use a computed offset refer to it. The offset is computed based on the assumption that the `value_` data member is placed directly after the impl_ data member in Flag. This change will allow us to migrate to `T`-specific storage in the generic case. This change decreases the overhead for int flags by 32 bytes. PiperOrigin-RevId: 303038099 -- f2b37722cd7a6d3a60ef9713f0d2bbff56f3ddbf by Derek Mauro : Minor correctness fix for an ABSL_HAVE_BUILTIN conditional PiperOrigin-RevId: 302980666 -- 39c079a6141ae1c5728af8bf33a39c8aff9deb9f by Abseil Team : Use ABSL_HARDENING_ASSERT in b-tree and SwissTable iterators. PiperOrigin-RevId: 302970075 -- 9668a044e080c789df32bcaa1ffb5100831cd9fa by Benjamin Barenblat : Correct `add_subdirectory` line in CMake googletest support Commit bcefbdcdf6ad85046ccacee0aeffba5404d3e528 added support for building with CMake against a local googletest checkout, but I missed a line when constructing the diff. Change the `add_subdirectory` line to reference the correct directories. PiperOrigin-RevId: 302947488 -- 0a3c10fabf80a43ca69ab8b1570030e55f2be741 by Andy Soffer : Remove unused distribution format traits. PiperOrigin-RevId: 302896176 -- 0478f2f6270e5ed64c0e28ec09556ca90b2d46a9 by Samuel Benzaquen : Fix for CWG:2310. PiperOrigin-RevId: 302734089 -- 3cb978dda5cae5905affdc0914dcc2d27671ed11 by Samuel Benzaquen : Fix the Allocate/Deallocate functions to use the same underlying allocator type. PiperOrigin-RevId: 302721804 -- ae38d3984fb68b4e3ddc165fa8d5c24d5936be52 by Matthew Brown : Internal Change PiperOrigin-RevId: 302717314 -- 7357cf7abd03cc60b6e82b5f28a8e34935c3b4dc by Andy Getzendanner : Fix typo: s/ABSL_HARDENED_ASSERT/ABSL_HARDENING_ASSERT/ PiperOrigin-RevId: 302532164 GitOrigin-RevId: 990253454819ce26ff1dda9ab4bbc145b61d01e4 Change-Id: Ie595a221c16e1e7e1255ad42e029b646c5f3e11d --- absl/flags/internal/flag.cc | 82 +++++++++----- absl/flags/internal/flag.h | 259 +++++++++++++++++++++++++++----------------- 2 files changed, 215 insertions(+), 126 deletions(-) (limited to 'absl/flags/internal') diff --git a/absl/flags/internal/flag.cc b/absl/flags/internal/flag.cc index 56a5ed2b..5b4499ab 100644 --- a/absl/flags/internal/flag.cc +++ b/absl/flags/internal/flag.cc @@ -25,6 +25,7 @@ #include #include "absl/base/attributes.h" +#include "absl/base/casts.h" #include "absl/base/config.h" #include "absl/base/const_init.h" #include "absl/base/optimization.h" @@ -135,18 +136,18 @@ void FlagImpl::Init() { (*default_value_.gen_func)(), DynValueDeleter{op_}); switch (ValueStorageKind()) { case FlagValueStorageKind::kHeapAllocated: - value_.dynamic = init_value.release(); + HeapAllocatedValue() = init_value.release(); break; case FlagValueStorageKind::kOneWordAtomic: { int64_t atomic_value; - std::memcpy(&atomic_value, init_value.get(), flags_internal::Sizeof(op_)); - value_.one_word_atomic.store(atomic_value, std::memory_order_release); + std::memcpy(&atomic_value, init_value.get(), Sizeof(op_)); + OneWordValue().store(atomic_value, std::memory_order_release); break; } case FlagValueStorageKind::kTwoWordsAtomic: { AlignedTwoWords atomic_value{0, 0}; - std::memcpy(&atomic_value, init_value.get(), flags_internal::Sizeof(op_)); - value_.two_words_atomic.store(atomic_value, std::memory_order_release); + std::memcpy(&atomic_value, init_value.get(), Sizeof(op_)); + TwoWordsValue().store(atomic_value, std::memory_order_release); break; } } @@ -198,18 +199,18 @@ std::unique_ptr FlagImpl::MakeInitValue() const { void FlagImpl::StoreValue(const void* src) { switch (ValueStorageKind()) { case FlagValueStorageKind::kHeapAllocated: - flags_internal::Copy(op_, src, value_.dynamic); + Copy(op_, src, HeapAllocatedValue()); break; case FlagValueStorageKind::kOneWordAtomic: { - int64_t one_word_val; - std::memcpy(&one_word_val, src, flags_internal::Sizeof(op_)); - value_.one_word_atomic.store(one_word_val, std::memory_order_release); + int64_t one_word_val = 0; + std::memcpy(&one_word_val, src, Sizeof(op_)); + OneWordValue().store(one_word_val, std::memory_order_release); break; } case FlagValueStorageKind::kTwoWordsAtomic: { AlignedTwoWords two_words_val{0, 0}; - std::memcpy(&two_words_val, src, flags_internal::Sizeof(op_)); - value_.two_words_atomic.store(two_words_val, std::memory_order_release); + std::memcpy(&two_words_val, src, Sizeof(op_)); + TwoWordsValue().store(two_words_val, std::memory_order_release); break; } } @@ -258,17 +259,19 @@ std::string FlagImpl::CurrentValue() const { switch (ValueStorageKind()) { case FlagValueStorageKind::kHeapAllocated: { absl::MutexLock l(guard); - return flags_internal::Unparse(op_, value_.dynamic); + return flags_internal::Unparse(op_, HeapAllocatedValue()); } case FlagValueStorageKind::kOneWordAtomic: { const auto one_word_val = - value_.one_word_atomic.load(std::memory_order_acquire); - return flags_internal::Unparse(op_, &one_word_val); + absl::bit_cast>( + OneWordValue().load(std::memory_order_acquire)); + return flags_internal::Unparse(op_, one_word_val.data()); } case FlagValueStorageKind::kTwoWordsAtomic: { const auto two_words_val = - value_.two_words_atomic.load(std::memory_order_acquire); - return flags_internal::Unparse(op_, &two_words_val); + absl::bit_cast>( + TwoWordsValue().load(std::memory_order_acquire)); + return flags_internal::Unparse(op_, two_words_val.data()); } } @@ -317,18 +320,18 @@ std::unique_ptr FlagImpl::SaveState() { switch (ValueStorageKind()) { case FlagValueStorageKind::kHeapAllocated: { return absl::make_unique( - this, flags_internal::Clone(op_, value_.dynamic), modified, + this, flags_internal::Clone(op_, HeapAllocatedValue()), modified, on_command_line, counter_); } case FlagValueStorageKind::kOneWordAtomic: { return absl::make_unique( - this, value_.one_word_atomic.load(std::memory_order_acquire), - modified, on_command_line, counter_); + this, OneWordValue().load(std::memory_order_acquire), modified, + on_command_line, counter_); } case FlagValueStorageKind::kTwoWordsAtomic: { return absl::make_unique( - this, value_.two_words_atomic.load(std::memory_order_acquire), - modified, on_command_line, counter_); + this, TwoWordsValue().load(std::memory_order_acquire), modified, + on_command_line, counter_); } } return nullptr; @@ -359,6 +362,28 @@ bool FlagImpl::RestoreState(const FlagState& flag_state) { return true; } +template +typename StorageT::value_type& FlagImpl::OffsetValue() const { + char* p = reinterpret_cast(const_cast(this)); + // The offset is deduced via Flag value type specific op_. + size_t offset = flags_internal::ValueOffset(op_); + + return reinterpret_cast(p + offset)->value; +} + +void*& FlagImpl::HeapAllocatedValue() const { + assert(ValueStorageKind() == FlagValueStorageKind::kHeapAllocated); + return OffsetValue(); +} +std::atomic& FlagImpl::OneWordValue() const { + assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic); + return OffsetValue(); +} +std::atomic& FlagImpl::TwoWordsValue() const { + assert(ValueStorageKind() == FlagValueStorageKind::kTwoWordsAtomic); + return OffsetValue(); +} + // Attempts to parse supplied `value` string using parsing routine in the `flag` // argument. If parsing successful, this function replaces the dst with newly // parsed value. In case if any error is encountered in either step, the error @@ -383,20 +408,19 @@ void FlagImpl::Read(void* dst) const { switch (ValueStorageKind()) { case FlagValueStorageKind::kHeapAllocated: { absl::MutexLock l(guard); - - flags_internal::CopyConstruct(op_, value_.dynamic, dst); + flags_internal::CopyConstruct(op_, HeapAllocatedValue(), dst); break; } case FlagValueStorageKind::kOneWordAtomic: { - const auto one_word_val = - value_.one_word_atomic.load(std::memory_order_acquire); - std::memcpy(dst, &one_word_val, flags_internal::Sizeof(op_)); + const int64_t one_word_val = + OneWordValue().load(std::memory_order_acquire); + std::memcpy(dst, &one_word_val, Sizeof(op_)); break; } case FlagValueStorageKind::kTwoWordsAtomic: { - const auto two_words_val = - value_.two_words_atomic.load(std::memory_order_acquire); - std::memcpy(dst, &two_words_val, flags_internal::Sizeof(op_)); + const AlignedTwoWords two_words_val = + TwoWordsValue().load(std::memory_order_acquire); + std::memcpy(dst, &two_words_val, Sizeof(op_)); break; } } diff --git a/absl/flags/internal/flag.h b/absl/flags/internal/flag.h index f27e558b..19119bbb 100644 --- a/absl/flags/internal/flag.h +++ b/absl/flags/internal/flag.h @@ -53,56 +53,13 @@ enum class FlagOp { kStaticTypeId, kParse, kUnparse, + kValueOffset, }; using FlagOpFn = void* (*)(FlagOp, const void*, void*, void*); -// Flag value specific operations routine. +// Forward declaration for Flag value specific operations. template -void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3) { - switch (op) { - case FlagOp::kDelete: - delete static_cast(v1); - return nullptr; - case FlagOp::kClone: - return new T(*static_cast(v1)); - case FlagOp::kCopy: - *static_cast(v2) = *static_cast(v1); - return nullptr; - case FlagOp::kCopyConstruct: - new (v2) T(*static_cast(v1)); - return nullptr; - case FlagOp::kSizeof: - return reinterpret_cast(sizeof(T)); - case FlagOp::kStaticTypeId: { - auto* static_id = &FlagStaticTypeIdGen; - - // Cast from function pointer to void* is not portable. - // We don't have an easy way to work around this, but it works fine - // on all the platforms we test and as long as size of pointers match - // we should be fine to do reinterpret cast. - static_assert(sizeof(void*) == sizeof(static_id), - "Flag's static type id does not work on this platform"); - return reinterpret_cast(static_id); - } - case FlagOp::kParse: { - // Initialize the temporary instance of type T based on current value in - // destination (which is going to be flag's default value). - T temp(*static_cast(v2)); - if (!absl::ParseFlag(*static_cast(v1), &temp, - static_cast(v3))) { - return nullptr; - } - *static_cast(v2) = std::move(temp); - return v2; - } - case FlagOp::kUnparse: - *static_cast(v2) = - absl::UnparseFlag(*static_cast(v1)); - return nullptr; - default: - return nullptr; - } -} +void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3); // Deletes memory interpreting obj as flag value type pointer. inline void Delete(FlagOpFn op, const void* obj) { @@ -144,6 +101,16 @@ inline FlagStaticTypeId StaticTypeId(FlagOpFn op) { return reinterpret_cast( op(FlagOp::kStaticTypeId, nullptr, nullptr, nullptr)); } +// Returns offset of the field value_ from the field impl_ inside of +// absl::Flag data. Given FlagImpl pointer p you can get the +// location of the corresponding value as: +// reinterpret_cast(p) + ValueOffset(). +inline ptrdiff_t ValueOffset(FlagOpFn op) { + // This sequence of casts reverses the sequence from + // `flags_internal::FlagOps()` + return static_cast(reinterpret_cast( + op(FlagOp::kValueOffset, nullptr, nullptr, nullptr))); +} /////////////////////////////////////////////////////////////////////////////// // Flag help auxiliary structs. @@ -239,6 +206,10 @@ using FlagUseOneWordStorage = std::integral_constant< struct alignas(16) AlignedTwoWords { int64_t first; int64_t second; + + bool IsInitialized() const { + return first != flags_internal::UninitializedFlagValue(); + } }; template @@ -248,8 +219,14 @@ using FlagUseTwoWordsStorage = std::integral_constant< #else // This is actually unused and only here to avoid ifdefs in other palces. struct AlignedTwoWords { - constexpr AlignedTwoWords() = default; - constexpr AlignedTwoWords(int64_t, int64_t) {} + constexpr AlignedTwoWords() noexcept : dummy() {} + constexpr AlignedTwoWords(int64_t, int64_t) noexcept : dummy() {} + char dummy; + + bool IsInitialized() const { + std::abort(); + return true; + } }; // This trait should be type dependent, otherwise SFINAE below will fail @@ -269,23 +246,70 @@ enum class FlagValueStorageKind : uint8_t { kTwoWordsAtomic = 2 }; -union FlagValue { - constexpr explicit FlagValue(int64_t v) : one_word_atomic(v) {} +template +static constexpr FlagValueStorageKind StorageKind() { + return FlagUseHeapStorage::value + ? FlagValueStorageKind::kHeapAllocated + : FlagUseOneWordStorage::value + ? FlagValueStorageKind::kOneWordAtomic + : FlagUseTwoWordsStorage::value + ? FlagValueStorageKind::kTwoWordsAtomic + : FlagValueStorageKind::kHeapAllocated; +} + +struct FlagHeapAllocatedValue { + using value_type = void*; + + value_type value; +}; + +struct FlagOneWordValue { + using value_type = std::atomic; + constexpr FlagOneWordValue() : value(UninitializedFlagValue()) {} - template - static constexpr FlagValueStorageKind Kind() { - return FlagUseHeapStorage::value - ? FlagValueStorageKind::kHeapAllocated - : FlagUseOneWordStorage::value - ? FlagValueStorageKind::kOneWordAtomic - : FlagUseTwoWordsStorage::value - ? FlagValueStorageKind::kTwoWordsAtomic - : FlagValueStorageKind::kHeapAllocated; + value_type value; +}; + +struct FlagTwoWordsValue { + using value_type = std::atomic; + constexpr FlagTwoWordsValue() + : value(AlignedTwoWords{UninitializedFlagValue(), 0}) {} + + value_type value; +}; + +template ()> +struct FlagValue; + +template +struct FlagValue + : FlagHeapAllocatedValue { + bool Get(T*) const { return false; } +}; + +template +struct FlagValue : FlagOneWordValue { + bool Get(T* dst) const { + int64_t one_word_val = value.load(std::memory_order_acquire); + if (ABSL_PREDICT_FALSE(one_word_val == UninitializedFlagValue())) { + return false; + } + std::memcpy(dst, static_cast(&one_word_val), sizeof(T)); + return true; } +}; - void* dynamic; - std::atomic one_word_atomic; - std::atomic two_words_atomic; +template +struct FlagValue : FlagTwoWordsValue { + bool Get(T* dst) const { + AlignedTwoWords two_words_val = value.load(std::memory_order_acquire); + if (ABSL_PREDICT_FALSE(!two_words_val.IsInitialized())) { + return false; + } + std::memcpy(dst, static_cast(&two_words_val), sizeof(T)); + return true; + } }; /////////////////////////////////////////////////////////////////////////////// @@ -333,35 +357,10 @@ class FlagImpl final : public flags_internal::CommandLineFlag { counter_(0), callback_(nullptr), default_value_(default_value_gen), - value_(flags_internal::UninitializedFlagValue()), data_guard_{} {} // Constant access methods void Read(void* dst) const override ABSL_LOCKS_EXCLUDED(*DataGuard()); - template ::value, - int>::type = 0> - void Get(T* dst) const { - Read(dst); - } - template ::value, - int>::type = 0> - void Get(T* dst) const { - int64_t one_word_val = - value_.one_word_atomic.load(std::memory_order_acquire); - if (ABSL_PREDICT_FALSE(one_word_val == UninitializedFlagValue())) { - DataGuard(); // Make sure flag initialized - one_word_val = value_.one_word_atomic.load(std::memory_order_acquire); - } - std::memcpy(dst, static_cast(&one_word_val), sizeof(T)); - } - template ::value, int>::type = 0> - void Get(T* dst) const { - DataGuard(); // Make sure flag initialized - const auto two_words_val = - value_.two_words_atomic.load(std::memory_order_acquire); - std::memcpy(dst, &two_words_val, sizeof(T)); - } // Mutating access methods void Write(const void* src) ABSL_LOCKS_EXCLUDED(*DataGuard()); @@ -391,6 +390,25 @@ class FlagImpl final : public flags_internal::CommandLineFlag { ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); // Flag initialization called via absl::call_once. void Init(); + + // Offset value access methods. One per storage kind. These methods to not + // respect const correctness, so be very carefull using them. + + // This is a shared helper routine which encapsulates most of the magic. Since + // it is only used inside the three routines below, which are defined in + // flag.cc, we can define it in that file as well. + template + typename StorageT::value_type& OffsetValue() const; + // This is an accessor for a value stored in heap allocated storage. + // Returns a mutable reference to a pointer to allow vlaue mutation. + void*& HeapAllocatedValue() const; + // This is an accessor for a value stored as one word atomic. Returns a + // mutable reference to an atomic value. + std::atomic& OneWordValue() const; + // This is an accessor for a value stored as two words atomic. Returns a + // mutable reference to an atomic value. + std::atomic& TwoWordsValue() const; + // Attempts to parse supplied `value` string. If parsing is successful, // returns new value. Otherwise returns nullptr. std::unique_ptr TryParse(absl::string_view value, @@ -488,13 +506,6 @@ class FlagImpl final : public flags_internal::CommandLineFlag { // these two cases. FlagDefaultSrc default_value_; - // Atomically mutable flag's state - - // Flag's value. This can be either the atomically stored small value or - // pointer to the heap allocated dynamic value. value_storage_kind_ is used - // to distinguish these cases. - FlagValue value_; - // This is reserved space for an absl::Mutex to guard flag data. It will be // initialized in FlagImpl::Init via placement new. // We can't use "absl::Mutex data_guard_", since this class is not literal. @@ -514,8 +525,9 @@ class Flag { public: constexpr Flag(const char* name, const char* filename, const FlagHelpArg help, const FlagDfltGenFunc default_value_gen) - : impl_(name, filename, &FlagOps, help, FlagValue::Kind(), - default_value_gen) {} + : impl_(name, filename, &FlagOps, help, + flags_internal::StorageKind(), default_value_gen), + value_() {} T Get() const { // See implementation notes in CommandLineFlag::Get(). @@ -530,7 +542,7 @@ class Flag { impl_.AssertValidType(&flags_internal::FlagStaticTypeIdGen); #endif - impl_.Get(&u.value); + if (!value_.Get(&u.value)) impl_.Read(&u.value); return std::move(u.value); } void Set(const T& v) { @@ -556,10 +568,63 @@ class Flag { private: template friend class FlagRegistrar; + // Flag's data + // The implementation depends on value_ field to be placed exactly after the + // impl_ field, so that impl_ can figure out the offset to the value and + // access it. FlagImpl impl_; + FlagValue value_; }; +/////////////////////////////////////////////////////////////////////////////// +// Implementation of Flag value specific operations routine. +template +void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3) { + switch (op) { + case FlagOp::kDelete: + delete static_cast(v1); + return nullptr; + case FlagOp::kClone: + return new T(*static_cast(v1)); + case FlagOp::kCopy: + *static_cast(v2) = *static_cast(v1); + return nullptr; + case FlagOp::kCopyConstruct: + new (v2) T(*static_cast(v1)); + return nullptr; + case FlagOp::kSizeof: + return reinterpret_cast(static_cast(sizeof(T))); + case FlagOp::kStaticTypeId: + return reinterpret_cast(&FlagStaticTypeIdGen); + case FlagOp::kParse: { + // Initialize the temporary instance of type T based on current value in + // destination (which is going to be flag's default value). + T temp(*static_cast(v2)); + if (!absl::ParseFlag(*static_cast(v1), &temp, + static_cast(v3))) { + return nullptr; + } + *static_cast(v2) = std::move(temp); + return v2; + } + case FlagOp::kUnparse: + *static_cast(v2) = + absl::UnparseFlag(*static_cast(v1)); + return nullptr; + case FlagOp::kValueOffset: { + // Round sizeof(FlagImp) to a multiple of alignof(FlagValue) to get the + // offset of the data. + ptrdiff_t round_to = alignof(FlagValue); + ptrdiff_t offset = + (sizeof(FlagImpl) + round_to - 1) / round_to * round_to; + return reinterpret_cast(offset); + } + } + return nullptr; +} + +/////////////////////////////////////////////////////////////////////////////// // This class facilitates Flag object registration and tail expression-based // flag definition, for example: // ABSL_FLAG(int, foo, 42, "Foo help").OnUpdate(NotifyFooWatcher); -- cgit v1.2.3