Export of internal Abseil changes

--
990253454819ce26ff1dda9ab4bbc145b61d01e4 by Xiaoyi Zhang <zhangxy@google.com>:

Import github PR https://github.com/abseil/abseil-cpp/pull/645

PiperOrigin-RevId: 303119797

--
5ac845cb7929b7d1eaf59a309afd811db5001175 by Abseil Team <absl-team@google.com>:

Fix internal exception spec compatibility error

PiperOrigin-RevId: 303104081

--
3290595dd866eecab3c7044e2e3ca0adb74f1bf5 by Gennadiy Rozental <rogeeff@google.com>:

Use FlagValue<T> to represent the value of a flag. Place it directly after
FlagImpl and use a computed offset refer to it.

The offset is computed based on the assumption that the `value_` data member
is placed directly after the impl_ data member in Flag<T>.

This change will allow us to migrate to `T`-specific storage in the generic case.

This change decreases the overhead for int flags by 32 bytes.

PiperOrigin-RevId: 303038099

--
f2b37722cd7a6d3a60ef9713f0d2bbff56f3ddbf by Derek Mauro <dmauro@google.com>:

Minor correctness fix for an ABSL_HAVE_BUILTIN conditional

PiperOrigin-RevId: 302980666

--
39c079a6141ae1c5728af8bf33a39c8aff9deb9f by Abseil Team <absl-team@google.com>:

Use ABSL_HARDENING_ASSERT in b-tree and SwissTable iterators.

PiperOrigin-RevId: 302970075

--
9668a044e080c789df32bcaa1ffb5100831cd9fa by Benjamin Barenblat <bbaren@google.com>:

Correct `add_subdirectory` line in CMake googletest support

Commit bcefbdcdf6ad85046ccacee0aeffba5404d3e528 added support for building with CMake against a local googletest checkout, but I missed a line when constructing the diff. Change the `add_subdirectory` line to reference the correct directories.

PiperOrigin-RevId: 302947488

--
0a3c10fabf80a43ca69ab8b1570030e55f2be741 by Andy Soffer <asoffer@google.com>:

Remove unused distribution format traits.

PiperOrigin-RevId: 302896176

--
0478f2f6270e5ed64c0e28ec09556ca90b2d46a9 by Samuel Benzaquen <sbenza@google.com>:

Fix for CWG:2310.

PiperOrigin-RevId: 302734089

--
3cb978dda5cae5905affdc0914dcc2d27671ed11 by Samuel Benzaquen <sbenza@google.com>:

Fix the Allocate/Deallocate functions to use the same underlying allocator type.

PiperOrigin-RevId: 302721804

--
ae38d3984fb68b4e3ddc165fa8d5c24d5936be52 by Matthew Brown <matthewbr@google.com>:

Internal Change

PiperOrigin-RevId: 302717314

--
7357cf7abd03cc60b6e82b5f28a8e34935c3b4dc by Andy Getzendanner <durandal@google.com>:

Fix typo: s/ABSL_HARDENED_ASSERT/ABSL_HARDENING_ASSERT/

PiperOrigin-RevId: 302532164
GitOrigin-RevId: 990253454819ce26ff1dda9ab4bbc145b61d01e4
Change-Id: Ie595a221c16e1e7e1255ad42e029b646c5f3e11d

3 files changed, 227 insertions, 138 deletions

diff --git a/absl/flags/flag_test.cc b/absl/flags/flag_test.cc
index 3a025576..377e3b2b 100644
--- a/absl/flags/flag_test.cc
+++ b/absl/flags/flag_test.cc
@@ -91,30 +91,30 @@ struct S2 {
 };
 
 TEST_F(FlagTest, Traits) {
-  EXPECT_EQ(flags::FlagValue::Kind<int>(),
+  EXPECT_EQ(flags::StorageKind<int>(),
             flags::FlagValueStorageKind::kOneWordAtomic);
-  EXPECT_EQ(flags::FlagValue::Kind<bool>(),
+  EXPECT_EQ(flags::StorageKind<bool>(),
             flags::FlagValueStorageKind::kOneWordAtomic);
-  EXPECT_EQ(flags::FlagValue::Kind<double>(),
+  EXPECT_EQ(flags::StorageKind<double>(),
             flags::FlagValueStorageKind::kOneWordAtomic);
-  EXPECT_EQ(flags::FlagValue::Kind<int64_t>(),
+  EXPECT_EQ(flags::StorageKind<int64_t>(),
             flags::FlagValueStorageKind::kOneWordAtomic);
 
 #if defined(ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD)
-  EXPECT_EQ(flags::FlagValue::Kind<S1>(),
+  EXPECT_EQ(flags::StorageKind<S1>(),
             flags::FlagValueStorageKind::kTwoWordsAtomic);
-  EXPECT_EQ(flags::FlagValue::Kind<S2>(),
+  EXPECT_EQ(flags::StorageKind<S2>(),
             flags::FlagValueStorageKind::kTwoWordsAtomic);
 #else
-  EXPECT_EQ(flags::FlagValue::Kind<S1>(),
+  EXPECT_EQ(flags::StorageKind<S1>(),
             flags::FlagValueStorageKind::kHeapAllocated);
-  EXPECT_EQ(flags::FlagValue::Kind<S2>(),
+  EXPECT_EQ(flags::StorageKind<S2>(),
             flags::FlagValueStorageKind::kHeapAllocated);
 #endif
 
-  EXPECT_EQ(flags::FlagValue::Kind<std::string>(),
+  EXPECT_EQ(flags::StorageKind<std::string>(),
             flags::FlagValueStorageKind::kHeapAllocated);
-  EXPECT_EQ(flags::FlagValue::Kind<std::vector<std::string>>(),
+  EXPECT_EQ(flags::StorageKind<std::vector<std::string>>(),
             flags::FlagValueStorageKind::kHeapAllocated);
 }
 
@@ -624,10 +624,10 @@ TEST_F(FlagTest, TestNonDefaultConstructibleType) {
   EXPECT_EQ(absl::GetFlag(FLAGS_ndc_flag2).value, 25);
 }
 
-// --------------------------------------------------------------------
-
 }  // namespace
 
+// --------------------------------------------------------------------
+
 ABSL_RETIRED_FLAG(bool, old_bool_flag, true, "old descr");
 ABSL_RETIRED_FLAG(int, old_int_flag, (int)std::sqrt(10), "old descr");
 ABSL_RETIRED_FLAG(std::string, old_str_flag, "", absl::StrCat("old ", "descr"));
diff --git a/absl/flags/internal/flag.cc b/absl/flags/internal/flag.cc
index 56a5ed2b..5b4499ab 100644
--- a/absl/flags/internal/flag.cc
+++ b/absl/flags/internal/flag.cc
@@ -25,6 +25,7 @@
 #include <vector>
 
 #include "absl/base/attributes.h"
+#include "absl/base/casts.h"
 #include "absl/base/config.h"
 #include "absl/base/const_init.h"
 #include "absl/base/optimization.h"
@@ -135,18 +136,18 @@ void FlagImpl::Init() {
       (*default_value_.gen_func)(), DynValueDeleter{op_});
   switch (ValueStorageKind()) {
     case FlagValueStorageKind::kHeapAllocated:
-      value_.dynamic = init_value.release();
+      HeapAllocatedValue() = init_value.release();
       break;
     case FlagValueStorageKind::kOneWordAtomic: {
       int64_t atomic_value;
-      std::memcpy(&atomic_value, init_value.get(), flags_internal::Sizeof(op_));
-      value_.one_word_atomic.store(atomic_value, std::memory_order_release);
+      std::memcpy(&atomic_value, init_value.get(), Sizeof(op_));
+      OneWordValue().store(atomic_value, std::memory_order_release);
       break;
     }
     case FlagValueStorageKind::kTwoWordsAtomic: {
       AlignedTwoWords atomic_value{0, 0};
-      std::memcpy(&atomic_value, init_value.get(), flags_internal::Sizeof(op_));
-      value_.two_words_atomic.store(atomic_value, std::memory_order_release);
+      std::memcpy(&atomic_value, init_value.get(), Sizeof(op_));
+      TwoWordsValue().store(atomic_value, std::memory_order_release);
       break;
     }
   }
@@ -198,18 +199,18 @@ std::unique_ptr<void, DynValueDeleter> FlagImpl::MakeInitValue() const {
 void FlagImpl::StoreValue(const void* src) {
   switch (ValueStorageKind()) {
     case FlagValueStorageKind::kHeapAllocated:
-      flags_internal::Copy(op_, src, value_.dynamic);
+      Copy(op_, src, HeapAllocatedValue());
       break;
     case FlagValueStorageKind::kOneWordAtomic: {
-      int64_t one_word_val;
-      std::memcpy(&one_word_val, src, flags_internal::Sizeof(op_));
-      value_.one_word_atomic.store(one_word_val, std::memory_order_release);
+      int64_t one_word_val = 0;
+      std::memcpy(&one_word_val, src, Sizeof(op_));
+      OneWordValue().store(one_word_val, std::memory_order_release);
       break;
     }
     case FlagValueStorageKind::kTwoWordsAtomic: {
       AlignedTwoWords two_words_val{0, 0};
-      std::memcpy(&two_words_val, src, flags_internal::Sizeof(op_));
-      value_.two_words_atomic.store(two_words_val, std::memory_order_release);
+      std::memcpy(&two_words_val, src, Sizeof(op_));
+      TwoWordsValue().store(two_words_val, std::memory_order_release);
       break;
     }
   }
@@ -258,17 +259,19 @@ std::string FlagImpl::CurrentValue() const {
   switch (ValueStorageKind()) {
     case FlagValueStorageKind::kHeapAllocated: {
       absl::MutexLock l(guard);
-      return flags_internal::Unparse(op_, value_.dynamic);
+      return flags_internal::Unparse(op_, HeapAllocatedValue());
     }
     case FlagValueStorageKind::kOneWordAtomic: {
       const auto one_word_val =
-          value_.one_word_atomic.load(std::memory_order_acquire);
-      return flags_internal::Unparse(op_, &one_word_val);
+          absl::bit_cast<std::array<char, sizeof(int64_t)>>(
+              OneWordValue().load(std::memory_order_acquire));
+      return flags_internal::Unparse(op_, one_word_val.data());
     }
     case FlagValueStorageKind::kTwoWordsAtomic: {
       const auto two_words_val =
-          value_.two_words_atomic.load(std::memory_order_acquire);
-      return flags_internal::Unparse(op_, &two_words_val);
+          absl::bit_cast<std::array<char, sizeof(AlignedTwoWords)>>(
+              TwoWordsValue().load(std::memory_order_acquire));
+      return flags_internal::Unparse(op_, two_words_val.data());
     }
   }
 
@@ -317,18 +320,18 @@ std::unique_ptr<FlagStateInterface> FlagImpl::SaveState() {
   switch (ValueStorageKind()) {
     case FlagValueStorageKind::kHeapAllocated: {
       return absl::make_unique<FlagState>(
-          this, flags_internal::Clone(op_, value_.dynamic), modified,
+          this, flags_internal::Clone(op_, HeapAllocatedValue()), modified,
           on_command_line, counter_);
     }
     case FlagValueStorageKind::kOneWordAtomic: {
       return absl::make_unique<FlagState>(
-          this, value_.one_word_atomic.load(std::memory_order_acquire),
-          modified, on_command_line, counter_);
+          this, OneWordValue().load(std::memory_order_acquire), modified,
+          on_command_line, counter_);
     }
     case FlagValueStorageKind::kTwoWordsAtomic: {
       return absl::make_unique<FlagState>(
-          this, value_.two_words_atomic.load(std::memory_order_acquire),
-          modified, on_command_line, counter_);
+          this, TwoWordsValue().load(std::memory_order_acquire), modified,
+          on_command_line, counter_);
     }
   }
   return nullptr;
@@ -359,6 +362,28 @@ bool FlagImpl::RestoreState(const FlagState& flag_state) {
   return true;
 }
 
+template <typename StorageT>
+typename StorageT::value_type& FlagImpl::OffsetValue() const {
+  char* p = reinterpret_cast<char*>(const_cast<FlagImpl*>(this));
+  // The offset is deduced via Flag value type specific op_.
+  size_t offset = flags_internal::ValueOffset(op_);
+
+  return reinterpret_cast<StorageT*>(p + offset)->value;
+}
+
+void*& FlagImpl::HeapAllocatedValue() const {
+  assert(ValueStorageKind() == FlagValueStorageKind::kHeapAllocated);
+  return OffsetValue<FlagHeapAllocatedValue>();
+}
+std::atomic<int64_t>& FlagImpl::OneWordValue() const {
+  assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic);
+  return OffsetValue<FlagOneWordValue>();
+}
+std::atomic<AlignedTwoWords>& FlagImpl::TwoWordsValue() const {
+  assert(ValueStorageKind() == FlagValueStorageKind::kTwoWordsAtomic);
+  return OffsetValue<FlagTwoWordsValue>();
+}
+
 // Attempts to parse supplied `value` string using parsing routine in the `flag`
 // argument. If parsing successful, this function replaces the dst with newly
 // parsed value. In case if any error is encountered in either step, the error
@@ -383,20 +408,19 @@ void FlagImpl::Read(void* dst) const {
   switch (ValueStorageKind()) {
     case FlagValueStorageKind::kHeapAllocated: {
       absl::MutexLock l(guard);
-
-      flags_internal::CopyConstruct(op_, value_.dynamic, dst);
+      flags_internal::CopyConstruct(op_, HeapAllocatedValue(), dst);
       break;
     }
     case FlagValueStorageKind::kOneWordAtomic: {
-      const auto one_word_val =
-          value_.one_word_atomic.load(std::memory_order_acquire);
-      std::memcpy(dst, &one_word_val, flags_internal::Sizeof(op_));
+      const int64_t one_word_val =
+          OneWordValue().load(std::memory_order_acquire);
+      std::memcpy(dst, &one_word_val, Sizeof(op_));
       break;
     }
     case FlagValueStorageKind::kTwoWordsAtomic: {
-      const auto two_words_val =
-          value_.two_words_atomic.load(std::memory_order_acquire);
-      std::memcpy(dst, &two_words_val, flags_internal::Sizeof(op_));
+      const AlignedTwoWords two_words_val =
+          TwoWordsValue().load(std::memory_order_acquire);
+      std::memcpy(dst, &two_words_val, Sizeof(op_));
       break;
     }
   }
diff --git a/absl/flags/internal/flag.h b/absl/flags/internal/flag.h
index f27e558b..19119bbb 100644
--- a/absl/flags/internal/flag.h
+++ b/absl/flags/internal/flag.h
@@ -53,56 +53,13 @@ enum class FlagOp {
   kStaticTypeId,
   kParse,
   kUnparse,
+  kValueOffset,
 };
 using FlagOpFn = void* (*)(FlagOp, const void*, void*, void*);
 
-// Flag value specific operations routine.
+// Forward declaration for Flag value specific operations.
 template <typename T>
-void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3) {
-  switch (op) {
-    case FlagOp::kDelete:
-      delete static_cast<const T*>(v1);
-      return nullptr;
-    case FlagOp::kClone:
-      return new T(*static_cast<const T*>(v1));
-    case FlagOp::kCopy:
-      *static_cast<T*>(v2) = *static_cast<const T*>(v1);
-      return nullptr;
-    case FlagOp::kCopyConstruct:
-      new (v2) T(*static_cast<const T*>(v1));
-      return nullptr;
-    case FlagOp::kSizeof:
-      return reinterpret_cast<void*>(sizeof(T));
-    case FlagOp::kStaticTypeId: {
-      auto* static_id = &FlagStaticTypeIdGen<T>;
-
-      // Cast from function pointer to void* is not portable.
-      // We don't have an easy way to work around this, but it works fine
-      // on all the platforms we test and as long as size of pointers match
-      // we should be fine to do reinterpret cast.
-      static_assert(sizeof(void*) == sizeof(static_id),
-                    "Flag's static type id does not work on this platform");
-      return reinterpret_cast<void*>(static_id);
-    }
-    case FlagOp::kParse: {
-      // Initialize the temporary instance of type T based on current value in
-      // destination (which is going to be flag's default value).
-      T temp(*static_cast<T*>(v2));
-      if (!absl::ParseFlag<T>(*static_cast<const absl::string_view*>(v1), &temp,
-                              static_cast<std::string*>(v3))) {
-        return nullptr;
-      }
-      *static_cast<T*>(v2) = std::move(temp);
-      return v2;
-    }
-    case FlagOp::kUnparse:
-      *static_cast<std::string*>(v2) =
-          absl::UnparseFlag<T>(*static_cast<const T*>(v1));
-      return nullptr;
-    default:
-      return nullptr;
-  }
-}
+void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3);
 
 // Deletes memory interpreting obj as flag value type pointer.
 inline void Delete(FlagOpFn op, const void* obj) {
@@ -144,6 +101,16 @@ inline FlagStaticTypeId StaticTypeId(FlagOpFn op) {
   return reinterpret_cast<FlagStaticTypeId>(
       op(FlagOp::kStaticTypeId, nullptr, nullptr, nullptr));
 }
+// Returns offset of the field value_ from the field impl_ inside of
+// absl::Flag<T> data. Given FlagImpl pointer p you can get the
+// location of the corresponding value as:
+//      reinterpret_cast<char*>(p) + ValueOffset().
+inline ptrdiff_t ValueOffset(FlagOpFn op) {
+  // This sequence of casts reverses the sequence from
+  // `flags_internal::FlagOps()`
+  return static_cast<ptrdiff_t>(reinterpret_cast<intptr_t>(
+      op(FlagOp::kValueOffset, nullptr, nullptr, nullptr)));
+}
 
 ///////////////////////////////////////////////////////////////////////////////
 // Flag help auxiliary structs.
@@ -239,6 +206,10 @@ using FlagUseOneWordStorage = std::integral_constant<
 struct alignas(16) AlignedTwoWords {
   int64_t first;
   int64_t second;
+
+  bool IsInitialized() const {
+    return first != flags_internal::UninitializedFlagValue();
+  }
 };
 
 template <typename T>
@@ -248,8 +219,14 @@ using FlagUseTwoWordsStorage = std::integral_constant<
 #else
 // This is actually unused and only here to avoid ifdefs in other palces.
 struct AlignedTwoWords {
-  constexpr AlignedTwoWords() = default;
-  constexpr AlignedTwoWords(int64_t, int64_t) {}
+  constexpr AlignedTwoWords() noexcept : dummy() {}
+  constexpr AlignedTwoWords(int64_t, int64_t) noexcept : dummy() {}
+  char dummy;
+
+  bool IsInitialized() const {
+    std::abort();
+    return true;
+  }
 };
 
 // This trait should be type dependent, otherwise SFINAE below will fail
@@ -269,23 +246,70 @@ enum class FlagValueStorageKind : uint8_t {
   kTwoWordsAtomic = 2
 };
 
-union FlagValue {
-  constexpr explicit FlagValue(int64_t v) : one_word_atomic(v) {}
+template <typename T>
+static constexpr FlagValueStorageKind StorageKind() {
+  return FlagUseHeapStorage<T>::value
+             ? FlagValueStorageKind::kHeapAllocated
+             : FlagUseOneWordStorage<T>::value
+                   ? FlagValueStorageKind::kOneWordAtomic
+                   : FlagUseTwoWordsStorage<T>::value
+                         ? FlagValueStorageKind::kTwoWordsAtomic
+                         : FlagValueStorageKind::kHeapAllocated;
+}
+
+struct FlagHeapAllocatedValue {
+  using value_type = void*;
+
+  value_type value;
+};
+
+struct FlagOneWordValue {
+  using value_type = std::atomic<int64_t>;
+  constexpr FlagOneWordValue() : value(UninitializedFlagValue()) {}
 
-  template <typename T>
-  static constexpr FlagValueStorageKind Kind() {
-    return FlagUseHeapStorage<T>::value
-               ? FlagValueStorageKind::kHeapAllocated
-               : FlagUseOneWordStorage<T>::value
-                     ? FlagValueStorageKind::kOneWordAtomic
-                     : FlagUseTwoWordsStorage<T>::value
-                           ? FlagValueStorageKind::kTwoWordsAtomic
-                           : FlagValueStorageKind::kHeapAllocated;
+  value_type value;
+};
+
+struct FlagTwoWordsValue {
+  using value_type = std::atomic<AlignedTwoWords>;
+  constexpr FlagTwoWordsValue()
+      : value(AlignedTwoWords{UninitializedFlagValue(), 0}) {}
+
+  value_type value;
+};
+
+template <typename T,
+          FlagValueStorageKind Kind = flags_internal::StorageKind<T>()>
+struct FlagValue;
+
+template <typename T>
+struct FlagValue<T, FlagValueStorageKind::kHeapAllocated>
+    : FlagHeapAllocatedValue {
+  bool Get(T*) const { return false; }
+};
+
+template <typename T>
+struct FlagValue<T, FlagValueStorageKind::kOneWordAtomic> : FlagOneWordValue {
+  bool Get(T* dst) const {
+    int64_t one_word_val = value.load(std::memory_order_acquire);
+    if (ABSL_PREDICT_FALSE(one_word_val == UninitializedFlagValue())) {
+      return false;
+    }
+    std::memcpy(dst, static_cast<const void*>(&one_word_val), sizeof(T));
+    return true;
   }
+};
 
-  void* dynamic;
-  std::atomic<int64_t> one_word_atomic;
-  std::atomic<flags_internal::AlignedTwoWords> two_words_atomic;
+template <typename T>
+struct FlagValue<T, FlagValueStorageKind::kTwoWordsAtomic> : FlagTwoWordsValue {
+  bool Get(T* dst) const {
+    AlignedTwoWords two_words_val = value.load(std::memory_order_acquire);
+    if (ABSL_PREDICT_FALSE(!two_words_val.IsInitialized())) {
+      return false;
+    }
+    std::memcpy(dst, static_cast<const void*>(&two_words_val), sizeof(T));
+    return true;
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -333,35 +357,10 @@ class FlagImpl final : public flags_internal::CommandLineFlag {
         counter_(0),
         callback_(nullptr),
         default_value_(default_value_gen),
-        value_(flags_internal::UninitializedFlagValue()),
         data_guard_{} {}
 
   // Constant access methods
   void Read(void* dst) const override ABSL_LOCKS_EXCLUDED(*DataGuard());
-  template <typename T, typename std::enable_if<FlagUseHeapStorage<T>::value,
-                                                int>::type = 0>
-  void Get(T* dst) const {
-    Read(dst);
-  }
-  template <typename T, typename std::enable_if<FlagUseOneWordStorage<T>::value,
-                                                int>::type = 0>
-  void Get(T* dst) const {
-    int64_t one_word_val =
-        value_.one_word_atomic.load(std::memory_order_acquire);
-    if (ABSL_PREDICT_FALSE(one_word_val == UninitializedFlagValue())) {
-      DataGuard();  // Make sure flag initialized
-      one_word_val = value_.one_word_atomic.load(std::memory_order_acquire);
-    }
-    std::memcpy(dst, static_cast<const void*>(&one_word_val), sizeof(T));
-  }
-  template <typename T, typename std::enable_if<
-                            FlagUseTwoWordsStorage<T>::value, int>::type = 0>
-  void Get(T* dst) const {
-    DataGuard();  // Make sure flag initialized
-    const auto two_words_val =
-        value_.two_words_atomic.load(std::memory_order_acquire);
-    std::memcpy(dst, &two_words_val, sizeof(T));
-  }
 
   // Mutating access methods
   void Write(const void* src) ABSL_LOCKS_EXCLUDED(*DataGuard());
@@ -391,6 +390,25 @@ class FlagImpl final : public flags_internal::CommandLineFlag {
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard());
   // Flag initialization called via absl::call_once.
   void Init();
+
+  // Offset value access methods. One per storage kind. These methods to not
+  // respect const correctness, so be very carefull using them.
+
+  // This is a shared helper routine which encapsulates most of the magic. Since
+  // it is only used inside the three routines below, which are defined in
+  // flag.cc, we can define it in that file as well.
+  template <typename StorageT>
+  typename StorageT::value_type& OffsetValue() const;
+  // This is an accessor for a value stored in heap allocated storage.
+  // Returns a mutable reference to a pointer to allow vlaue mutation.
+  void*& HeapAllocatedValue() const;
+  // This is an accessor for a value stored as one word atomic. Returns a
+  // mutable reference to an atomic value.
+  std::atomic<int64_t>& OneWordValue() const;
+  // This is an accessor for a value stored as two words atomic. Returns a
+  // mutable reference to an atomic value.
+  std::atomic<AlignedTwoWords>& TwoWordsValue() const;
+
   // Attempts to parse supplied `value` string. If parsing is successful,
   // returns new value. Otherwise returns nullptr.
   std::unique_ptr<void, DynValueDeleter> TryParse(absl::string_view value,
@@ -488,13 +506,6 @@ class FlagImpl final : public flags_internal::CommandLineFlag {
   // these two cases.
   FlagDefaultSrc default_value_;
 
-  // Atomically mutable flag's state
-
-  // Flag's value. This can be either the atomically stored small value or
-  // pointer to the heap allocated dynamic value. value_storage_kind_ is used
-  // to distinguish these cases.
-  FlagValue value_;
-
   // This is reserved space for an absl::Mutex to guard flag data. It will be
   // initialized in FlagImpl::Init via placement new.
   // We can't use "absl::Mutex data_guard_", since this class is not literal.
@@ -514,8 +525,9 @@ class Flag {
  public:
   constexpr Flag(const char* name, const char* filename, const FlagHelpArg help,
                  const FlagDfltGenFunc default_value_gen)
-      : impl_(name, filename, &FlagOps<T>, help, FlagValue::Kind<T>(),
-              default_value_gen) {}
+      : impl_(name, filename, &FlagOps<T>, help,
+              flags_internal::StorageKind<T>(), default_value_gen),
+        value_() {}
 
   T Get() const {
     // See implementation notes in CommandLineFlag::Get().
@@ -530,7 +542,7 @@ class Flag {
     impl_.AssertValidType(&flags_internal::FlagStaticTypeIdGen<T>);
 #endif
 
-    impl_.Get(&u.value);
+    if (!value_.Get(&u.value)) impl_.Read(&u.value);
     return std::move(u.value);
   }
   void Set(const T& v) {
@@ -556,10 +568,63 @@ class Flag {
  private:
   template <typename U, bool do_register>
   friend class FlagRegistrar;
+
   // Flag's data
+  // The implementation depends on value_ field to be placed exactly after the
+  // impl_ field, so that impl_ can figure out the offset to the value and
+  // access it.
   FlagImpl impl_;
+  FlagValue<T> value_;
 };
 
+///////////////////////////////////////////////////////////////////////////////
+// Implementation of Flag value specific operations routine.
+template <typename T>
+void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3) {
+  switch (op) {
+    case FlagOp::kDelete:
+      delete static_cast<const T*>(v1);
+      return nullptr;
+    case FlagOp::kClone:
+      return new T(*static_cast<const T*>(v1));
+    case FlagOp::kCopy:
+      *static_cast<T*>(v2) = *static_cast<const T*>(v1);
+      return nullptr;
+    case FlagOp::kCopyConstruct:
+      new (v2) T(*static_cast<const T*>(v1));
+      return nullptr;
+    case FlagOp::kSizeof:
+      return reinterpret_cast<void*>(static_cast<uintptr_t>(sizeof(T)));
+    case FlagOp::kStaticTypeId:
+      return reinterpret_cast<void*>(&FlagStaticTypeIdGen<T>);
+    case FlagOp::kParse: {
+      // Initialize the temporary instance of type T based on current value in
+      // destination (which is going to be flag's default value).
+      T temp(*static_cast<T*>(v2));
+      if (!absl::ParseFlag<T>(*static_cast<const absl::string_view*>(v1), &temp,
+                              static_cast<std::string*>(v3))) {
+        return nullptr;
+      }
+      *static_cast<T*>(v2) = std::move(temp);
+      return v2;
+    }
+    case FlagOp::kUnparse:
+      *static_cast<std::string*>(v2) =
+          absl::UnparseFlag<T>(*static_cast<const T*>(v1));
+      return nullptr;
+    case FlagOp::kValueOffset: {
+      // Round sizeof(FlagImp) to a multiple of alignof(FlagValue<T>) to get the
+      // offset of the data.
+      ptrdiff_t round_to = alignof(FlagValue<T>);
+      ptrdiff_t offset =
+          (sizeof(FlagImpl) + round_to - 1) / round_to * round_to;
+      return reinterpret_cast<void*>(offset);
+    }
+  }
+  return nullptr;
+}
+
+///////////////////////////////////////////////////////////////////////////////
 // This class facilitates Flag object registration and tail expression-based
 // flag definition, for example:
 // ABSL_FLAG(int, foo, 42, "Foo help").OnUpdate(NotifyFooWatcher);