1 files changed, 112 insertions, 65 deletions

diff --git a/absl/flags/internal/flag.cc b/absl/flags/internal/flag.cc
index 1502e7f1..55892d77 100644
--- a/absl/flags/internal/flag.cc
+++ b/absl/flags/internal/flag.cc
@@ -30,6 +30,7 @@
 #include "absl/base/call_once.h"
 #include "absl/base/casts.h"
 #include "absl/base/config.h"
+#include "absl/base/dynamic_annotations.h"
 #include "absl/base/optimization.h"
 #include "absl/flags/config.h"
 #include "absl/flags/internal/commandlineflag.h"
@@ -96,7 +97,8 @@ class FlagState : public flags_internal::FlagStateInterface {
         counter_(counter) {}
 
   ~FlagState() override {
-    if (flag_impl_.ValueStorageKind() != FlagValueStorageKind::kAlignedBuffer)
+    if (flag_impl_.ValueStorageKind() != FlagValueStorageKind::kAlignedBuffer &&
+        flag_impl_.ValueStorageKind() != FlagValueStorageKind::kSequenceLocked)
       return;
     flags_internal::Delete(flag_impl_.op_, value_.heap_allocated);
   }
@@ -118,11 +120,9 @@ class FlagState : public flags_internal::FlagStateInterface {
   union SavedValue {
     explicit SavedValue(void* v) : heap_allocated(v) {}
     explicit SavedValue(int64_t v) : one_word(v) {}
-    explicit SavedValue(flags_internal::AlignedTwoWords v) : two_words(v) {}
 
     void* heap_allocated;
     int64_t one_word;
-    flags_internal::AlignedTwoWords two_words;
   } value_;
   bool modified_;
   bool on_command_line_;
@@ -146,12 +146,7 @@ void FlagImpl::Init() {
   auto def_kind = static_cast<FlagDefaultKind>(def_kind_);
 
   switch (ValueStorageKind()) {
-    case FlagValueStorageKind::kAlignedBuffer:
-      // For this storage kind the default_value_ always points to gen_func
-      // during initialization.
-      assert(def_kind == FlagDefaultKind::kGenFunc);
-      (*default_value_.gen_func)(AlignedBufferValue());
-      break;
+    case FlagValueStorageKind::kValueAndInitBit:
     case FlagValueStorageKind::kOneWordAtomic: {
       alignas(int64_t) std::array<char, sizeof(int64_t)> buf{};
       if (def_kind == FlagDefaultKind::kGenFunc) {
@@ -160,21 +155,33 @@ void FlagImpl::Init() {
         assert(def_kind != FlagDefaultKind::kDynamicValue);
         std::memcpy(buf.data(), &default_value_, Sizeof(op_));
       }
+      if (ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit) {
+        // We presume here the memory layout of FlagValueAndInitBit struct.
+        uint8_t initialized = 1;
+        std::memcpy(buf.data() + Sizeof(op_), &initialized,
+                    sizeof(initialized));
+      }
+      // Type can contain valid uninitialized bits, e.g. padding.
+      ABSL_ANNOTATE_MEMORY_IS_INITIALIZED(buf.data(), buf.size());
       OneWordValue().store(absl::bit_cast<int64_t>(buf),
                            std::memory_order_release);
       break;
     }
-    case FlagValueStorageKind::kTwoWordsAtomic: {
+    case FlagValueStorageKind::kSequenceLocked: {
       // For this storage kind the default_value_ always points to gen_func
       // during initialization.
       assert(def_kind == FlagDefaultKind::kGenFunc);
-      alignas(AlignedTwoWords) std::array<char, sizeof(AlignedTwoWords)> buf{};
-      (*default_value_.gen_func)(buf.data());
-      auto atomic_value = absl::bit_cast<AlignedTwoWords>(buf);
-      TwoWordsValue().store(atomic_value, std::memory_order_release);
+      (*default_value_.gen_func)(AtomicBufferValue());
       break;
     }
+    case FlagValueStorageKind::kAlignedBuffer:
+      // For this storage kind the default_value_ always points to gen_func
+      // during initialization.
+      assert(def_kind == FlagDefaultKind::kGenFunc);
+      (*default_value_.gen_func)(AlignedBufferValue());
+      break;
   }
+  seq_lock_.MarkInitialized();
 }
 
 absl::Mutex* FlagImpl::DataGuard() const {
@@ -201,7 +208,7 @@ void FlagImpl::AssertValidType(FlagFastTypeId rhs_type_id,
 
   if (lhs_runtime_type_id == rhs_runtime_type_id) return;
 
-#if defined(ABSL_FLAGS_INTERNAL_HAS_RTTI)
+#ifdef ABSL_INTERNAL_HAS_RTTI
   if (*lhs_runtime_type_id == *rhs_runtime_type_id) return;
 #endif
 
@@ -229,25 +236,25 @@ std::unique_ptr<void, DynValueDeleter> FlagImpl::MakeInitValue() const {
 
 void FlagImpl::StoreValue(const void* src) {
   switch (ValueStorageKind()) {
-    case FlagValueStorageKind::kAlignedBuffer:
-      Copy(op_, src, AlignedBufferValue());
-      break;
+    case FlagValueStorageKind::kValueAndInitBit:
     case FlagValueStorageKind::kOneWordAtomic: {
-      int64_t one_word_val = 0;
+      // Load the current value to avoid setting 'init' bit manualy.
+      int64_t one_word_val = OneWordValue().load(std::memory_order_acquire);
       std::memcpy(&one_word_val, src, Sizeof(op_));
       OneWordValue().store(one_word_val, std::memory_order_release);
+      seq_lock_.IncrementModificationCount();
       break;
     }
-    case FlagValueStorageKind::kTwoWordsAtomic: {
-      AlignedTwoWords two_words_val{0, 0};
-      std::memcpy(&two_words_val, src, Sizeof(op_));
-      TwoWordsValue().store(two_words_val, std::memory_order_release);
+    case FlagValueStorageKind::kSequenceLocked: {
+      seq_lock_.Write(AtomicBufferValue(), src, Sizeof(op_));
       break;
     }
+    case FlagValueStorageKind::kAlignedBuffer:
+      Copy(op_, src, AlignedBufferValue());
+      seq_lock_.IncrementModificationCount();
+      break;
   }
-
   modified_ = true;
-  ++counter_;
   InvokeCallback();
 }
 
@@ -266,6 +273,10 @@ FlagFastTypeId FlagImpl::TypeId() const {
   return flags_internal::FastTypeId(op_);
 }
 
+int64_t FlagImpl::ModificationCount() const {
+  return seq_lock_.ModificationCount();
+}
+
 bool FlagImpl::IsSpecifiedOnCommandLine() const {
   absl::MutexLock l(DataGuard());
   return on_command_line_;
@@ -281,21 +292,22 @@ std::string FlagImpl::DefaultValue() const {
 std::string FlagImpl::CurrentValue() const {
   auto* guard = DataGuard();  // Make sure flag initialized
   switch (ValueStorageKind()) {
-    case FlagValueStorageKind::kAlignedBuffer: {
-      absl::MutexLock l(guard);
-      return flags_internal::Unparse(op_, AlignedBufferValue());
-    }
+    case FlagValueStorageKind::kValueAndInitBit:
     case FlagValueStorageKind::kOneWordAtomic: {
       const auto one_word_val =
           absl::bit_cast<std::array<char, sizeof(int64_t)>>(
               OneWordValue().load(std::memory_order_acquire));
       return flags_internal::Unparse(op_, one_word_val.data());
     }
-    case FlagValueStorageKind::kTwoWordsAtomic: {
-      const auto two_words_val =
-          absl::bit_cast<std::array<char, sizeof(AlignedTwoWords)>>(
-              TwoWordsValue().load(std::memory_order_acquire));
-      return flags_internal::Unparse(op_, two_words_val.data());
+    case FlagValueStorageKind::kSequenceLocked: {
+      std::unique_ptr<void, DynValueDeleter> cloned(flags_internal::Alloc(op_),
+                                                    DynValueDeleter{op_});
+      ReadSequenceLockedData(cloned.get());
+      return flags_internal::Unparse(op_, cloned.get());
+    }
+    case FlagValueStorageKind::kAlignedBuffer: {
+      absl::MutexLock l(guard);
+      return flags_internal::Unparse(op_, AlignedBufferValue());
     }
   }
 
@@ -342,20 +354,26 @@ std::unique_ptr<FlagStateInterface> FlagImpl::SaveState() {
   bool modified = modified_;
   bool on_command_line = on_command_line_;
   switch (ValueStorageKind()) {
-    case FlagValueStorageKind::kAlignedBuffer: {
-      return absl::make_unique<FlagState>(
-          *this, flags_internal::Clone(op_, AlignedBufferValue()), modified,
-          on_command_line, counter_);
-    }
+    case FlagValueStorageKind::kValueAndInitBit:
     case FlagValueStorageKind::kOneWordAtomic: {
       return absl::make_unique<FlagState>(
           *this, OneWordValue().load(std::memory_order_acquire), modified,
-          on_command_line, counter_);
+          on_command_line, ModificationCount());
+    }
+    case FlagValueStorageKind::kSequenceLocked: {
+      void* cloned = flags_internal::Alloc(op_);
+      // Read is guaranteed to be successful because we hold the lock.
+      bool success =
+          seq_lock_.TryRead(cloned, AtomicBufferValue(), Sizeof(op_));
+      assert(success);
+      static_cast<void>(success);
+      return absl::make_unique<FlagState>(*this, cloned, modified,
+                                          on_command_line, ModificationCount());
     }
-    case FlagValueStorageKind::kTwoWordsAtomic: {
+    case FlagValueStorageKind::kAlignedBuffer: {
       return absl::make_unique<FlagState>(
-          *this, TwoWordsValue().load(std::memory_order_acquire), modified,
-          on_command_line, counter_);
+          *this, flags_internal::Clone(op_, AlignedBufferValue()), modified,
+          on_command_line, ModificationCount());
     }
   }
   return nullptr;
@@ -363,20 +381,18 @@ std::unique_ptr<FlagStateInterface> FlagImpl::SaveState() {
 
 bool FlagImpl::RestoreState(const FlagState& flag_state) {
   absl::MutexLock l(DataGuard());
-
-  if (flag_state.counter_ == counter_) {
+  if (flag_state.counter_ == ModificationCount()) {
     return false;
   }
 
   switch (ValueStorageKind()) {
-    case FlagValueStorageKind::kAlignedBuffer:
-      StoreValue(flag_state.value_.heap_allocated);
-      break;
+    case FlagValueStorageKind::kValueAndInitBit:
     case FlagValueStorageKind::kOneWordAtomic:
       StoreValue(&flag_state.value_.one_word);
       break;
-    case FlagValueStorageKind::kTwoWordsAtomic:
-      StoreValue(&flag_state.value_.two_words);
+    case FlagValueStorageKind::kSequenceLocked:
+    case FlagValueStorageKind::kAlignedBuffer:
+      StoreValue(flag_state.value_.heap_allocated);
       break;
   }
 
@@ -400,14 +416,15 @@ void* FlagImpl::AlignedBufferValue() const {
   return OffsetValue<void>();
 }
 
-std::atomic<int64_t>& FlagImpl::OneWordValue() const {
-  assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic);
-  return OffsetValue<FlagOneWordValue>()->value;
+std::atomic<uint64_t>* FlagImpl::AtomicBufferValue() const {
+  assert(ValueStorageKind() == FlagValueStorageKind::kSequenceLocked);
+  return OffsetValue<std::atomic<uint64_t>>();
 }
 
-std::atomic<AlignedTwoWords>& FlagImpl::TwoWordsValue() const {
-  assert(ValueStorageKind() == FlagValueStorageKind::kTwoWordsAtomic);
-  return OffsetValue<FlagTwoWordsValue>()->value;
+std::atomic<int64_t>& FlagImpl::OneWordValue() const {
+  assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic ||
+         ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit);
+  return OffsetValue<FlagOneWordValue>()->value;
 }
 
 // Attempts to parse supplied `value` string using parsing routine in the `flag`
@@ -432,26 +449,56 @@ std::unique_ptr<void, DynValueDeleter> FlagImpl::TryParse(
 void FlagImpl::Read(void* dst) const {
   auto* guard = DataGuard();  // Make sure flag initialized
   switch (ValueStorageKind()) {
-    case FlagValueStorageKind::kAlignedBuffer: {
-      absl::MutexLock l(guard);
-      flags_internal::CopyConstruct(op_, AlignedBufferValue(), dst);
-      break;
-    }
+    case FlagValueStorageKind::kValueAndInitBit:
     case FlagValueStorageKind::kOneWordAtomic: {
       const int64_t one_word_val =
           OneWordValue().load(std::memory_order_acquire);
       std::memcpy(dst, &one_word_val, Sizeof(op_));
       break;
     }
-    case FlagValueStorageKind::kTwoWordsAtomic: {
-      const AlignedTwoWords two_words_val =
-          TwoWordsValue().load(std::memory_order_acquire);
-      std::memcpy(dst, &two_words_val, Sizeof(op_));
+    case FlagValueStorageKind::kSequenceLocked: {
+      ReadSequenceLockedData(dst);
+      break;
+    }
+    case FlagValueStorageKind::kAlignedBuffer: {
+      absl::MutexLock l(guard);
+      flags_internal::CopyConstruct(op_, AlignedBufferValue(), dst);
       break;
     }
   }
 }
 
+int64_t FlagImpl::ReadOneWord() const {
+  assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic ||
+         ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit);
+  auto* guard = DataGuard();  // Make sure flag initialized
+  (void)guard;
+  return OneWordValue().load(std::memory_order_acquire);
+}
+
+bool FlagImpl::ReadOneBool() const {
+  assert(ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit);
+  auto* guard = DataGuard();  // Make sure flag initialized
+  (void)guard;
+  return absl::bit_cast<FlagValueAndInitBit<bool>>(
+             OneWordValue().load(std::memory_order_acquire))
+      .value;
+}
+
+void FlagImpl::ReadSequenceLockedData(void* dst) const {
+  int size = Sizeof(op_);
+  // Attempt to read using the sequence lock.
+  if (ABSL_PREDICT_TRUE(seq_lock_.TryRead(dst, AtomicBufferValue(), size))) {
+    return;
+  }
+  // We failed due to contention. Acquire the lock to prevent contention
+  // and try again.
+  absl::ReaderMutexLock l(DataGuard());
+  bool success = seq_lock_.TryRead(dst, AtomicBufferValue(), size);
+  assert(success);
+  static_cast<void>(success);
+}
+
 void FlagImpl::Write(const void* src) {
   absl::MutexLock l(DataGuard());