Export of internal Abseil changes

--
04cb3b22497190170aa5b774e98080c5de2ba60b by Abseil Team <absl-team@google.com>:

Alternative bit mixer for LowLevelHash on ARM

LowLevelHash's bit-mixer is inefficient on ARM because it calculates a 128-bit product of two 64-bit numbers. On ARM, this requires a sequence of two instructions with a high combined latency and poor throughput. This change provides alternative bit-mixing code for ARM that uses only 64-bit arithmetic (multiplication, xor, and left-shifts) and speeds things up considerably.

The bit-mixing code for ARM was inspired by by Woothash[1] and xxh3[1]. Once I landed on a sequence of operations that provided good mixing, I used a test harness to search for the combination of shift / rotate factors that provided the best mixing, as indicated by SMHasher hash quality tests. The new mixing code passes 13 out of 15 of the hash quality test suites in SMHasher, with the two failures being in the noise range: e.g. 1 collision vs. zero expected in a keyset of ~8m keys.

  [1]: https://github.com/tommyettinger/waterhash/blob/49f5cf0b63b9/woothash.h#L16-L20
  [2]: https://github.com/Cyan4973/xxHash/blob/6853ddc36e46/xxhash.h#L3240-L3265

PiperOrigin-RevId: 391833008

--
17a4de1f9d623155c75b19285d414cd55a487cd6 by Saleem Abdulrasool <abdulras@google.com>:

debugging: add support for unwinding on RISCV Linux

This adds partial support for unwinding the RISCV call stack.  It is largely
duplicated from the AArch64 support with alterations for the ELF RISCV psABI.
This covers RISCV64 and RISCV32, though not the ILP32E calling convention.

PiperOrigin-RevId: 391818522

--
32c93e449327b2cea32b32f6365e84b420fe1ed3 by Gennadiy Rozental <rogeeff@google.com>:

New storage for types smaller than 8 bytes.

Also adding new read interface for types smaller than or rqual to 8 bytes to avoid passing the pointer.

PiperOrigin-RevId: 391726822

--
e987ac08a7787801cbfc7d7c96649e97fa8cff1a by Abseil Team <absl-team@google.com>:

Extern template `find_first_non_full` to reduce linkage size for TU with single not inlined function.

PiperOrigin-RevId: 391718862

--
73af9bfcb5bf045089133e18bbd20eb5bb699172 by Gennadiy Rozental <rogeeff@google.com>:

Make most non-mutable most int128 methods and friend free functions constexpr.

Some functions are implemented offline (at least in some configurations) and can't be made constexpr. Mutable methods can't be made constexpr until we drop c++11 support.

Fixes #978

PiperOrigin-RevId: 391706535
GitOrigin-RevId: 04cb3b22497190170aa5b774e98080c5de2ba60b
Change-Id: If051fad5ff004e2e82fa53618fc04a6fe3d2d4be

3 files changed, 157 insertions, 42 deletions

diff --git a/absl/flags/flag_test.cc b/absl/flags/flag_test.cc
index ba81317b..6e974a5b 100644
--- a/absl/flags/flag_test.cc
+++ b/absl/flags/flag_test.cc
@@ -103,9 +103,9 @@ struct S2 {
 
 TEST_F(FlagTest, Traits) {
   EXPECT_EQ(flags::StorageKind<int>(),
-            flags::FlagValueStorageKind::kOneWordAtomic);
+            flags::FlagValueStorageKind::kValueAndInitBit);
   EXPECT_EQ(flags::StorageKind<bool>(),
-            flags::FlagValueStorageKind::kOneWordAtomic);
+            flags::FlagValueStorageKind::kValueAndInitBit);
   EXPECT_EQ(flags::StorageKind<double>(),
             flags::FlagValueStorageKind::kOneWordAtomic);
   EXPECT_EQ(flags::StorageKind<int64_t>(),
@@ -724,6 +724,8 @@ ABSL_FLAG(CustomUDT, test_flag_custom_udt, CustomUDT(), "test flag custom UDT");
 namespace {
 
 TEST_F(FlagTest, TestCustomUDT) {
+  EXPECT_EQ(flags::StorageKind<CustomUDT>(),
+            flags::FlagValueStorageKind::kOneWordAtomic);
   EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_custom_udt), CustomUDT(1, 1));
   absl::SetFlag(&FLAGS_test_flag_custom_udt, CustomUDT(2, 3));
   EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_custom_udt), CustomUDT(2, 3));
@@ -944,3 +946,34 @@ TEST_F(FlagTest, TestNonTriviallyCopyableUDT) {
 }
 
 }  // namespace
+
+// --------------------------------------------------------------------
+
+namespace {
+
+enum TestE { A = 1, B = 2, C = 3 };
+
+struct EnumWrapper {
+  EnumWrapper() : e(A) {}
+
+  TestE e;
+};
+
+bool AbslParseFlag(absl::string_view, EnumWrapper*, std::string*) {
+  return true;
+}
+std::string AbslUnparseFlag(const EnumWrapper&) { return ""; }
+
+}  // namespace
+
+ABSL_FLAG(EnumWrapper, test_enum_wrapper_flag, {}, "help");
+
+TEST_F(FlagTest, TesTypeWrappingEnum) {
+  EnumWrapper value = absl::GetFlag(FLAGS_test_enum_wrapper_flag);
+  EXPECT_EQ(value.e, A);
+
+  value.e = B;
+  absl::SetFlag(&FLAGS_test_enum_wrapper_flag, value);
+  value = absl::GetFlag(FLAGS_test_enum_wrapper_flag);
+  EXPECT_EQ(value.e, B);
+}
diff --git a/absl/flags/internal/flag.cc b/absl/flags/internal/flag.cc
index f83c1fe7..1515022d 100644
--- a/absl/flags/internal/flag.cc
+++ b/absl/flags/internal/flag.cc
@@ -145,12 +145,7 @@ void FlagImpl::Init() {
   auto def_kind = static_cast<FlagDefaultKind>(def_kind_);
 
   switch (ValueStorageKind()) {
-    case FlagValueStorageKind::kAlignedBuffer:
-      // For this storage kind the default_value_ always points to gen_func
-      // during initialization.
-      assert(def_kind == FlagDefaultKind::kGenFunc);
-      (*default_value_.gen_func)(AlignedBufferValue());
-      break;
+    case FlagValueStorageKind::kValueAndInitBit:
     case FlagValueStorageKind::kOneWordAtomic: {
       alignas(int64_t) std::array<char, sizeof(int64_t)> buf{};
       if (def_kind == FlagDefaultKind::kGenFunc) {
@@ -159,6 +154,12 @@ void FlagImpl::Init() {
         assert(def_kind != FlagDefaultKind::kDynamicValue);
         std::memcpy(buf.data(), &default_value_, Sizeof(op_));
       }
+      if (ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit) {
+        // We presume here the memory layout of FlagValueAndInitBit struct.
+        uint8_t initialized = 1;
+        std::memcpy(buf.data() + Sizeof(op_), &initialized,
+                    sizeof(initialized));
+      }
       OneWordValue().store(absl::bit_cast<int64_t>(buf),
                            std::memory_order_release);
       break;
@@ -170,6 +171,12 @@ void FlagImpl::Init() {
       (*default_value_.gen_func)(AtomicBufferValue());
       break;
     }
+    case FlagValueStorageKind::kAlignedBuffer:
+      // For this storage kind the default_value_ always points to gen_func
+      // during initialization.
+      assert(def_kind == FlagDefaultKind::kGenFunc);
+      (*default_value_.gen_func)(AlignedBufferValue());
+      break;
   }
   seq_lock_.MarkInitialized();
 }
@@ -226,12 +233,10 @@ std::unique_ptr<void, DynValueDeleter> FlagImpl::MakeInitValue() const {
 
 void FlagImpl::StoreValue(const void* src) {
   switch (ValueStorageKind()) {
-    case FlagValueStorageKind::kAlignedBuffer:
-      Copy(op_, src, AlignedBufferValue());
-      seq_lock_.IncrementModificationCount();
-      break;
+    case FlagValueStorageKind::kValueAndInitBit:
     case FlagValueStorageKind::kOneWordAtomic: {
-      int64_t one_word_val = 0;
+      // Load the current value to avoid setting 'init' bit manualy.
+      int64_t one_word_val = OneWordValue().load(std::memory_order_acquire);
       std::memcpy(&one_word_val, src, Sizeof(op_));
       OneWordValue().store(one_word_val, std::memory_order_release);
       seq_lock_.IncrementModificationCount();
@@ -241,6 +246,10 @@ void FlagImpl::StoreValue(const void* src) {
       seq_lock_.Write(AtomicBufferValue(), src, Sizeof(op_));
       break;
     }
+    case FlagValueStorageKind::kAlignedBuffer:
+      Copy(op_, src, AlignedBufferValue());
+      seq_lock_.IncrementModificationCount();
+      break;
   }
   modified_ = true;
   InvokeCallback();
@@ -280,10 +289,7 @@ std::string FlagImpl::DefaultValue() const {
 std::string FlagImpl::CurrentValue() const {
   auto* guard = DataGuard();  // Make sure flag initialized
   switch (ValueStorageKind()) {
-    case FlagValueStorageKind::kAlignedBuffer: {
-      absl::MutexLock l(guard);
-      return flags_internal::Unparse(op_, AlignedBufferValue());
-    }
+    case FlagValueStorageKind::kValueAndInitBit:
     case FlagValueStorageKind::kOneWordAtomic: {
       const auto one_word_val =
           absl::bit_cast<std::array<char, sizeof(int64_t)>>(
@@ -296,6 +302,10 @@ std::string FlagImpl::CurrentValue() const {
       ReadSequenceLockedData(cloned.get());
       return flags_internal::Unparse(op_, cloned.get());
     }
+    case FlagValueStorageKind::kAlignedBuffer: {
+      absl::MutexLock l(guard);
+      return flags_internal::Unparse(op_, AlignedBufferValue());
+    }
   }
 
   return "";
@@ -341,11 +351,7 @@ std::unique_ptr<FlagStateInterface> FlagImpl::SaveState() {
   bool modified = modified_;
   bool on_command_line = on_command_line_;
   switch (ValueStorageKind()) {
-    case FlagValueStorageKind::kAlignedBuffer: {
-      return absl::make_unique<FlagState>(
-          *this, flags_internal::Clone(op_, AlignedBufferValue()), modified,
-          on_command_line, ModificationCount());
-    }
+    case FlagValueStorageKind::kValueAndInitBit:
     case FlagValueStorageKind::kOneWordAtomic: {
       return absl::make_unique<FlagState>(
           *this, OneWordValue().load(std::memory_order_acquire), modified,
@@ -361,6 +367,11 @@ std::unique_ptr<FlagStateInterface> FlagImpl::SaveState() {
       return absl::make_unique<FlagState>(*this, cloned, modified,
                                           on_command_line, ModificationCount());
     }
+    case FlagValueStorageKind::kAlignedBuffer: {
+      return absl::make_unique<FlagState>(
+          *this, flags_internal::Clone(op_, AlignedBufferValue()), modified,
+          on_command_line, ModificationCount());
+    }
   }
   return nullptr;
 }
@@ -372,13 +383,14 @@ bool FlagImpl::RestoreState(const FlagState& flag_state) {
   }
 
   switch (ValueStorageKind()) {
-    case FlagValueStorageKind::kAlignedBuffer:
-    case FlagValueStorageKind::kSequenceLocked:
-      StoreValue(flag_state.value_.heap_allocated);
-      break;
+    case FlagValueStorageKind::kValueAndInitBit:
     case FlagValueStorageKind::kOneWordAtomic:
       StoreValue(&flag_state.value_.one_word);
       break;
+    case FlagValueStorageKind::kSequenceLocked:
+    case FlagValueStorageKind::kAlignedBuffer:
+      StoreValue(flag_state.value_.heap_allocated);
+      break;
   }
 
   modified_ = flag_state.modified_;
@@ -407,7 +419,8 @@ std::atomic<uint64_t>* FlagImpl::AtomicBufferValue() const {
 }
 
 std::atomic<int64_t>& FlagImpl::OneWordValue() const {
-  assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic);
+  assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic ||
+         ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit);
   return OffsetValue<FlagOneWordValue>()->value;
 }
 
@@ -433,11 +446,7 @@ std::unique_ptr<void, DynValueDeleter> FlagImpl::TryParse(
 void FlagImpl::Read(void* dst) const {
   auto* guard = DataGuard();  // Make sure flag initialized
   switch (ValueStorageKind()) {
-    case FlagValueStorageKind::kAlignedBuffer: {
-      absl::MutexLock l(guard);
-      flags_internal::CopyConstruct(op_, AlignedBufferValue(), dst);
-      break;
-    }
+    case FlagValueStorageKind::kValueAndInitBit:
     case FlagValueStorageKind::kOneWordAtomic: {
       const int64_t one_word_val =
           OneWordValue().load(std::memory_order_acquire);
@@ -448,9 +457,31 @@ void FlagImpl::Read(void* dst) const {
       ReadSequenceLockedData(dst);
       break;
     }
+    case FlagValueStorageKind::kAlignedBuffer: {
+      absl::MutexLock l(guard);
+      flags_internal::CopyConstruct(op_, AlignedBufferValue(), dst);
+      break;
+    }
   }
 }
 
+int64_t FlagImpl::ReadOneWord() const {
+  assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic ||
+         ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit);
+  auto* guard = DataGuard();  // Make sure flag initialized
+  (void)guard;
+  return OneWordValue().load(std::memory_order_acquire);
+}
+
+bool FlagImpl::ReadOneBool() const {
+  assert(ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit);
+  auto* guard = DataGuard();  // Make sure flag initialized
+  (void)guard;
+  return absl::bit_cast<FlagValueAndInitBit<bool>>(
+             OneWordValue().load(std::memory_order_acquire))
+      .value;
+}
+
 void FlagImpl::ReadSequenceLockedData(void* dst) const {
   int size = Sizeof(op_);
   // Attempt to read using the sequence lock.
diff --git a/absl/flags/internal/flag.h b/absl/flags/internal/flag.h
index e6bade0a..8636fadc 100644
--- a/absl/flags/internal/flag.h
+++ b/absl/flags/internal/flag.h
@@ -29,6 +29,7 @@
 
 #include "absl/base/attributes.h"
 #include "absl/base/call_once.h"
+#include "absl/base/casts.h"
 #include "absl/base/config.h"
 #include "absl/base/optimization.h"
 #include "absl/base/thread_annotations.h"
@@ -305,48 +306,71 @@ constexpr FlagDefaultArg DefaultArg(char) {
 constexpr int64_t UninitializedFlagValue() { return 0xababababababababll; }
 
 template <typename T>
+using FlagUseValueAndInitBitStorage = std::integral_constant<
+    bool, absl::type_traits_internal::is_trivially_copyable<T>::value &&
+              std::is_default_constructible<T>::value && (sizeof(T) < 8)>;
+
+template <typename T>
 using FlagUseOneWordStorage = std::integral_constant<
     bool, absl::type_traits_internal::is_trivially_copyable<T>::value &&
               (sizeof(T) <= 8)>;
 
 template <class T>
-using FlagShouldUseSequenceLock = std::integral_constant<
+using FlagUseSequenceLockStorage = std::integral_constant<
     bool, absl::type_traits_internal::is_trivially_copyable<T>::value &&
               (sizeof(T) > 8)>;
 
 enum class FlagValueStorageKind : uint8_t {
-  kAlignedBuffer = 0,
+  kValueAndInitBit = 0,
   kOneWordAtomic = 1,
   kSequenceLocked = 2,
+  kAlignedBuffer = 3,
 };
 
 template <typename T>
 static constexpr FlagValueStorageKind StorageKind() {
-  return FlagUseOneWordStorage<T>::value ? FlagValueStorageKind::kOneWordAtomic
-         : FlagShouldUseSequenceLock<T>::value
+  return FlagUseValueAndInitBitStorage<T>::value
+             ? FlagValueStorageKind::kValueAndInitBit
+         : FlagUseOneWordStorage<T>::value
+             ? FlagValueStorageKind::kOneWordAtomic
+         : FlagUseSequenceLockStorage<T>::value
              ? FlagValueStorageKind::kSequenceLocked
              : FlagValueStorageKind::kAlignedBuffer;
 }
 
 struct FlagOneWordValue {
-  constexpr FlagOneWordValue() : value(UninitializedFlagValue()) {}
-
+  constexpr explicit FlagOneWordValue(int64_t v) : value(v) {}
   std::atomic<int64_t> value;
 };
 
+template <typename T>
+struct alignas(8) FlagValueAndInitBit {
+  T value;
+  // Use an int instead of a bool to guarantee that a non-zero value has
+  // a bit set.
+  uint8_t init;
+};
+
 template <typename T,
           FlagValueStorageKind Kind = flags_internal::StorageKind<T>()>
 struct FlagValue;
 
 template <typename T>
-struct FlagValue<T, FlagValueStorageKind::kAlignedBuffer> {
-  bool Get(const SequenceLock&, T&) const { return false; }
-
-  alignas(T) char value[sizeof(T)];
+struct FlagValue<T, FlagValueStorageKind::kValueAndInitBit> : FlagOneWordValue {
+  constexpr FlagValue() : FlagOneWordValue(0) {}
+  bool Get(const SequenceLock&, T& dst) const {
+    int64_t storage = value.load(std::memory_order_acquire);
+    if (ABSL_PREDICT_FALSE(storage == 0)) {
+      return false;
+    }
+    dst = absl::bit_cast<FlagValueAndInitBit<T>>(storage).value;
+    return true;
+  }
 };
 
 template <typename T>
 struct FlagValue<T, FlagValueStorageKind::kOneWordAtomic> : FlagOneWordValue {
+  constexpr FlagValue() : FlagOneWordValue(UninitializedFlagValue()) {}
   bool Get(const SequenceLock&, T& dst) const {
     int64_t one_word_val = value.load(std::memory_order_acquire);
     if (ABSL_PREDICT_FALSE(one_word_val == UninitializedFlagValue())) {
@@ -370,6 +394,13 @@ struct FlagValue<T, FlagValueStorageKind::kSequenceLocked> {
       std::atomic<uint64_t>) std::atomic<uint64_t> value_words[kNumWords];
 };
 
+template <typename T>
+struct FlagValue<T, FlagValueStorageKind::kAlignedBuffer> {
+  bool Get(const SequenceLock&, T&) const { return false; }
+
+  alignas(T) char value[sizeof(T)];
+};
+
 ///////////////////////////////////////////////////////////////////////////////
 // Flag callback auxiliary structs.
 
@@ -415,7 +446,27 @@ class FlagImpl final : public CommandLineFlag {
         data_guard_{} {}
 
   // Constant access methods
+  int64_t ReadOneWord() const ABSL_LOCKS_EXCLUDED(*DataGuard());
+  bool ReadOneBool() const ABSL_LOCKS_EXCLUDED(*DataGuard());
   void Read(void* dst) const override ABSL_LOCKS_EXCLUDED(*DataGuard());
+  void Read(bool* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) {
+    *value = ReadOneBool();
+  }
+  template <typename T,
+            absl::enable_if_t<flags_internal::StorageKind<T>() ==
+                                  FlagValueStorageKind::kOneWordAtomic,
+                              int> = 0>
+  void Read(T* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) {
+    int64_t v = ReadOneWord();
+    std::memcpy(value, static_cast<const void*>(&v), sizeof(T));
+  }
+  template <typename T,
+            typename std::enable_if<flags_internal::StorageKind<T>() ==
+                                        FlagValueStorageKind::kValueAndInitBit,
+                                    int>::type = 0>
+  void Read(T* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) {
+    *value = absl::bit_cast<FlagValueAndInitBit<T>>(ReadOneWord()).value;
+  }
 
   // Mutating access methods
   void Write(const void* src) ABSL_LOCKS_EXCLUDED(*DataGuard());