Export of internal Abseil changes

--
04cb3b22497190170aa5b774e98080c5de2ba60b by Abseil Team <absl-team@google.com>:

Alternative bit mixer for LowLevelHash on ARM

LowLevelHash's bit-mixer is inefficient on ARM because it calculates a 128-bit product of two 64-bit numbers. On ARM, this requires a sequence of two instructions with a high combined latency and poor throughput. This change provides alternative bit-mixing code for ARM that uses only 64-bit arithmetic (multiplication, xor, and left-shifts) and speeds things up considerably.

The bit-mixing code for ARM was inspired by by Woothash[1] and xxh3[1]. Once I landed on a sequence of operations that provided good mixing, I used a test harness to search for the combination of shift / rotate factors that provided the best mixing, as indicated by SMHasher hash quality tests. The new mixing code passes 13 out of 15 of the hash quality test suites in SMHasher, with the two failures being in the noise range: e.g. 1 collision vs. zero expected in a keyset of ~8m keys.

  [1]: https://github.com/tommyettinger/waterhash/blob/49f5cf0b63b9/woothash.h#L16-L20
  [2]: https://github.com/Cyan4973/xxHash/blob/6853ddc36e46/xxhash.h#L3240-L3265

PiperOrigin-RevId: 391833008

--
17a4de1f9d623155c75b19285d414cd55a487cd6 by Saleem Abdulrasool <abdulras@google.com>:

debugging: add support for unwinding on RISCV Linux

This adds partial support for unwinding the RISCV call stack.  It is largely
duplicated from the AArch64 support with alterations for the ELF RISCV psABI.
This covers RISCV64 and RISCV32, though not the ILP32E calling convention.

PiperOrigin-RevId: 391818522

--
32c93e449327b2cea32b32f6365e84b420fe1ed3 by Gennadiy Rozental <rogeeff@google.com>:

New storage for types smaller than 8 bytes.

Also adding new read interface for types smaller than or rqual to 8 bytes to avoid passing the pointer.

PiperOrigin-RevId: 391726822

--
e987ac08a7787801cbfc7d7c96649e97fa8cff1a by Abseil Team <absl-team@google.com>:

Extern template `find_first_non_full` to reduce linkage size for TU with single not inlined function.

PiperOrigin-RevId: 391718862

--
73af9bfcb5bf045089133e18bbd20eb5bb699172 by Gennadiy Rozental <rogeeff@google.com>:

Make most non-mutable most int128 methods and friend free functions constexpr.

Some functions are implemented offline (at least in some configurations) and can't be made constexpr. Mutable methods can't be made constexpr until we drop c++11 support.

Fixes #978

PiperOrigin-RevId: 391706535
GitOrigin-RevId: 04cb3b22497190170aa5b774e98080c5de2ba60b
Change-Id: If051fad5ff004e2e82fa53618fc04a6fe3d2d4be

3 files changed, 153 insertions, 152 deletions

diff --git a/absl/numeric/int128.h b/absl/numeric/int128.h
index a9cbe489..c7ad96be 100644
--- a/absl/numeric/int128.h
+++ b/absl/numeric/int128.h
@@ -586,10 +586,10 @@ inline uint128& uint128::operator=(int128 v) {
 
 // Arithmetic operators.
 
-uint128 operator<<(uint128 lhs, int amount);
-uint128 operator>>(uint128 lhs, int amount);
-uint128 operator+(uint128 lhs, uint128 rhs);
-uint128 operator-(uint128 lhs, uint128 rhs);
+constexpr uint128 operator<<(uint128 lhs, int amount);
+constexpr uint128 operator>>(uint128 lhs, int amount);
+constexpr uint128 operator+(uint128 lhs, uint128 rhs);
+constexpr uint128 operator-(uint128 lhs, uint128 rhs);
 uint128 operator*(uint128 lhs, uint128 rhs);
 uint128 operator/(uint128 lhs, uint128 rhs);
 uint128 operator%(uint128 lhs, uint128 rhs);
@@ -786,7 +786,7 @@ inline uint128::operator long double() const {
 
 // Comparison operators.
 
-inline bool operator==(uint128 lhs, uint128 rhs) {
+constexpr bool operator==(uint128 lhs, uint128 rhs) {
 #if defined(ABSL_HAVE_INTRINSIC_INT128)
   return static_cast<unsigned __int128>(lhs) ==
          static_cast<unsigned __int128>(rhs);
@@ -796,11 +796,9 @@ inline bool operator==(uint128 lhs, uint128 rhs) {
 #endif
 }
 
-inline bool operator!=(uint128 lhs, uint128 rhs) {
-  return !(lhs == rhs);
-}
+constexpr bool operator!=(uint128 lhs, uint128 rhs) { return !(lhs == rhs); }
 
-inline bool operator<(uint128 lhs, uint128 rhs) {
+constexpr bool operator<(uint128 lhs, uint128 rhs) {
 #ifdef ABSL_HAVE_INTRINSIC_INT128
   return static_cast<unsigned __int128>(lhs) <
          static_cast<unsigned __int128>(rhs);
@@ -811,11 +809,11 @@ inline bool operator<(uint128 lhs, uint128 rhs) {
 #endif
 }
 
-inline bool operator>(uint128 lhs, uint128 rhs) { return rhs < lhs; }
+constexpr bool operator>(uint128 lhs, uint128 rhs) { return rhs < lhs; }
 
-inline bool operator<=(uint128 lhs, uint128 rhs) { return !(rhs < lhs); }
+constexpr bool operator<=(uint128 lhs, uint128 rhs) { return !(rhs < lhs); }
 
-inline bool operator>=(uint128 lhs, uint128 rhs) { return !(lhs < rhs); }
+constexpr bool operator>=(uint128 lhs, uint128 rhs) { return !(lhs < rhs); }
 
 // Unary operators.
 
@@ -827,14 +825,13 @@ constexpr inline int128 operator+(int128 val) {
   return val;
 }
 
-inline uint128 operator-(uint128 val) {
+constexpr uint128 operator-(uint128 val) {
 #if defined(ABSL_HAVE_INTRINSIC_INT128)
   return -static_cast<unsigned __int128>(val);
 #else
-  uint64_t hi = ~Uint128High64(val);
-  uint64_t lo = ~Uint128Low64(val) + 1;
-  if (lo == 0) ++hi;  // carry
-  return MakeUint128(hi, lo);
+  return MakeUint128(
+      ~Uint128High64(val) + static_cast<unsigned long>(Uint128Low64(val) == 0),
+      ~Uint128Low64(val) + 1);
 #endif
 }
 
@@ -903,67 +900,77 @@ inline uint128& uint128::operator^=(uint128 other) {
 
 // Arithmetic operators.
 
-inline uint128 operator<<(uint128 lhs, int amount) {
+constexpr uint128 operator<<(uint128 lhs, int amount) {
 #ifdef ABSL_HAVE_INTRINSIC_INT128
   return static_cast<unsigned __int128>(lhs) << amount;
 #else
   // uint64_t shifts of >= 64 are undefined, so we will need some
   // special-casing.
-  if (amount < 64) {
-    if (amount != 0) {
-      return MakeUint128(
-          (Uint128High64(lhs) << amount) | (Uint128Low64(lhs) >> (64 - amount)),
-          Uint128Low64(lhs) << amount);
-    }
-    return lhs;
-  }
-  return MakeUint128(Uint128Low64(lhs) << (amount - 64), 0);
+  return amount >= 64 ? MakeUint128(Uint128Low64(lhs) << (amount - 64), 0)
+         : amount == 0 ? lhs
+                       : MakeUint128((Uint128High64(lhs) << amount) |
+                                         (Uint128Low64(lhs) >> (64 - amount)),
+                                     Uint128Low64(lhs) << amount);
 #endif
 }
 
-inline uint128 operator>>(uint128 lhs, int amount) {
+constexpr uint128 operator>>(uint128 lhs, int amount) {
 #ifdef ABSL_HAVE_INTRINSIC_INT128
   return static_cast<unsigned __int128>(lhs) >> amount;
 #else
   // uint64_t shifts of >= 64 are undefined, so we will need some
   // special-casing.
-  if (amount < 64) {
-    if (amount != 0) {
-      return MakeUint128(Uint128High64(lhs) >> amount,
-                         (Uint128Low64(lhs) >> amount) |
-                             (Uint128High64(lhs) << (64 - amount)));
-    }
-    return lhs;
-  }
-  return MakeUint128(0, Uint128High64(lhs) >> (amount - 64));
+  return amount >= 64 ? MakeUint128(0, Uint128High64(lhs) >> (amount - 64))
+         : amount == 0 ? lhs
+                       : MakeUint128(Uint128High64(lhs) >> amount,
+                                     (Uint128Low64(lhs) >> amount) |
+                                         (Uint128High64(lhs) << (64 - amount)));
 #endif
 }
 
-inline uint128 operator+(uint128 lhs, uint128 rhs) {
+#if !defined(ABSL_HAVE_INTRINSIC_INT128)
+namespace int128_internal {
+constexpr uint128 AddResult(uint128 result, uint128 lhs) {
+  // check for carry
+  return (Uint128Low64(result) < Uint128Low64(lhs))
+             ? MakeUint128(Uint128High64(result) + 1, Uint128Low64(result))
+             : result;
+}
+}  // namespace int128_internal
+#endif
+
+constexpr uint128 operator+(uint128 lhs, uint128 rhs) {
 #if defined(ABSL_HAVE_INTRINSIC_INT128)
   return static_cast<unsigned __int128>(lhs) +
          static_cast<unsigned __int128>(rhs);
 #else
-  uint128 result = MakeUint128(Uint128High64(lhs) + Uint128High64(rhs),
-                               Uint128Low64(lhs) + Uint128Low64(rhs));
-  if (Uint128Low64(result) < Uint128Low64(lhs)) {  // check for carry
-    return MakeUint128(Uint128High64(result) + 1, Uint128Low64(result));
-  }
-  return result;
+  return int128_internal::AddResult(
+      MakeUint128(Uint128High64(lhs) + Uint128High64(rhs),
+                  Uint128Low64(lhs) + Uint128Low64(rhs)),
+      lhs);
 #endif
 }
 
-inline uint128 operator-(uint128 lhs, uint128 rhs) {
+#if !defined(ABSL_HAVE_INTRINSIC_INT128)
+namespace int128_internal {
+constexpr uint128 SubstructResult(uint128 result, uint128 lhs, uint128 rhs) {
+  // check for carry
+  return (Uint128Low64(lhs) < Uint128Low64(rhs))
+             ? MakeUint128(Uint128High64(result) - 1, Uint128Low64(result))
+             : result;
+}
+}  // namespace int128_internal
+#endif
+
+constexpr uint128 operator-(uint128 lhs, uint128 rhs) {
 #if defined(ABSL_HAVE_INTRINSIC_INT128)
   return static_cast<unsigned __int128>(lhs) -
          static_cast<unsigned __int128>(rhs);
 #else
-  uint128 result = MakeUint128(Uint128High64(lhs) - Uint128High64(rhs),
-                               Uint128Low64(lhs) - Uint128Low64(rhs));
-  if (Uint128Low64(lhs) < Uint128Low64(rhs)) {  // check for carry
-    return MakeUint128(Uint128High64(result) - 1, Uint128Low64(result));
-  }
-  return result;
+  return int128_internal::SubstructResult(
+      MakeUint128(Uint128High64(lhs) - Uint128High64(rhs),
+                  Uint128Low64(lhs) - Uint128Low64(rhs)),
+      lhs, rhs);
 #endif
 }
 
@@ -1063,17 +1070,17 @@ inline int128& int128::operator=(unsigned long long v) {
 }
 
 // Arithmetic operators.
-
-int128 operator+(int128 lhs, int128 rhs);
-int128 operator-(int128 lhs, int128 rhs);
+constexpr int128 operator-(int128 v);
+constexpr int128 operator+(int128 lhs, int128 rhs);
+constexpr int128 operator-(int128 lhs, int128 rhs);
 int128 operator*(int128 lhs, int128 rhs);
 int128 operator/(int128 lhs, int128 rhs);
 int128 operator%(int128 lhs, int128 rhs);
-int128 operator|(int128 lhs, int128 rhs);
-int128 operator&(int128 lhs, int128 rhs);
-int128 operator^(int128 lhs, int128 rhs);
-int128 operator<<(int128 lhs, int amount);
-int128 operator>>(int128 lhs, int amount);
+constexpr int128 operator|(int128 lhs, int128 rhs);
+constexpr int128 operator&(int128 lhs, int128 rhs);
+constexpr int128 operator^(int128 lhs, int128 rhs);
+constexpr int128 operator<<(int128 lhs, int amount);
+constexpr int128 operator>>(int128 lhs, int amount);
 
 inline int128& int128::operator+=(int128 other) {
   *this = *this + other;
@@ -1125,6 +1132,9 @@ inline int128& int128::operator>>=(int amount) {
   return *this;
 }
 
+// Forward declaration for comparison operators.
+constexpr bool operator!=(int128 lhs, int128 rhs);
+
 namespace int128_internal {
 
 // Casts from unsigned to signed while preserving the underlying binary
diff --git a/absl/numeric/int128_have_intrinsic.inc b/absl/numeric/int128_have_intrinsic.inc
index d6c76dd3..3945fa29 100644
--- a/absl/numeric/int128_have_intrinsic.inc
+++ b/absl/numeric/int128_have_intrinsic.inc
@@ -155,7 +155,7 @@ constexpr int128::operator unsigned __int128() const {
 #if defined(__clang__) && !defined(__ppc64__)
 inline int128::operator float() const { return static_cast<float>(v_); }
 
-inline int128::operator double () const { return static_cast<double>(v_); }
+inline int128::operator double() const { return static_cast<double>(v_); }
 
 inline int128::operator long double() const {
   return static_cast<long double>(v_);
@@ -163,8 +163,8 @@ inline int128::operator long double() const {
 
 #else  // Clang on PowerPC
 // Forward declaration for conversion operators to floating point types.
-int128 operator-(int128 v);
-bool operator!=(int128 lhs, int128 rhs);
+constexpr int128 operator-(int128 v);
+constexpr bool operator!=(int128 lhs, int128 rhs);
 
 inline int128::operator float() const {
   // We must convert the absolute value and then negate as needed, because
@@ -199,51 +199,45 @@ inline int128::operator long double() const {
 
 // Comparison operators.
 
-inline bool operator==(int128 lhs, int128 rhs) {
+constexpr bool operator==(int128 lhs, int128 rhs) {
   return static_cast<__int128>(lhs) == static_cast<__int128>(rhs);
 }
 
-inline bool operator!=(int128 lhs, int128 rhs) {
+constexpr bool operator!=(int128 lhs, int128 rhs) {
   return static_cast<__int128>(lhs) != static_cast<__int128>(rhs);
 }
 
-inline bool operator<(int128 lhs, int128 rhs) {
+constexpr bool operator<(int128 lhs, int128 rhs) {
   return static_cast<__int128>(lhs) < static_cast<__int128>(rhs);
 }
 
-inline bool operator>(int128 lhs, int128 rhs) {
+constexpr bool operator>(int128 lhs, int128 rhs) {
   return static_cast<__int128>(lhs) > static_cast<__int128>(rhs);
 }
 
-inline bool operator<=(int128 lhs, int128 rhs) {
+constexpr bool operator<=(int128 lhs, int128 rhs) {
   return static_cast<__int128>(lhs) <= static_cast<__int128>(rhs);
 }
 
-inline bool operator>=(int128 lhs, int128 rhs) {
+constexpr bool operator>=(int128 lhs, int128 rhs) {
   return static_cast<__int128>(lhs) >= static_cast<__int128>(rhs);
 }
 
 // Unary operators.
 
-inline int128 operator-(int128 v) {
-  return -static_cast<__int128>(v);
-}
+constexpr int128 operator-(int128 v) { return -static_cast<__int128>(v); }
 
-inline bool operator!(int128 v) {
-  return !static_cast<__int128>(v);
-}
+constexpr bool operator!(int128 v) { return !static_cast<__int128>(v); }
 
-inline int128 operator~(int128 val) {
-  return ~static_cast<__int128>(val);
-}
+constexpr int128 operator~(int128 val) { return ~static_cast<__int128>(val); }
 
 // Arithmetic operators.
 
-inline int128 operator+(int128 lhs, int128 rhs) {
+constexpr int128 operator+(int128 lhs, int128 rhs) {
   return static_cast<__int128>(lhs) + static_cast<__int128>(rhs);
 }
 
-inline int128 operator-(int128 lhs, int128 rhs) {
+constexpr int128 operator-(int128 lhs, int128 rhs) {
   return static_cast<__int128>(lhs) - static_cast<__int128>(rhs);
 }
 
@@ -281,22 +275,22 @@ inline int128& int128::operator--() {
   return *this;
 }
 
-inline int128 operator|(int128 lhs, int128 rhs) {
+constexpr int128 operator|(int128 lhs, int128 rhs) {
   return static_cast<__int128>(lhs) | static_cast<__int128>(rhs);
 }
 
-inline int128 operator&(int128 lhs, int128 rhs) {
+constexpr int128 operator&(int128 lhs, int128 rhs) {
   return static_cast<__int128>(lhs) & static_cast<__int128>(rhs);
 }
 
-inline int128 operator^(int128 lhs, int128 rhs) {
+constexpr int128 operator^(int128 lhs, int128 rhs) {
   return static_cast<__int128>(lhs) ^ static_cast<__int128>(rhs);
 }
 
-inline int128 operator<<(int128 lhs, int amount) {
+constexpr int128 operator<<(int128 lhs, int amount) {
   return static_cast<__int128>(lhs) << amount;
 }
 
-inline int128 operator>>(int128 lhs, int amount) {
+constexpr int128 operator>>(int128 lhs, int amount) {
   return static_cast<__int128>(lhs) >> amount;
 }
diff --git a/absl/numeric/int128_no_intrinsic.inc b/absl/numeric/int128_no_intrinsic.inc
index c753771a..66f6809f 100644
--- a/absl/numeric/int128_no_intrinsic.inc
+++ b/absl/numeric/int128_no_intrinsic.inc
@@ -134,10 +134,6 @@ constexpr int128::operator unsigned long long() const {  // NOLINT(runtime/int)
   return static_cast<unsigned long long>(lo_);           // NOLINT(runtime/int)
 }
 
-// Forward declaration for conversion operators to floating point types.
-int128 operator-(int128 v);
-bool operator!=(int128 lhs, int128 rhs);
-
 inline int128::operator float() const {
   // We must convert the absolute value and then negate as needed, because
   // floating point types are typically sign-magnitude. Otherwise, the
@@ -169,76 +165,80 @@ inline int128::operator long double() const {
 
 // Comparison operators.
 
-inline bool operator==(int128 lhs, int128 rhs) {
+constexpr bool operator==(int128 lhs, int128 rhs) {
   return (Int128Low64(lhs) == Int128Low64(rhs) &&
           Int128High64(lhs) == Int128High64(rhs));
 }
 
-inline bool operator!=(int128 lhs, int128 rhs) {
-  return !(lhs == rhs);
-}
+constexpr bool operator!=(int128 lhs, int128 rhs) { return !(lhs == rhs); }
 
-inline bool operator<(int128 lhs, int128 rhs) {
+constexpr bool operator<(int128 lhs, int128 rhs) {
   return (Int128High64(lhs) == Int128High64(rhs))
              ? (Int128Low64(lhs) < Int128Low64(rhs))
              : (Int128High64(lhs) < Int128High64(rhs));
 }
 
-inline bool operator>(int128 lhs, int128 rhs) {
+constexpr bool operator>(int128 lhs, int128 rhs) {
   return (Int128High64(lhs) == Int128High64(rhs))
              ? (Int128Low64(lhs) > Int128Low64(rhs))
              : (Int128High64(lhs) > Int128High64(rhs));
 }
 
-inline bool operator<=(int128 lhs, int128 rhs) {
-  return !(lhs > rhs);
-}
+constexpr bool operator<=(int128 lhs, int128 rhs) { return !(lhs > rhs); }
 
-inline bool operator>=(int128 lhs, int128 rhs) {
-  return !(lhs < rhs);
-}
+constexpr bool operator>=(int128 lhs, int128 rhs) { return !(lhs < rhs); }
 
 // Unary operators.
 
-inline int128 operator-(int128 v) {
-  int64_t hi = ~Int128High64(v);
-  uint64_t lo = ~Int128Low64(v) + 1;
-  if (lo == 0) ++hi;  // carry
-  return MakeInt128(hi, lo);
+constexpr int128 operator-(int128 v) {
+  return MakeInt128(~Int128High64(v) + (Int128Low64(v) == 0),
+                    ~Int128Low64(v) + 1);
 }
 
-inline bool operator!(int128 v) {
+constexpr bool operator!(int128 v) {
   return !Int128Low64(v) && !Int128High64(v);
 }
 
-inline int128 operator~(int128 val) {
+constexpr int128 operator~(int128 val) {
   return MakeInt128(~Int128High64(val), ~Int128Low64(val));
 }
 
 // Arithmetic operators.
 
-inline int128 operator+(int128 lhs, int128 rhs) {
-  int128 result = MakeInt128(Int128High64(lhs) + Int128High64(rhs),
-                             Int128Low64(lhs) + Int128Low64(rhs));
-  if (Int128Low64(result) < Int128Low64(lhs)) {  // check for carry
-    return MakeInt128(Int128High64(result) + 1, Int128Low64(result));
-  }
-  return result;
+namespace int128_internal {
+constexpr int128 SignedAddResult(int128 result, int128 lhs) {
+  // check for carry
+  return (Int128Low64(result) < Int128Low64(lhs))
+             ? MakeInt128(Int128High64(result) + 1, Int128Low64(result))
+             : result;
+}
+}  // namespace int128_internal
+constexpr int128 operator+(int128 lhs, int128 rhs) {
+  return int128_internal::SignedAddResult(
+      MakeInt128(Int128High64(lhs) + Int128High64(rhs),
+                 Int128Low64(lhs) + Int128Low64(rhs)),
+      lhs);
 }
 
-inline int128 operator-(int128 lhs, int128 rhs) {
-  int128 result = MakeInt128(Int128High64(lhs) - Int128High64(rhs),
-                             Int128Low64(lhs) - Int128Low64(rhs));
-  if (Int128Low64(lhs) < Int128Low64(rhs)) {  // check for carry
-    return MakeInt128(Int128High64(result) - 1, Int128Low64(result));
-  }
-  return result;
+namespace int128_internal {
+constexpr int128 SignedSubstructResult(int128 result, int128 lhs, int128 rhs) {
+  // check for carry
+  return (Int128Low64(lhs) < Int128Low64(rhs))
+             ? MakeInt128(Int128High64(result) - 1, Int128Low64(result))
+             : result;
+}
+}  // namespace int128_internal
+constexpr int128 operator-(int128 lhs, int128 rhs) {
+  return int128_internal::SignedSubstructResult(
+      MakeInt128(Int128High64(lhs) - Int128High64(rhs),
+                 Int128Low64(lhs) - Int128Low64(rhs)),
+      lhs, rhs);
 }
 
 inline int128 operator*(int128 lhs, int128 rhs) {
-  uint128 result = uint128(lhs) * rhs;
-  return MakeInt128(int128_internal::BitCastToSigned(Uint128High64(result)),
-                    Uint128Low64(result));
+  return MakeInt128(
+      int128_internal::BitCastToSigned(Uint128High64(uint128(lhs) * rhs)),
+      Uint128Low64(uint128(lhs) * rhs));
 }
 
 inline int128 int128::operator++(int) {
@@ -263,46 +263,43 @@ inline int128& int128::operator--() {
   return *this;
 }
 
-inline int128 operator|(int128 lhs, int128 rhs) {
+constexpr int128 operator|(int128 lhs, int128 rhs) {
   return MakeInt128(Int128High64(lhs) | Int128High64(rhs),
                     Int128Low64(lhs) | Int128Low64(rhs));
 }
 
-inline int128 operator&(int128 lhs, int128 rhs) {
+constexpr int128 operator&(int128 lhs, int128 rhs) {
   return MakeInt128(Int128High64(lhs) & Int128High64(rhs),
                     Int128Low64(lhs) & Int128Low64(rhs));
 }
 
-inline int128 operator^(int128 lhs, int128 rhs) {
+constexpr int128 operator^(int128 lhs, int128 rhs) {
   return MakeInt128(Int128High64(lhs) ^ Int128High64(rhs),
                     Int128Low64(lhs) ^ Int128Low64(rhs));
 }
 
-inline int128 operator<<(int128 lhs, int amount) {
+constexpr int128 operator<<(int128 lhs, int amount) {
   // uint64_t shifts of >= 64 are undefined, so we need some special-casing.
-  if (amount < 64) {
-    if (amount != 0) {
-      return MakeInt128(
-          (Int128High64(lhs) << amount) |
-              static_cast<int64_t>(Int128Low64(lhs) >> (64 - amount)),
-          Int128Low64(lhs) << amount);
-    }
-    return lhs;
-  }
-  return MakeInt128(static_cast<int64_t>(Int128Low64(lhs) << (amount - 64)), 0);
-}
-
-inline int128 operator>>(int128 lhs, int amount) {
+  return amount >= 64
+             ? MakeInt128(
+                   static_cast<int64_t>(Int128Low64(lhs) << (amount - 64)), 0)
+         : amount == 0
+             ? lhs
+             : MakeInt128(
+                   (Int128High64(lhs) << amount) |
+                       static_cast<int64_t>(Int128Low64(lhs) >> (64 - amount)),
+                   Int128Low64(lhs) << amount);
+}
+
+constexpr int128 operator>>(int128 lhs, int amount) {
   // uint64_t shifts of >= 64 are undefined, so we need some special-casing.
-  if (amount < 64) {
-    if (amount != 0) {
-      return MakeInt128(
-          Int128High64(lhs) >> amount,
-          (Int128Low64(lhs) >> amount) |
-              (static_cast<uint64_t>(Int128High64(lhs)) << (64 - amount)));
-    }
-    return lhs;
-  }
-  return MakeInt128(0,
-                    static_cast<uint64_t>(Int128High64(lhs) >> (amount - 64)));
+  return amount >= 64
+             ? MakeInt128(
+                   0, static_cast<uint64_t>(Int128High64(lhs) >> (amount - 64)))
+         : amount == 0
+             ? lhs
+             : MakeInt128(Int128High64(lhs) >> amount,
+                          (Int128Low64(lhs) >> amount) |
+                              (static_cast<uint64_t>(Int128High64(lhs))
+                               << (64 - amount)));
 }