1 files changed, 70 insertions, 12 deletions

diff --git a/absl/strings/internal/cord_internal.h b/absl/strings/internal/cord_internal.h
index 00b9baa9..aa91a691 100644
--- a/absl/strings/internal/cord_internal.h
+++ b/absl/strings/internal/cord_internal.h
@@ -33,14 +33,17 @@ namespace cord_internal {
 // Wraps std::atomic for reference counting.
 class Refcount {
  public:
-  Refcount() : count_{1} {}
-  ~Refcount() {}
+  constexpr Refcount() : count_{kRefIncrement} {}
+  struct Immortal {};
+  explicit constexpr Refcount(Immortal) : count_(kImmortalTag) {}
 
-  // Increments the reference count by 1. Imposes no memory ordering.
-  inline void Increment() { count_.fetch_add(1, std::memory_order_relaxed); }
+  // Increments the reference count. Imposes no memory ordering.
+  inline void Increment() {
+    count_.fetch_add(kRefIncrement, std::memory_order_relaxed);
+  }
 
   // Asserts that the current refcount is greater than 0. If the refcount is
-  // greater than 1, decrements the reference count by 1.
+  // greater than 1, decrements the reference count.
   //
   // Returns false if there are no references outstanding; true otherwise.
   // Inserts barriers to ensure that state written before this method returns
@@ -48,19 +51,24 @@ class Refcount {
   // false.
   inline bool Decrement() {
     int32_t refcount = count_.load(std::memory_order_acquire);
-    assert(refcount > 0);
-    return refcount != 1 && count_.fetch_sub(1, std::memory_order_acq_rel) != 1;
+    assert(refcount > 0 || refcount & kImmortalTag);
+    return refcount != kRefIncrement &&
+           count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel) !=
+               kRefIncrement;
   }
 
   // Same as Decrement but expect that refcount is greater than 1.
   inline bool DecrementExpectHighRefcount() {
-    int32_t refcount = count_.fetch_sub(1, std::memory_order_acq_rel);
-    assert(refcount > 0);
-    return refcount != 1;
+    int32_t refcount =
+        count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel);
+    assert(refcount > 0 || refcount & kImmortalTag);
+    return refcount != kRefIncrement;
   }
 
   // Returns the current reference count using acquire semantics.
-  inline int32_t Get() const { return count_.load(std::memory_order_acquire); }
+  inline int32_t Get() const {
+    return count_.load(std::memory_order_acquire) >> kImmortalShift;
+  }
 
   // Returns whether the atomic integer is 1.
   // If the reference count is used in the conventional way, a
@@ -70,9 +78,27 @@ class Refcount {
   // performs the memory barrier needed for the owning thread
   // to act on the object, knowing that it has exclusive access to the
   // object.
-  inline bool IsOne() { return count_.load(std::memory_order_acquire) == 1; }
+  inline bool IsOne() {
+    return count_.load(std::memory_order_acquire) == kRefIncrement;
+  }
+
+  bool IsImmortal() const {
+    return (count_.load(std::memory_order_relaxed) & kImmortalTag) != 0;
+  }
 
  private:
+  // We reserve the bottom bit to tag a reference count as immortal.
+  // By making it `1` we ensure that we never reach `0` when adding/subtracting
+  // `2`, thus it never looks as if it should be destroyed.
+  // These are used for the StringConstant constructor where we do not increase
+  // the refcount at construction time (due to constinit requirements) but we
+  // will still decrease it at destruction time to avoid branching on Unref.
+  enum {
+    kImmortalShift = 1,
+    kRefIncrement = 1 << kImmortalShift,
+    kImmortalTag = kRefIncrement - 1
+  };
+
   std::atomic<int32_t> count_;
 };
 
@@ -97,6 +123,10 @@ enum CordRepKind {
 };
 
 struct CordRep {
+  CordRep() = default;
+  constexpr CordRep(Refcount::Immortal immortal, size_t l)
+      : length(l), refcount(immortal), tag(EXTERNAL), data{} {}
+
   // The following three fields have to be less than 32 bytes since
   // that is the smallest supported flat node size.
   size_t length;
@@ -135,6 +165,12 @@ using ExternalReleaserInvoker = void (*)(CordRepExternal*);
 // External CordReps are allocated together with a type erased releaser. The
 // releaser is stored in the memory directly following the CordRepExternal.
 struct CordRepExternal : public CordRep {
+  CordRepExternal() = default;
+  explicit constexpr CordRepExternal(absl::string_view str)
+      : CordRep(Refcount::Immortal{}, str.size()),
+        base(str.data()),
+        releaser_invoker(nullptr) {}
+
   const char* base;
   // Pointer to function that knows how to call and destroy the releaser.
   ExternalReleaserInvoker releaser_invoker;
@@ -178,6 +214,14 @@ struct CordRepExternalImpl
   }
 };
 
+template <typename Str>
+struct ConstInitExternalStorage {
+  ABSL_CONST_INIT static CordRepExternal value;
+};
+
+template <typename Str>
+CordRepExternal ConstInitExternalStorage<Str>::value(Str::value);
+
 enum {
   kMaxInline = 15,
   // Tag byte & kMaxInline means we are storing a pointer.
@@ -195,9 +239,23 @@ struct AsTree {
   char padding[kMaxInline + 1 - sizeof(absl::cord_internal::CordRep*) - 1];
   char tagged_size;
 };
+
+constexpr char GetOrNull(absl::string_view data, size_t pos) {
+  return pos < data.size() ? data[pos] : '\0';
+}
+
 union InlineData {
   constexpr InlineData() : as_chars{} {}
   explicit constexpr InlineData(AsTree tree) : as_tree(tree) {}
+  explicit constexpr InlineData(absl::string_view chars)
+      : as_chars{GetOrNull(chars, 0),  GetOrNull(chars, 1),
+                 GetOrNull(chars, 2),  GetOrNull(chars, 3),
+                 GetOrNull(chars, 4),  GetOrNull(chars, 5),
+                 GetOrNull(chars, 6),  GetOrNull(chars, 7),
+                 GetOrNull(chars, 8),  GetOrNull(chars, 9),
+                 GetOrNull(chars, 10), GetOrNull(chars, 11),
+                 GetOrNull(chars, 12), GetOrNull(chars, 13),
+                 GetOrNull(chars, 14), static_cast<char>(chars.size())} {}
 
   AsTree as_tree;
   char as_chars[kMaxInline + 1];