Export of internal Abseil changes

--
6e9f93888bbe6718997ed90bbd049f1f3572b066 by Abseil Team <absl-team@google.com>:

Fix status to be safe for self move assignment.

PiperOrigin-RevId: 346815392

--
35cae74a977f258e81dfbe925fb5a34cb6632036 by Gennadiy Rozental <rogeeff@google.com>:

Eliminate unnecessary access to the global vars.

PiperOrigin-RevId: 346777168

--
e7e786c243069060f5d6c1c945decb4b0b83f95b by Andy Getzendanner <durandal@google.com>:

Internal change.

PiperOrigin-RevId: 346685656

--
4ccd41c48f1a83cfa20b3ea534f743dd7d788376 by Abseil Team <absl-team@google.com>:

Move CordRep Ref() and Unref() logic into cord_internal.h

This change moves Ref() and Unref() logic out of cord.cc into cord_internal. The main purpose is to make upcoming ring buffer changes easier to isolate from existing cord.cc code. Notice that this removes the nullptr check from Unref() and now requires it to be non null (which held true most times). We may need to rethink if the 'unref unlikely one' is the common case: are cordreps most likely shared? This may be something between 'root' and non root nodes, i.e., is it more likely for leaf / flat nodes in large cords to be shared than top level cordreps being shared? Vice versa? Benchmarks say that we mostly shouldn't care, the caveat being that atomic ops seem more expensive on upcoming archs (arcadia) so we should error on the side of an extra IsOne() branch saving us single owned atomic ops.

PiperOrigin-RevId: 346676121

--
f0babab103b9e60d61ba09482d468985e43eceb3 by Samuel Benzaquen <sbenza@google.com>:

Fix iterator based constructor and `.insert` members to only require
EmplaceConstructible as the standard specifies.

PiperOrigin-RevId: 346616707

--
8f48eedda02277f9c96a88ed7726e34b557cce20 by Evan Brown <ezb@google.com>:

Fix a bug in binary_search_impl when there's a transparent, three-way comparator that has different equivalence classes for different lookup types.

Add a new can_have_multiple_equivalent_keys method to share the common logic for these cases.

PiperOrigin-RevId: 346605948

--
649183cb3cc9383431de9c81fb1c0f885d4001ae by Abseil Team <absl-team@google.com>:

Add benchmark for accessing a Duration flag.

PiperOrigin-RevId: 346594843

--
fefdb046520871af63ce2229e2f7cccfc0483dea by Abseil Team <absl-team@google.com>:

Restructure CordReader for upcoming ring buffer changes.

PiperOrigin-RevId: 346410642

--
8b2f50e7da0ebab06ead5f94e366e984ca23cb6a by Abseil Team <absl-team@google.com>:

Wire in an internal-only flag to toggle upcoming ring buffer changes on/off for experimentation.

PiperOrigin-RevId: 346199111
GitOrigin-RevId: 6e9f93888bbe6718997ed90bbd049f1f3572b066
Change-Id: I8f34866b25a79209cb5448bbb28dd3044111d2e9

6 files changed, 246 insertions, 173 deletions

diff --git a/absl/strings/BUILD.bazel b/absl/strings/BUILD.bazel
index a1579a4a..aab3a286 100644
--- a/absl/strings/BUILD.bazel
+++ b/absl/strings/BUILD.bazel
@@ -267,16 +267,23 @@ cc_test(
 
 cc_library(
     name = "cord_internal",
+    srcs = [
+        "internal/cord_internal.cc",
+    ],
     hdrs = [
         "internal/cord_internal.h",
         "internal/cord_rep_flat.h",
     ],
     copts = ABSL_DEFAULT_COPTS,
-    visibility = ["//visibility:private"],
+    visibility = [
+        "//visibility:private",
+    ],
     deps = [
         ":strings",
         "//absl/base:base_internal",
+        "//absl/base:core_headers",
         "//absl/container:compressed_tuple",
+        "//absl/container:inlined_vector",
         "//absl/meta:type_traits",
     ],
 )
@@ -304,6 +311,7 @@ cc_library(
         "//absl/functional:function_ref",
         "//absl/meta:type_traits",
         "//absl/types:optional",
+        "//absl/types:variant",
     ],
 )
 
diff --git a/absl/strings/CMakeLists.txt b/absl/strings/CMakeLists.txt
index af0b57d8..54c10151 100644
--- a/absl/strings/CMakeLists.txt
+++ b/absl/strings/CMakeLists.txt
@@ -556,6 +556,7 @@ absl_cc_library(
     "cord.h"
   SRCS
     "cord.cc"
+    "internal/cord_internal.cc"
     "internal/cord_internal.h"
     "internal/cord_rep_flat.h"
   COPTS
@@ -570,6 +571,7 @@ absl_cc_library(
     absl::function_ref
     absl::inlined_vector
     absl::optional
+    absl::variant
     absl::raw_logging_internal
     absl::strings
     absl::strings_internal
diff --git a/absl/strings/cord.cc b/absl/strings/cord.cc
index badeb610..f475042c 100644
--- a/absl/strings/cord.cc
+++ b/absl/strings/cord.cc
@@ -60,51 +60,8 @@ using ::absl::cord_internal::EXTERNAL;
 using ::absl::cord_internal::FLAT;
 using ::absl::cord_internal::SUBSTRING;
 
-namespace cord_internal {
-
-inline CordRepConcat* CordRep::concat() {
-  assert(tag == CONCAT);
-  return static_cast<CordRepConcat*>(this);
-}
-
-inline const CordRepConcat* CordRep::concat() const {
-  assert(tag == CONCAT);
-  return static_cast<const CordRepConcat*>(this);
-}
-
-inline CordRepSubstring* CordRep::substring() {
-  assert(tag == SUBSTRING);
-  return static_cast<CordRepSubstring*>(this);
-}
-
-inline const CordRepSubstring* CordRep::substring() const {
-  assert(tag == SUBSTRING);
-  return static_cast<const CordRepSubstring*>(this);
-}
-
-inline CordRepExternal* CordRep::external() {
-  assert(tag == EXTERNAL);
-  return static_cast<CordRepExternal*>(this);
-}
-
-inline const CordRepExternal* CordRep::external() const {
-  assert(tag == EXTERNAL);
-  return static_cast<const CordRepExternal*>(this);
-}
-
-inline CordRepFlat* CordRep::flat() {
-  assert(tag >= FLAT);
-  return static_cast<CordRepFlat*>(this);
-}
-inline const CordRepFlat* CordRep::flat() const {
-  assert(tag >= FLAT);
-  return static_cast<const CordRepFlat*>(this);
-}
-
-}  // namespace cord_internal
-
-// Prefer copying blocks of at most this size, otherwise reference count.
-static const size_t kMaxBytesToCopy = 511;
+using ::absl::cord_internal::kInlinedVectorSize;
+using ::absl::cord_internal::kMaxBytesToCopy;
 
 constexpr uint64_t Fibonacci(unsigned char n, uint64_t a = 0, uint64_t b = 1) {
   return n == 0 ? a : Fibonacci(n - 1, b, a + b);
@@ -136,17 +93,6 @@ static constexpr uint64_t min_length[] = {
 
 static const int kMinLengthSize = ABSL_ARRAYSIZE(min_length);
 
-// The inlined size to use with absl::InlinedVector.
-//
-// Note: The InlinedVectors in this file (and in cord.h) do not need to use
-// the same value for their inlined size. The fact that they do is historical.
-// It may be desirable for each to use a different inlined size optimized for
-// that InlinedVector's usage.
-//
-// TODO(jgm): Benchmark to see if there's a more optimal value than 47 for
-// the inlined vector size (47 exists for backward compatibility).
-static const int kInlinedVectorSize = 47;
-
 static inline bool IsRootBalanced(CordRep* node) {
   if (node->tag != CONCAT) {
     return true;
@@ -182,86 +128,6 @@ static inline CordRep* VerifyTree(CordRep* node) {
   return node;
 }
 
-// --------------------------------------------------------------------
-// Memory management
-
-inline CordRep* Ref(CordRep* rep) {
-  if (rep != nullptr) {
-    rep->refcount.Increment();
-  }
-  return rep;
-}
-
-// This internal routine is called from the cold path of Unref below. Keeping it
-// in a separate routine allows good inlining of Unref into many profitable call
-// sites. However, the call to this function can be highly disruptive to the
-// register pressure in those callers. To minimize the cost to callers, we use
-// a special LLVM calling convention that preserves most registers. This allows
-// the call to this routine in cold paths to not disrupt the caller's register
-// pressure. This calling convention is not available on all platforms; we
-// intentionally allow LLVM to ignore the attribute rather than attempting to
-// hardcode the list of supported platforms.
-#if defined(__clang__) && !defined(__i386__)
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wattributes"
-__attribute__((preserve_most))
-#pragma clang diagnostic pop
-#endif
-static void UnrefInternal(CordRep* rep) {
-  assert(rep != nullptr);
-
-  absl::InlinedVector<CordRep*, kInlinedVectorSize> pending;
-  while (true) {
-    assert(!rep->refcount.IsImmortal());
-    if (rep->tag == CONCAT) {
-      CordRepConcat* rep_concat = rep->concat();
-      CordRep* right = rep_concat->right;
-      if (!right->refcount.Decrement()) {
-        pending.push_back(right);
-      }
-      CordRep* left = rep_concat->left;
-      delete rep_concat;
-      rep = nullptr;
-      if (!left->refcount.Decrement()) {
-        rep = left;
-        continue;
-      }
-    } else if (rep->tag == EXTERNAL) {
-      CordRepExternal::Delete(rep);
-      rep = nullptr;
-    } else if (rep->tag == SUBSTRING) {
-      CordRepSubstring* rep_substring = rep->substring();
-      CordRep* child = rep_substring->child;
-      delete rep_substring;
-      rep = nullptr;
-      if (!child->refcount.Decrement()) {
-        rep = child;
-        continue;
-      }
-    } else {
-      CordRepFlat::Delete(rep);
-      rep = nullptr;
-    }
-
-    if (!pending.empty()) {
-      rep = pending.back();
-      pending.pop_back();
-    } else {
-      break;
-    }
-  }
-}
-
-inline void Unref(CordRep* rep) {
-  // Fast-path for two common, hot cases: a null rep and a shared root.
-  if (ABSL_PREDICT_TRUE(rep == nullptr ||
-                        rep->refcount.DecrementExpectHighRefcount())) {
-    return;
-  }
-
-  UnrefInternal(rep);
-}
-
 // Return the depth of a node
 static int Depth(const CordRep* rep) {
   if (rep->tag == CONCAT) {
@@ -285,12 +151,14 @@ static void SetConcatChildren(CordRepConcat* concat, CordRep* left,
 // The returned node has a refcount of 1.
 static CordRep* RawConcat(CordRep* left, CordRep* right) {
   // Avoid making degenerate concat nodes (one child is empty)
-  if (left == nullptr || left->length == 0) {
-    Unref(left);
+  if (left == nullptr) return right;
+  if (right == nullptr) return left;
+  if (left->length == 0) {
+    CordRep::Unref(left);
     return right;
   }
-  if (right == nullptr || right->length == 0) {
-    Unref(right);
+  if (right->length == 0) {
+    CordRep::Unref(right);
     return left;
   }
 
@@ -364,7 +232,7 @@ void InitializeCordRepExternal(absl::string_view data, CordRepExternal* rep) {
 static CordRep* NewSubstring(CordRep* child, size_t offset, size_t length) {
   // Never create empty substring nodes
   if (length == 0) {
-    Unref(child);
+    CordRep::Unref(child);
     return nullptr;
   } else {
     CordRepSubstring* rep = new CordRepSubstring();
@@ -562,13 +430,13 @@ void Cord::InlineRep::AssignSlow(const Cord::InlineRep& src) {
 
   data_ = src.data_;
   if (is_tree()) {
-    Ref(tree());
+    CordRep::Ref(tree());
   }
 }
 
 void Cord::InlineRep::ClearSlow() {
   if (is_tree()) {
-    Unref(tree());
+    CordRep::Unref(tree());
   }
   ResetToEmpty();
 }
@@ -577,7 +445,9 @@ void Cord::InlineRep::ClearSlow() {
 // Constructors and destructors
 
 Cord::Cord(const Cord& src) : contents_(src.contents_) {
-  Ref(contents_.tree());  // Does nothing if contents_ has embedded data
+  if (CordRep* tree = contents_.tree()) {
+    CordRep::Ref(tree);
+  }
 }
 
 Cord::Cord(absl::string_view src) {
@@ -623,14 +493,18 @@ template Cord::Cord(std::string&& src);
 // The destruction code is separate so that the compiler can determine
 // that it does not need to call the destructor on a moved-from Cord.
 void Cord::DestroyCordSlow() {
-  Unref(VerifyTree(contents_.tree()));
+  if (CordRep* tree = contents_.tree()) {
+    CordRep::Unref(VerifyTree(tree));
+  }
 }
 
 // --------------------------------------------------------------------
 // Mutators
 
 void Cord::Clear() {
-  Unref(contents_.clear());
+  if (CordRep* tree = contents_.clear()) {
+    CordRep::Unref(tree);
+  }
 }
 
 Cord& Cord::operator=(absl::string_view src) {
@@ -641,7 +515,7 @@ Cord& Cord::operator=(absl::string_view src) {
   if (length <= InlineRep::kMaxInline) {
     // Embed into this->contents_
     contents_.set_data(data, length, true);
-    Unref(tree);
+    if (tree) CordRep::Unref(tree);
     return *this;
   }
   if (tree != nullptr && tree->tag >= FLAT &&
@@ -654,7 +528,7 @@ Cord& Cord::operator=(absl::string_view src) {
     return *this;
   }
   contents_.set_tree(NewTree(data, length, 0));
-  Unref(tree);
+  if (tree) CordRep::Unref(tree);
   return *this;
 }
 
@@ -731,7 +605,7 @@ void Cord::InlineRep::AppendArray(const char* src_data, size_t src_size) {
 }
 
 inline CordRep* Cord::TakeRep() const& {
-  return Ref(contents_.tree());
+  return CordRep::Ref(contents_.tree());
 }
 
 inline CordRep* Cord::TakeRep() && {
@@ -775,6 +649,7 @@ inline void Cord::AppendImpl(C&& src) {
     return;
   }
 
+  // Guaranteed to be a tree (kMaxBytesToCopy > kInlinedSize)
   contents_.AppendTree(std::forward<C>(src).TakeRep());
 }
 
@@ -796,7 +671,7 @@ template void Cord::Append(std::string&& src);
 void Cord::Prepend(const Cord& src) {
   CordRep* src_tree = src.contents_.tree();
   if (src_tree != nullptr) {
-    Ref(src_tree);
+    CordRep::Ref(src_tree);
     contents_.PrependTree(src_tree);
     return;
   }
@@ -835,7 +710,7 @@ template void Cord::Prepend(std::string&& src);
 
 static CordRep* RemovePrefixFrom(CordRep* node, size_t n) {
   if (n >= node->length) return nullptr;
-  if (n == 0) return Ref(node);
+  if (n == 0) return CordRep::Ref(node);
   absl::InlinedVector<CordRep*, kInlinedVectorSize> rhs_stack;
 
   while (node->tag == CONCAT) {
@@ -853,7 +728,7 @@ static CordRep* RemovePrefixFrom(CordRep* node, size_t n) {
   assert(n <= node->length);
 
   if (n == 0) {
-    Ref(node);
+    CordRep::Ref(node);
   } else {
     size_t start = n;
     size_t len = node->length - n;
@@ -862,10 +737,10 @@ static CordRep* RemovePrefixFrom(CordRep* node, size_t n) {
       start += node->substring()->start;
       node = node->substring()->child;
     }
-    node = NewSubstring(Ref(node), start, len);
+    node = NewSubstring(CordRep::Ref(node), start, len);
   }
   while (!rhs_stack.empty()) {
-    node = Concat(node, Ref(rhs_stack.back()));
+    node = Concat(node, CordRep::Ref(rhs_stack.back()));
     rhs_stack.pop_back();
   }
   return node;
@@ -876,7 +751,7 @@ static CordRep* RemovePrefixFrom(CordRep* node, size_t n) {
 // edited in place iff that node and all its ancestors have a refcount of 1.
 static CordRep* RemoveSuffixFrom(CordRep* node, size_t n) {
   if (n >= node->length) return nullptr;
-  if (n == 0) return Ref(node);
+  if (n == 0) return CordRep::Ref(node);
   absl::InlinedVector<CordRep*, kInlinedVectorSize> lhs_stack;
   bool inplace_ok = node->refcount.IsOne();
 
@@ -896,11 +771,11 @@ static CordRep* RemoveSuffixFrom(CordRep* node, size_t n) {
   assert(n <= node->length);
 
   if (n == 0) {
-    Ref(node);
+    CordRep::Ref(node);
   } else if (inplace_ok && node->tag != EXTERNAL) {
     // Consider making a new buffer if the current node capacity is much
     // larger than the new length.
-    Ref(node);
+    CordRep::Ref(node);
     node->length -= n;
   } else {
     size_t start = 0;
@@ -909,10 +784,10 @@ static CordRep* RemoveSuffixFrom(CordRep* node, size_t n) {
       start = node->substring()->start;
       node = node->substring()->child;
     }
-    node = NewSubstring(Ref(node), start, len);
+    node = NewSubstring(CordRep::Ref(node), start, len);
   }
   while (!lhs_stack.empty()) {
-    node = Concat(Ref(lhs_stack.back()), node);
+    node = Concat(CordRep::Ref(lhs_stack.back()), node);
     lhs_stack.pop_back();
   }
   return node;
@@ -927,7 +802,7 @@ void Cord::RemovePrefix(size_t n) {
     contents_.remove_prefix(n);
   } else {
     CordRep* newrep = RemovePrefixFrom(tree, n);
-    Unref(tree);
+    CordRep::Unref(tree);
     contents_.replace_tree(VerifyTree(newrep));
   }
 }
@@ -941,7 +816,7 @@ void Cord::RemoveSuffix(size_t n) {
     contents_.reduce_size(n);
   } else {
     CordRep* newrep = RemoveSuffixFrom(tree, n);
-    Unref(tree);
+    CordRep::Unref(tree);
     contents_.replace_tree(VerifyTree(newrep));
   }
 }
@@ -974,13 +849,13 @@ static CordRep* NewSubRange(CordRep* node, size_t pos, size_t n) {
       results.pop_back();
       results.push_back(Concat(left, right));
     } else if (pos == 0 && n == node->length) {
-      results.push_back(Ref(node));
+      results.push_back(CordRep::Ref(node));
     } else if (node->tag != CONCAT) {
       if (node->tag == SUBSTRING) {
         pos += node->substring()->start;
         node = node->substring()->child;
       }
-      results.push_back(NewSubstring(Ref(node), pos, n));
+      results.push_back(NewSubstring(CordRep::Ref(node), pos, n));
     } else if (pos + n <= node->concat()->left->length) {
       todo.push_back(SubRange(node->concat()->left, pos, n));
     } else if (pos >= node->concat()->left->length) {
@@ -1058,9 +933,9 @@ class CordForest {
           concat_node->left = concat_freelist_;
           concat_freelist_ = concat_node;
         } else {
-          Ref(concat_node->right);
-          Ref(concat_node->left);
-          Unref(concat_node);
+          CordRep::Ref(concat_node->right);
+          CordRep::Ref(concat_node->left);
+          CordRep::Unref(concat_node);
         }
       } else {
         AddNode(node);
@@ -1488,7 +1363,7 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
   if (n < current_chunk_.size()) {
     // Range to read is a proper subrange of the current chunk.
     assert(current_leaf_ != nullptr);
-    CordRep* subnode = Ref(current_leaf_);
+    CordRep* subnode = CordRep::Ref(current_leaf_);
     const char* data =
         subnode->tag == EXTERNAL ? subnode->external()->base : subnode->data;
     subnode = NewSubstring(subnode, current_chunk_.data() - data, n);
@@ -1500,7 +1375,7 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
   // Range to read begins with a proper subrange of the current chunk.
   assert(!current_chunk_.empty());
   assert(current_leaf_ != nullptr);
-  CordRep* subnode = Ref(current_leaf_);
+  CordRep* subnode = CordRep::Ref(current_leaf_);
   if (current_chunk_.size() < subnode->length) {
     const char* data =
         subnode->tag == EXTERNAL ? subnode->external()->base : subnode->data;
@@ -1526,7 +1401,7 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
     // children starting from current_subtree_ if this loop exits while staying
     // below current_subtree_; etc.; alternatively, push parents instead of
     // right children on the stack).
-    subnode = Concat(subnode, Ref(node));
+    subnode = Concat(subnode, CordRep::Ref(node));
     n -= node->length;
     bytes_remaining_ -= node->length;
     node = nullptr;
@@ -1548,7 +1423,7 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
       node = node->concat()->left;
     } else {
       // Read left, descend right.
-      subnode = Concat(subnode, Ref(node->concat()->left));
+      subnode = Concat(subnode, CordRep::Ref(node->concat()->left));
       n -= node->concat()->left->length;
       bytes_remaining_ -= node->concat()->left->length;
       node = node->concat()->right;
@@ -1567,7 +1442,9 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
   // chunk.
   assert(node->tag == EXTERNAL || node->tag >= FLAT);
   assert(length > n);
-  if (n > 0) subnode = Concat(subnode, NewSubstring(Ref(node), offset, n));
+  if (n > 0) {
+    subnode = Concat(subnode, NewSubstring(CordRep::Ref(node), offset, n));
+  }
   const char* data =
       node->tag == EXTERNAL ? node->external()->base : node->data;
   current_chunk_ = absl::string_view(data + offset + n, length - n);
@@ -1691,7 +1568,9 @@ absl::string_view Cord::FlattenSlowPath() {
                                             s.size());
         });
   }
-  Unref(contents_.tree());
+  if (CordRep* tree = contents_.tree()) {
+    CordRep::Unref(tree);
+  }
   contents_.set_tree(new_rep);
   return absl::string_view(new_buffer, total_size);
 }
diff --git a/absl/strings/cord.h b/absl/strings/cord.h
index 5d5c897e..b9cdc435 100644
--- a/absl/strings/cord.h
+++ b/absl/strings/cord.h
@@ -82,6 +82,7 @@
 #include "absl/strings/internal/string_constant.h"
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
+#include "absl/types/variant.h"
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
diff --git a/absl/strings/internal/cord_internal.cc b/absl/strings/internal/cord_internal.cc
new file mode 100644
index 00000000..35f4d235
--- /dev/null
+++ b/absl/strings/internal/cord_internal.cc
@@ -0,0 +1,76 @@
+// Copyright 2020 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "absl/strings/internal/cord_internal.h"
+
+#include <atomic>
+#include <cassert>
+#include <memory>
+
+#include "absl/container/inlined_vector.h"
+#include "absl/strings/internal/cord_rep_flat.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace cord_internal {
+
+ABSL_CONST_INIT std::atomic<bool> cord_ring_buffer_enabled(false);
+
+void CordRep::Destroy(CordRep* rep) {
+  assert(rep != nullptr);
+
+  absl::InlinedVector<CordRep*, Constants::kInlinedVectorSize> pending;
+  while (true) {
+    assert(!rep->refcount.IsImmortal());
+    if (rep->tag == CONCAT) {
+      CordRepConcat* rep_concat = rep->concat();
+      CordRep* right = rep_concat->right;
+      if (!right->refcount.Decrement()) {
+        pending.push_back(right);
+      }
+      CordRep* left = rep_concat->left;
+      delete rep_concat;
+      rep = nullptr;
+      if (!left->refcount.Decrement()) {
+        rep = left;
+        continue;
+      }
+    } else if (rep->tag == EXTERNAL) {
+      CordRepExternal::Delete(rep);
+      rep = nullptr;
+    } else if (rep->tag == SUBSTRING) {
+      CordRepSubstring* rep_substring = rep->substring();
+      CordRep* child = rep_substring->child;
+      delete rep_substring;
+      rep = nullptr;
+      if (!child->refcount.Decrement()) {
+        rep = child;
+        continue;
+      }
+    } else {
+      CordRepFlat::Delete(rep);
+      rep = nullptr;
+    }
+
+    if (!pending.empty()) {
+      rep = pending.back();
+      pending.pop_back();
+    } else {
+      break;
+    }
+  }
+}
+
+}  // namespace cord_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
diff --git a/absl/strings/internal/cord_internal.h b/absl/strings/internal/cord_internal.h
index 6fb75c4f..f1af8e6e 100644
--- a/absl/strings/internal/cord_internal.h
+++ b/absl/strings/internal/cord_internal.h
@@ -22,6 +22,7 @@
 #include <type_traits>
 
 #include "absl/base/internal/invoke.h"
+#include "absl/base/optimization.h"
 #include "absl/container/internal/compressed_tuple.h"
 #include "absl/meta/type_traits.h"
 #include "absl/strings/string_view.h"
@@ -30,6 +31,28 @@ namespace absl {
 ABSL_NAMESPACE_BEGIN
 namespace cord_internal {
 
+extern std::atomic<bool> cord_ring_buffer_enabled;
+
+inline void enable_cord_ring_buffer(bool enable) {
+  cord_ring_buffer_enabled.store(enable, std::memory_order_relaxed);
+}
+
+enum Constants {
+  // The inlined size to use with absl::InlinedVector.
+  //
+  // Note: The InlinedVectors in this file (and in cord.h) do not need to use
+  // the same value for their inlined size. The fact that they do is historical.
+  // It may be desirable for each to use a different inlined size optimized for
+  // that InlinedVector's usage.
+  //
+  // TODO(jgm): Benchmark to see if there's a more optimal value than 47 for
+  // the inlined vector size (47 exists for backward compatibility).
+  kInlinedVectorSize = 47,
+
+  // Prefer copying blocks of at most this size, otherwise reference count.
+  kMaxBytesToCopy = 511
+};
+
 // Wraps std::atomic for reference counting.
 class Refcount {
  public:
@@ -151,6 +174,34 @@ struct CordRep {
   inline const CordRepExternal* external() const;
   inline CordRepFlat* flat();
   inline const CordRepFlat* flat() const;
+
+  // --------------------------------------------------------------------
+  // Memory management
+
+  // This internal routine is called from the cold path of Unref below. Keeping
+  // it in a separate routine allows good inlining of Unref into many profitable
+  // call sites. However, the call to this function can be highly disruptive to
+  // the register pressure in those callers. To minimize the cost to callers, we
+  // use a special LLVM calling convention that preserves most registers. This
+  // allows the call to this routine in cold paths to not disrupt the caller's
+  // register pressure. This calling convention is not available on all
+  // platforms; we intentionally allow LLVM to ignore the attribute rather than
+  // attempting to hardcode the list of supported platforms.
+#if defined(__clang__) && !defined(__i386__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wattributes"
+  __attribute__((preserve_most))
+#pragma clang diagnostic pop
+#endif
+  static void Destroy(CordRep* rep);
+
+  // Increments the reference count of `rep`.
+  // Requires `rep` to be a non-null pointer value.
+  static inline CordRep* Ref(CordRep* rep);
+
+  // Decrements the reference count of `rep`. Destroys rep if count reaches
+  // zero. Requires `rep` to be a non-null pointer value.
+  static inline void Unref(CordRep* rep);
 };
 
 struct CordRepConcat : public CordRep {
@@ -284,7 +335,63 @@ static_assert(sizeof(InlineData) == kMaxInline + 1, "");
 static_assert(sizeof(AsTree) == sizeof(InlineData), "");
 static_assert(offsetof(AsTree, tagged_size) == kMaxInline, "");
 
+inline CordRepConcat* CordRep::concat() {
+  assert(tag == CONCAT);
+  return static_cast<CordRepConcat*>(this);
+}
+
+inline const CordRepConcat* CordRep::concat() const {
+  assert(tag == CONCAT);
+  return static_cast<const CordRepConcat*>(this);
+}
+
+inline CordRepSubstring* CordRep::substring() {
+  assert(tag == SUBSTRING);
+  return static_cast<CordRepSubstring*>(this);
+}
+
+inline const CordRepSubstring* CordRep::substring() const {
+  assert(tag == SUBSTRING);
+  return static_cast<const CordRepSubstring*>(this);
+}
+
+inline CordRepExternal* CordRep::external() {
+  assert(tag == EXTERNAL);
+  return static_cast<CordRepExternal*>(this);
+}
+
+inline const CordRepExternal* CordRep::external() const {
+  assert(tag == EXTERNAL);
+  return static_cast<const CordRepExternal*>(this);
+}
+
+inline CordRepFlat* CordRep::flat() {
+  assert(tag >= FLAT && tag <= MAX_FLAT_TAG);
+  return reinterpret_cast<CordRepFlat*>(this);
+}
+
+inline const CordRepFlat* CordRep::flat() const {
+  assert(tag >= FLAT && tag <= MAX_FLAT_TAG);
+  return reinterpret_cast<const CordRepFlat*>(this);
+}
+
+inline CordRep* CordRep::Ref(CordRep* rep) {
+  assert(rep != nullptr);
+  rep->refcount.Increment();
+  return rep;
+}
+
+inline void CordRep::Unref(CordRep* rep) {
+  assert(rep != nullptr);
+  // Expect refcount to be 0. Avoiding the cost of an atomic decrement should
+  // typically outweigh the cost of an extra branch checking for ref == 1.
+  if (ABSL_PREDICT_FALSE(!rep->refcount.DecrementExpectHighRefcount())) {
+    Destroy(rep);
+  }
+}
+
 }  // namespace cord_internal
+
 ABSL_NAMESPACE_END
 }  // namespace absl
 #endif  // ABSL_STRINGS_INTERNAL_CORD_INTERNAL_H_