diff options
Diffstat (limited to 'absl/strings/cord.cc')
| -rw-r--r-- | absl/strings/cord.cc | 1563 |
1 files changed, 469 insertions, 1094 deletions
diff --git a/absl/strings/cord.cc b/absl/strings/cord.cc index 93533757..85a67a08 100644 --- a/absl/strings/cord.cc +++ b/absl/strings/cord.cc @@ -34,10 +34,16 @@ #include "absl/base/port.h" #include "absl/container/fixed_array.h" #include "absl/container/inlined_vector.h" +#include "absl/strings/cord_buffer.h" #include "absl/strings/escaping.h" +#include "absl/strings/internal/cord_data_edge.h" #include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" +#include "absl/strings/internal/cord_rep_crc.h" #include "absl/strings/internal/cord_rep_flat.h" -#include "absl/strings/internal/cord_rep_ring.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_scope.h" +#include "absl/strings/internal/cordz_update_tracker.h" #include "absl/strings/internal/resize_uninitialized.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" @@ -48,73 +54,19 @@ namespace absl { ABSL_NAMESPACE_BEGIN using ::absl::cord_internal::CordRep; -using ::absl::cord_internal::CordRepConcat; +using ::absl::cord_internal::CordRepBtree; +using ::absl::cord_internal::CordRepCrc; using ::absl::cord_internal::CordRepExternal; using ::absl::cord_internal::CordRepFlat; -using ::absl::cord_internal::CordRepRing; using ::absl::cord_internal::CordRepSubstring; -using ::absl::cord_internal::kMinFlatLength; +using ::absl::cord_internal::CordzUpdateTracker; +using ::absl::cord_internal::InlineData; using ::absl::cord_internal::kMaxFlatLength; - -using ::absl::cord_internal::CONCAT; -using ::absl::cord_internal::EXTERNAL; -using ::absl::cord_internal::FLAT; -using ::absl::cord_internal::RING; -using ::absl::cord_internal::SUBSTRING; +using ::absl::cord_internal::kMinFlatLength; using ::absl::cord_internal::kInlinedVectorSize; using ::absl::cord_internal::kMaxBytesToCopy; -constexpr uint64_t Fibonacci(unsigned char n, uint64_t a = 0, uint64_t b = 1) { - return n == 0 ? a : Fibonacci(n - 1, b, a + b); -} - -static_assert(Fibonacci(63) == 6557470319842, - "Fibonacci values computed incorrectly"); - -// Minimum length required for a given depth tree -- a tree is considered -// balanced if -// length(t) >= min_length[depth(t)] -// The root node depth is allowed to become twice as large to reduce rebalancing -// for larger strings (see IsRootBalanced). -static constexpr uint64_t min_length[] = { - Fibonacci(2), Fibonacci(3), Fibonacci(4), Fibonacci(5), - Fibonacci(6), Fibonacci(7), Fibonacci(8), Fibonacci(9), - Fibonacci(10), Fibonacci(11), Fibonacci(12), Fibonacci(13), - Fibonacci(14), Fibonacci(15), Fibonacci(16), Fibonacci(17), - Fibonacci(18), Fibonacci(19), Fibonacci(20), Fibonacci(21), - Fibonacci(22), Fibonacci(23), Fibonacci(24), Fibonacci(25), - Fibonacci(26), Fibonacci(27), Fibonacci(28), Fibonacci(29), - Fibonacci(30), Fibonacci(31), Fibonacci(32), Fibonacci(33), - Fibonacci(34), Fibonacci(35), Fibonacci(36), Fibonacci(37), - Fibonacci(38), Fibonacci(39), Fibonacci(40), Fibonacci(41), - Fibonacci(42), Fibonacci(43), Fibonacci(44), Fibonacci(45), - Fibonacci(46), Fibonacci(47), - 0xffffffffffffffffull, // Avoid overflow -}; - -static const int kMinLengthSize = ABSL_ARRAYSIZE(min_length); - -static inline bool cord_ring_enabled() { - return cord_internal::cord_ring_buffer_enabled.load( - std::memory_order_relaxed); -} - -static inline bool IsRootBalanced(CordRep* node) { - if (node->tag != CONCAT) { - return true; - } else if (node->concat()->depth() <= 15) { - return true; - } else if (node->concat()->depth() > kMinLengthSize) { - return false; - } else { - // Allow depth to become twice as large as implied by fibonacci rule to - // reduce rebalancing for larger strings. - return (node->length >= min_length[node->concat()->depth() / 2]); - } -} - -static CordRep* Rebalance(CordRep* node); static void DumpNode(CordRep* rep, bool include_data, std::ostream* os, int indent = 0); static bool VerifyNode(CordRep* root, CordRep* start_node, @@ -136,119 +88,32 @@ static inline CordRep* VerifyTree(CordRep* node) { return node; } -// Return the depth of a node -static int Depth(const CordRep* rep) { - if (rep->tag == CONCAT) { - return rep->concat()->depth(); - } else { - return 0; - } -} - -static void SetConcatChildren(CordRepConcat* concat, CordRep* left, - CordRep* right) { - concat->left = left; - concat->right = right; - - concat->length = left->length + right->length; - concat->set_depth(1 + std::max(Depth(left), Depth(right))); -} - -// Create a concatenation of the specified nodes. -// Does not change the refcounts of "left" and "right". -// The returned node has a refcount of 1. -static CordRep* RawConcat(CordRep* left, CordRep* right) { - // Avoid making degenerate concat nodes (one child is empty) - if (left == nullptr) return right; - if (right == nullptr) return left; - if (left->length == 0) { - CordRep::Unref(left); - return right; - } - if (right->length == 0) { - CordRep::Unref(right); - return left; - } - - CordRepConcat* rep = new CordRepConcat(); - rep->tag = CONCAT; - SetConcatChildren(rep, left, right); - - return rep; -} - -static CordRep* Concat(CordRep* left, CordRep* right) { - CordRep* rep = RawConcat(left, right); - if (rep != nullptr && !IsRootBalanced(rep)) { - rep = Rebalance(rep); - } - return VerifyTree(rep); -} - -// Make a balanced tree out of an array of leaf nodes. -static CordRep* MakeBalancedTree(CordRep** reps, size_t n) { - // Make repeated passes over the array, merging adjacent pairs - // until we are left with just a single node. - while (n > 1) { - size_t dst = 0; - for (size_t src = 0; src < n; src += 2) { - if (src + 1 < n) { - reps[dst] = Concat(reps[src], reps[src + 1]); - } else { - reps[dst] = reps[src]; - } - dst++; - } - n = dst; - } - - return reps[0]; -} - static CordRepFlat* CreateFlat(const char* data, size_t length, - size_t alloc_hint) { + size_t alloc_hint) { CordRepFlat* flat = CordRepFlat::New(length + alloc_hint); flat->length = length; memcpy(flat->Data(), data, length); return flat; } -// Creates a new flat or ringbuffer out of the specified array. +// Creates a new flat or Btree out of the specified array. // The returned node has a refcount of 1. -static CordRep* RingNewTree(const char* data, size_t length, - size_t alloc_hint) { +static CordRep* NewBtree(const char* data, size_t length, size_t alloc_hint) { if (length <= kMaxFlatLength) { return CreateFlat(data, length, alloc_hint); } CordRepFlat* flat = CreateFlat(data, kMaxFlatLength, 0); data += kMaxFlatLength; length -= kMaxFlatLength; - size_t extra = (length - 1) / kMaxFlatLength + 1; - auto* root = CordRepRing::Create(flat, extra); - return CordRepRing::Append(root, {data, length}, alloc_hint); + auto* root = CordRepBtree::Create(flat); + return CordRepBtree::Append(root, {data, length}, alloc_hint); } // Create a new tree out of the specified array. // The returned node has a refcount of 1. -static CordRep* NewTree(const char* data, - size_t length, - size_t alloc_hint) { +static CordRep* NewTree(const char* data, size_t length, size_t alloc_hint) { if (length == 0) return nullptr; - if (cord_ring_enabled()) { - return RingNewTree(data, length, alloc_hint); - } - absl::FixedArray<CordRep*> reps((length - 1) / kMaxFlatLength + 1); - size_t n = 0; - do { - const size_t len = std::min(length, kMaxFlatLength); - CordRepFlat* rep = CordRepFlat::New(len + alloc_hint); - rep->length = len; - memcpy(rep->Data(), data, len); - reps[n++] = VerifyTree(rep); - data += len; - length -= len; - } while (length != 0); - return MakeBalancedTree(reps.data(), n); + return NewBtree(data, length, alloc_hint); } namespace cord_internal { @@ -263,32 +128,46 @@ void InitializeCordRepExternal(absl::string_view data, CordRepExternal* rep) { } // namespace cord_internal -static CordRep* NewSubstring(CordRep* child, size_t offset, size_t length) { - // Never create empty substring nodes - if (length == 0) { - CordRep::Unref(child); - return nullptr; - } else { - CordRepSubstring* rep = new CordRepSubstring(); - assert((offset + length) <= child->length); - rep->length = length; - rep->tag = SUBSTRING; - rep->start = offset; - rep->child = child; - return VerifyTree(rep); +// Creates a CordRep from the provided string. If the string is large enough, +// and not wasteful, we move the string into an external cord rep, preserving +// the already allocated string contents. +// Requires the provided string length to be larger than `kMaxInline`. +static CordRep* CordRepFromString(std::string&& src) { + assert(src.length() > cord_internal::kMaxInline); + if ( + // String is short: copy data to avoid external block overhead. + src.size() <= kMaxBytesToCopy || + // String is wasteful: copy data to avoid pinning too much unused memory. + src.size() < src.capacity() / 2 + ) { + return NewTree(src.data(), src.size(), 0); } + + struct StringReleaser { + void operator()(absl::string_view /* data */) {} + std::string data; + }; + const absl::string_view original_data = src; + auto* rep = + static_cast<::absl::cord_internal::CordRepExternalImpl<StringReleaser>*>( + absl::cord_internal::NewExternalRep(original_data, + StringReleaser{std::move(src)})); + // Moving src may have invalidated its data pointer, so adjust it. + rep->base = rep->template get<0>().data.data(); + return rep; } // -------------------------------------------------------------------- // Cord::InlineRep functions +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL constexpr unsigned char Cord::InlineRep::kMaxInline; +#endif -inline void Cord::InlineRep::set_data(const char* data, size_t n, - bool nullify_tail) { +inline void Cord::InlineRep::set_data(const char* data, size_t n) { static_assert(kMaxInline == 15, "set_data is hard-coded for a length of 15"); - cord_internal::SmallMemmove(data_.as_chars(), data, n, nullify_tail); + cord_internal::SmallMemmove<true>(data_.as_chars(), data, n); set_inline_size(n); } @@ -299,20 +178,6 @@ inline char* Cord::InlineRep::set_data(size_t n) { return data_.as_chars(); } -inline CordRep* Cord::InlineRep::force_tree(size_t extra_hint) { - if (data_.is_tree()) { - return data_.as_tree(); - } - - size_t len = inline_size(); - CordRepFlat* result = CordRepFlat::New(len + extra_hint); - result->length = len; - static_assert(kMinFlatLength >= sizeof(data_), ""); - memcpy(result->Data(), data_.as_chars(), sizeof(data_)); - set_tree(result); - return result; -} - inline void Cord::InlineRep::reduce_size(size_t n) { size_t tag = inline_size(); assert(tag <= kMaxInline); @@ -328,31 +193,68 @@ inline void Cord::InlineRep::remove_prefix(size_t n) { reduce_size(n); } -// Returns `rep` converted into a CordRepRing. -// Directly returns `rep` if `rep` is already a CordRepRing. -static CordRepRing* ForceRing(CordRep* rep, size_t extra) { - return (rep->tag == RING) ? rep->ring() : CordRepRing::Create(rep, extra); +// Returns `rep` converted into a CordRepBtree. +// Directly returns `rep` if `rep` is already a CordRepBtree. +static CordRepBtree* ForceBtree(CordRep* rep) { + return rep->IsBtree() + ? rep->btree() + : CordRepBtree::Create(cord_internal::RemoveCrcNode(rep)); +} + +void Cord::InlineRep::AppendTreeToInlined(CordRep* tree, + MethodIdentifier method) { + assert(!is_tree()); + if (!data_.is_empty()) { + CordRepFlat* flat = MakeFlatWithExtraCapacity(0); + tree = CordRepBtree::Append(CordRepBtree::Create(flat), tree); + } + EmplaceTree(tree, method); +} + +void Cord::InlineRep::AppendTreeToTree(CordRep* tree, MethodIdentifier method) { + assert(is_tree()); + const CordzUpdateScope scope(data_.cordz_info(), method); + tree = CordRepBtree::Append(ForceBtree(data_.as_tree()), tree); + SetTree(tree, scope); } -void Cord::InlineRep::AppendTree(CordRep* tree) { - if (tree == nullptr) return; - if (data_.is_empty()) { - set_tree(tree); - } else if (cord_ring_enabled()) { - set_tree(CordRepRing::Append(ForceRing(force_tree(0), 1), tree)); +void Cord::InlineRep::AppendTree(CordRep* tree, MethodIdentifier method) { + assert(tree != nullptr); + assert(tree->length != 0); + assert(!tree->IsCrc()); + if (data_.is_tree()) { + AppendTreeToTree(tree, method); } else { - set_tree(Concat(force_tree(0), tree)); + AppendTreeToInlined(tree, method); } } -void Cord::InlineRep::PrependTree(CordRep* tree) { +void Cord::InlineRep::PrependTreeToInlined(CordRep* tree, + MethodIdentifier method) { + assert(!is_tree()); + if (!data_.is_empty()) { + CordRepFlat* flat = MakeFlatWithExtraCapacity(0); + tree = CordRepBtree::Prepend(CordRepBtree::Create(flat), tree); + } + EmplaceTree(tree, method); +} + +void Cord::InlineRep::PrependTreeToTree(CordRep* tree, + MethodIdentifier method) { + assert(is_tree()); + const CordzUpdateScope scope(data_.cordz_info(), method); + tree = CordRepBtree::Prepend(ForceBtree(data_.as_tree()), tree); + SetTree(tree, scope); +} + +void Cord::InlineRep::PrependTree(CordRep* tree, MethodIdentifier method) { assert(tree != nullptr); - if (data_.is_empty()) { - set_tree(tree); - } else if (cord_ring_enabled()) { - set_tree(CordRepRing::Prepend(ForceRing(force_tree(0), 1), tree)); + assert(tree->length != 0); + assert(!tree->IsCrc()); + if (data_.is_tree()) { + PrependTreeToTree(tree, method); } else { - set_tree(Concat(tree, force_tree(0))); + PrependTreeToInlined(tree, method); } } @@ -362,8 +264,8 @@ void Cord::InlineRep::PrependTree(CordRep* tree) { // written to region and the actual size increase will be written to size. static inline bool PrepareAppendRegion(CordRep* root, char** region, size_t* size, size_t max_length) { - if (root->tag == RING && root->refcount.IsOne()) { - Span<char> span = root->ring()->GetAppendBuffer(max_length); + if (root->IsBtree() && root->refcount.IsOne()) { + Span<char> span = root->btree()->GetAppendBuffer(max_length); if (!span.empty()) { *region = span.data(); *size = span.size(); @@ -371,13 +273,8 @@ static inline bool PrepareAppendRegion(CordRep* root, char** region, } } - // Search down the right-hand path for a non-full FLAT node. CordRep* dst = root; - while (dst->tag == CONCAT && dst->refcount.IsOne()) { - dst = dst->concat()->right; - } - - if (dst->tag < FLAT || !dst->refcount.IsOne()) { + if (!dst->IsFlat() || !dst->refcount.IsOne()) { *region = nullptr; *size = 0; return false; @@ -391,12 +288,7 @@ static inline bool PrepareAppendRegion(CordRep* root, char** region, return false; } - size_t size_increase = std::min(capacity - in_use, max_length); - - // We need to update the length fields for all nodes, including the leaf node. - for (CordRep* rep = root; rep != dst; rep = rep->concat()->right) { - rep->length += size_increase; - } + const size_t size_increase = std::min(capacity - in_use, max_length); dst->length += size_increase; *region = dst->flat()->Data() + in_use; @@ -404,148 +296,56 @@ static inline bool PrepareAppendRegion(CordRep* root, char** region, return true; } -void Cord::InlineRep::GetAppendRegion(char** region, size_t* size, - size_t max_length) { - if (max_length == 0) { - *region = nullptr; - *size = 0; - return; - } - - // Try to fit in the inline buffer if possible. - if (!is_tree()) { - size_t inline_length = inline_size(); - if (max_length <= kMaxInline - inline_length) { - *region = data_.as_chars() + inline_length; - *size = max_length; - set_inline_size(inline_length + max_length); - return; - } - } - - CordRep* root = force_tree(max_length); - - if (PrepareAppendRegion(root, region, size, max_length)) { - return; - } - - // Allocate new node. - CordRepFlat* new_node = - CordRepFlat::New(std::max(static_cast<size_t>(root->length), max_length)); - new_node->length = std::min(new_node->Capacity(), max_length); - *region = new_node->Data(); - *size = new_node->length; - - if (cord_ring_enabled()) { - replace_tree(CordRepRing::Append(ForceRing(root, 1), new_node)); - return; - } - replace_tree(Concat(root, new_node)); -} - -void Cord::InlineRep::GetAppendRegion(char** region, size_t* size) { - const size_t max_length = std::numeric_limits<size_t>::max(); - - // Try to fit in the inline buffer if possible. - if (!data_.is_tree()) { - size_t inline_length = inline_size(); - if (inline_length < kMaxInline) { - *region = data_.as_chars() + inline_length; - *size = kMaxInline - inline_length; - set_inline_size(kMaxInline); - return; - } - } - - CordRep* root = force_tree(max_length); - - if (PrepareAppendRegion(root, region, size, max_length)) { - return; - } - - // Allocate new node. - CordRepFlat* new_node = CordRepFlat::New(root->length); - new_node->length = new_node->Capacity(); - *region = new_node->Data(); - *size = new_node->length; - - if (cord_ring_enabled()) { - replace_tree(CordRepRing::Append(ForceRing(root, 1), new_node)); +void Cord::InlineRep::AssignSlow(const Cord::InlineRep& src) { + assert(&src != this); + assert(is_tree() || src.is_tree()); + auto constexpr method = CordzUpdateTracker::kAssignCord; + if (ABSL_PREDICT_TRUE(!is_tree())) { + EmplaceTree(CordRep::Ref(src.as_tree()), src.data_, method); return; } - replace_tree(Concat(root, new_node)); -} -// If the rep is a leaf, this will increment the value at total_mem_usage and -// will return true. -static bool RepMemoryUsageLeaf(const CordRep* rep, size_t* total_mem_usage) { - if (rep->tag >= FLAT) { - *total_mem_usage += rep->flat()->AllocatedSize(); - return true; - } - if (rep->tag == EXTERNAL) { - *total_mem_usage += sizeof(CordRepConcat) + rep->length; - return true; - } - return false; -} - -void Cord::InlineRep::AssignSlow(const Cord::InlineRep& src) { - ClearSlow(); - - data_ = src.data_; - if (is_tree()) { - data_.set_profiled(false); - CordRep::Ref(tree()); - clear_cordz_info(); + CordRep* tree = as_tree(); + if (CordRep* src_tree = src.tree()) { + // Leave any existing `cordz_info` in place, and let MaybeTrackCord() + // decide if this cord should be (or remains to be) sampled or not. + data_.set_tree(CordRep::Ref(src_tree)); + CordzInfo::MaybeTrackCord(data_, src.data_, method); + } else { + CordzInfo::MaybeUntrackCord(data_.cordz_info()); + data_ = src.data_; } + CordRep::Unref(tree); } -void Cord::InlineRep::ClearSlow() { +void Cord::InlineRep::UnrefTree() { if (is_tree()) { + CordzInfo::MaybeUntrackCord(data_.cordz_info()); CordRep::Unref(tree()); } - ResetToEmpty(); } // -------------------------------------------------------------------- // Constructors and destructors -Cord::Cord(absl::string_view src) { +Cord::Cord(absl::string_view src, MethodIdentifier method) + : contents_(InlineData::kDefaultInit) { const size_t n = src.size(); if (n <= InlineRep::kMaxInline) { - contents_.set_data(src.data(), n, false); + contents_.set_data(src.data(), n); } else { - contents_.set_tree(NewTree(src.data(), n, 0)); + CordRep* rep = NewTree(src.data(), n, 0); + contents_.EmplaceTree(rep, method); } } template <typename T, Cord::EnableIfString<T>> -Cord::Cord(T&& src) { - if ( - // String is short: copy data to avoid external block overhead. - src.size() <= kMaxBytesToCopy || - // String is wasteful: copy data to avoid pinning too much unused memory. - src.size() < src.capacity() / 2 - ) { - if (src.size() <= InlineRep::kMaxInline) { - contents_.set_data(src.data(), src.size(), false); - } else { - contents_.set_tree(NewTree(src.data(), src.size(), 0)); - } +Cord::Cord(T&& src) : contents_(InlineData::kDefaultInit) { + if (src.size() <= InlineRep::kMaxInline) { + contents_.set_data(src.data(), src.size()); } else { - struct StringReleaser { - void operator()(absl::string_view /* data */) {} - std::string data; - }; - const absl::string_view original_data = src; - auto* rep = static_cast< - ::absl::cord_internal::CordRepExternalImpl<StringReleaser>*>( - absl::cord_internal::NewExternalRep( - original_data, StringReleaser{std::forward<T>(src)})); - // Moving src may have invalidated its data pointer, so adjust it. - rep->base = rep->template get<0>().data.data(); - contents_.set_tree(rep); + CordRep* rep = CordRepFromString(std::forward<T>(src)); + contents_.EmplaceTree(rep, CordzUpdateTracker::kConstructorString); } } @@ -554,9 +354,9 @@ template Cord::Cord(std::string&& src); // The destruction code is separate so that the compiler can determine // that it does not need to call the destructor on a moved-from Cord. void Cord::DestroyCordSlow() { - if (CordRep* tree = contents_.tree()) { - CordRep::Unref(VerifyTree(tree)); - } + assert(contents_.is_tree()); + CordzInfo::MaybeUntrackCord(contents_.cordz_info()); + CordRep::Unref(VerifyTree(contents_.as_tree())); } // -------------------------------------------------------------------- @@ -568,109 +368,101 @@ void Cord::Clear() { } } -Cord& Cord::operator=(absl::string_view src) { +Cord& Cord::AssignLargeString(std::string&& src) { + auto constexpr method = CordzUpdateTracker::kAssignString; + assert(src.size() > kMaxBytesToCopy); + CordRep* rep = CordRepFromString(std::move(src)); + if (CordRep* tree = contents_.tree()) { + CordzUpdateScope scope(contents_.cordz_info(), method); + contents_.SetTree(rep, scope); + CordRep::Unref(tree); + } else { + contents_.EmplaceTree(rep, method); + } + return *this; +} +Cord& Cord::operator=(absl::string_view src) { + auto constexpr method = CordzUpdateTracker::kAssignString; const char* data = src.data(); size_t length = src.size(); CordRep* tree = contents_.tree(); if (length <= InlineRep::kMaxInline) { - // Embed into this->contents_ - contents_.set_data(data, length, true); - if (tree) CordRep::Unref(tree); - return *this; - } - if (tree != nullptr && tree->tag >= FLAT && - tree->flat()->Capacity() >= length && - tree->refcount.IsOne()) { - // Copy in place if the existing FLAT node is reusable. - memmove(tree->flat()->Data(), data, length); - tree->length = length; - VerifyTree(tree); + // Embed into this->contents_, which is somewhat subtle: + // - MaybeUntrackCord must be called before Unref(tree). + // - MaybeUntrackCord must be called before set_data() clobbers cordz_info. + // - set_data() must be called before Unref(tree) as it may reference tree. + if (tree != nullptr) CordzInfo::MaybeUntrackCord(contents_.cordz_info()); + contents_.set_data(data, length); + if (tree != nullptr) CordRep::Unref(tree); return *this; } - contents_.set_tree(NewTree(data, length, 0)); - if (tree) CordRep::Unref(tree); - return *this; -} - -template <typename T, Cord::EnableIfString<T>> -Cord& Cord::operator=(T&& src) { - if (src.size() <= kMaxBytesToCopy) { - *this = absl::string_view(src); + if (tree != nullptr) { + CordzUpdateScope scope(contents_.cordz_info(), method); + if (tree->IsFlat() && tree->flat()->Capacity() >= length && + tree->refcount.IsOne()) { + // Copy in place if the existing FLAT node is reusable. + memmove(tree->flat()->Data(), data, length); + tree->length = length; + VerifyTree(tree); + return *this; + } + contents_.SetTree(NewTree(data, length, 0), scope); + CordRep::Unref(tree); } else { - *this = Cord(std::forward<T>(src)); + contents_.EmplaceTree(NewTree(data, length, 0), method); } return *this; } -template Cord& Cord::operator=(std::string&& src); - // TODO(sanjay): Move to Cord::InlineRep section of file. For now, // we keep it here to make diffs easier. -void Cord::InlineRep::AppendArray(const char* src_data, size_t src_size) { - if (src_size == 0) return; // memcpy(_, nullptr, 0) is undefined. +void Cord::InlineRep::AppendArray(absl::string_view src, + MethodIdentifier method) { + if (src.empty()) return; // memcpy(_, nullptr, 0) is undefined. size_t appended = 0; - CordRep* root = nullptr; - if (is_tree()) { - root = data_.as_tree(); + CordRep* rep = tree(); + const CordRep* const root = rep; + CordzUpdateScope scope(root ? cordz_info() : nullptr, method); + if (root != nullptr) { + rep = cord_internal::RemoveCrcNode(rep); char* region; - if (PrepareAppendRegion(root, ®ion, &appended, src_size)) { - memcpy(region, src_data, appended); + if (PrepareAppendRegion(rep, ®ion, &appended, src.size())) { + memcpy(region, src.data(), appended); } } else { // Try to fit in the inline buffer if possible. size_t inline_length = inline_size(); - if (src_size <= kMaxInline - inline_length) { + if (src.size() <= kMaxInline - inline_length) { // Append new data to embedded array - memcpy(data_.as_chars() + inline_length, src_data, src_size); - set_inline_size(inline_length + src_size); + memcpy(data_.as_chars() + inline_length, src.data(), src.size()); + set_inline_size(inline_length + src.size()); return; } - // It is possible that src_data == data_, but when we transition from an - // InlineRep to a tree we need to assign data_ = root via set_tree. To - // avoid corrupting the source data before we copy it, delay calling - // set_tree until after we've copied data. - // We are going from an inline size to beyond inline size. Make the new size - // either double the inlined size, or the added size + 10%. - const size_t size1 = inline_length * 2 + src_size; - const size_t size2 = inline_length + src_size / 10; - root = CordRepFlat::New(std::max<size_t>(size1, size2)); - appended = std::min( - src_size, root->flat()->Capacity() - inline_length); - memcpy(root->flat()->Data(), data_.as_chars(), inline_length); - memcpy(root->flat()->Data() + inline_length, src_data, appended); - root->length = inline_length + appended; - set_tree(root); + // Allocate flat to be a perfect fit on first append exceeding inlined size. + // Subsequent growth will use amortized growth until we reach maximum flat + // size. + rep = CordRepFlat::New(inline_length + src.size()); + appended = std::min(src.size(), rep->flat()->Capacity() - inline_length); + memcpy(rep->flat()->Data(), data_.as_chars(), inline_length); + memcpy(rep->flat()->Data() + inline_length, src.data(), appended); + rep->length = inline_length + appended; } - src_data += appended; - src_size -= appended; - if (src_size == 0) { + src.remove_prefix(appended); + if (src.empty()) { + CommitTree(root, rep, scope, method); return; } - if (cord_ring_enabled()) { - absl::string_view data(src_data, src_size); - root = ForceRing(root, (data.size() - 1) / kMaxFlatLength + 1); - replace_tree(CordRepRing::Append(root->ring(), data)); - return; - } + // TODO(b/192061034): keep legacy 10% growth rate: consider other rates. + rep = ForceBtree(rep); + const size_t min_growth = std::max<size_t>(rep->length / 10, src.size()); + rep = CordRepBtree::Append(rep->btree(), src, min_growth - src.size()); - // Use new block(s) for any remaining bytes that were not handled above. - // Alloc extra memory only if the right child of the root of the new tree is - // going to be a FLAT node, which will permit further inplace appends. - size_t length = src_size; - if (src_size < kMaxFlatLength) { - // The new length is either - // - old size + 10% - // - old_size + src_size - // This will cause a reasonable conservative step-up in size that is still - // large enough to avoid excessive amounts of small fragments being added. - length = std::max<size_t>(root->length / 10, src_size); - } - set_tree(Concat(root, NewTree(src_data, src_size, length - src_size))); + CommitTree(root, rep, scope, method); } inline CordRep* Cord::TakeRep() const& { @@ -685,10 +477,18 @@ inline CordRep* Cord::TakeRep() && { template <typename C> inline void Cord::AppendImpl(C&& src) { + auto constexpr method = CordzUpdateTracker::kAppendCord; if (empty()) { - // In case of an empty destination avoid allocating a new node, do not copy - // data. - *this = std::forward<C>(src); + // Since destination is empty, we can avoid allocating a node, + if (src.contents_.is_tree()) { + // by taking the tree directly + CordRep* rep = + cord_internal::RemoveCrcNode(std::forward<C>(src).TakeRep()); + contents_.EmplaceTree(rep, method); + } else { + // or copying over inline data + contents_.data_ = src.contents_.data_; + } return; } @@ -698,12 +498,12 @@ inline void Cord::AppendImpl(C&& src) { CordRep* src_tree = src.contents_.tree(); if (src_tree == nullptr) { // src has embedded data. - contents_.AppendArray(src.contents_.data(), src_size); + contents_.AppendArray({src.contents_.data(), src_size}, method); return; } - if (src_tree->tag >= FLAT) { + if (src_tree->IsFlat()) { // src tree just has one flat node. - contents_.AppendArray(src_tree->flat()->Data(), src_size); + contents_.AppendArray({src_tree->flat()->Data(), src_size}, method); return; } if (&src == this) { @@ -719,19 +519,65 @@ inline void Cord::AppendImpl(C&& src) { } // Guaranteed to be a tree (kMaxBytesToCopy > kInlinedSize) - contents_.AppendTree(std::forward<C>(src).TakeRep()); + CordRep* rep = cord_internal::RemoveCrcNode(std::forward<C>(src).TakeRep()); + contents_.AppendTree(rep, CordzUpdateTracker::kAppendCord); +} + +static CordRep::ExtractResult ExtractAppendBuffer(CordRep* rep, + size_t min_capacity) { + switch (rep->tag) { + case cord_internal::BTREE: + return CordRepBtree::ExtractAppendBuffer(rep->btree(), min_capacity); + default: + if (rep->IsFlat() && rep->refcount.IsOne() && + rep->flat()->Capacity() - rep->length >= min_capacity) { + return {nullptr, rep}; + } + return {rep, nullptr}; + } } -void Cord::Append(const Cord& src) { AppendImpl(src); } +static CordBuffer CreateAppendBuffer(InlineData& data, size_t capacity) { + // Watch out for overflow, people can ask for size_t::max(). + const size_t size = data.inline_size(); + capacity = (std::min)(std::numeric_limits<size_t>::max() - size, capacity); + CordBuffer buffer = CordBuffer::CreateWithDefaultLimit(size + capacity); + cord_internal::SmallMemmove(buffer.data(), data.as_chars(), size); + buffer.SetLength(size); + data = {}; + return buffer; +} -void Cord::Append(Cord&& src) { AppendImpl(std::move(src)); } +CordBuffer Cord::GetAppendBufferSlowPath(size_t capacity, size_t min_capacity) { + auto constexpr method = CordzUpdateTracker::kGetAppendBuffer; + CordRep* tree = contents_.tree(); + if (tree != nullptr) { + CordzUpdateScope scope(contents_.cordz_info(), method); + CordRep::ExtractResult result = ExtractAppendBuffer(tree, min_capacity); + if (result.extracted != nullptr) { + contents_.SetTreeOrEmpty(result.tree, scope); + return CordBuffer(result.extracted->flat()); + } + return CordBuffer::CreateWithDefaultLimit(capacity); + } + return CreateAppendBuffer(contents_.data_, capacity); +} + +void Cord::Append(const Cord& src) { + AppendImpl(src); +} + +void Cord::Append(Cord&& src) { + AppendImpl(std::move(src)); +} template <typename T, Cord::EnableIfString<T>> void Cord::Append(T&& src) { if (src.size() <= kMaxBytesToCopy) { Append(absl::string_view(src)); } else { - Append(Cord(std::forward<T>(src))); + CordRep* rep = CordRepFromString(std::forward<T>(src)); + contents_.AppendTree(rep, CordzUpdateTracker::kAppendString); } } @@ -741,7 +587,8 @@ void Cord::Prepend(const Cord& src) { CordRep* src_tree = src.contents_.tree(); if (src_tree != nullptr) { CordRep::Ref(src_tree); - contents_.PrependTree(src_tree); + contents_.PrependTree(cord_internal::RemoveCrcNode(src_tree), + CordzUpdateTracker::kPrependCord); return; } @@ -750,7 +597,7 @@ void Cord::Prepend(const Cord& src) { return Prepend(src_contents); } -void Cord::Prepend(absl::string_view src) { +void Cord::PrependArray(absl::string_view src, MethodIdentifier method) { if (src.empty()) return; // memcpy(_, nullptr, 0) is undefined. if (!contents_.is_tree()) { size_t cur_size = contents_.inline_size(); @@ -764,105 +611,49 @@ void Cord::Prepend(absl::string_view src) { return; } } - contents_.PrependTree(NewTree(src.data(), src.size(), 0)); + CordRep* rep = NewTree(src.data(), src.size(), 0); + contents_.PrependTree(rep, method); } -template <typename T, Cord::EnableIfString<T>> -inline void Cord::Prepend(T&& src) { - if (src.size() <= kMaxBytesToCopy) { - Prepend(absl::string_view(src)); +void Cord::AppendPrecise(absl::string_view src, MethodIdentifier method) { + assert(!src.empty()); + assert(src.size() <= cord_internal::kMaxFlatLength); + if (contents_.remaining_inline_capacity() >= src.size()) { + const size_t inline_length = contents_.inline_size(); + memcpy(contents_.data_.as_chars() + inline_length, src.data(), src.size()); + contents_.set_inline_size(inline_length + src.size()); } else { - Prepend(Cord(std::forward<T>(src))); + contents_.AppendTree(CordRepFlat::Create(src), method); } } -template void Cord::Prepend(std::string&& src); - -static CordRep* RemovePrefixFrom(CordRep* node, size_t n) { - if (n >= node->length) return nullptr; - if (n == 0) return CordRep::Ref(node); - absl::InlinedVector<CordRep*, kInlinedVectorSize> rhs_stack; - - while (node->tag == CONCAT) { - assert(n <= node->length); - if (n < node->concat()->left->length) { - // Push right to stack, descend left. - rhs_stack.push_back(node->concat()->right); - node = node->concat()->left; - } else { - // Drop left, descend right. - n -= node->concat()->left->length; - node = node->concat()->right; - } - } - assert(n <= node->length); - - if (n == 0) { - CordRep::Ref(node); +void Cord::PrependPrecise(absl::string_view src, MethodIdentifier method) { + assert(!src.empty()); + assert(src.size() <= cord_internal::kMaxFlatLength); + if (contents_.remaining_inline_capacity() >= src.size()) { + const size_t inline_length = contents_.inline_size(); + char data[InlineRep::kMaxInline + 1] = {0}; + memcpy(data, src.data(), src.size()); + memcpy(data + src.size(), contents_.data(), inline_length); + memcpy(contents_.data_.as_chars(), data, InlineRep::kMaxInline + 1); + contents_.set_inline_size(inline_length + src.size()); } else { - size_t start = n; - size_t len = node->length - n; - if (node->tag == SUBSTRING) { - // Consider in-place update of node, similar to in RemoveSuffixFrom(). - start += node->substring()->start; - node = node->substring()->child; - } - node = NewSubstring(CordRep::Ref(node), start, len); - } - while (!rhs_stack.empty()) { - node = Concat(node, CordRep::Ref(rhs_stack.back())); - rhs_stack.pop_back(); + contents_.PrependTree(CordRepFlat::Create(src), method); } - return node; } -// RemoveSuffixFrom() is very similar to RemovePrefixFrom(), with the -// exception that removing a suffix has an optimization where a node may be -// edited in place iff that node and all its ancestors have a refcount of 1. -static CordRep* RemoveSuffixFrom(CordRep* node, size_t n) { - if (n >= node->length) return nullptr; - if (n == 0) return CordRep::Ref(node); - absl::InlinedVector<CordRep*, kInlinedVectorSize> lhs_stack; - bool inplace_ok = node->refcount.IsOne(); - - while (node->tag == CONCAT) { - assert(n <= node->length); - if (n < node->concat()->right->length) { - // Push left to stack, descend right. - lhs_stack.push_back(node->concat()->left); - node = node->concat()->right; - } else { - // Drop right, descend left. - n -= node->concat()->right->length; - node = node->concat()->left; - } - inplace_ok = inplace_ok && node->refcount.IsOne(); - } - assert(n <= node->length); - - if (n == 0) { - CordRep::Ref(node); - } else if (inplace_ok && node->tag != EXTERNAL) { - // Consider making a new buffer if the current node capacity is much - // larger than the new length. - CordRep::Ref(node); - node->length -= n; +template <typename T, Cord::EnableIfString<T>> +inline void Cord::Prepend(T&& src) { + if (src.size() <= kMaxBytesToCopy) { + Prepend(absl::string_view(src)); } else { - size_t start = 0; - size_t len = node->length - n; - if (node->tag == SUBSTRING) { - start = node->substring()->start; - node = node->substring()->child; - } - node = NewSubstring(CordRep::Ref(node), start, len); + CordRep* rep = CordRepFromString(std::forward<T>(src)); + contents_.PrependTree(rep, CordzUpdateTracker::kPrependString); } - while (!lhs_stack.empty()) { - node = Concat(CordRep::Ref(lhs_stack.back()), node); - lhs_stack.pop_back(); - } - return node; } +template void Cord::Prepend(std::string&& src); + void Cord::RemovePrefix(size_t n) { ABSL_INTERNAL_CHECK(n <= size(), absl::StrCat("Requested prefix size ", n, @@ -870,12 +661,26 @@ void Cord::RemovePrefix(size_t n) { CordRep* tree = contents_.tree(); if (tree == nullptr) { contents_.remove_prefix(n); - } else if (tree->tag == RING) { - contents_.replace_tree(CordRepRing::RemovePrefix(tree->ring(), n)); } else { - CordRep* newrep = RemovePrefixFrom(tree, n); - CordRep::Unref(tree); - contents_.replace_tree(VerifyTree(newrep)); + auto constexpr method = CordzUpdateTracker::kRemovePrefix; + CordzUpdateScope scope(contents_.cordz_info(), method); + tree = cord_internal::RemoveCrcNode(tree); + if (n >= tree->length) { + CordRep::Unref(tree); + tree = nullptr; + } else if (tree->IsBtree()) { + CordRep* old = tree; + tree = tree->btree()->SubTree(n, tree->length - n); + CordRep::Unref(old); + } else if (tree->IsSubstring() && tree->refcount.IsOne()) { + tree->substring()->start += n; + tree->length -= n; + } else { + CordRep* rep = CordRepSubstring::Substring(tree, n, tree->length - n); + CordRep::Unref(tree); + tree = rep; + } + contents_.SetTreeOrEmpty(tree, scope); } } @@ -886,64 +691,25 @@ void Cord::RemoveSuffix(size_t n) { CordRep* tree = contents_.tree(); if (tree == nullptr) { contents_.reduce_size(n); - } else if (tree->tag == RING) { - contents_.replace_tree(CordRepRing::RemoveSuffix(tree->ring(), n)); } else { - CordRep* newrep = RemoveSuffixFrom(tree, n); - CordRep::Unref(tree); - contents_.replace_tree(VerifyTree(newrep)); - } -} - -// Work item for NewSubRange(). -struct SubRange { - SubRange(CordRep* a_node, size_t a_pos, size_t a_n) - : node(a_node), pos(a_pos), n(a_n) {} - CordRep* node; // nullptr means concat last 2 results. - size_t pos; - size_t n; -}; - -static CordRep* NewSubRange(CordRep* node, size_t pos, size_t n) { - absl::InlinedVector<CordRep*, kInlinedVectorSize> results; - absl::InlinedVector<SubRange, kInlinedVectorSize> todo; - todo.push_back(SubRange(node, pos, n)); - do { - const SubRange& sr = todo.back(); - node = sr.node; - pos = sr.pos; - n = sr.n; - todo.pop_back(); - - if (node == nullptr) { - assert(results.size() >= 2); - CordRep* right = results.back(); - results.pop_back(); - CordRep* left = results.back(); - results.pop_back(); - results.push_back(Concat(left, right)); - } else if (pos == 0 && n == node->length) { - results.push_back(CordRep::Ref(node)); - } else if (node->tag != CONCAT) { - if (node->tag == SUBSTRING) { - pos += node->substring()->start; - node = node->substring()->child; - } - results.push_back(NewSubstring(CordRep::Ref(node), pos, n)); - } else if (pos + n <= node->concat()->left->length) { - todo.push_back(SubRange(node->concat()->left, pos, n)); - } else if (pos >= node->concat()->left->length) { - pos -= node->concat()->left->length; - todo.push_back(SubRange(node->concat()->right, pos, n)); + auto constexpr method = CordzUpdateTracker::kRemoveSuffix; + CordzUpdateScope scope(contents_.cordz_info(), method); + tree = cord_internal::RemoveCrcNode(tree); + if (n >= tree->length) { + CordRep::Unref(tree); + tree = nullptr; + } else if (tree->IsBtree()) { + tree = CordRepBtree::RemoveSuffix(tree->btree(), n); + } else if (!tree->IsExternal() && tree->refcount.IsOne()) { + assert(tree->IsFlat() || tree->IsSubstring()); + tree->length -= n; } else { - size_t left_n = node->concat()->left->length - pos; - todo.push_back(SubRange(nullptr, 0, 0)); // Concat() - todo.push_back(SubRange(node->concat()->right, 0, n - left_n)); - todo.push_back(SubRange(node->concat()->left, pos, left_n)); + CordRep* rep = CordRepSubstring::Substring(tree, 0, tree->length - n); + CordRep::Unref(tree); + tree = rep; } - } while (!todo.empty()); - assert(results.size() == 1); - return results[0]; + contents_.SetTreeOrEmpty(tree, scope); + } } Cord Cord::Subcord(size_t pos, size_t new_size) const { @@ -951,17 +717,18 @@ Cord Cord::Subcord(size_t pos, size_t new_size) const { size_t length = size(); if (pos > length) pos = length; if (new_size > length - pos) new_size = length - pos; + if (new_size == 0) return sub_cord; + CordRep* tree = contents_.tree(); if (tree == nullptr) { - // sub_cord is newly constructed, no need to re-zero-out the tail of - // contents_ memory. - sub_cord.contents_.set_data(contents_.data() + pos, new_size, false); - } else if (new_size == 0) { - // We want to return empty subcord, so nothing to do. - } else if (new_size <= InlineRep::kMaxInline) { + sub_cord.contents_.set_data(contents_.data() + pos, new_size); + return sub_cord; + } + + if (new_size <= InlineRep::kMaxInline) { + char* dest = sub_cord.contents_.data_.as_chars(); Cord::ChunkIterator it = chunk_begin(); it.AdvanceBytes(pos); - char* dest = sub_cord.contents_.data_.as_chars(); size_t remaining_size = new_size; while (remaining_size > it->size()) { cord_internal::SmallMemmove(dest, it->data(), it->size()); @@ -971,153 +738,18 @@ Cord Cord::Subcord(size_t pos, size_t new_size) const { } cord_internal::SmallMemmove(dest, it->data(), remaining_size); sub_cord.contents_.set_inline_size(new_size); - } else if (tree->tag == RING) { - tree = CordRepRing::SubRing(CordRep::Ref(tree)->ring(), pos, new_size); - sub_cord.contents_.set_tree(tree); - } else { - sub_cord.contents_.set_tree(NewSubRange(tree, pos, new_size)); - } - return sub_cord; -} - -// -------------------------------------------------------------------- -// Balancing - -class CordForest { - public: - explicit CordForest(size_t length) - : root_length_(length), trees_(kMinLengthSize, nullptr) {} - - void Build(CordRep* cord_root) { - std::vector<CordRep*> pending = {cord_root}; - - while (!pending.empty()) { - CordRep* node = pending.back(); - pending.pop_back(); - CheckNode(node); - if (ABSL_PREDICT_FALSE(node->tag != CONCAT)) { - AddNode(node); - continue; - } - - CordRepConcat* concat_node = node->concat(); - if (concat_node->depth() >= kMinLengthSize || - concat_node->length < min_length[concat_node->depth()]) { - pending.push_back(concat_node->right); - pending.push_back(concat_node->left); - - if (concat_node->refcount.IsOne()) { - concat_node->left = concat_freelist_; - concat_freelist_ = concat_node; - } else { - CordRep::Ref(concat_node->right); - CordRep::Ref(concat_node->left); - CordRep::Unref(concat_node); - } - } else { - AddNode(node); - } - } - } - - CordRep* ConcatNodes() { - CordRep* sum = nullptr; - for (auto* node : trees_) { - if (node == nullptr) continue; - - sum = PrependNode(node, sum); - root_length_ -= node->length; - if (root_length_ == 0) break; - } - ABSL_INTERNAL_CHECK(sum != nullptr, "Failed to locate sum node"); - return VerifyTree(sum); - } - - private: - CordRep* AppendNode(CordRep* node, CordRep* sum) { - return (sum == nullptr) ? node : MakeConcat(sum, node); - } - - CordRep* PrependNode(CordRep* node, CordRep* sum) { - return (sum == nullptr) ? node : MakeConcat(node, sum); - } - - void AddNode(CordRep* node) { - CordRep* sum = nullptr; - - // Collect together everything with which we will merge with node - int i = 0; - for (; node->length > min_length[i + 1]; ++i) { - auto& tree_at_i = trees_[i]; - - if (tree_at_i == nullptr) continue; - sum = PrependNode(tree_at_i, sum); - tree_at_i = nullptr; - } - - sum = AppendNode(node, sum); - - // Insert sum into appropriate place in the forest - for (; sum->length >= min_length[i]; ++i) { - auto& tree_at_i = trees_[i]; - if (tree_at_i == nullptr) continue; - - sum = MakeConcat(tree_at_i, sum); - tree_at_i = nullptr; - } - - // min_length[0] == 1, which means sum->length >= min_length[0] - assert(i > 0); - trees_[i - 1] = sum; - } - - // Make concat node trying to resue existing CordRepConcat nodes we - // already collected in the concat_freelist_. - CordRep* MakeConcat(CordRep* left, CordRep* right) { - if (concat_freelist_ == nullptr) return RawConcat(left, right); - - CordRepConcat* rep = concat_freelist_; - if (concat_freelist_->left == nullptr) { - concat_freelist_ = nullptr; - } else { - concat_freelist_ = concat_freelist_->left->concat(); - } - SetConcatChildren(rep, left, right); - - return rep; + return sub_cord; } - static void CheckNode(CordRep* node) { - ABSL_INTERNAL_CHECK(node->length != 0u, ""); - if (node->tag == CONCAT) { - ABSL_INTERNAL_CHECK(node->concat()->left != nullptr, ""); - ABSL_INTERNAL_CHECK(node->concat()->right != nullptr, ""); - ABSL_INTERNAL_CHECK(node->length == (node->concat()->left->length + - node->concat()->right->length), - ""); - } - } - - size_t root_length_; - - // use an inlined vector instead of a flat array to get bounds checking - absl::InlinedVector<CordRep*, kInlinedVectorSize> trees_; - - // List of concat nodes we can re-use for Cord balancing. - CordRepConcat* concat_freelist_ = nullptr; -}; - -static CordRep* Rebalance(CordRep* node) { - VerifyTree(node); - assert(node->tag == CONCAT); - - if (node->length == 0) { - return nullptr; + tree = cord_internal::SkipCrcNode(tree); + if (tree->IsBtree()) { + tree = tree->btree()->SubTree(pos, new_size); + } else { + tree = CordRepSubstring::Substring(tree, pos, new_size); } - - CordForest forest(node->length); - forest.Build(node); - return forest.ConcatNodes(); + sub_cord.contents_.EmplaceTree(tree, contents_.data_, + CordzUpdateTracker::kSubCord); + return sub_cord; } // -------------------------------------------------------------------- @@ -1159,29 +791,29 @@ bool ComputeCompareResult<bool>(int memcmp_res) { } // namespace -// Helper routine. Locates the first flat chunk of the Cord without -// initializing the iterator. +// Helper routine. Locates the first flat or external chunk of the Cord without +// initializing the iterator, and returns a string_view referencing the data. inline absl::string_view Cord::InlineRep::FindFlatStartPiece() const { if (!is_tree()) { return absl::string_view(data_.as_chars(), data_.inline_size()); } - CordRep* node = tree(); - if (node->tag >= FLAT) { + CordRep* node = cord_internal::SkipCrcNode(tree()); + if (node->IsFlat()) { return absl::string_view(node->flat()->Data(), node->length); } - if (node->tag == EXTERNAL) { + if (node->IsExternal()) { return absl::string_view(node->external()->base, node->length); } - if (node->tag == RING) { - return node->ring()->entry_data(node->ring()->head()); - } - - // Walk down the left branches until we hit a non-CONCAT node. - while (node->tag == CONCAT) { - node = node->concat()->left; + if (node->IsBtree()) { + CordRepBtree* tree = node->btree(); + int height = tree->height(); + while (--height >= 0) { + tree = tree->Edge(CordRepBtree::kFront)->btree(); + } + return tree->Data(tree->begin()); } // Get the child node if we encounter a SUBSTRING. @@ -1189,20 +821,42 @@ inline absl::string_view Cord::InlineRep::FindFlatStartPiece() const { size_t length = node->length; assert(length != 0); - if (node->tag == SUBSTRING) { + if (node->IsSubstring()) { offset = node->substring()->start; node = node->substring()->child; } - if (node->tag >= FLAT) { + if (node->IsFlat()) { return absl::string_view(node->flat()->Data() + offset, length); } - assert((node->tag == EXTERNAL) && "Expect FLAT or EXTERNAL node here"); + assert(node->IsExternal() && "Expect FLAT or EXTERNAL node here"); return absl::string_view(node->external()->base + offset, length); } +void Cord::SetExpectedChecksum(uint32_t crc) { + auto constexpr method = CordzUpdateTracker::kSetExpectedChecksum; + if (empty()) return; + + if (!contents_.is_tree()) { + CordRep* rep = contents_.MakeFlatWithExtraCapacity(0); + rep = CordRepCrc::New(rep, crc); + contents_.EmplaceTree(rep, method); + } else { + const CordzUpdateScope scope(contents_.data_.cordz_info(), method); + CordRep* rep = CordRepCrc::New(contents_.data_.as_tree(), crc); + contents_.SetTree(rep, scope); + } +} + +absl::optional<uint32_t> Cord::ExpectedChecksum() const { + if (!contents_.is_tree() || !contents_.tree()->IsCrc()) { + return absl::nullopt; + } + return contents_.tree()->crc()->crc; +} + inline int Cord::CompareSlowPath(absl::string_view rhs, size_t compared_size, size_t size_to_compare) const { auto advance = [](Cord::ChunkIterator* it, absl::string_view* chunk) { @@ -1378,46 +1032,11 @@ void Cord::CopyToArraySlowPath(char* dst) const { } } -Cord::ChunkIterator& Cord::ChunkIterator::AdvanceStack() { - auto& stack_of_right_children = stack_of_right_children_; - if (stack_of_right_children.empty()) { - assert(!current_chunk_.empty()); // Called on invalid iterator. - // We have reached the end of the Cord. - return *this; - } - - // Process the next node on the stack. - CordRep* node = stack_of_right_children.back(); - stack_of_right_children.pop_back(); - - // Walk down the left branches until we hit a non-CONCAT node. Save the - // right children to the stack for subsequent traversal. - while (node->tag == CONCAT) { - stack_of_right_children.push_back(node->concat()->right); - node = node->concat()->left; - } - - // Get the child node if we encounter a SUBSTRING. - size_t offset = 0; - size_t length = node->length; - if (node->tag == SUBSTRING) { - offset = node->substring()->start; - node = node->substring()->child; - } - - assert(node->tag == EXTERNAL || node->tag >= FLAT); - assert(length != 0); - const char* data = - node->tag == EXTERNAL ? node->external()->base : node->flat()->Data(); - current_chunk_ = absl::string_view(data + offset, length); - current_leaf_ = node; - return *this; -} - Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) { ABSL_HARDENING_ASSERT(bytes_remaining_ >= n && "Attempted to iterate past `end()`"); Cord subcord; + auto constexpr method = CordzUpdateTracker::kCordReader; if (n <= InlineRep::kMaxInline) { // Range to read fits in inline data. Flatten it. @@ -1437,184 +1056,51 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) { return subcord; } - if (ring_reader_) { + if (btree_reader_) { size_t chunk_size = current_chunk_.size(); if (n <= chunk_size && n <= kMaxBytesToCopy) { - subcord = Cord(current_chunk_.substr(0, n)); - } else { - auto* ring = CordRep::Ref(ring_reader_.ring())->ring(); - size_t offset = ring_reader_.length() - bytes_remaining_; - subcord.contents_.set_tree(CordRepRing::SubRing(ring, offset, n)); - } - if (n < chunk_size) { - bytes_remaining_ -= n; - current_chunk_.remove_prefix(n); + subcord = Cord(current_chunk_.substr(0, n), method); + if (n < chunk_size) { + current_chunk_.remove_prefix(n); + } else { + current_chunk_ = btree_reader_.Next(); + } } else { - AdvanceBytesRing(n); + CordRep* rep; + current_chunk_ = btree_reader_.Read(n, chunk_size, rep); + subcord.contents_.EmplaceTree(rep, method); } + bytes_remaining_ -= n; return subcord; } - auto& stack_of_right_children = stack_of_right_children_; - if (n < current_chunk_.size()) { - // Range to read is a proper subrange of the current chunk. - assert(current_leaf_ != nullptr); - CordRep* subnode = CordRep::Ref(current_leaf_); - const char* data = subnode->tag == EXTERNAL ? subnode->external()->base - : subnode->flat()->Data(); - subnode = NewSubstring(subnode, current_chunk_.data() - data, n); - subcord.contents_.set_tree(VerifyTree(subnode)); - RemoveChunkPrefix(n); - return subcord; - } - - // Range to read begins with a proper subrange of the current chunk. - assert(!current_chunk_.empty()); + // Short circuit if reading the entire data edge. assert(current_leaf_ != nullptr); - CordRep* subnode = CordRep::Ref(current_leaf_); - if (current_chunk_.size() < subnode->length) { - const char* data = subnode->tag == EXTERNAL ? subnode->external()->base - : subnode->flat()->Data(); - subnode = NewSubstring(subnode, current_chunk_.data() - data, - current_chunk_.size()); - } - n -= current_chunk_.size(); - bytes_remaining_ -= current_chunk_.size(); - - // Process the next node(s) on the stack, reading whole subtrees depending on - // their length and how many bytes we are advancing. - CordRep* node = nullptr; - while (!stack_of_right_children.empty()) { - node = stack_of_right_children.back(); - stack_of_right_children.pop_back(); - if (node->length > n) break; - // TODO(qrczak): This might unnecessarily recreate existing concat nodes. - // Avoiding that would need pretty complicated logic (instead of - // current_leaf, keep current_subtree_ which points to the highest node - // such that the current leaf can be found on the path of left children - // starting from current_subtree_; delay creating subnode while node is - // below current_subtree_; find the proper node along the path of left - // children starting from current_subtree_ if this loop exits while staying - // below current_subtree_; etc.; alternatively, push parents instead of - // right children on the stack). - subnode = Concat(subnode, CordRep::Ref(node)); - n -= node->length; - bytes_remaining_ -= node->length; - node = nullptr; - } - - if (node == nullptr) { - // We have reached the end of the Cord. - assert(bytes_remaining_ == 0); - subcord.contents_.set_tree(VerifyTree(subnode)); + if (n == current_leaf_->length) { + bytes_remaining_ = 0; + current_chunk_ = {}; + CordRep* tree = CordRep::Ref(current_leaf_); + subcord.contents_.EmplaceTree(VerifyTree(tree), method); return subcord; } - // Walk down the appropriate branches until we hit a non-CONCAT node. Save the - // right children to the stack for subsequent traversal. - while (node->tag == CONCAT) { - if (node->concat()->left->length > n) { - // Push right, descend left. - stack_of_right_children.push_back(node->concat()->right); - node = node->concat()->left; - } else { - // Read left, descend right. - subnode = Concat(subnode, CordRep::Ref(node->concat()->left)); - n -= node->concat()->left->length; - bytes_remaining_ -= node->concat()->left->length; - node = node->concat()->right; - } - } + // From this point on, we need a partial substring node. + // Get pointer to the underlying flat or external data payload and + // compute data pointer and offset into current flat or external. + CordRep* payload = current_leaf_->IsSubstring() + ? current_leaf_->substring()->child + : current_leaf_; + const char* data = payload->IsExternal() ? payload->external()->base + : payload->flat()->Data(); + const size_t offset = current_chunk_.data() - data; - // Get the child node if we encounter a SUBSTRING. - size_t offset = 0; - size_t length = node->length; - if (node->tag == SUBSTRING) { - offset = node->substring()->start; - node = node->substring()->child; - } - - // Range to read ends with a proper (possibly empty) subrange of the current - // chunk. - assert(node->tag == EXTERNAL || node->tag >= FLAT); - assert(length > n); - if (n > 0) { - subnode = Concat(subnode, NewSubstring(CordRep::Ref(node), offset, n)); - } - const char* data = - node->tag == EXTERNAL ? node->external()->base : node->flat()->Data(); - current_chunk_ = absl::string_view(data + offset + n, length - n); - current_leaf_ = node; + auto* tree = CordRepSubstring::Substring(payload, offset, n); + subcord.contents_.EmplaceTree(VerifyTree(tree), method); bytes_remaining_ -= n; - subcord.contents_.set_tree(VerifyTree(subnode)); + current_chunk_.remove_prefix(n); return subcord; } -void Cord::ChunkIterator::AdvanceBytesSlowPath(size_t n) { - assert(bytes_remaining_ >= n && "Attempted to iterate past `end()`"); - assert(n >= current_chunk_.size()); // This should only be called when - // iterating to a new node. - - n -= current_chunk_.size(); - bytes_remaining_ -= current_chunk_.size(); - - if (stack_of_right_children_.empty()) { - // We have reached the end of the Cord. - assert(bytes_remaining_ == 0); - return; - } - - // Process the next node(s) on the stack, skipping whole subtrees depending on - // their length and how many bytes we are advancing. - CordRep* node = nullptr; - auto& stack_of_right_children = stack_of_right_children_; - while (!stack_of_right_children.empty()) { - node = stack_of_right_children.back(); - stack_of_right_children.pop_back(); - if (node->length > n) break; - n -= node->length; - bytes_remaining_ -= node->length; - node = nullptr; - } - - if (node == nullptr) { - // We have reached the end of the Cord. - assert(bytes_remaining_ == 0); - return; - } - - // Walk down the appropriate branches until we hit a non-CONCAT node. Save the - // right children to the stack for subsequent traversal. - while (node->tag == CONCAT) { - if (node->concat()->left->length > n) { - // Push right, descend left. - stack_of_right_children.push_back(node->concat()->right); - node = node->concat()->left; - } else { - // Skip left, descend right. - n -= node->concat()->left->length; - bytes_remaining_ -= node->concat()->left->length; - node = node->concat()->right; - } - } - - // Get the child node if we encounter a SUBSTRING. - size_t offset = 0; - size_t length = node->length; - if (node->tag == SUBSTRING) { - offset = node->substring()->start; - node = node->substring()->child; - } - - assert(node->tag == EXTERNAL || node->tag >= FLAT); - assert(length > n); - const char* data = - node->tag == EXTERNAL ? node->external()->base : node->flat()->Data(); - current_chunk_ = absl::string_view(data + offset + n, length - n); - current_leaf_ = node; - bytes_remaining_ -= n; -} - char Cord::operator[](size_t i) const { ABSL_HARDENING_ASSERT(i < size()); size_t offset = i; @@ -1622,30 +1108,21 @@ char Cord::operator[](size_t i) const { if (rep == nullptr) { return contents_.data()[i]; } + rep = cord_internal::SkipCrcNode(rep); while (true) { assert(rep != nullptr); assert(offset < rep->length); - if (rep->tag >= FLAT) { + if (rep->IsFlat()) { // Get the "i"th character directly from the flat array. return rep->flat()->Data()[offset]; - } else if (rep->tag == RING) { - return rep->ring()->GetCharacter(offset); - } else if (rep->tag == EXTERNAL) { + } else if (rep->IsBtree()) { + return rep->btree()->GetCharacter(offset); + } else if (rep->IsExternal()) { // Get the "i"th character from the external array. return rep->external()->base[offset]; - } else if (rep->tag == CONCAT) { - // Recursively branch to the side of the concatenation that the "i"th - // character is on. - size_t left_length = rep->concat()->left->length; - if (offset < left_length) { - rep = rep->concat()->left; - } else { - offset -= left_length; - rep = rep->concat()->right; - } } else { // This must be a substring a node, so bypass it to get to the child. - assert(rep->tag == SUBSTRING); + assert(rep->IsSubstring()); offset += rep->substring()->start; rep = rep->substring()->child; } @@ -1653,6 +1130,7 @@ char Cord::operator[](size_t i) const { } absl::string_view Cord::FlattenSlowPath() { + assert(contents_.is_tree()); size_t total_size = size(); CordRep* new_rep; char* new_buffer; @@ -1673,31 +1151,36 @@ absl::string_view Cord::FlattenSlowPath() { s.size()); }); } - if (CordRep* tree = contents_.tree()) { - CordRep::Unref(tree); - } - contents_.set_tree(new_rep); + CordzUpdateScope scope(contents_.cordz_info(), CordzUpdateTracker::kFlatten); + CordRep::Unref(contents_.as_tree()); + contents_.SetTree(new_rep, scope); return absl::string_view(new_buffer, total_size); } /* static */ bool Cord::GetFlatAux(CordRep* rep, absl::string_view* fragment) { assert(rep != nullptr); - if (rep->tag >= FLAT) { + rep = cord_internal::SkipCrcNode(rep); + if (rep->IsFlat()) { *fragment = absl::string_view(rep->flat()->Data(), rep->length); return true; - } else if (rep->tag == EXTERNAL) { + } else if (rep->IsExternal()) { *fragment = absl::string_view(rep->external()->base, rep->length); return true; - } else if (rep->tag == SUBSTRING) { + } else if (rep->IsBtree()) { + return rep->btree()->IsFlat(fragment); + } else if (rep->IsSubstring()) { CordRep* child = rep->substring()->child; - if (child->tag >= FLAT) { + if (child->IsFlat()) { *fragment = absl::string_view( child->flat()->Data() + rep->substring()->start, rep->length); return true; - } else if (child->tag == EXTERNAL) { + } else if (child->IsExternal()) { *fragment = absl::string_view( child->external()->base + rep->substring()->start, rep->length); return true; + } else if (child->IsBtree()) { + return child->btree()->IsFlat(rep->substring()->start, rep->length, + fragment); } } return false; @@ -1706,7 +1189,10 @@ absl::string_view Cord::FlattenSlowPath() { /* static */ void Cord::ForEachChunkAux( absl::cord_internal::CordRep* rep, absl::FunctionRef<void(absl::string_view)> callback) { - if (rep->tag == RING) { + assert(rep != nullptr); + rep = cord_internal::SkipCrcNode(rep); + + if (rep->IsBtree()) { ChunkIterator it(rep), end; while (it != end) { callback(*it); @@ -1715,44 +1201,13 @@ absl::string_view Cord::FlattenSlowPath() { return; } - assert(rep != nullptr); - int stack_pos = 0; - constexpr int stack_max = 128; - // Stack of right branches for tree traversal - absl::cord_internal::CordRep* stack[stack_max]; - absl::cord_internal::CordRep* current_node = rep; - while (true) { - if (current_node->tag == CONCAT) { - if (stack_pos == stack_max) { - // There's no more room on our stack array to add another right branch, - // and the idea is to avoid allocations, so call this function - // recursively to navigate this subtree further. (This is not something - // we expect to happen in practice). - ForEachChunkAux(current_node, callback); - - // Pop the next right branch and iterate. - current_node = stack[--stack_pos]; - continue; - } else { - // Save the right branch for later traversal and continue down the left - // branch. - stack[stack_pos++] = current_node->concat()->right; - current_node = current_node->concat()->left; - continue; - } - } - // This is a leaf node, so invoke our callback. - absl::string_view chunk; - bool success = GetFlatAux(current_node, &chunk); - assert(success); - if (success) { - callback(chunk); - } - if (stack_pos == 0) { - // end of traversal - return; - } - current_node = stack[--stack_pos]; + // This is a leaf node, so invoke our callback. + absl::cord_internal::CordRep* current_node = cord_internal::SkipCrcNode(rep); + absl::string_view chunk; + bool success = GetFlatAux(current_node, &chunk); + assert(success); + if (success) { + callback(chunk); } } @@ -1767,40 +1222,28 @@ static void DumpNode(CordRep* rep, bool include_data, std::ostream* os, *os << " ["; if (include_data) *os << static_cast<void*>(rep); *os << "]"; - *os << " " << (IsRootBalanced(rep) ? 'b' : 'u'); *os << " " << std::setw(indent) << ""; - if (rep->tag == CONCAT) { - *os << "CONCAT depth=" << Depth(rep) << "\n"; + if (rep->IsCrc()) { + *os << "CRC crc=" << rep->crc()->crc << "\n"; indent += kIndentStep; - indents.push_back(indent); - stack.push_back(rep->concat()->right); - rep = rep->concat()->left; - } else if (rep->tag == SUBSTRING) { + rep = rep->crc()->child; + } else if (rep->IsSubstring()) { *os << "SUBSTRING @ " << rep->substring()->start << "\n"; indent += kIndentStep; rep = rep->substring()->child; } else { // Leaf or ring - if (rep->tag == EXTERNAL) { + if (rep->IsExternal()) { *os << "EXTERNAL ["; if (include_data) *os << absl::CEscape(std::string(rep->external()->base, rep->length)); *os << "]\n"; - } else if (rep->tag >= FLAT) { - *os << "FLAT cap=" << rep->flat()->Capacity() - << " ["; + } else if (rep->IsFlat()) { + *os << "FLAT cap=" << rep->flat()->Capacity() << " ["; if (include_data) *os << absl::CEscape(std::string(rep->flat()->Data(), rep->length)); *os << "]\n"; } else { - assert(rep->tag == RING); - auto* ring = rep->ring(); - *os << "RING, entries = " << ring->entries() << "\n"; - CordRepRing::index_type head = ring->head(); - do { - DumpNode(ring->entry_child(head), include_data, os, - indent + kIndentStep); - head = ring->advance(head);; - } while (head != ring->tail()); + CordRepBtree::Dump(rep, /*label=*/ "", include_data, *os); } if (stack.empty()) break; rep = stack.back(); @@ -1820,7 +1263,7 @@ static std::string ReportError(CordRep* root, CordRep* node) { } static bool VerifyNode(CordRep* root, CordRep* start_node, - bool full_validation) { + bool /* full_validation */) { absl::InlinedVector<CordRep*, 2> worklist; worklist.push_back(start_node); do { @@ -1830,100 +1273,33 @@ static bool VerifyNode(CordRep* root, CordRep* start_node, ABSL_INTERNAL_CHECK(node != nullptr, ReportError(root, node)); if (node != root) { ABSL_INTERNAL_CHECK(node->length != 0, ReportError(root, node)); + ABSL_INTERNAL_CHECK(!node->IsCrc(), ReportError(root, node)); } - if (node->tag == CONCAT) { - ABSL_INTERNAL_CHECK(node->concat()->left != nullptr, - ReportError(root, node)); - ABSL_INTERNAL_CHECK(node->concat()->right != nullptr, - ReportError(root, node)); - ABSL_INTERNAL_CHECK((node->length == node->concat()->left->length + - node->concat()->right->length), + if (node->IsFlat()) { + ABSL_INTERNAL_CHECK(node->length <= node->flat()->Capacity(), ReportError(root, node)); - if (full_validation) { - worklist.push_back(node->concat()->right); - worklist.push_back(node->concat()->left); - } - } else if (node->tag >= FLAT) { - ABSL_INTERNAL_CHECK( - node->length <= node->flat()->Capacity(), - ReportError(root, node)); - } else if (node->tag == EXTERNAL) { + } else if (node->IsExternal()) { ABSL_INTERNAL_CHECK(node->external()->base != nullptr, ReportError(root, node)); - } else if (node->tag == SUBSTRING) { + } else if (node->IsSubstring()) { ABSL_INTERNAL_CHECK( node->substring()->start < node->substring()->child->length, ReportError(root, node)); ABSL_INTERNAL_CHECK(node->substring()->start + node->length <= node->substring()->child->length, ReportError(root, node)); + } else if (node->IsCrc()) { + ABSL_INTERNAL_CHECK(node->crc()->child != nullptr, + ReportError(root, node)); + ABSL_INTERNAL_CHECK(node->crc()->length == node->crc()->child->length, + ReportError(root, node)); + worklist.push_back(node->crc()->child); } } while (!worklist.empty()); return true; } -// Traverses the tree and computes the total memory allocated. -/* static */ size_t Cord::MemoryUsageAux(const CordRep* rep) { - size_t total_mem_usage = 0; - - // Allow a quick exit for the common case that the root is a leaf. - if (RepMemoryUsageLeaf(rep, &total_mem_usage)) { - return total_mem_usage; - } - - // Iterate over the tree. cur_node is never a leaf node and leaf nodes will - // never be appended to tree_stack. This reduces overhead from manipulating - // tree_stack. - absl::InlinedVector<const CordRep*, kInlinedVectorSize> tree_stack; - const CordRep* cur_node = rep; - while (true) { - const CordRep* next_node = nullptr; - - if (cur_node->tag == CONCAT) { - total_mem_usage += sizeof(CordRepConcat); - const CordRep* left = cur_node->concat()->left; - if (!RepMemoryUsageLeaf(left, &total_mem_usage)) { - next_node = left; - } - - const CordRep* right = cur_node->concat()->right; - if (!RepMemoryUsageLeaf(right, &total_mem_usage)) { - if (next_node) { - tree_stack.push_back(next_node); - } - next_node = right; - } - } else if (cur_node->tag == RING) { - total_mem_usage += CordRepRing::AllocSize(cur_node->ring()->capacity()); - const CordRepRing* ring = cur_node->ring(); - CordRepRing::index_type pos = ring->head(), tail = ring->tail(); - do { - CordRep* node = ring->entry_child(pos); - assert(node->tag >= FLAT || node->tag == EXTERNAL); - RepMemoryUsageLeaf(node, &total_mem_usage); - } while ((pos = ring->advance(pos)) != tail); - } else { - // Since cur_node is not a leaf or a concat node it must be a substring. - assert(cur_node->tag == SUBSTRING); - total_mem_usage += sizeof(CordRepSubstring); - next_node = cur_node->substring()->child; - if (RepMemoryUsageLeaf(next_node, &total_mem_usage)) { - next_node = nullptr; - } - } - - if (!next_node) { - if (tree_stack.empty()) { - return total_mem_usage; - } - next_node = tree_stack.back(); - tree_stack.pop_back(); - } - cur_node = next_node; - } -} - std::ostream& operator<<(std::ostream& out, const Cord& cord) { for (absl::string_view chunk : cord.Chunks()) { out.write(chunk.data(), chunk.size()); @@ -1941,7 +1317,6 @@ uint8_t CordTestAccess::LengthToTag(size_t s) { ABSL_INTERNAL_CHECK(s <= kMaxFlatLength, absl::StrCat("Invalid length ", s)); return cord_internal::AllocatedSizeToTag(s + cord_internal::kFlatOverhead); } -size_t CordTestAccess::SizeofCordRepConcat() { return sizeof(CordRepConcat); } size_t CordTestAccess::SizeofCordRepExternal() { return sizeof(CordRepExternal); } |