diff options
Diffstat (limited to 'absl/strings/internal')
-rw-r--r-- | absl/strings/internal/cordz_handle.cc | 15 | ||||
-rw-r--r-- | absl/strings/internal/cordz_handle.h | 7 | ||||
-rw-r--r-- | absl/strings/internal/cordz_info.cc | 78 | ||||
-rw-r--r-- | absl/strings/internal/cordz_info.h | 59 | ||||
-rw-r--r-- | absl/strings/internal/cordz_info_test.cc | 30 | ||||
-rw-r--r-- | absl/strings/internal/cordz_sample_token_test.cc | 18 | ||||
-rw-r--r-- | absl/strings/internal/cordz_update_scope.h | 8 |
7 files changed, 127 insertions, 88 deletions
diff --git a/absl/strings/internal/cordz_handle.cc b/absl/strings/internal/cordz_handle.cc index 22d07978..5297ec8e 100644 --- a/absl/strings/internal/cordz_handle.cc +++ b/absl/strings/internal/cordz_handle.cc @@ -16,16 +16,19 @@ #include <atomic> #include "absl/base/internal/raw_logging.h" // For ABSL_RAW_CHECK +#include "absl/base/internal/spinlock.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace cord_internal { +using ::absl::base_internal::SpinLockHolder; + ABSL_CONST_INIT CordzHandle::Queue CordzHandle::global_queue_(absl::kConstInit); CordzHandle::CordzHandle(bool is_snapshot) : is_snapshot_(is_snapshot) { if (is_snapshot) { - MutexLock lock(&queue_->mutex); + SpinLockHolder lock(&queue_->mutex); CordzHandle* dq_tail = queue_->dq_tail.load(std::memory_order_acquire); if (dq_tail != nullptr) { dq_prev_ = dq_tail; @@ -40,7 +43,7 @@ CordzHandle::~CordzHandle() { if (is_snapshot_) { std::vector<CordzHandle*> to_delete; { - absl::MutexLock lock(&queue_->mutex); + SpinLockHolder lock(&queue_->mutex); CordzHandle* next = dq_next_; if (dq_prev_ == nullptr) { // We were head of the queue, delete every CordzHandle until we reach @@ -70,7 +73,7 @@ void CordzHandle::Delete(CordzHandle* handle) { handle->ODRCheck(); Queue* const queue = handle->queue_; if (!handle->is_snapshot_ && !queue->IsEmpty()) { - MutexLock lock(&queue->mutex); + SpinLockHolder lock(&queue->mutex); CordzHandle* dq_tail = queue->dq_tail.load(std::memory_order_acquire); if (dq_tail != nullptr) { handle->dq_prev_ = dq_tail; @@ -85,7 +88,7 @@ void CordzHandle::Delete(CordzHandle* handle) { std::vector<const CordzHandle*> CordzHandle::DiagnosticsGetDeleteQueue() { std::vector<const CordzHandle*> handles; - MutexLock lock(&global_queue_.mutex); + SpinLockHolder lock(&global_queue_.mutex); CordzHandle* dq_tail = global_queue_.dq_tail.load(std::memory_order_acquire); for (const CordzHandle* p = dq_tail; p; p = p->dq_prev_) { handles.push_back(p); @@ -100,7 +103,7 @@ bool CordzHandle::DiagnosticsHandleIsSafeToInspect( if (handle == nullptr) return true; if (handle->is_snapshot_) return false; bool snapshot_found = false; - MutexLock lock(&queue_->mutex); + SpinLockHolder lock(&queue_->mutex); for (const CordzHandle* p = queue_->dq_tail; p; p = p->dq_prev_) { if (p == handle) return !snapshot_found; if (p == this) snapshot_found = true; @@ -117,7 +120,7 @@ CordzHandle::DiagnosticsGetSafeToInspectDeletedHandles() { return handles; } - MutexLock lock(&queue_->mutex); + SpinLockHolder lock(&queue_->mutex); for (const CordzHandle* p = dq_next_; p != nullptr; p = p->dq_next_) { if (!p->is_snapshot()) { handles.push_back(p); diff --git a/absl/strings/internal/cordz_handle.h b/absl/strings/internal/cordz_handle.h index 71563c8b..93076cfd 100644 --- a/absl/strings/internal/cordz_handle.h +++ b/absl/strings/internal/cordz_handle.h @@ -20,6 +20,7 @@ #include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/spinlock.h" #include "absl/synchronization/mutex.h" namespace absl { @@ -70,9 +71,11 @@ class CordzHandle { // Global queue data. CordzHandle stores a pointer to the global queue // instance to harden against ODR violations. struct Queue { - constexpr explicit Queue(absl::ConstInitType) : mutex(absl::kConstInit) {} + constexpr explicit Queue(absl::ConstInitType) + : mutex(absl::kConstInit, + absl::base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL) {} - absl::Mutex mutex; + absl::base_internal::SpinLock mutex; std::atomic<CordzHandle*> dq_tail ABSL_GUARDED_BY(mutex){nullptr}; // Returns true if this delete queue is empty. This method does not acquire diff --git a/absl/strings/internal/cordz_info.cc b/absl/strings/internal/cordz_info.cc index d2200680..0461ec46 100644 --- a/absl/strings/internal/cordz_info.cc +++ b/absl/strings/internal/cordz_info.cc @@ -15,6 +15,7 @@ #include "absl/strings/internal/cordz_info.h" #include "absl/base/config.h" +#include "absl/base/internal/spinlock.h" #include "absl/debugging/stacktrace.h" #include "absl/strings/internal/cord_internal.h" #include "absl/strings/internal/cordz_handle.h" @@ -27,22 +28,34 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace cord_internal { +using ::absl::base_internal::SpinLockHolder; + constexpr int CordzInfo::kMaxStackDepth; -ABSL_CONST_INIT std::atomic<CordzInfo*> CordzInfo::ci_head_{nullptr}; -ABSL_CONST_INIT absl::Mutex CordzInfo::ci_mutex_(absl::kConstInit); +ABSL_CONST_INIT CordzInfo::List CordzInfo::global_list_{absl::kConstInit}; CordzInfo* CordzInfo::Head(const CordzSnapshot& snapshot) { ABSL_ASSERT(snapshot.is_snapshot()); - ABSL_ASSERT(snapshot.DiagnosticsHandleIsSafeToInspect(ci_head_unsafe())); - return ci_head_unsafe(); + + // We can do an 'unsafe' load of 'head', as we are guaranteed that the + // instance it points to is kept alive by the provided CordzSnapshot, so we + // can simply return the current value using an acquire load. + // We do enforce in DEBUG builds that the 'head' value is present in the + // delete queue: ODR violations may lead to 'snapshot' and 'global_list_' + // being in different libraries / modules. + CordzInfo* head = global_list_.head.load(std::memory_order_acquire); + ABSL_ASSERT(snapshot.DiagnosticsHandleIsSafeToInspect(head)); + return head; } CordzInfo* CordzInfo::Next(const CordzSnapshot& snapshot) const { ABSL_ASSERT(snapshot.is_snapshot()); + + // Similar to the 'Head()' function, we do not need a mutex here. + CordzInfo* next = ci_next_.load(std::memory_order_acquire); ABSL_ASSERT(snapshot.DiagnosticsHandleIsSafeToInspect(this)); - ABSL_ASSERT(snapshot.DiagnosticsHandleIsSafeToInspect(ci_next_unsafe())); - return ci_next_unsafe(); + ABSL_ASSERT(snapshot.DiagnosticsHandleIsSafeToInspect(next)); + return next; } void CordzInfo::TrackCord(InlineData& cord, MethodIdentifier method) { @@ -63,14 +76,6 @@ void CordzInfo::TrackCord(InlineData& cord, const InlineData& src, cordz_info->Track(); } -void CordzInfo::UntrackCord(CordzInfo* cordz_info) { - assert(cordz_info); - if (cordz_info) { - cordz_info->Untrack(); - CordzHandle::Delete(cordz_info); - } -} - CordzInfo::MethodIdentifier CordzInfo::GetParentMethod(const CordzInfo* src) { if (src == nullptr) return MethodIdentifier::kUnknown; return src->parent_method_ != MethodIdentifier::kUnknown ? src->parent_method_ @@ -110,14 +115,14 @@ CordzInfo::~CordzInfo() { } void CordzInfo::Track() { - absl::MutexLock l(&ci_mutex_); + SpinLockHolder l(&list_->mutex); - CordzInfo* const head = ci_head_.load(std::memory_order_acquire); + CordzInfo* const head = list_->head.load(std::memory_order_acquire); if (head != nullptr) { head->ci_prev_.store(this, std::memory_order_release); } ci_next_.store(head, std::memory_order_release); - ci_head_.store(this, std::memory_order_release); + list_->head.store(this, std::memory_order_release); } void CordzInfo::Untrack() { @@ -128,24 +133,28 @@ void CordzInfo::Untrack() { rep_ = nullptr; } - absl::MutexLock l(&ci_mutex_); - - CordzInfo* const head = ci_head_.load(std::memory_order_acquire); - CordzInfo* const next = ci_next_.load(std::memory_order_acquire); - CordzInfo* const prev = ci_prev_.load(std::memory_order_acquire); - - if (next) { - ABSL_ASSERT(next->ci_prev_.load(std::memory_order_acquire) == this); - next->ci_prev_.store(prev, std::memory_order_release); - } - if (prev) { - ABSL_ASSERT(head != this); - ABSL_ASSERT(prev->ci_next_.load(std::memory_order_acquire) == this); - prev->ci_next_.store(next, std::memory_order_release); - } else { - ABSL_ASSERT(head == this); - ci_head_.store(next, std::memory_order_release); + ODRCheck(); + { + SpinLockHolder l(&list_->mutex); + + CordzInfo* const head = list_->head.load(std::memory_order_acquire); + CordzInfo* const next = ci_next_.load(std::memory_order_acquire); + CordzInfo* const prev = ci_prev_.load(std::memory_order_acquire); + + if (next) { + ABSL_ASSERT(next->ci_prev_.load(std::memory_order_acquire) == this); + next->ci_prev_.store(prev, std::memory_order_release); + } + if (prev) { + ABSL_ASSERT(head != this); + ABSL_ASSERT(prev->ci_next_.load(std::memory_order_acquire) == this); + prev->ci_next_.store(next, std::memory_order_release); + } else { + ABSL_ASSERT(head == this); + list_->head.store(next, std::memory_order_release); + } } + CordzHandle::Delete(this); } void CordzInfo::Lock(MethodIdentifier method) @@ -163,7 +172,6 @@ void CordzInfo::Unlock() ABSL_UNLOCK_FUNCTION(mutex_) { mutex_.Unlock(); if (!tracked) { Untrack(); - CordzHandle::Delete(this); } } diff --git a/absl/strings/internal/cordz_info.h b/absl/strings/internal/cordz_info.h index 4cf57664..f7682cb3 100644 --- a/absl/strings/internal/cordz_info.h +++ b/absl/strings/internal/cordz_info.h @@ -20,6 +20,8 @@ #include <functional> #include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/spinlock.h" #include "absl/base/thread_annotations.h" #include "absl/strings/internal/cord_internal.h" #include "absl/strings/internal/cordz_functions.h" @@ -79,17 +81,22 @@ class ABSL_LOCKABLE CordzInfo : public CordzHandle { // and before the root cordrep of the sampled cord is unreffed. // This function may extend the lifetime of the cordrep in cases where the // CordInfo instance is being held by a concurrent collection thread. - static void UntrackCord(CordzInfo* cordz_info); + void Untrack(); + + // Invokes UntrackCord() on `info` if `info` is not null. + static void MaybeUntrackCord(CordzInfo* info); CordzInfo() = delete; CordzInfo(const CordzInfo&) = delete; CordzInfo& operator=(const CordzInfo&) = delete; // Retrieves the oldest existing CordzInfo. - static CordzInfo* Head(const CordzSnapshot& snapshot); + static CordzInfo* Head(const CordzSnapshot& snapshot) + ABSL_NO_THREAD_SAFETY_ANALYSIS; // Retrieves the next oldest existing CordzInfo older than 'this' instance. - CordzInfo* Next(const CordzSnapshot& snapshot) const; + CordzInfo* Next(const CordzSnapshot& snapshot) const + ABSL_NO_THREAD_SAFETY_ANALYSIS; // Locks this instance for the update identified by `method`. // Increases the count for `method` in `update_tracker`. @@ -141,6 +148,17 @@ class ABSL_LOCKABLE CordzInfo : public CordzHandle { } private: + // Global cordz info list. CordzInfo stores a pointer to the global list + // instance to harden against ODR violations. + struct List { + constexpr explicit List(absl::ConstInitType) + : mutex(absl::kConstInit, + absl::base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL) {} + + absl::base_internal::SpinLock mutex; + std::atomic<CordzInfo*> head ABSL_GUARDED_BY(mutex){nullptr}; + }; + static constexpr int kMaxStackDepth = 64; explicit CordzInfo(CordRep* rep, const CordzInfo* src, @@ -148,7 +166,6 @@ class ABSL_LOCKABLE CordzInfo : public CordzHandle { ~CordzInfo() override; void Track(); - void Untrack(); // Returns the parent method from `src`, which is either `parent_method_` or // `method_` depending on `parent_method_` being kUnknown. @@ -161,23 +178,20 @@ class ABSL_LOCKABLE CordzInfo : public CordzHandle { // Returns 0 if `src` is null. static int FillParentStack(const CordzInfo* src, void** stack); - // 'Unsafe' head/next/prev accessors not requiring the lock being held. - // These are used exclusively for iterations (Head / Next) where we enforce - // a token being held, so reading an 'old' / deleted pointer is fine. - static CordzInfo* ci_head_unsafe() ABSL_NO_THREAD_SAFETY_ANALYSIS { - return ci_head_.load(std::memory_order_acquire); - } - CordzInfo* ci_next_unsafe() const ABSL_NO_THREAD_SAFETY_ANALYSIS { - return ci_next_.load(std::memory_order_acquire); - } - CordzInfo* ci_prev_unsafe() const ABSL_NO_THREAD_SAFETY_ANALYSIS { - return ci_prev_.load(std::memory_order_acquire); + void ODRCheck() const { +#ifndef NDEBUG + ABSL_RAW_CHECK(list_ == &global_list_, "ODR violation in Cord"); +#endif } - static absl::Mutex ci_mutex_; - static std::atomic<CordzInfo*> ci_head_ ABSL_GUARDED_BY(ci_mutex_); - std::atomic<CordzInfo*> ci_prev_ ABSL_GUARDED_BY(ci_mutex_){nullptr}; - std::atomic<CordzInfo*> ci_next_ ABSL_GUARDED_BY(ci_mutex_){nullptr}; + ABSL_CONST_INIT static List global_list_; + List* const list_ = &global_list_; + + // ci_prev_ and ci_next_ require the global list mutex to be held. + // Unfortunately we can't use thread annotations such that the thread safety + // analysis understands that list_ and global_list_ are one and the same. + std::atomic<CordzInfo*> ci_prev_{nullptr}; + std::atomic<CordzInfo*> ci_next_{nullptr}; mutable absl::Mutex mutex_; CordRep* rep_ ABSL_GUARDED_BY(mutex_); @@ -209,6 +223,13 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CordzInfo::MaybeTrackCord( } } +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CordzInfo::MaybeUntrackCord( + CordzInfo* info) { + if (ABSL_PREDICT_FALSE(info)) { + info->Untrack(); + } +} + inline void CordzInfo::AssertHeld() ABSL_ASSERT_EXCLUSIVE_LOCK(mutex_) { #ifndef NDEBUG mutex_.AssertHeld(); diff --git a/absl/strings/internal/cordz_info_test.cc b/absl/strings/internal/cordz_info_test.cc index 76aa20b0..5eaf4b9c 100644 --- a/absl/strings/internal/cordz_info_test.cc +++ b/absl/strings/internal/cordz_info_test.cc @@ -70,7 +70,7 @@ TEST(CordzInfoTest, TrackCord) { EXPECT_FALSE(info->is_snapshot()); EXPECT_THAT(CordzInfo::Head(CordzSnapshot()), Eq(info)); EXPECT_THAT(info->GetCordRepForTesting(), Eq(data.rep.rep)); - CordzInfo::UntrackCord(info); + info->Untrack(); } TEST(CordzInfoTest, UntrackCord) { @@ -79,7 +79,7 @@ TEST(CordzInfoTest, UntrackCord) { CordzInfo* info = data.data.cordz_info(); CordzSnapshot snapshot; - CordzInfo::UntrackCord(info); + info->Untrack(); EXPECT_THAT(CordzInfo::Head(CordzSnapshot()), Eq(nullptr)); EXPECT_THAT(info->GetCordRepForTesting(), Eq(nullptr)); EXPECT_THAT(DeleteQueue(), ElementsAre(info, &snapshot)); @@ -96,7 +96,7 @@ TEST(CordzInfoTest, SetCordRep) { info->Unlock(); EXPECT_THAT(info->GetCordRepForTesting(), Eq(rep.rep)); - CordzInfo::UntrackCord(info); + info->Untrack(); } TEST(CordzInfoTest, SetCordRepNullUntracksCordOnUnlock) { @@ -121,7 +121,7 @@ TEST(CordzInfoTest, SetCordRepRequiresMutex) { CordzInfo* info = data.data.cordz_info(); TestCordRep rep; EXPECT_DEBUG_DEATH(info->SetCordRep(rep.rep), ".*"); - CordzInfo::UntrackCord(info); + info->Untrack(); } #endif // GTEST_HAS_DEATH_TEST @@ -143,11 +143,11 @@ TEST(CordzInfoTest, TrackUntrackHeadFirstV2) { EXPECT_THAT(info2->Next(snapshot), Eq(info1)); EXPECT_THAT(info1->Next(snapshot), Eq(nullptr)); - CordzInfo::UntrackCord(info2); + info2->Untrack(); ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info1)); EXPECT_THAT(info1->Next(snapshot), Eq(nullptr)); - CordzInfo::UntrackCord(info1); + info1->Untrack(); ASSERT_THAT(CordzInfo::Head(snapshot), Eq(nullptr)); } @@ -168,11 +168,11 @@ TEST(CordzInfoTest, TrackUntrackTailFirstV2) { EXPECT_THAT(info2->Next(snapshot), Eq(info1)); EXPECT_THAT(info1->Next(snapshot), Eq(nullptr)); - CordzInfo::UntrackCord(info1); + info1->Untrack(); ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info2)); EXPECT_THAT(info2->Next(snapshot), Eq(nullptr)); - CordzInfo::UntrackCord(info2); + info2->Untrack(); ASSERT_THAT(CordzInfo::Head(snapshot), Eq(nullptr)); } @@ -208,7 +208,7 @@ TEST(CordzInfoTest, StackV2) { // got_stack. EXPECT_THAT(got_stack, HasSubstr(expected_stack)); - CordzInfo::UntrackCord(info); + info->Untrack(); } // Local helper functions to get different stacks for child and parent. @@ -231,7 +231,7 @@ TEST(CordzInfoTest, GetStatistics) { EXPECT_THAT(statistics.parent_method, Eq(kUnknownMethod)); EXPECT_THAT(statistics.update_tracker.Value(kTrackCordMethod), Eq(1)); - CordzInfo::UntrackCord(info); + info->Untrack(); } TEST(CordzInfoTest, LockCountsMethod) { @@ -246,7 +246,7 @@ TEST(CordzInfoTest, LockCountsMethod) { CordzStatistics statistics = info->GetCordzStatistics(); EXPECT_THAT(statistics.update_tracker.Value(kUpdateMethod), Eq(2)); - CordzInfo::UntrackCord(info); + info->Untrack(); } TEST(CordzInfoTest, FromParent) { @@ -265,8 +265,8 @@ TEST(CordzInfoTest, FromParent) { EXPECT_THAT(statistics.parent_method, Eq(kTrackCordMethod)); EXPECT_THAT(statistics.update_tracker.Value(kChildMethod), Eq(1)); - CordzInfo::UntrackCord(info_parent); - CordzInfo::UntrackCord(info_child); + info_parent->Untrack(); + info_child->Untrack(); } TEST(CordzInfoTest, FromParentInlined) { @@ -279,7 +279,7 @@ TEST(CordzInfoTest, FromParentInlined) { EXPECT_THAT(statistics.method, Eq(kChildMethod)); EXPECT_THAT(statistics.parent_method, Eq(kUnknownMethod)); EXPECT_THAT(statistics.update_tracker.Value(kChildMethod), Eq(1)); - CordzInfo::UntrackCord(info); + info->Untrack(); } TEST(CordzInfoTest, RecordMetrics) { @@ -293,7 +293,7 @@ TEST(CordzInfoTest, RecordMetrics) { CordzStatistics actual = info->GetCordzStatistics(); EXPECT_EQ(actual.size, expected.size); - CordzInfo::UntrackCord(info); + info->Untrack(); } } // namespace diff --git a/absl/strings/internal/cordz_sample_token_test.cc b/absl/strings/internal/cordz_sample_token_test.cc index 80f41ca1..9f54301d 100644 --- a/absl/strings/internal/cordz_sample_token_test.cc +++ b/absl/strings/internal/cordz_sample_token_test.cc @@ -96,9 +96,9 @@ TEST(CordzSampleTokenTest, Iterator) { EXPECT_THAT(found, ElementsAre(info3, info2, info1)); - CordzInfo::UntrackCord(info1); - CordzInfo::UntrackCord(info2); - CordzInfo::UntrackCord(info3); + info1->Untrack(); + info2->Untrack(); + info3->Untrack(); } TEST(CordzSampleTokenTest, IteratorEquality) { @@ -135,9 +135,9 @@ TEST(CordzSampleTokenTest, IteratorEquality) { // Both lhs and rhs are done, so they are on nullptr. EXPECT_THAT(lhs, Eq(rhs)); - CordzInfo::UntrackCord(info1); - CordzInfo::UntrackCord(info2); - CordzInfo::UntrackCord(info3); + info1->Untrack(); + info2->Untrack(); + info3->Untrack(); } TEST(CordzSampleTokenTest, MultiThreaded) { @@ -166,7 +166,7 @@ TEST(CordzSampleTokenTest, MultiThreaded) { // Track/untrack. if (cord.data.is_profiled()) { // 1) Untrack - CordzInfo::UntrackCord(cord.data.cordz_info()); + cord.data.cordz_info()->Untrack(); cord.data.clear_cordz_info();; } else { // 2) Track @@ -192,9 +192,7 @@ TEST(CordzSampleTokenTest, MultiThreaded) { } } for (TestCordData& cord : cords) { - if (cord.data.is_profiled()) { - CordzInfo::UntrackCord(cord.data.cordz_info()); - } + CordzInfo::MaybeUntrackCord(cord.data.cordz_info()); } }); } diff --git a/absl/strings/internal/cordz_update_scope.h b/absl/strings/internal/cordz_update_scope.h index 018359a8..57ba75de 100644 --- a/absl/strings/internal/cordz_update_scope.h +++ b/absl/strings/internal/cordz_update_scope.h @@ -40,6 +40,12 @@ class ABSL_SCOPED_LOCKABLE CordzUpdateScope { } } + // CordzUpdateScope can not be copied or assigned to. + CordzUpdateScope(CordzUpdateScope&& rhs) = delete; + CordzUpdateScope(const CordzUpdateScope&) = delete; + CordzUpdateScope& operator=(CordzUpdateScope&& rhs) = delete; + CordzUpdateScope& operator=(const CordzUpdateScope&) = delete; + ~CordzUpdateScope() ABSL_UNLOCK_FUNCTION() { if (ABSL_PREDICT_FALSE(info_)) { info_->Unlock(); @@ -55,7 +61,7 @@ class ABSL_SCOPED_LOCKABLE CordzUpdateScope { CordzInfo* info() const { return info_; } private: - CordzInfo* const info_; + CordzInfo* info_; }; } // namespace cord_internal |