diff options
Diffstat (limited to 'absl/synchronization/mutex.cc')
| -rw-r--r-- | absl/synchronization/mutex.cc | 33 |
1 files changed, 24 insertions, 9 deletions
diff --git a/absl/synchronization/mutex.cc b/absl/synchronization/mutex.cc index c7968f06..aafeb674 100644 --- a/absl/synchronization/mutex.cc +++ b/absl/synchronization/mutex.cc @@ -59,6 +59,7 @@ using absl::base_internal::CurrentThreadIdentityIfPresent; using absl::base_internal::PerThreadSynch; +using absl::base_internal::SchedulingGuard; using absl::base_internal::ThreadIdentity; using absl::synchronization_internal::GetOrCreateCurrentThreadIdentity; using absl::synchronization_internal::GraphCycles; @@ -141,7 +142,9 @@ static const MutexGlobals& GetMutexGlobals() { namespace { enum DelayMode { AGGRESSIVE, GENTLE }; }; -static int Delay(int32_t c, DelayMode mode) { + +namespace synchronization_internal { +int MutexDelay(int32_t c, int mode) { // If this a uniprocessor, only yield/sleep. Otherwise, if the mode is // aggressive then spin many times before yielding. If the mode is // gentle then spin only a few times before yielding. Aggressive spinning is @@ -153,6 +156,7 @@ static int Delay(int32_t c, DelayMode mode) { // Spin. c++; } else { + SchedulingGuard::ScopedEnable enable_rescheduling; ABSL_TSAN_MUTEX_PRE_DIVERT(nullptr, 0); if (c == limit) { // Yield once. @@ -167,6 +171,7 @@ static int Delay(int32_t c, DelayMode mode) { } return c; } +} // namespace synchronization_internal // --------------------------Generic atomic ops // Ensure that "(*pv & bits) == bits" by doing an atomic update of "*pv" to @@ -1051,6 +1056,7 @@ static PerThreadSynch *DequeueAllWakeable(PerThreadSynch *head, // Try to remove thread s from the list of waiters on this mutex. // Does nothing if s is not on the waiter list. void Mutex::TryRemove(PerThreadSynch *s) { + SchedulingGuard::ScopedDisable disable_rescheduling; intptr_t v = mu_.load(std::memory_order_relaxed); // acquire spinlock & lock if ((v & (kMuWait | kMuSpin | kMuWriter | kMuReader)) == kMuWait && @@ -1115,7 +1121,7 @@ ABSL_XRAY_LOG_ARGS(1) void Mutex::Block(PerThreadSynch *s) { this->TryRemove(s); int c = 0; while (s->next != nullptr) { - c = Delay(c, GENTLE); + c = synchronization_internal::MutexDelay(c, GENTLE); this->TryRemove(s); } if (kDebugMode) { @@ -1894,6 +1900,7 @@ static void CheckForMutexCorruption(intptr_t v, const char* label) { } void Mutex::LockSlowLoop(SynchWaitParams *waitp, int flags) { + SchedulingGuard::ScopedDisable disable_rescheduling; int c = 0; intptr_t v = mu_.load(std::memory_order_relaxed); if ((v & kMuEvent) != 0) { @@ -1995,7 +2002,8 @@ void Mutex::LockSlowLoop(SynchWaitParams *waitp, int flags) { ABSL_RAW_CHECK( waitp->thread->waitp == nullptr || waitp->thread->suppress_fatal_errors, "detected illegal recursion into Mutex code"); - c = Delay(c, GENTLE); // delay, then try again + // delay, then try again + c = synchronization_internal::MutexDelay(c, GENTLE); } ABSL_RAW_CHECK( waitp->thread->waitp == nullptr || waitp->thread->suppress_fatal_errors, @@ -2013,6 +2021,7 @@ void Mutex::LockSlowLoop(SynchWaitParams *waitp, int flags) { // or it is in the process of blocking on a condition variable; it must requeue // itself on the mutex/condvar to wait for its condition to become true. ABSL_ATTRIBUTE_NOINLINE void Mutex::UnlockSlow(SynchWaitParams *waitp) { + SchedulingGuard::ScopedDisable disable_rescheduling; intptr_t v = mu_.load(std::memory_order_relaxed); this->AssertReaderHeld(); CheckForMutexCorruption(v, "Unlock"); @@ -2289,7 +2298,8 @@ ABSL_ATTRIBUTE_NOINLINE void Mutex::UnlockSlow(SynchWaitParams *waitp) { mu_.store(nv, std::memory_order_release); break; // out of for(;;)-loop } - c = Delay(c, AGGRESSIVE); // aggressive here; no one can proceed till we do + // aggressive here; no one can proceed till we do + c = synchronization_internal::MutexDelay(c, AGGRESSIVE); } // end of for(;;)-loop if (wake_list != kPerThreadSynchNull) { @@ -2328,6 +2338,7 @@ void Mutex::Trans(MuHow how) { // It will later acquire the mutex with high probability. Otherwise, we // enqueue thread w on this mutex. void Mutex::Fer(PerThreadSynch *w) { + SchedulingGuard::ScopedDisable disable_rescheduling; int c = 0; ABSL_RAW_CHECK(w->waitp->cond == nullptr, "Mutex::Fer while waiting on Condition"); @@ -2377,7 +2388,7 @@ void Mutex::Fer(PerThreadSynch *w) { return; } } - c = Delay(c, GENTLE); + c = synchronization_internal::MutexDelay(c, GENTLE); } } @@ -2426,6 +2437,7 @@ CondVar::~CondVar() { // Remove thread s from the list of waiters on this condition variable. void CondVar::Remove(PerThreadSynch *s) { + SchedulingGuard::ScopedDisable disable_rescheduling; intptr_t v; int c = 0; for (v = cv_.load(std::memory_order_relaxed);; @@ -2454,7 +2466,8 @@ void CondVar::Remove(PerThreadSynch *s) { std::memory_order_release); return; } else { - c = Delay(c, GENTLE); // try again after a delay + // try again after a delay + c = synchronization_internal::MutexDelay(c, GENTLE); } } } @@ -2487,7 +2500,7 @@ static void CondVarEnqueue(SynchWaitParams *waitp) { !cv_word->compare_exchange_weak(v, v | kCvSpin, std::memory_order_acquire, std::memory_order_relaxed)) { - c = Delay(c, GENTLE); + c = synchronization_internal::MutexDelay(c, GENTLE); v = cv_word->load(std::memory_order_relaxed); } ABSL_RAW_CHECK(waitp->thread->waitp == nullptr, "waiting when shouldn't be"); @@ -2586,6 +2599,7 @@ void CondVar::Wakeup(PerThreadSynch *w) { } void CondVar::Signal() { + SchedulingGuard::ScopedDisable disable_rescheduling; ABSL_TSAN_MUTEX_PRE_SIGNAL(nullptr, 0); intptr_t v; int c = 0; @@ -2618,7 +2632,7 @@ void CondVar::Signal() { ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0); return; } else { - c = Delay(c, GENTLE); + c = synchronization_internal::MutexDelay(c, GENTLE); } } ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0); @@ -2655,7 +2669,8 @@ void CondVar::SignalAll () { ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0); return; } else { - c = Delay(c, GENTLE); // try again after a delay + // try again after a delay + c = synchronization_internal::MutexDelay(c, GENTLE); } } ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0); |