From 1cf6469b372a0ab2787ad95f1ebb611c81b968b3 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Fri, 8 Sep 2023 05:55:49 -0700 Subject: absl: inline and de-dup Mutex::Await/LockWhen/CondVar::Wait Mutex::Await/LockWhen/CondVar::Wait duplicate code, and cause additional calls at runtime and code bloat. Inline thin wrappers that just convert argument types and add a single de-duped implementation for these methods. This reduces code size, shaves off 55K from the mutex_test in release build, and should make things marginally faster. $ nm -nS mutex_test | egrep "(_ZN4absl5Mutex.*(Await|LockWhen))|(_ZN4absl7CondVar.*Wait)" before: 00000000000912c0 00000000000001a8 T _ZN4absl7CondVar4WaitEPNS_5MutexE 00000000000988c0 0000000000000c36 T _ZN4absl7CondVar16WaitWithDeadlineEPNS_5MutexENS_4TimeE 000000000009a6e0 0000000000000041 T _ZN4absl5Mutex19LockWhenWithTimeoutERKNS_9ConditionENS_8DurationE 00000000000a28c0 0000000000000779 T _ZN4absl5Mutex17AwaitWithDeadlineERKNS_9ConditionENS_4TimeE 00000000000cf4e0 0000000000000011 T _ZN4absl5Mutex8LockWhenERKNS_9ConditionE 00000000000cf500 0000000000000041 T _ZN4absl5Mutex20LockWhenWithDeadlineERKNS_9ConditionENS_4TimeE 00000000000cf560 0000000000000011 T _ZN4absl5Mutex14ReaderLockWhenERKNS_9ConditionE 00000000000cf580 0000000000000041 T _ZN4absl5Mutex26ReaderLockWhenWithDeadlineERKNS_9ConditionENS_4TimeE 00000000000cf5e0 0000000000000766 T _ZN4absl5Mutex5AwaitERKNS_9ConditionE 00000000000cfd60 00000000000007b5 T _ZN4absl5Mutex16AwaitWithTimeoutERKNS_9ConditionENS_8DurationE 00000000000d0700 00000000000003cf T _ZN4absl7CondVar15WaitWithTimeoutEPNS_5MutexENS_8DurationE 000000000011c280 0000000000000041 T _ZN4absl5Mutex25ReaderLockWhenWithTimeoutERKNS_9ConditionENS_8DurationE after: 000000000009c300 00000000000007ed T _ZN4absl7CondVar10WaitCommonEPNS_5MutexENS_24synchronization_internal13KernelTimeoutE 00000000000a03c0 00000000000006fe T _ZN4absl5Mutex11AwaitCommonERKNS_9ConditionENS_24synchronization_internal13KernelTimeoutE 000000000011ae00 0000000000000025 T _ZN4absl5Mutex14LockWhenCommonERKNS_9ConditionENS_24synchronization_internal13KernelTimeoutEb PiperOrigin-RevId: 563729364 Change-Id: Ic6b43761f76719c01e03d43cc0e0c419e41a85c1 --- absl/synchronization/mutex.cc | 111 +++++------------------------------------- absl/synchronization/mutex.h | 56 ++++++++++++++++----- 2 files changed, 56 insertions(+), 111 deletions(-) diff --git a/absl/synchronization/mutex.cc b/absl/synchronization/mutex.cc index dc34d8a9..d63ec5ec 100644 --- a/absl/synchronization/mutex.cc +++ b/absl/synchronization/mutex.cc @@ -1522,104 +1522,25 @@ void Mutex::ReaderLock() { ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0); } -void Mutex::LockWhen(const Condition& cond) { - ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); - GraphId id = DebugOnlyDeadlockCheck(this); - this->LockSlow(kExclusive, &cond, 0); - DebugOnlyLockEnter(this, id); - ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); -} - -bool Mutex::LockWhenWithTimeout(const Condition& cond, absl::Duration timeout) { - ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); - GraphId id = DebugOnlyDeadlockCheck(this); - bool res = LockSlowWithDeadline(kExclusive, &cond, KernelTimeout(timeout), 0); - DebugOnlyLockEnter(this, id); - ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); - return res; -} - -bool Mutex::LockWhenWithDeadline(const Condition& cond, absl::Time deadline) { - ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); - GraphId id = DebugOnlyDeadlockCheck(this); - bool res = - LockSlowWithDeadline(kExclusive, &cond, KernelTimeout(deadline), 0); - DebugOnlyLockEnter(this, id); - ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); - return res; -} - -void Mutex::ReaderLockWhen(const Condition& cond) { - ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock); - GraphId id = DebugOnlyDeadlockCheck(this); - this->LockSlow(kShared, &cond, 0); - DebugOnlyLockEnter(this, id); - ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0); -} - -bool Mutex::ReaderLockWhenWithTimeout(const Condition& cond, - absl::Duration timeout) { - ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock); - GraphId id = DebugOnlyDeadlockCheck(this); - bool res = LockSlowWithDeadline(kShared, &cond, KernelTimeout(timeout), 0); - DebugOnlyLockEnter(this, id); - ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0); - return res; -} - -bool Mutex::ReaderLockWhenWithDeadline(const Condition& cond, - absl::Time deadline) { - ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock); +bool Mutex::LockWhenCommon(const Condition& cond, + synchronization_internal::KernelTimeout t, + bool write) { + MuHow how = write ? kExclusive : kShared; + ABSL_TSAN_MUTEX_PRE_LOCK(this, TsanFlags(how)); GraphId id = DebugOnlyDeadlockCheck(this); - bool res = LockSlowWithDeadline(kShared, &cond, KernelTimeout(deadline), 0); + bool res = LockSlowWithDeadline(how, &cond, t, 0); DebugOnlyLockEnter(this, id); - ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0); + ABSL_TSAN_MUTEX_POST_LOCK(this, TsanFlags(how), 0); return res; } -void Mutex::Await(const Condition& cond) { - if (cond.Eval()) { // condition already true; nothing to do - if (kDebugMode) { - this->AssertReaderHeld(); - } - } else { // normal case - ABSL_RAW_CHECK(this->AwaitCommon(cond, KernelTimeout::Never()), - "condition untrue on return from Await"); - } -} - -bool Mutex::AwaitWithTimeout(const Condition& cond, absl::Duration timeout) { - if (cond.Eval()) { // condition already true; nothing to do - if (kDebugMode) { - this->AssertReaderHeld(); - } - return true; +bool Mutex::AwaitCommon(const Condition& cond, KernelTimeout t) { + if (kDebugMode) { + this->AssertReaderHeld(); } - - KernelTimeout t{timeout}; - bool res = this->AwaitCommon(cond, t); - ABSL_RAW_CHECK(res || t.has_timeout(), - "condition untrue on return from Await"); - return res; -} - -bool Mutex::AwaitWithDeadline(const Condition& cond, absl::Time deadline) { if (cond.Eval()) { // condition already true; nothing to do - if (kDebugMode) { - this->AssertReaderHeld(); - } return true; } - - KernelTimeout t{deadline}; - bool res = this->AwaitCommon(cond, t); - ABSL_RAW_CHECK(res || t.has_timeout(), - "condition untrue on return from Await"); - return res; -} - -bool Mutex::AwaitCommon(const Condition& cond, KernelTimeout t) { - this->AssertReaderHeld(); MuHow how = (mu_.load(std::memory_order_relaxed) & kMuWriter) ? kExclusive : kShared; ABSL_TSAN_MUTEX_PRE_UNLOCK(this, TsanFlags(how)); @@ -1634,6 +1555,8 @@ bool Mutex::AwaitCommon(const Condition& cond, KernelTimeout t) { bool res = waitp.cond != nullptr || // => cond known true from LockSlowLoop EvalConditionAnnotated(&cond, this, true, false, how == kShared); ABSL_TSAN_MUTEX_POST_LOCK(this, TsanFlags(how), 0); + ABSL_RAW_CHECK(res || t.has_timeout(), + "condition untrue on return from Await"); return res; } @@ -2654,16 +2577,6 @@ bool CondVar::WaitCommon(Mutex* mutex, KernelTimeout t) { return rc; } -bool CondVar::WaitWithTimeout(Mutex* mu, absl::Duration timeout) { - return WaitCommon(mu, KernelTimeout(timeout)); -} - -bool CondVar::WaitWithDeadline(Mutex* mu, absl::Time deadline) { - return WaitCommon(mu, KernelTimeout(deadline)); -} - -void CondVar::Wait(Mutex* mu) { WaitCommon(mu, KernelTimeout::Never()); } - // Wake thread w // If it was a timed wait, w will be waiting on w->cv // Otherwise, if it was not a Mutex mutex, w will be waiting on w->sem diff --git a/absl/synchronization/mutex.h b/absl/synchronization/mutex.h index f2bafa97..6459053b 100644 --- a/absl/synchronization/mutex.h +++ b/absl/synchronization/mutex.h @@ -324,7 +324,9 @@ class ABSL_LOCKABLE Mutex { // `true`, `Await()` *may* skip the release/re-acquire step. // // `Await()` requires that this thread holds this `Mutex` in some mode. - void Await(const Condition& cond); + void Await(const Condition& cond) { + AwaitCommon(cond, synchronization_internal::KernelTimeout::Never()); + } // Mutex::LockWhen() // Mutex::ReaderLockWhen() @@ -334,9 +336,15 @@ class ABSL_LOCKABLE Mutex { // be acquired, then atomically acquires this `Mutex`. `LockWhen()` is // logically equivalent to `*Lock(); Await();` though they may have different // performance characteristics. - void LockWhen(const Condition& cond) ABSL_EXCLUSIVE_LOCK_FUNCTION(); + void LockWhen(const Condition& cond) ABSL_EXCLUSIVE_LOCK_FUNCTION() { + LockWhenCommon(cond, synchronization_internal::KernelTimeout::Never(), + true); + } - void ReaderLockWhen(const Condition& cond) ABSL_SHARED_LOCK_FUNCTION(); + void ReaderLockWhen(const Condition& cond) ABSL_SHARED_LOCK_FUNCTION() { + LockWhenCommon(cond, synchronization_internal::KernelTimeout::Never(), + false); + } void WriterLockWhen(const Condition& cond) ABSL_EXCLUSIVE_LOCK_FUNCTION() { this->LockWhen(cond); @@ -363,9 +371,13 @@ class ABSL_LOCKABLE Mutex { // Negative timeouts are equivalent to a zero timeout. // // This method requires that this thread holds this `Mutex` in some mode. - bool AwaitWithTimeout(const Condition& cond, absl::Duration timeout); + bool AwaitWithTimeout(const Condition& cond, absl::Duration timeout) { + return AwaitCommon(cond, synchronization_internal::KernelTimeout{timeout}); + } - bool AwaitWithDeadline(const Condition& cond, absl::Time deadline); + bool AwaitWithDeadline(const Condition& cond, absl::Time deadline) { + return AwaitCommon(cond, synchronization_internal::KernelTimeout{deadline}); + } // Mutex::LockWhenWithTimeout() // Mutex::ReaderLockWhenWithTimeout() @@ -379,9 +391,15 @@ class ABSL_LOCKABLE Mutex { // // Negative timeouts are equivalent to a zero timeout. bool LockWhenWithTimeout(const Condition& cond, absl::Duration timeout) - ABSL_EXCLUSIVE_LOCK_FUNCTION(); + ABSL_EXCLUSIVE_LOCK_FUNCTION() { + return LockWhenCommon( + cond, synchronization_internal::KernelTimeout{timeout}, true); + } bool ReaderLockWhenWithTimeout(const Condition& cond, absl::Duration timeout) - ABSL_SHARED_LOCK_FUNCTION(); + ABSL_SHARED_LOCK_FUNCTION() { + return LockWhenCommon( + cond, synchronization_internal::KernelTimeout{timeout}, false); + } bool WriterLockWhenWithTimeout(const Condition& cond, absl::Duration timeout) ABSL_EXCLUSIVE_LOCK_FUNCTION() { return this->LockWhenWithTimeout(cond, timeout); @@ -399,9 +417,15 @@ class ABSL_LOCKABLE Mutex { // // Deadlines in the past are equivalent to an immediate deadline. bool LockWhenWithDeadline(const Condition& cond, absl::Time deadline) - ABSL_EXCLUSIVE_LOCK_FUNCTION(); + ABSL_EXCLUSIVE_LOCK_FUNCTION() { + return LockWhenCommon( + cond, synchronization_internal::KernelTimeout{deadline}, true); + } bool ReaderLockWhenWithDeadline(const Condition& cond, absl::Time deadline) - ABSL_SHARED_LOCK_FUNCTION(); + ABSL_SHARED_LOCK_FUNCTION() { + return LockWhenCommon( + cond, synchronization_internal::KernelTimeout{deadline}, false); + } bool WriterLockWhenWithDeadline(const Condition& cond, absl::Time deadline) ABSL_EXCLUSIVE_LOCK_FUNCTION() { return this->LockWhenWithDeadline(cond, deadline); @@ -500,6 +524,8 @@ class ABSL_LOCKABLE Mutex { // Common code between Await() and AwaitWithTimeout/Deadline() bool AwaitCommon(const Condition& cond, synchronization_internal::KernelTimeout t); + bool LockWhenCommon(const Condition& cond, + synchronization_internal::KernelTimeout t, bool write); // Attempt to remove thread s from queue. void TryRemove(base_internal::PerThreadSynch* s); // Block a thread on mutex. @@ -888,7 +914,9 @@ class CondVar { // spurious wakeup), then reacquires the `Mutex` and returns. // // Requires and ensures that the current thread holds the `Mutex`. - void Wait(Mutex* mu); + void Wait(Mutex* mu) { + WaitCommon(mu, synchronization_internal::KernelTimeout::Never()); + } // CondVar::WaitWithTimeout() // @@ -903,7 +931,9 @@ class CondVar { // to return `true` or `false`. // // Requires and ensures that the current thread holds the `Mutex`. - bool WaitWithTimeout(Mutex* mu, absl::Duration timeout); + bool WaitWithTimeout(Mutex* mu, absl::Duration timeout) { + return WaitCommon(mu, synchronization_internal::KernelTimeout(timeout)); + } // CondVar::WaitWithDeadline() // @@ -920,7 +950,9 @@ class CondVar { // to return `true` or `false`. // // Requires and ensures that the current thread holds the `Mutex`. - bool WaitWithDeadline(Mutex* mu, absl::Time deadline); + bool WaitWithDeadline(Mutex* mu, absl::Time deadline) { + return WaitCommon(mu, synchronization_internal::KernelTimeout(deadline)); + } // CondVar::Signal() // -- cgit v1.2.3