diff options
Diffstat (limited to 'absl/synchronization')
28 files changed, 357 insertions, 266 deletions
diff --git a/absl/synchronization/BUILD.bazel b/absl/synchronization/BUILD.bazel index fca8cb69..3f876b9f 100644 --- a/absl/synchronization/BUILD.bazel +++ b/absl/synchronization/BUILD.bazel @@ -14,6 +14,7 @@ # limitations under the License. # +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -42,8 +43,25 @@ cc_library( deps = [ "//absl/base", "//absl/base:base_internal", + "//absl/base:config", "//absl/base:core_headers", "//absl/base:malloc_internal", + "//absl/base:raw_logging_internal", + ], +) + +cc_library( + name = "kernel_timeout_internal", + hdrs = ["internal/kernel_timeout.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/synchronization:__pkg__", + ], + deps = [ + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/time", ], ) @@ -63,7 +81,6 @@ cc_library( "barrier.h", "blocking_counter.h", "internal/create_thread_identity.h", - "internal/kernel_timeout.h", "internal/mutex_nonprod.inc", "internal/per_thread_sem.h", "internal/waiter.h", @@ -77,6 +94,7 @@ cc_library( }) + ABSL_DEFAULT_LINKOPTS, deps = [ ":graphcycles_internal", + ":kernel_timeout_internal", "//absl/base", "//absl/base:atomic_hook", "//absl/base:base_internal", @@ -84,6 +102,7 @@ cc_library( "//absl/base:core_headers", "//absl/base:dynamic_annotations", "//absl/base:malloc_internal", + "//absl/base:raw_logging_internal", "//absl/debugging:stacktrace", "//absl/debugging:symbolize", "//absl/time", @@ -124,8 +143,8 @@ cc_test( linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":graphcycles_internal", - "//absl/base", "//absl/base:core_headers", + "//absl/base:raw_logging_internal", "@com_google_googletest//:gtest_main", ], ) @@ -140,7 +159,7 @@ cc_test( ], deps = [ ":graphcycles_internal", - "//absl/base", + "//absl/base:raw_logging_internal", "@com_github_google_benchmark//:benchmark_main", ], ) @@ -171,6 +190,7 @@ cc_test( ":thread_pool", "//absl/base", "//absl/base:core_headers", + "//absl/base:raw_logging_internal", "//absl/memory", "//absl/time", "@com_google_googletest//:gtest_main", @@ -243,7 +263,6 @@ cc_test( deps = [ ":per_thread_sem_test_common", ":synchronization", - "//absl/base", "//absl/strings", "//absl/time", "@com_google_googletest//:gtest_main", @@ -260,7 +279,7 @@ cc_test( tags = ["no_test_ios_x86_64"], deps = [ ":synchronization", - "//absl/base", "//absl/base:core_headers", + "//absl/base:raw_logging_internal", ], ) diff --git a/absl/synchronization/CMakeLists.txt b/absl/synchronization/CMakeLists.txt index 4b708823..dfe5d05d 100644 --- a/absl/synchronization/CMakeLists.txt +++ b/absl/synchronization/CMakeLists.txt @@ -26,8 +26,23 @@ absl_cc_library( DEPS absl::base absl::base_internal + absl::config absl::core_headers absl::malloc_internal + absl::raw_logging_internal +) + +absl_cc_library( + NAME + kernel_timeout_internal + HDRS + "internal/kernel_timeout.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::raw_logging_internal + absl::time ) absl_cc_library( @@ -37,7 +52,6 @@ absl_cc_library( "barrier.h" "blocking_counter.h" "internal/create_thread_identity.h" - "internal/kernel_timeout.h" "internal/mutex_nonprod.inc" "internal/per_thread_sem.h" "internal/waiter.h" @@ -55,6 +69,7 @@ absl_cc_library( ${ABSL_DEFAULT_COPTS} DEPS absl::graphcycles_internal + absl::kernel_timeout_internal absl::atomic_hook absl::base absl::base_internal @@ -62,6 +77,7 @@ absl_cc_library( absl::core_headers absl::dynamic_annotations absl::malloc_internal + absl::raw_logging_internal absl::stacktrace absl::symbolize absl::time @@ -104,8 +120,8 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::graphcycles_internal - absl::base absl::core_headers + absl::raw_logging_internal gmock_main ) @@ -135,6 +151,7 @@ absl_cc_test( absl::base absl::core_headers absl::memory + absl::raw_logging_internal absl::time gmock_main ) @@ -178,7 +195,6 @@ absl_cc_test( DEPS absl::per_thread_sem_test_common absl::synchronization - absl::base absl::strings absl::time gmock_main @@ -193,6 +209,6 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::synchronization - absl::base absl::core_headers + absl::raw_logging_internal ) diff --git a/absl/synchronization/barrier.cc b/absl/synchronization/barrier.cc index 72089c52..0dfd795e 100644 --- a/absl/synchronization/barrier.cc +++ b/absl/synchronization/barrier.cc @@ -18,7 +18,7 @@ #include "absl/synchronization/mutex.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN // Return whether int *arg is zero. static bool IsZero(void *arg) { @@ -48,5 +48,5 @@ bool Barrier::Block() { return this->num_to_exit_ == 0; } -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/synchronization/barrier.h b/absl/synchronization/barrier.h index 53d5ca26..d8e75440 100644 --- a/absl/synchronization/barrier.h +++ b/absl/synchronization/barrier.h @@ -23,7 +23,7 @@ #include "absl/synchronization/mutex.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN // Barrier // @@ -70,10 +70,10 @@ class Barrier { private: Mutex lock_; - int num_to_block_ GUARDED_BY(lock_); - int num_to_exit_ GUARDED_BY(lock_); + int num_to_block_ ABSL_GUARDED_BY(lock_); + int num_to_exit_ ABSL_GUARDED_BY(lock_); }; -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_SYNCHRONIZATION_BARRIER_H_ diff --git a/absl/synchronization/blocking_counter.cc b/absl/synchronization/blocking_counter.cc index c6968973..3cea7aed 100644 --- a/absl/synchronization/blocking_counter.cc +++ b/absl/synchronization/blocking_counter.cc @@ -17,7 +17,7 @@ #include "absl/base/internal/raw_logging.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN // Return whether int *arg is zero. static bool IsZero(void *arg) { @@ -53,5 +53,5 @@ void BlockingCounter::Wait() { // after we return from this method. } -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/synchronization/blocking_counter.h b/absl/synchronization/blocking_counter.h index 5dab5a94..1f53f9f2 100644 --- a/absl/synchronization/blocking_counter.h +++ b/absl/synchronization/blocking_counter.h @@ -24,7 +24,7 @@ #include "absl/synchronization/mutex.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN // BlockingCounter // @@ -89,11 +89,11 @@ class BlockingCounter { private: Mutex lock_; - int count_ GUARDED_BY(lock_); - int num_waiting_ GUARDED_BY(lock_); + int count_ ABSL_GUARDED_BY(lock_); + int num_waiting_ ABSL_GUARDED_BY(lock_); }; -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_SYNCHRONIZATION_BLOCKING_COUNTER_H_ diff --git a/absl/synchronization/blocking_counter_test.cc b/absl/synchronization/blocking_counter_test.cc index 62d98738..2926224a 100644 --- a/absl/synchronization/blocking_counter_test.cc +++ b/absl/synchronization/blocking_counter_test.cc @@ -22,7 +22,7 @@ #include "absl/time/time.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace { void PauseAndDecreaseCounter(BlockingCounter* counter, int* done) { @@ -64,5 +64,5 @@ TEST(BlockingCounterTest, BasicFunctionality) { } } // namespace -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/synchronization/internal/create_thread_identity.cc b/absl/synchronization/internal/create_thread_identity.cc index 65f6d8fc..fa0070a9 100644 --- a/absl/synchronization/internal/create_thread_identity.cc +++ b/absl/synchronization/internal/create_thread_identity.cc @@ -27,7 +27,7 @@ #include "absl/synchronization/internal/per_thread_sem.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace synchronization_internal { // ThreadIdentity storage is persistent, we maintain a free-list of previously @@ -38,7 +38,7 @@ static base_internal::ThreadIdentity* thread_identity_freelist; // A per-thread destructor for reclaiming associated ThreadIdentity objects. // Since we must preserve their storage we cache them for re-use. -static void ReclaimThreadIdentity(void* v) { +void ReclaimThreadIdentity(void* v) { base_internal::ThreadIdentity* identity = static_cast<base_internal::ThreadIdentity*>(v); @@ -48,6 +48,8 @@ static void ReclaimThreadIdentity(void* v) { base_internal::LowLevelAlloc::Free(identity->per_thread_synch.all_locks); } + PerThreadSem::Destroy(identity); + // We must explicitly clear the current thread's identity: // (a) Subsequent (unrelated) per-thread destructors may require an identity. // We must guarantee a new identity is used in this case (this instructor @@ -85,7 +87,6 @@ static void ResetThreadIdentity(base_internal::ThreadIdentity* identity) { pts->wake = false; pts->cond_waiter = false; pts->all_locks = nullptr; - identity->waiter_state = {}; identity->blocked_count_ptr = nullptr; identity->ticker.store(0, std::memory_order_relaxed); identity->wait_start.store(0, std::memory_order_relaxed); @@ -133,7 +134,7 @@ base_internal::ThreadIdentity* CreateThreadIdentity() { } } // namespace synchronization_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_LOW_LEVEL_ALLOC_MISSING diff --git a/absl/synchronization/internal/create_thread_identity.h b/absl/synchronization/internal/create_thread_identity.h index d743cc3b..e121f683 100644 --- a/absl/synchronization/internal/create_thread_identity.h +++ b/absl/synchronization/internal/create_thread_identity.h @@ -29,13 +29,17 @@ #include "absl/base/port.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace synchronization_internal { // Allocates and attaches a ThreadIdentity object for the calling thread. // For private use only. base_internal::ThreadIdentity* CreateThreadIdentity(); +// A per-thread destructor for reclaiming associated ThreadIdentity objects. +// For private use only. +void ReclaimThreadIdentity(void* v); + // Returns the ThreadIdentity object representing the calling thread; guaranteed // to be unique for its lifetime. The returned object will remain valid for the // program's lifetime; although it may be re-assigned to a subsequent thread. @@ -50,7 +54,7 @@ inline base_internal::ThreadIdentity* GetOrCreateCurrentThreadIdentity() { } } // namespace synchronization_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_SYNCHRONIZATION_INTERNAL_CREATE_THREAD_IDENTITY_H_ diff --git a/absl/synchronization/internal/graphcycles.cc b/absl/synchronization/internal/graphcycles.cc index f4fbeadd..6a2bcdf6 100644 --- a/absl/synchronization/internal/graphcycles.cc +++ b/absl/synchronization/internal/graphcycles.cc @@ -44,7 +44,7 @@ // Do not use STL. This module does not use standard memory allocation. namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace synchronization_internal { namespace { @@ -691,7 +691,7 @@ int GraphCycles::GetStackTrace(GraphId id, void*** ptr) { } } // namespace synchronization_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_LOW_LEVEL_ALLOC_MISSING diff --git a/absl/synchronization/internal/graphcycles.h b/absl/synchronization/internal/graphcycles.h index 208527c3..ceba33e4 100644 --- a/absl/synchronization/internal/graphcycles.h +++ b/absl/synchronization/internal/graphcycles.h @@ -36,12 +36,14 @@ // InsertEdge() is very fast when the edge already exists, and reasonably fast // otherwise. // FindPath() is linear in the size of the graph. -// The current implemenation uses O(|V|+|E|) space. +// The current implementation uses O(|V|+|E|) space. #include <cstdint> +#include "absl/base/config.h" + namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace synchronization_internal { // Opaque identifier for a graph node. @@ -133,7 +135,7 @@ class GraphCycles { }; } // namespace synchronization_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif diff --git a/absl/synchronization/internal/graphcycles_test.cc b/absl/synchronization/internal/graphcycles_test.cc index fca86219..74eaffe7 100644 --- a/absl/synchronization/internal/graphcycles_test.cc +++ b/absl/synchronization/internal/graphcycles_test.cc @@ -25,7 +25,7 @@ #include "absl/base/macros.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace synchronization_internal { // We emulate a GraphCycles object with a node vector and an edge vector. @@ -460,5 +460,5 @@ TEST_F(GraphCyclesTest, ManyEdges) { } } // namespace synchronization_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/synchronization/internal/kernel_timeout.h b/absl/synchronization/internal/kernel_timeout.h index e0f01e06..d6ac5db0 100644 --- a/absl/synchronization/internal/kernel_timeout.h +++ b/absl/synchronization/internal/kernel_timeout.h @@ -34,7 +34,7 @@ #include "absl/time/time.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace synchronization_internal { class Futex; @@ -149,7 +149,7 @@ class KernelTimeout { }; } // namespace synchronization_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_SYNCHRONIZATION_INTERNAL_KERNEL_TIMEOUT_H_ diff --git a/absl/synchronization/internal/mutex_nonprod.cc b/absl/synchronization/internal/mutex_nonprod.cc index aa1ed83b..4590b98d 100644 --- a/absl/synchronization/internal/mutex_nonprod.cc +++ b/absl/synchronization/internal/mutex_nonprod.cc @@ -31,7 +31,7 @@ #include "absl/time/time.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace synchronization_internal { namespace { @@ -316,5 +316,5 @@ bool Condition::Eval() const { void RegisterSymbolizer(bool (*)(const void*, char*, int)) {} -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/synchronization/internal/mutex_nonprod.inc b/absl/synchronization/internal/mutex_nonprod.inc index ac10879b..a1502e72 100644 --- a/absl/synchronization/internal/mutex_nonprod.inc +++ b/absl/synchronization/internal/mutex_nonprod.inc @@ -36,7 +36,7 @@ #endif namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN class Condition; namespace synchronization_internal { @@ -252,10 +252,10 @@ class SynchronizationStorage { absl::once_flag once_; - // An aligned space for T. - typename std::aligned_storage<sizeof(T), alignof(T)>::type space_; + // An aligned space for the T. + alignas(T) unsigned char space_[sizeof(T)]; }; } // namespace synchronization_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/synchronization/internal/per_thread_sem.cc b/absl/synchronization/internal/per_thread_sem.cc index 284a5df4..821ca9b4 100644 --- a/absl/synchronization/internal/per_thread_sem.cc +++ b/absl/synchronization/internal/per_thread_sem.cc @@ -25,7 +25,7 @@ #include "absl/synchronization/internal/waiter.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace synchronization_internal { void PerThreadSem::SetThreadBlockedCounter(std::atomic<int> *counter) { @@ -41,12 +41,16 @@ std::atomic<int> *PerThreadSem::GetThreadBlockedCounter() { } void PerThreadSem::Init(base_internal::ThreadIdentity *identity) { - Waiter::GetWaiter(identity)->Init(); + new (Waiter::GetWaiter(identity)) Waiter(); identity->ticker.store(0, std::memory_order_relaxed); identity->wait_start.store(0, std::memory_order_relaxed); identity->is_idle.store(false, std::memory_order_relaxed); } +void PerThreadSem::Destroy(base_internal::ThreadIdentity *identity) { + Waiter::GetWaiter(identity)->~Waiter(); +} + void PerThreadSem::Tick(base_internal::ThreadIdentity *identity) { const int ticker = identity->ticker.fetch_add(1, std::memory_order_relaxed) + 1; @@ -59,7 +63,7 @@ void PerThreadSem::Tick(base_internal::ThreadIdentity *identity) { } } // namespace synchronization_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl extern "C" { diff --git a/absl/synchronization/internal/per_thread_sem.h b/absl/synchronization/internal/per_thread_sem.h index 5bb0978b..8ab43915 100644 --- a/absl/synchronization/internal/per_thread_sem.h +++ b/absl/synchronization/internal/per_thread_sem.h @@ -32,7 +32,7 @@ #include "absl/synchronization/internal/kernel_timeout.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN class Mutex; @@ -66,6 +66,10 @@ class PerThreadSem { // REQUIRES: May only be called by ThreadIdentity. static void Init(base_internal::ThreadIdentity* identity); + // Destroy the PerThreadSem associated with "identity". + // REQUIRES: May only be called by ThreadIdentity. + static void Destroy(base_internal::ThreadIdentity* identity); + // Increments "identity"'s count. static inline void Post(base_internal::ThreadIdentity* identity); @@ -78,10 +82,11 @@ class PerThreadSem { friend class PerThreadSemTest; friend class absl::Mutex; friend absl::base_internal::ThreadIdentity* CreateThreadIdentity(); + friend void ReclaimThreadIdentity(void* v); }; } // namespace synchronization_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl // In some build configurations we pass --detect-odr-violations to the diff --git a/absl/synchronization/internal/per_thread_sem_test.cc b/absl/synchronization/internal/per_thread_sem_test.cc index 93bc4244..b5a2f6d4 100644 --- a/absl/synchronization/internal/per_thread_sem_test.cc +++ b/absl/synchronization/internal/per_thread_sem_test.cc @@ -33,7 +33,7 @@ // primitives which might use PerThreadSem, most notably absl::Mutex. namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace synchronization_internal { class SimpleSemaphore { @@ -176,5 +176,5 @@ TEST_F(PerThreadSemTest, Timeouts) { } // namespace } // namespace synchronization_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/synchronization/internal/thread_pool.h b/absl/synchronization/internal/thread_pool.h index 8941be68..0cb96dac 100644 --- a/absl/synchronization/internal/thread_pool.h +++ b/absl/synchronization/internal/thread_pool.h @@ -26,7 +26,7 @@ #include "absl/synchronization/mutex.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace synchronization_internal { // A simple ThreadPool implementation for tests. @@ -61,7 +61,7 @@ class ThreadPool { } private: - bool WorkAvailable() const EXCLUSIVE_LOCKS_REQUIRED(mu_) { + bool WorkAvailable() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { return !queue_.empty(); } @@ -82,12 +82,12 @@ class ThreadPool { } absl::Mutex mu_; - std::queue<std::function<void()>> queue_ GUARDED_BY(mu_); + std::queue<std::function<void()>> queue_ ABSL_GUARDED_BY(mu_); std::vector<std::thread> threads_; }; } // namespace synchronization_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_SYNCHRONIZATION_INTERNAL_THREAD_POOL_H_ diff --git a/absl/synchronization/internal/waiter.cc b/absl/synchronization/internal/waiter.cc index 17c6a506..2949f5a8 100644 --- a/absl/synchronization/internal/waiter.cc +++ b/absl/synchronization/internal/waiter.cc @@ -49,7 +49,7 @@ #include "absl/synchronization/internal/kernel_timeout.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace synchronization_internal { static void MaybeBecomeIdle() { @@ -123,16 +123,21 @@ class Futex { } }; -void Waiter::Init() { +Waiter::Waiter() { futex_.store(0, std::memory_order_relaxed); } +Waiter::~Waiter() = default; + bool Waiter::Wait(KernelTimeout t) { // Loop until we can atomically decrement futex from a positive // value, waiting on a futex while we believe it is zero. + // Note that, since the thread ticker is just reset, we don't need to check + // whether the thread is idle on the very first pass of the loop. + bool first_pass = true; while (true) { int32_t x = futex_.load(std::memory_order_relaxed); - if (x != 0) { + while (x != 0) { if (!futex_.compare_exchange_weak(x, x - 1, std::memory_order_acquire, std::memory_order_relaxed)) { @@ -141,6 +146,8 @@ bool Waiter::Wait(KernelTimeout t) { return true; // Consumed a wakeup, we are done. } + + if (!first_pass) MaybeBecomeIdle(); const int err = Futex::WaitUntil(&futex_, 0, t); if (err != 0) { if (err == -EINTR || err == -EWOULDBLOCK) { @@ -151,14 +158,13 @@ bool Waiter::Wait(KernelTimeout t) { ABSL_RAW_LOG(FATAL, "Futex operation failed with error %d\n", err); } } - - MaybeBecomeIdle(); + first_pass = false; } } void Waiter::Post() { if (futex_.fetch_add(1, std::memory_order_release) == 0) { - // We incremented from 0, need to wake a potential waker. + // We incremented from 0, need to wake a potential waiter. Poke(); } } @@ -196,7 +202,7 @@ class PthreadMutexHolder { pthread_mutex_t *mu_; }; -void Waiter::Init() { +Waiter::Waiter() { const int err = pthread_mutex_init(&mu_, 0); if (err != 0) { ABSL_RAW_LOG(FATAL, "pthread_mutex_init failed: %d", err); @@ -207,8 +213,20 @@ void Waiter::Init() { ABSL_RAW_LOG(FATAL, "pthread_cond_init failed: %d", err2); } - waiter_count_.store(0, std::memory_order_relaxed); - wakeup_count_.store(0, std::memory_order_relaxed); + waiter_count_ = 0; + wakeup_count_ = 0; +} + +Waiter::~Waiter() { + const int err = pthread_mutex_destroy(&mu_); + if (err != 0) { + ABSL_RAW_LOG(FATAL, "pthread_mutex_destroy failed: %d", err); + } + + const int err2 = pthread_cond_destroy(&cv_); + if (err2 != 0) { + ABSL_RAW_LOG(FATAL, "pthread_cond_destroy failed: %d", err2); + } } bool Waiter::Wait(KernelTimeout t) { @@ -218,21 +236,13 @@ bool Waiter::Wait(KernelTimeout t) { } PthreadMutexHolder h(&mu_); - waiter_count_.fetch_add(1, std::memory_order_relaxed); + ++waiter_count_; // Loop until we find a wakeup to consume or timeout. - while (true) { - int x = wakeup_count_.load(std::memory_order_relaxed); - if (x != 0) { - if (!wakeup_count_.compare_exchange_weak(x, x - 1, - std::memory_order_acquire, - std::memory_order_relaxed)) { - continue; // Raced with someone, retry. - } - // Successfully consumed a wakeup, we're done. - waiter_count_.fetch_sub(1, std::memory_order_relaxed); - return true; - } - + // Note that, since the thread ticker is just reset, we don't need to check + // whether the thread is idle on the very first pass of the loop. + bool first_pass = true; + while (wakeup_count_ == 0) { + if (!first_pass) MaybeBecomeIdle(); // No wakeups available, time to wait. if (!t.has_timeout()) { const int err = pthread_cond_wait(&cv_, &mu_); @@ -242,46 +252,56 @@ bool Waiter::Wait(KernelTimeout t) { } else { const int err = pthread_cond_timedwait(&cv_, &mu_, &abs_timeout); if (err == ETIMEDOUT) { - waiter_count_.fetch_sub(1, std::memory_order_relaxed); + --waiter_count_; return false; } if (err != 0) { - ABSL_RAW_LOG(FATAL, "pthread_cond_wait failed: %d", err); + ABSL_RAW_LOG(FATAL, "pthread_cond_timedwait failed: %d", err); } } - MaybeBecomeIdle(); + first_pass = false; } + // Consume a wakeup and we're done. + --wakeup_count_; + --waiter_count_; + return true; } void Waiter::Post() { - wakeup_count_.fetch_add(1, std::memory_order_release); - Poke(); + PthreadMutexHolder h(&mu_); + ++wakeup_count_; + InternalCondVarPoke(); } void Waiter::Poke() { - if (waiter_count_.load(std::memory_order_relaxed) == 0) { - return; - } - // Potentially a waker. Take the lock and check again. PthreadMutexHolder h(&mu_); - if (waiter_count_.load(std::memory_order_relaxed) == 0) { - return; - } - const int err = pthread_cond_signal(&cv_); - if (err != 0) { - ABSL_RAW_LOG(FATAL, "pthread_cond_signal failed: %d", err); + InternalCondVarPoke(); +} + +void Waiter::InternalCondVarPoke() { + if (waiter_count_ != 0) { + const int err = pthread_cond_signal(&cv_); + if (ABSL_PREDICT_FALSE(err != 0)) { + ABSL_RAW_LOG(FATAL, "pthread_cond_signal failed: %d", err); + } } } #elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_SEM -void Waiter::Init() { +Waiter::Waiter() { if (sem_init(&sem_, 0, 0) != 0) { ABSL_RAW_LOG(FATAL, "sem_init failed with errno %d\n", errno); } wakeups_.store(0, std::memory_order_relaxed); } +Waiter::~Waiter() { + if (sem_destroy(&sem_) != 0) { + ABSL_RAW_LOG(FATAL, "sem_destroy failed with errno %d\n", errno); + } +} + bool Waiter::Wait(KernelTimeout t) { struct timespec abs_timeout; if (t.has_timeout()) { @@ -289,9 +309,12 @@ bool Waiter::Wait(KernelTimeout t) { } // Loop until we timeout or consume a wakeup. + // Note that, since the thread ticker is just reset, we don't need to check + // whether the thread is idle on the very first pass of the loop. + bool first_pass = true; while (true) { int x = wakeups_.load(std::memory_order_relaxed); - if (x != 0) { + while (x != 0) { if (!wakeups_.compare_exchange_weak(x, x - 1, std::memory_order_acquire, std::memory_order_relaxed)) { @@ -301,6 +324,7 @@ bool Waiter::Wait(KernelTimeout t) { return true; } + if (!first_pass) MaybeBecomeIdle(); // Nothing to consume, wait (looping on EINTR). while (true) { if (!t.has_timeout()) { @@ -314,13 +338,16 @@ bool Waiter::Wait(KernelTimeout t) { ABSL_RAW_LOG(FATAL, "sem_timedwait failed: %d", errno); } } - MaybeBecomeIdle(); + first_pass = false; } } void Waiter::Post() { - wakeups_.fetch_add(1, std::memory_order_release); // Post a wakeup. - Poke(); + // Post a wakeup. + if (wakeups_.fetch_add(1, std::memory_order_release) == 0) { + // We incremented from 0, need to wake a potential waiter. + Poke(); + } } void Waiter::Poke() { @@ -341,31 +368,29 @@ class Waiter::WinHelper { return reinterpret_cast<CONDITION_VARIABLE *>(&w->cv_storage_); } - static_assert(sizeof(SRWLOCK) == sizeof(Waiter::SRWLockStorage), - "SRWLockStorage does not have the same size as SRWLOCK"); + static_assert(sizeof(SRWLOCK) == sizeof(void *), + "`mu_storage_` does not have the same size as SRWLOCK"); + static_assert(alignof(SRWLOCK) == alignof(void *), + "`mu_storage_` does not have the same alignment as SRWLOCK"); + + static_assert(sizeof(CONDITION_VARIABLE) == sizeof(void *), + "`ABSL_CONDITION_VARIABLE_STORAGE` does not have the same size " + "as `CONDITION_VARIABLE`"); static_assert( - alignof(SRWLOCK) == alignof(Waiter::SRWLockStorage), - "SRWLockStorage does not have the same alignment as SRWLOCK"); - - static_assert(sizeof(CONDITION_VARIABLE) == - sizeof(Waiter::ConditionVariableStorage), - "ABSL_CONDITION_VARIABLE_STORAGE does not have the same size " - "as CONDITION_VARIABLE"); - static_assert(alignof(CONDITION_VARIABLE) == - alignof(Waiter::ConditionVariableStorage), - "ConditionVariableStorage does not have the same " - "alignment as CONDITION_VARIABLE"); - - // The SRWLOCK and CONDITION_VARIABLE types must be trivially constuctible + alignof(CONDITION_VARIABLE) == alignof(void *), + "`cv_storage_` does not have the same alignment as `CONDITION_VARIABLE`"); + + // The SRWLOCK and CONDITION_VARIABLE types must be trivially constructible // and destructible because we never call their constructors or destructors. static_assert(std::is_trivially_constructible<SRWLOCK>::value, - "The SRWLOCK type must be trivially constructible"); - static_assert(std::is_trivially_constructible<CONDITION_VARIABLE>::value, - "The CONDITION_VARIABLE type must be trivially constructible"); + "The `SRWLOCK` type must be trivially constructible"); + static_assert( + std::is_trivially_constructible<CONDITION_VARIABLE>::value, + "The `CONDITION_VARIABLE` type must be trivially constructible"); static_assert(std::is_trivially_destructible<SRWLOCK>::value, - "The SRWLOCK type must be trivially destructible"); + "The `SRWLOCK` type must be trivially destructible"); static_assert(std::is_trivially_destructible<CONDITION_VARIABLE>::value, - "The CONDITION_VARIABLE type must be trivially destructible"); + "The `CONDITION_VARIABLE` type must be trivially destructible"); }; class LockHolder { @@ -385,36 +410,33 @@ class LockHolder { SRWLOCK* mu_; }; -void Waiter::Init() { +Waiter::Waiter() { auto *mu = ::new (static_cast<void *>(&mu_storage_)) SRWLOCK; auto *cv = ::new (static_cast<void *>(&cv_storage_)) CONDITION_VARIABLE; InitializeSRWLock(mu); InitializeConditionVariable(cv); - waiter_count_.store(0, std::memory_order_relaxed); - wakeup_count_.store(0, std::memory_order_relaxed); + waiter_count_ = 0; + wakeup_count_ = 0; } +// SRW locks and condition variables do not need to be explicitly destroyed. +// https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-initializesrwlock +// https://stackoverflow.com/questions/28975958/why-does-windows-have-no-deleteconditionvariable-function-to-go-together-with +Waiter::~Waiter() = default; + bool Waiter::Wait(KernelTimeout t) { SRWLOCK *mu = WinHelper::GetLock(this); CONDITION_VARIABLE *cv = WinHelper::GetCond(this); LockHolder h(mu); - waiter_count_.fetch_add(1, std::memory_order_relaxed); + ++waiter_count_; // Loop until we find a wakeup to consume or timeout. - while (true) { - int x = wakeup_count_.load(std::memory_order_relaxed); - if (x != 0) { - if (!wakeup_count_.compare_exchange_weak(x, x - 1, - std::memory_order_acquire, - std::memory_order_relaxed)) { - continue; // Raced with someone, retry. - } - // Successfully consumed a wakeup, we're done. - waiter_count_.fetch_sub(1, std::memory_order_relaxed); - return true; - } - + // Note that, since the thread ticker is just reset, we don't need to check + // whether the thread is idle on the very first pass of the loop. + bool first_pass = true; + while (wakeup_count_ == 0) { + if (!first_pass) MaybeBecomeIdle(); // No wakeups available, time to wait. if (!SleepConditionVariableSRW(cv, mu, t.InMillisecondsFromNow(), 0)) { // GetLastError() returns a Win32 DWORD, but we assign to @@ -422,32 +444,35 @@ bool Waiter::Wait(KernelTimeout t) { // initialization guarantees this is not a narrowing conversion. const unsigned long err{GetLastError()}; // NOLINT(runtime/int) if (err == ERROR_TIMEOUT) { - waiter_count_.fetch_sub(1, std::memory_order_relaxed); + --waiter_count_; return false; } else { ABSL_RAW_LOG(FATAL, "SleepConditionVariableSRW failed: %lu", err); } } - - MaybeBecomeIdle(); + first_pass = false; } + // Consume a wakeup and we're done. + --wakeup_count_; + --waiter_count_; + return true; } void Waiter::Post() { - wakeup_count_.fetch_add(1, std::memory_order_release); - Poke(); + LockHolder h(WinHelper::GetLock(this)); + ++wakeup_count_; + InternalCondVarPoke(); } void Waiter::Poke() { - if (waiter_count_.load(std::memory_order_relaxed) == 0) { - return; - } - // Potentially a waker. Take the lock and check again. LockHolder h(WinHelper::GetLock(this)); - if (waiter_count_.load(std::memory_order_relaxed) == 0) { - return; + InternalCondVarPoke(); +} + +void Waiter::InternalCondVarPoke() { + if (waiter_count_ != 0) { + WakeConditionVariable(WinHelper::GetCond(this)); } - WakeConditionVariable(WinHelper::GetCond(this)); } #else @@ -455,5 +480,5 @@ void Waiter::Poke() { #endif } // namespace synchronization_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/synchronization/internal/waiter.h b/absl/synchronization/internal/waiter.h index 06032642..a6e6d4c7 100644 --- a/absl/synchronization/internal/waiter.h +++ b/absl/synchronization/internal/waiter.h @@ -18,10 +18,16 @@ #include "absl/base/config.h" -#ifndef _WIN32 +#ifdef _WIN32 +#include <sdkddkver.h> +#else #include <pthread.h> #endif +#ifdef __linux__ +#include <linux/futex.h> +#endif + #ifdef ABSL_HAVE_SEMAPHORE_H #include <semaphore.h> #endif @@ -40,9 +46,14 @@ #if defined(ABSL_FORCE_WAITER_MODE) #define ABSL_WAITER_MODE ABSL_FORCE_WAITER_MODE -#elif defined(_WIN32) +#elif defined(_WIN32) && _WIN32_WINNT >= _WIN32_WINNT_VISTA #define ABSL_WAITER_MODE ABSL_WAITER_MODE_WIN32 -#elif defined(__linux__) +#elif defined(__BIONIC__) +// Bionic supports all the futex operations we need even when some of the futex +// definitions are missing. +#define ABSL_WAITER_MODE ABSL_WAITER_MODE_FUTEX +#elif defined(__linux__) && defined(FUTEX_CLOCK_REALTIME) +// FUTEX_CLOCK_REALTIME requires Linux >= 2.6.28. #define ABSL_WAITER_MODE ABSL_WAITER_MODE_FUTEX #elif defined(ABSL_HAVE_SEMAPHORE_H) #define ABSL_WAITER_MODE ABSL_WAITER_MODE_SEM @@ -51,20 +62,21 @@ #endif namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace synchronization_internal { // Waiter is an OS-specific semaphore. class Waiter { public: - // No constructor, instances use the reserved space in ThreadIdentity. - // All initialization logic belongs in `Init()`. - Waiter() = delete; + // Prepare any data to track waits. + Waiter(); + + // Not copyable or movable Waiter(const Waiter&) = delete; Waiter& operator=(const Waiter&) = delete; - // Prepare any data to track waits. - void Init(); + // Destroy any data to track waits. + ~Waiter(); // Blocks the calling thread until a matching call to `Post()` or // `t` has passed. Returns `true` if woken (`Post()` called), @@ -105,10 +117,13 @@ class Waiter { static_assert(sizeof(int32_t) == sizeof(futex_), "Wrong size for futex"); #elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_CONDVAR + // REQUIRES: mu_ must be held. + void InternalCondVarPoke(); + pthread_mutex_t mu_; pthread_cond_t cv_; - std::atomic<int> waiter_count_; - std::atomic<int> wakeup_count_; // Unclaimed wakeups, written under lock. + int waiter_count_; + int wakeup_count_; // Unclaimed wakeups. #elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_SEM sem_t sem_; @@ -118,26 +133,19 @@ class Waiter { std::atomic<int> wakeups_; #elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_WIN32 - // The Windows API has lots of choices for synchronization - // primivitives. We are using SRWLOCK and CONDITION_VARIABLE - // because they don't require a destructor to release system - // resources. - // - // However, we can't include Windows.h in our headers, so we use aligned - // storage buffers to define the storage. - using SRWLockStorage = - typename std::aligned_storage<sizeof(void*), alignof(void*)>::type; - using ConditionVariableStorage = - typename std::aligned_storage<sizeof(void*), alignof(void*)>::type; - // WinHelper - Used to define utilities for accessing the lock and // condition variable storage once the types are complete. class WinHelper; - SRWLockStorage mu_storage_; - ConditionVariableStorage cv_storage_; - std::atomic<int> waiter_count_; - std::atomic<int> wakeup_count_; + // REQUIRES: WinHelper::GetLock(this) must be held. + void InternalCondVarPoke(); + + // We can't include Windows.h in our headers, so we use aligned charachter + // buffers to define the storage of SRWLOCK and CONDITION_VARIABLE. + alignas(void*) unsigned char mu_storage_[sizeof(void*)]; + alignas(void*) unsigned char cv_storage_[sizeof(void*)]; + int waiter_count_; + int wakeup_count_; #else #error Unknown ABSL_WAITER_MODE @@ -145,7 +153,7 @@ class Waiter { }; } // namespace synchronization_internal -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_SYNCHRONIZATION_INTERNAL_WAITER_H_ diff --git a/absl/synchronization/lifetime_test.cc b/absl/synchronization/lifetime_test.cc index 0279c8f8..cc973a32 100644 --- a/absl/synchronization/lifetime_test.cc +++ b/absl/synchronization/lifetime_test.cc @@ -122,6 +122,11 @@ class OnDestruction { Function fn_; }; +// These tests require that the compiler correctly supports C++11 constant +// initialization... but MSVC has a known regression since v19.10: +// https://developercommunity.visualstudio.com/content/problem/336946/class-with-constexpr-constructor-not-using-static.html +// TODO(epastor): Limit the affected range once MSVC fixes this bug. +#if defined(__clang__) || !(defined(_MSC_VER) && _MSC_VER > 1900) // kConstInit // Test early usage. (Declaration comes first; definitions must appear after // the test runner.) @@ -143,14 +148,15 @@ ABSL_CONST_INIT absl::Mutex early_const_init_mutex(absl::kConstInit); // constructors of globals "happen at link time"; memory is pre-initialized, // before the constructors of either grab_lock or check_still_locked are run.) extern absl::Mutex const_init_sanity_mutex; -OnConstruction grab_lock([]() NO_THREAD_SAFETY_ANALYSIS { +OnConstruction grab_lock([]() ABSL_NO_THREAD_SAFETY_ANALYSIS { const_init_sanity_mutex.Lock(); }); ABSL_CONST_INIT absl::Mutex const_init_sanity_mutex(absl::kConstInit); -OnConstruction check_still_locked([]() NO_THREAD_SAFETY_ANALYSIS { +OnConstruction check_still_locked([]() ABSL_NO_THREAD_SAFETY_ANALYSIS { const_init_sanity_mutex.AssertHeld(); const_init_sanity_mutex.Unlock(); }); +#endif // defined(__clang__) || !(defined(_MSC_VER) && _MSC_VER > 1900) // Test shutdown usage. (Declarations come first; definitions must appear after // the test runner.) diff --git a/absl/synchronization/mutex.cc b/absl/synchronization/mutex.cc index 07f220f5..e0879b05 100644 --- a/absl/synchronization/mutex.cc +++ b/absl/synchronization/mutex.cc @@ -71,7 +71,7 @@ ABSL_ATTRIBUTE_WEAK void AbslInternalMutexYield() { std::this_thread::yield(); } } // extern "C" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN namespace { @@ -107,13 +107,16 @@ static_assert( sizeof(MutexGlobals) == ABSL_CACHELINE_SIZE, "MutexGlobals must occupy an entire cacheline to prevent false sharing"); -ABSL_CONST_INIT absl::base_internal::AtomicHook<void (*)(int64_t wait_cycles)> - submit_profile_data; -ABSL_CONST_INIT absl::base_internal::AtomicHook< - void (*)(const char *msg, const void *obj, int64_t wait_cycles)> mutex_tracer; -ABSL_CONST_INIT absl::base_internal::AtomicHook< - void (*)(const char *msg, const void *cv)> cond_var_tracer; -ABSL_CONST_INIT absl::base_internal::AtomicHook< +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES + absl::base_internal::AtomicHook<void (*)(int64_t wait_cycles)> + submit_profile_data; +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES absl::base_internal::AtomicHook<void (*)( + const char *msg, const void *obj, int64_t wait_cycles)> + mutex_tracer; +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES + absl::base_internal::AtomicHook<void (*)(const char *msg, const void *cv)> + cond_var_tracer; +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES absl::base_internal::AtomicHook< bool (*)(const void *pc, char *out, int out_size)> symbolizer(absl::Symbolize); @@ -208,8 +211,8 @@ static absl::base_internal::SpinLock deadlock_graph_mu( absl::base_internal::kLinkerInitialized); // graph used to detect deadlocks. -static GraphCycles *deadlock_graph GUARDED_BY(deadlock_graph_mu) - PT_GUARDED_BY(deadlock_graph_mu); +static GraphCycles *deadlock_graph ABSL_GUARDED_BY(deadlock_graph_mu) + ABSL_PT_GUARDED_BY(deadlock_graph_mu); //------------------------------------------------------------------ // An event mechanism for debugging mutex use. @@ -280,10 +283,10 @@ static const uint32_t kNSynchEvent = 1031; static struct SynchEvent { // this is a trivial hash table for the events // struct is freed when refcount reaches 0 - int refcount GUARDED_BY(synch_event_mu); + int refcount ABSL_GUARDED_BY(synch_event_mu); // buckets have linear, 0-terminated chains - SynchEvent *next GUARDED_BY(synch_event_mu); + SynchEvent *next ABSL_GUARDED_BY(synch_event_mu); // Constant after initialization uintptr_t masked_addr; // object at this address is called "name" @@ -296,8 +299,8 @@ static struct SynchEvent { // this is a trivial hash table for the events bool log; // logging turned on // Constant after initialization - char name[1]; // actually longer---null-terminated std::string -} *synch_event[kNSynchEvent] GUARDED_BY(synch_event_mu); + char name[1]; // actually longer---NUL-terminated std::string +} * synch_event[kNSynchEvent] ABSL_GUARDED_BY(synch_event_mu); // Ensure that the object at "addr" has a SynchEvent struct associated with it, // set "bits" in the word there (waiting until lockbit is clear before doing @@ -1144,7 +1147,7 @@ PerThreadSynch *Mutex::Wakeup(PerThreadSynch *w) { } static GraphId GetGraphIdLocked(Mutex *mu) - EXCLUSIVE_LOCKS_REQUIRED(deadlock_graph_mu) { + ABSL_EXCLUSIVE_LOCKS_REQUIRED(deadlock_graph_mu) { if (!deadlock_graph) { // (re)create the deadlock graph. deadlock_graph = new (base_internal::LowLevelAlloc::Alloc(sizeof(*deadlock_graph))) @@ -1153,7 +1156,7 @@ static GraphId GetGraphIdLocked(Mutex *mu) return deadlock_graph->GetId(mu); } -static GraphId GetGraphId(Mutex *mu) LOCKS_EXCLUDED(deadlock_graph_mu) { +static GraphId GetGraphId(Mutex *mu) ABSL_LOCKS_EXCLUDED(deadlock_graph_mu) { deadlock_graph_mu.Lock(); GraphId id = GetGraphIdLocked(mu); deadlock_graph_mu.Unlock(); @@ -2721,5 +2724,5 @@ bool Condition::GuaranteedEqual(const Condition *a, const Condition *b) { a->arg_ == b->arg_ && a->method_ == b->method_; } -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/synchronization/mutex.h b/absl/synchronization/mutex.h index d6890099..8c70c4ce 100644 --- a/absl/synchronization/mutex.h +++ b/absl/synchronization/mutex.h @@ -82,7 +82,7 @@ #endif namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN class Condition; struct SynchWaitParams; @@ -136,7 +136,7 @@ struct SynchWaitParams; // // See also `MutexLock`, below, for scoped `Mutex` acquisition. -class LOCKABLE Mutex { +class ABSL_LOCKABLE Mutex { public: // Creates a `Mutex` that is not held by anyone. This constructor is // typically used for Mutexes allocated on the heap or the stack. @@ -165,27 +165,27 @@ class LOCKABLE Mutex { // // Blocks the calling thread, if necessary, until this `Mutex` is free, and // then acquires it exclusively. (This lock is also known as a "write lock.") - void Lock() EXCLUSIVE_LOCK_FUNCTION(); + void Lock() ABSL_EXCLUSIVE_LOCK_FUNCTION(); // Mutex::Unlock() // // Releases this `Mutex` and returns it from the exclusive/write state to the // free state. Caller must hold the `Mutex` exclusively. - void Unlock() UNLOCK_FUNCTION(); + void Unlock() ABSL_UNLOCK_FUNCTION(); // Mutex::TryLock() // // If the mutex can be acquired without blocking, does so exclusively and // returns `true`. Otherwise, returns `false`. Returns `true` with high // probability if the `Mutex` was free. - bool TryLock() EXCLUSIVE_TRYLOCK_FUNCTION(true); + bool TryLock() ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(true); // Mutex::AssertHeld() // // Return immediately if this thread holds the `Mutex` exclusively (in write // mode). Otherwise, may report an error (typically by crashing with a // diagnostic), or may return immediately. - void AssertHeld() const ASSERT_EXCLUSIVE_LOCK(); + void AssertHeld() const ABSL_ASSERT_EXCLUSIVE_LOCK(); // --------------------------------------------------------------------------- // Reader-Writer Locking @@ -226,28 +226,28 @@ class LOCKABLE Mutex { // `ReaderLock()` will block if some other thread has an exclusive/writer lock // on the mutex. - void ReaderLock() SHARED_LOCK_FUNCTION(); + void ReaderLock() ABSL_SHARED_LOCK_FUNCTION(); // Mutex::ReaderUnlock() // // Releases a read share of this `Mutex`. `ReaderUnlock` may return a mutex to // the free state if this thread holds the last reader lock on the mutex. Note // that you cannot call `ReaderUnlock()` on a mutex held in write mode. - void ReaderUnlock() UNLOCK_FUNCTION(); + void ReaderUnlock() ABSL_UNLOCK_FUNCTION(); // Mutex::ReaderTryLock() // // If the mutex can be acquired without blocking, acquires this mutex for // shared access and returns `true`. Otherwise, returns `false`. Returns // `true` with high probability if the `Mutex` was free or shared. - bool ReaderTryLock() SHARED_TRYLOCK_FUNCTION(true); + bool ReaderTryLock() ABSL_SHARED_TRYLOCK_FUNCTION(true); // Mutex::AssertReaderHeld() // // Returns immediately if this thread holds the `Mutex` in at least shared // mode (read mode). Otherwise, may report an error (typically by // crashing with a diagnostic), or may return immediately. - void AssertReaderHeld() const ASSERT_SHARED_LOCK(); + void AssertReaderHeld() const ABSL_ASSERT_SHARED_LOCK(); // Mutex::WriterLock() // Mutex::WriterUnlock() @@ -258,11 +258,11 @@ class LOCKABLE Mutex { // These methods may be used (along with the complementary `Reader*()` // methods) to distingish simple exclusive `Mutex` usage (`Lock()`, // etc.) from reader/writer lock usage. - void WriterLock() EXCLUSIVE_LOCK_FUNCTION() { this->Lock(); } + void WriterLock() ABSL_EXCLUSIVE_LOCK_FUNCTION() { this->Lock(); } - void WriterUnlock() UNLOCK_FUNCTION() { this->Unlock(); } + void WriterUnlock() ABSL_UNLOCK_FUNCTION() { this->Unlock(); } - bool WriterTryLock() EXCLUSIVE_TRYLOCK_FUNCTION(true) { + bool WriterTryLock() ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(true) { return this->TryLock(); } @@ -316,11 +316,11 @@ class LOCKABLE Mutex { // be acquired, then atomically acquires this `Mutex`. `LockWhen()` is // logically equivalent to `*Lock(); Await();` though they may have different // performance characteristics. - void LockWhen(const Condition &cond) EXCLUSIVE_LOCK_FUNCTION(); + void LockWhen(const Condition &cond) ABSL_EXCLUSIVE_LOCK_FUNCTION(); - void ReaderLockWhen(const Condition &cond) SHARED_LOCK_FUNCTION(); + void ReaderLockWhen(const Condition &cond) ABSL_SHARED_LOCK_FUNCTION(); - void WriterLockWhen(const Condition &cond) EXCLUSIVE_LOCK_FUNCTION() { + void WriterLockWhen(const Condition &cond) ABSL_EXCLUSIVE_LOCK_FUNCTION() { this->LockWhen(cond); } @@ -362,11 +362,11 @@ class LOCKABLE Mutex { // // Negative timeouts are equivalent to a zero timeout. bool LockWhenWithTimeout(const Condition &cond, absl::Duration timeout) - EXCLUSIVE_LOCK_FUNCTION(); + ABSL_EXCLUSIVE_LOCK_FUNCTION(); bool ReaderLockWhenWithTimeout(const Condition &cond, absl::Duration timeout) - SHARED_LOCK_FUNCTION(); + ABSL_SHARED_LOCK_FUNCTION(); bool WriterLockWhenWithTimeout(const Condition &cond, absl::Duration timeout) - EXCLUSIVE_LOCK_FUNCTION() { + ABSL_EXCLUSIVE_LOCK_FUNCTION() { return this->LockWhenWithTimeout(cond, timeout); } @@ -382,11 +382,11 @@ class LOCKABLE Mutex { // // Deadlines in the past are equivalent to an immediate deadline. bool LockWhenWithDeadline(const Condition &cond, absl::Time deadline) - EXCLUSIVE_LOCK_FUNCTION(); + ABSL_EXCLUSIVE_LOCK_FUNCTION(); bool ReaderLockWhenWithDeadline(const Condition &cond, absl::Time deadline) - SHARED_LOCK_FUNCTION(); + ABSL_SHARED_LOCK_FUNCTION(); bool WriterLockWhenWithDeadline(const Condition &cond, absl::Time deadline) - EXCLUSIVE_LOCK_FUNCTION() { + ABSL_EXCLUSIVE_LOCK_FUNCTION() { return this->LockWhenWithDeadline(cond, deadline); } @@ -536,9 +536,9 @@ class LOCKABLE Mutex { // private: // Mutex lock_; // }; -class SCOPED_LOCKABLE MutexLock { +class ABSL_SCOPED_LOCKABLE MutexLock { public: - explicit MutexLock(Mutex *mu) EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { + explicit MutexLock(Mutex *mu) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { this->mu_->Lock(); } @@ -547,7 +547,7 @@ class SCOPED_LOCKABLE MutexLock { MutexLock& operator=(const MutexLock&) = delete; MutexLock& operator=(MutexLock&&) = delete; - ~MutexLock() UNLOCK_FUNCTION() { this->mu_->Unlock(); } + ~MutexLock() ABSL_UNLOCK_FUNCTION() { this->mu_->Unlock(); } private: Mutex *const mu_; @@ -557,10 +557,9 @@ class SCOPED_LOCKABLE MutexLock { // // The `ReaderMutexLock` is a helper class, like `MutexLock`, which acquires and // releases a shared lock on a `Mutex` via RAII. -class SCOPED_LOCKABLE ReaderMutexLock { +class ABSL_SCOPED_LOCKABLE ReaderMutexLock { public: - explicit ReaderMutexLock(Mutex *mu) SHARED_LOCK_FUNCTION(mu) - : mu_(mu) { + explicit ReaderMutexLock(Mutex *mu) ABSL_SHARED_LOCK_FUNCTION(mu) : mu_(mu) { mu->ReaderLock(); } @@ -569,9 +568,7 @@ class SCOPED_LOCKABLE ReaderMutexLock { ReaderMutexLock& operator=(const ReaderMutexLock&) = delete; ReaderMutexLock& operator=(ReaderMutexLock&&) = delete; - ~ReaderMutexLock() UNLOCK_FUNCTION() { - this->mu_->ReaderUnlock(); - } + ~ReaderMutexLock() ABSL_UNLOCK_FUNCTION() { this->mu_->ReaderUnlock(); } private: Mutex *const mu_; @@ -581,9 +578,9 @@ class SCOPED_LOCKABLE ReaderMutexLock { // // The `WriterMutexLock` is a helper class, like `MutexLock`, which acquires and // releases a write (exclusive) lock on a `Mutex` via RAII. -class SCOPED_LOCKABLE WriterMutexLock { +class ABSL_SCOPED_LOCKABLE WriterMutexLock { public: - explicit WriterMutexLock(Mutex *mu) EXCLUSIVE_LOCK_FUNCTION(mu) + explicit WriterMutexLock(Mutex *mu) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { mu->WriterLock(); } @@ -593,9 +590,7 @@ class SCOPED_LOCKABLE WriterMutexLock { WriterMutexLock& operator=(const WriterMutexLock&) = delete; WriterMutexLock& operator=(WriterMutexLock&&) = delete; - ~WriterMutexLock() UNLOCK_FUNCTION() { - this->mu_->WriterUnlock(); - } + ~WriterMutexLock() ABSL_UNLOCK_FUNCTION() { this->mu_->WriterUnlock(); } private: Mutex *const mu_; @@ -634,7 +629,7 @@ class SCOPED_LOCKABLE WriterMutexLock { // Example: // // // assume count_ is not internal reference count -// int count_ GUARDED_BY(mu_); +// int count_ ABSL_GUARDED_BY(mu_); // // mu_.LockWhen(Condition(+[](int* count) { return *count == 0; }, // &count_)); @@ -861,13 +856,18 @@ class CondVar { // MutexLockMaybe // // MutexLockMaybe is like MutexLock, but is a no-op when mu is null. -class SCOPED_LOCKABLE MutexLockMaybe { +class ABSL_SCOPED_LOCKABLE MutexLockMaybe { public: - explicit MutexLockMaybe(Mutex *mu) EXCLUSIVE_LOCK_FUNCTION(mu) - : mu_(mu) { if (this->mu_ != nullptr) { this->mu_->Lock(); } } - ~MutexLockMaybe() UNLOCK_FUNCTION() { + explicit MutexLockMaybe(Mutex *mu) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) + : mu_(mu) { + if (this->mu_ != nullptr) { + this->mu_->Lock(); + } + } + ~MutexLockMaybe() ABSL_UNLOCK_FUNCTION() { if (this->mu_ != nullptr) { this->mu_->Unlock(); } } + private: Mutex *const mu_; MutexLockMaybe(const MutexLockMaybe&) = delete; @@ -880,17 +880,17 @@ class SCOPED_LOCKABLE MutexLockMaybe { // // ReleasableMutexLock is like MutexLock, but permits `Release()` of its // mutex before destruction. `Release()` may be called at most once. -class SCOPED_LOCKABLE ReleasableMutexLock { +class ABSL_SCOPED_LOCKABLE ReleasableMutexLock { public: - explicit ReleasableMutexLock(Mutex *mu) EXCLUSIVE_LOCK_FUNCTION(mu) + explicit ReleasableMutexLock(Mutex *mu) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { this->mu_->Lock(); } - ~ReleasableMutexLock() UNLOCK_FUNCTION() { + ~ReleasableMutexLock() ABSL_UNLOCK_FUNCTION() { if (this->mu_ != nullptr) { this->mu_->Unlock(); } } - void Release() UNLOCK_FUNCTION(); + void Release() ABSL_UNLOCK_FUNCTION(); private: Mutex *mu_; @@ -1001,7 +1001,7 @@ void RegisterCondVarTracer(void (*fn)(const char *msg, const void *cv)); // // 'pc' is the program counter being symbolized, 'out' is the buffer to write // into, and 'out_size' is the size of the buffer. This function can return -// false if symbolizing failed, or true if a null-terminated symbol was written +// false if symbolizing failed, or true if a NUL-terminated symbol was written // to 'out.' // // This has the same memory ordering concerns as RegisterMutexProfiler() above. @@ -1040,7 +1040,7 @@ enum class OnDeadlockCycle { // the manner chosen here. void SetMutexDeadlockDetectionMode(OnDeadlockCycle mode); -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl // In some build configurations we pass --detect-odr-violations to the diff --git a/absl/synchronization/mutex_test.cc b/absl/synchronization/mutex_test.cc index 9851ac19..afb363af 100644 --- a/absl/synchronization/mutex_test.cc +++ b/absl/synchronization/mutex_test.cc @@ -425,10 +425,10 @@ TEST(Mutex, CondVarWaitSignalsAwait) { // Use a struct so the lock annotations apply. struct { absl::Mutex barrier_mu; - bool barrier GUARDED_BY(barrier_mu) = false; + bool barrier ABSL_GUARDED_BY(barrier_mu) = false; absl::Mutex release_mu; - bool release GUARDED_BY(release_mu) = false; + bool release ABSL_GUARDED_BY(release_mu) = false; absl::CondVar released_cv; } state; @@ -466,10 +466,10 @@ TEST(Mutex, CondVarWaitWithTimeoutSignalsAwait) { // Use a struct so the lock annotations apply. struct { absl::Mutex barrier_mu; - bool barrier GUARDED_BY(barrier_mu) = false; + bool barrier ABSL_GUARDED_BY(barrier_mu) = false; absl::Mutex release_mu; - bool release GUARDED_BY(release_mu) = false; + bool release ABSL_GUARDED_BY(release_mu) = false; absl::CondVar released_cv; } state; @@ -770,7 +770,7 @@ static void GetReadLock(ReaderDecrementBugStruct *x) { // Test for reader counter being decremented incorrectly by waiter // with false condition. -TEST(Mutex, MutexReaderDecrementBug) NO_THREAD_SAFETY_ANALYSIS { +TEST(Mutex, MutexReaderDecrementBug) ABSL_NO_THREAD_SAFETY_ANALYSIS { ReaderDecrementBugStruct x; x.cond = false; x.waiting_on_cond = false; @@ -819,7 +819,7 @@ TEST(Mutex, MutexReaderDecrementBug) NO_THREAD_SAFETY_ANALYSIS { // TSAN reports errors when locked Mutexes are destroyed. TEST(Mutex, DISABLED_LockedMutexDestructionBug) NO_THREAD_SAFETY_ANALYSIS { #else -TEST(Mutex, LockedMutexDestructionBug) NO_THREAD_SAFETY_ANALYSIS { +TEST(Mutex, LockedMutexDestructionBug) ABSL_NO_THREAD_SAFETY_ANALYSIS { #endif for (int i = 0; i != 10; i++) { // Create, lock and destroy 10 locks. @@ -1101,7 +1101,7 @@ TEST(Mutex, DeadlockDetectorBazelWarning) { // annotation-based static thread-safety analysis is not currently // predicate-aware and cannot tell if the two for-loops that acquire and // release the locks have the same predicates. -TEST(Mutex, DeadlockDetectorStessTest) NO_THREAD_SAFETY_ANALYSIS { +TEST(Mutex, DeadlockDetectorStessTest) ABSL_NO_THREAD_SAFETY_ANALYSIS { // Stress test: Here we create a large number of locks and use all of them. // If a deadlock detector keeps a full graph of lock acquisition order, // it will likely be too slow for this test to pass. @@ -1123,7 +1123,7 @@ TEST(Mutex, DeadlockDetectorStessTest) NO_THREAD_SAFETY_ANALYSIS { // TSAN reports errors when locked Mutexes are destroyed. TEST(Mutex, DISABLED_DeadlockIdBug) NO_THREAD_SAFETY_ANALYSIS { #else -TEST(Mutex, DeadlockIdBug) NO_THREAD_SAFETY_ANALYSIS { +TEST(Mutex, DeadlockIdBug) ABSL_NO_THREAD_SAFETY_ANALYSIS { #endif // Test a scenario where a cached deadlock graph node id in the // list of held locks is not invalidated when the corresponding diff --git a/absl/synchronization/notification.cc b/absl/synchronization/notification.cc index d691cfca..e91b9038 100644 --- a/absl/synchronization/notification.cc +++ b/absl/synchronization/notification.cc @@ -22,7 +22,7 @@ #include "absl/time/time.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN void Notification::Notify() { MutexLock l(&this->mutex_); @@ -45,15 +45,6 @@ Notification::~Notification() { MutexLock l(&this->mutex_); } -static inline bool HasBeenNotifiedInternal( - const std::atomic<bool> *notified_yet) { - return notified_yet->load(std::memory_order_acquire); -} - -bool Notification::HasBeenNotified() const { - return HasBeenNotifiedInternal(&this->notified_yet_); -} - void Notification::WaitForNotification() const { if (!HasBeenNotifiedInternal(&this->notified_yet_)) { this->mutex_.LockWhen(Condition(&HasBeenNotifiedInternal, @@ -83,5 +74,5 @@ bool Notification::WaitForNotificationWithDeadline(absl::Time deadline) const { return notified; } -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/synchronization/notification.h b/absl/synchronization/notification.h index 8ed7f12a..9a354ca2 100644 --- a/absl/synchronization/notification.h +++ b/absl/synchronization/notification.h @@ -57,7 +57,7 @@ #include "absl/time/time.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN // ----------------------------------------------------------------------------- // Notification @@ -74,7 +74,9 @@ class Notification { // Notification::HasBeenNotified() // // Returns the value of the notification's internal "notified" state. - bool HasBeenNotified() const; + bool HasBeenNotified() const { + return HasBeenNotifiedInternal(&this->notified_yet_); + } // Notification::WaitForNotification() // @@ -106,11 +108,16 @@ class Notification { void Notify(); private: + static inline bool HasBeenNotifiedInternal( + const std::atomic<bool>* notified_yet) { + return notified_yet->load(std::memory_order_acquire); + } + mutable Mutex mutex_; std::atomic<bool> notified_yet_; // written under mutex_ }; -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl #endif // ABSL_SYNCHRONIZATION_NOTIFICATION_H_ diff --git a/absl/synchronization/notification_test.cc b/absl/synchronization/notification_test.cc index a64674ca..100ea76f 100644 --- a/absl/synchronization/notification_test.cc +++ b/absl/synchronization/notification_test.cc @@ -21,7 +21,7 @@ #include "absl/synchronization/mutex.h" namespace absl { -inline namespace lts_2019_08_08 { +ABSL_NAMESPACE_BEGIN // A thread-safe class that holds a counter. class ThreadSafeCounter { @@ -129,5 +129,5 @@ TEST(NotificationTest, SanityTest) { BasicTests(true, &local_notification2); } -} // inline namespace lts_2019_08_08 +ABSL_NAMESPACE_END } // namespace absl |