summaryrefslogtreecommitdiff
path: root/absl/synchronization/internal
diff options
context:
space:
mode:
Diffstat (limited to 'absl/synchronization/internal')
-rw-r--r--absl/synchronization/internal/create_thread_identity.cc9
-rw-r--r--absl/synchronization/internal/create_thread_identity.h8
-rw-r--r--absl/synchronization/internal/graphcycles.cc4
-rw-r--r--absl/synchronization/internal/graphcycles.h8
-rw-r--r--absl/synchronization/internal/graphcycles_test.cc4
-rw-r--r--absl/synchronization/internal/kernel_timeout.h4
-rw-r--r--absl/synchronization/internal/mutex_nonprod.cc4
-rw-r--r--absl/synchronization/internal/mutex_nonprod.inc8
-rw-r--r--absl/synchronization/internal/per_thread_sem.cc10
-rw-r--r--absl/synchronization/internal/per_thread_sem.h9
-rw-r--r--absl/synchronization/internal/per_thread_sem_test.cc4
-rw-r--r--absl/synchronization/internal/thread_pool.h8
-rw-r--r--absl/synchronization/internal/waiter.cc211
-rw-r--r--absl/synchronization/internal/waiter.h64
14 files changed, 202 insertions, 153 deletions
diff --git a/absl/synchronization/internal/create_thread_identity.cc b/absl/synchronization/internal/create_thread_identity.cc
index 65f6d8fc..fa0070a9 100644
--- a/absl/synchronization/internal/create_thread_identity.cc
+++ b/absl/synchronization/internal/create_thread_identity.cc
@@ -27,7 +27,7 @@
#include "absl/synchronization/internal/per_thread_sem.h"
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
// ThreadIdentity storage is persistent, we maintain a free-list of previously
@@ -38,7 +38,7 @@ static base_internal::ThreadIdentity* thread_identity_freelist;
// A per-thread destructor for reclaiming associated ThreadIdentity objects.
// Since we must preserve their storage we cache them for re-use.
-static void ReclaimThreadIdentity(void* v) {
+void ReclaimThreadIdentity(void* v) {
base_internal::ThreadIdentity* identity =
static_cast<base_internal::ThreadIdentity*>(v);
@@ -48,6 +48,8 @@ static void ReclaimThreadIdentity(void* v) {
base_internal::LowLevelAlloc::Free(identity->per_thread_synch.all_locks);
}
+ PerThreadSem::Destroy(identity);
+
// We must explicitly clear the current thread's identity:
// (a) Subsequent (unrelated) per-thread destructors may require an identity.
// We must guarantee a new identity is used in this case (this instructor
@@ -85,7 +87,6 @@ static void ResetThreadIdentity(base_internal::ThreadIdentity* identity) {
pts->wake = false;
pts->cond_waiter = false;
pts->all_locks = nullptr;
- identity->waiter_state = {};
identity->blocked_count_ptr = nullptr;
identity->ticker.store(0, std::memory_order_relaxed);
identity->wait_start.store(0, std::memory_order_relaxed);
@@ -133,7 +134,7 @@ base_internal::ThreadIdentity* CreateThreadIdentity() {
}
} // namespace synchronization_internal
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_LOW_LEVEL_ALLOC_MISSING
diff --git a/absl/synchronization/internal/create_thread_identity.h b/absl/synchronization/internal/create_thread_identity.h
index d743cc3b..e121f683 100644
--- a/absl/synchronization/internal/create_thread_identity.h
+++ b/absl/synchronization/internal/create_thread_identity.h
@@ -29,13 +29,17 @@
#include "absl/base/port.h"
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
// Allocates and attaches a ThreadIdentity object for the calling thread.
// For private use only.
base_internal::ThreadIdentity* CreateThreadIdentity();
+// A per-thread destructor for reclaiming associated ThreadIdentity objects.
+// For private use only.
+void ReclaimThreadIdentity(void* v);
+
// Returns the ThreadIdentity object representing the calling thread; guaranteed
// to be unique for its lifetime. The returned object will remain valid for the
// program's lifetime; although it may be re-assigned to a subsequent thread.
@@ -50,7 +54,7 @@ inline base_internal::ThreadIdentity* GetOrCreateCurrentThreadIdentity() {
}
} // namespace synchronization_internal
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_SYNCHRONIZATION_INTERNAL_CREATE_THREAD_IDENTITY_H_
diff --git a/absl/synchronization/internal/graphcycles.cc b/absl/synchronization/internal/graphcycles.cc
index f4fbeadd..6a2bcdf6 100644
--- a/absl/synchronization/internal/graphcycles.cc
+++ b/absl/synchronization/internal/graphcycles.cc
@@ -44,7 +44,7 @@
// Do not use STL. This module does not use standard memory allocation.
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
namespace {
@@ -691,7 +691,7 @@ int GraphCycles::GetStackTrace(GraphId id, void*** ptr) {
}
} // namespace synchronization_internal
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_LOW_LEVEL_ALLOC_MISSING
diff --git a/absl/synchronization/internal/graphcycles.h b/absl/synchronization/internal/graphcycles.h
index 208527c3..ceba33e4 100644
--- a/absl/synchronization/internal/graphcycles.h
+++ b/absl/synchronization/internal/graphcycles.h
@@ -36,12 +36,14 @@
// InsertEdge() is very fast when the edge already exists, and reasonably fast
// otherwise.
// FindPath() is linear in the size of the graph.
-// The current implemenation uses O(|V|+|E|) space.
+// The current implementation uses O(|V|+|E|) space.
#include <cstdint>
+#include "absl/base/config.h"
+
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
// Opaque identifier for a graph node.
@@ -133,7 +135,7 @@ class GraphCycles {
};
} // namespace synchronization_internal
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
#endif
diff --git a/absl/synchronization/internal/graphcycles_test.cc b/absl/synchronization/internal/graphcycles_test.cc
index fca86219..74eaffe7 100644
--- a/absl/synchronization/internal/graphcycles_test.cc
+++ b/absl/synchronization/internal/graphcycles_test.cc
@@ -25,7 +25,7 @@
#include "absl/base/macros.h"
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
// We emulate a GraphCycles object with a node vector and an edge vector.
@@ -460,5 +460,5 @@ TEST_F(GraphCyclesTest, ManyEdges) {
}
} // namespace synchronization_internal
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
diff --git a/absl/synchronization/internal/kernel_timeout.h b/absl/synchronization/internal/kernel_timeout.h
index e0f01e06..d6ac5db0 100644
--- a/absl/synchronization/internal/kernel_timeout.h
+++ b/absl/synchronization/internal/kernel_timeout.h
@@ -34,7 +34,7 @@
#include "absl/time/time.h"
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
class Futex;
@@ -149,7 +149,7 @@ class KernelTimeout {
};
} // namespace synchronization_internal
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_SYNCHRONIZATION_INTERNAL_KERNEL_TIMEOUT_H_
diff --git a/absl/synchronization/internal/mutex_nonprod.cc b/absl/synchronization/internal/mutex_nonprod.cc
index aa1ed83b..4590b98d 100644
--- a/absl/synchronization/internal/mutex_nonprod.cc
+++ b/absl/synchronization/internal/mutex_nonprod.cc
@@ -31,7 +31,7 @@
#include "absl/time/time.h"
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
namespace {
@@ -316,5 +316,5 @@ bool Condition::Eval() const {
void RegisterSymbolizer(bool (*)(const void*, char*, int)) {}
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
diff --git a/absl/synchronization/internal/mutex_nonprod.inc b/absl/synchronization/internal/mutex_nonprod.inc
index ac10879b..a1502e72 100644
--- a/absl/synchronization/internal/mutex_nonprod.inc
+++ b/absl/synchronization/internal/mutex_nonprod.inc
@@ -36,7 +36,7 @@
#endif
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
class Condition;
namespace synchronization_internal {
@@ -252,10 +252,10 @@ class SynchronizationStorage {
absl::once_flag once_;
- // An aligned space for T.
- typename std::aligned_storage<sizeof(T), alignof(T)>::type space_;
+ // An aligned space for the T.
+ alignas(T) unsigned char space_[sizeof(T)];
};
} // namespace synchronization_internal
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
diff --git a/absl/synchronization/internal/per_thread_sem.cc b/absl/synchronization/internal/per_thread_sem.cc
index 284a5df4..821ca9b4 100644
--- a/absl/synchronization/internal/per_thread_sem.cc
+++ b/absl/synchronization/internal/per_thread_sem.cc
@@ -25,7 +25,7 @@
#include "absl/synchronization/internal/waiter.h"
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
void PerThreadSem::SetThreadBlockedCounter(std::atomic<int> *counter) {
@@ -41,12 +41,16 @@ std::atomic<int> *PerThreadSem::GetThreadBlockedCounter() {
}
void PerThreadSem::Init(base_internal::ThreadIdentity *identity) {
- Waiter::GetWaiter(identity)->Init();
+ new (Waiter::GetWaiter(identity)) Waiter();
identity->ticker.store(0, std::memory_order_relaxed);
identity->wait_start.store(0, std::memory_order_relaxed);
identity->is_idle.store(false, std::memory_order_relaxed);
}
+void PerThreadSem::Destroy(base_internal::ThreadIdentity *identity) {
+ Waiter::GetWaiter(identity)->~Waiter();
+}
+
void PerThreadSem::Tick(base_internal::ThreadIdentity *identity) {
const int ticker =
identity->ticker.fetch_add(1, std::memory_order_relaxed) + 1;
@@ -59,7 +63,7 @@ void PerThreadSem::Tick(base_internal::ThreadIdentity *identity) {
}
} // namespace synchronization_internal
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
extern "C" {
diff --git a/absl/synchronization/internal/per_thread_sem.h b/absl/synchronization/internal/per_thread_sem.h
index 5bb0978b..8ab43915 100644
--- a/absl/synchronization/internal/per_thread_sem.h
+++ b/absl/synchronization/internal/per_thread_sem.h
@@ -32,7 +32,7 @@
#include "absl/synchronization/internal/kernel_timeout.h"
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
class Mutex;
@@ -66,6 +66,10 @@ class PerThreadSem {
// REQUIRES: May only be called by ThreadIdentity.
static void Init(base_internal::ThreadIdentity* identity);
+ // Destroy the PerThreadSem associated with "identity".
+ // REQUIRES: May only be called by ThreadIdentity.
+ static void Destroy(base_internal::ThreadIdentity* identity);
+
// Increments "identity"'s count.
static inline void Post(base_internal::ThreadIdentity* identity);
@@ -78,10 +82,11 @@ class PerThreadSem {
friend class PerThreadSemTest;
friend class absl::Mutex;
friend absl::base_internal::ThreadIdentity* CreateThreadIdentity();
+ friend void ReclaimThreadIdentity(void* v);
};
} // namespace synchronization_internal
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
// In some build configurations we pass --detect-odr-violations to the
diff --git a/absl/synchronization/internal/per_thread_sem_test.cc b/absl/synchronization/internal/per_thread_sem_test.cc
index 93bc4244..b5a2f6d4 100644
--- a/absl/synchronization/internal/per_thread_sem_test.cc
+++ b/absl/synchronization/internal/per_thread_sem_test.cc
@@ -33,7 +33,7 @@
// primitives which might use PerThreadSem, most notably absl::Mutex.
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
class SimpleSemaphore {
@@ -176,5 +176,5 @@ TEST_F(PerThreadSemTest, Timeouts) {
} // namespace
} // namespace synchronization_internal
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
diff --git a/absl/synchronization/internal/thread_pool.h b/absl/synchronization/internal/thread_pool.h
index 8941be68..0cb96dac 100644
--- a/absl/synchronization/internal/thread_pool.h
+++ b/absl/synchronization/internal/thread_pool.h
@@ -26,7 +26,7 @@
#include "absl/synchronization/mutex.h"
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
// A simple ThreadPool implementation for tests.
@@ -61,7 +61,7 @@ class ThreadPool {
}
private:
- bool WorkAvailable() const EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+ bool WorkAvailable() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
return !queue_.empty();
}
@@ -82,12 +82,12 @@ class ThreadPool {
}
absl::Mutex mu_;
- std::queue<std::function<void()>> queue_ GUARDED_BY(mu_);
+ std::queue<std::function<void()>> queue_ ABSL_GUARDED_BY(mu_);
std::vector<std::thread> threads_;
};
} // namespace synchronization_internal
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_SYNCHRONIZATION_INTERNAL_THREAD_POOL_H_
diff --git a/absl/synchronization/internal/waiter.cc b/absl/synchronization/internal/waiter.cc
index 17c6a506..2949f5a8 100644
--- a/absl/synchronization/internal/waiter.cc
+++ b/absl/synchronization/internal/waiter.cc
@@ -49,7 +49,7 @@
#include "absl/synchronization/internal/kernel_timeout.h"
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
static void MaybeBecomeIdle() {
@@ -123,16 +123,21 @@ class Futex {
}
};
-void Waiter::Init() {
+Waiter::Waiter() {
futex_.store(0, std::memory_order_relaxed);
}
+Waiter::~Waiter() = default;
+
bool Waiter::Wait(KernelTimeout t) {
// Loop until we can atomically decrement futex from a positive
// value, waiting on a futex while we believe it is zero.
+ // Note that, since the thread ticker is just reset, we don't need to check
+ // whether the thread is idle on the very first pass of the loop.
+ bool first_pass = true;
while (true) {
int32_t x = futex_.load(std::memory_order_relaxed);
- if (x != 0) {
+ while (x != 0) {
if (!futex_.compare_exchange_weak(x, x - 1,
std::memory_order_acquire,
std::memory_order_relaxed)) {
@@ -141,6 +146,8 @@ bool Waiter::Wait(KernelTimeout t) {
return true; // Consumed a wakeup, we are done.
}
+
+ if (!first_pass) MaybeBecomeIdle();
const int err = Futex::WaitUntil(&futex_, 0, t);
if (err != 0) {
if (err == -EINTR || err == -EWOULDBLOCK) {
@@ -151,14 +158,13 @@ bool Waiter::Wait(KernelTimeout t) {
ABSL_RAW_LOG(FATAL, "Futex operation failed with error %d\n", err);
}
}
-
- MaybeBecomeIdle();
+ first_pass = false;
}
}
void Waiter::Post() {
if (futex_.fetch_add(1, std::memory_order_release) == 0) {
- // We incremented from 0, need to wake a potential waker.
+ // We incremented from 0, need to wake a potential waiter.
Poke();
}
}
@@ -196,7 +202,7 @@ class PthreadMutexHolder {
pthread_mutex_t *mu_;
};
-void Waiter::Init() {
+Waiter::Waiter() {
const int err = pthread_mutex_init(&mu_, 0);
if (err != 0) {
ABSL_RAW_LOG(FATAL, "pthread_mutex_init failed: %d", err);
@@ -207,8 +213,20 @@ void Waiter::Init() {
ABSL_RAW_LOG(FATAL, "pthread_cond_init failed: %d", err2);
}
- waiter_count_.store(0, std::memory_order_relaxed);
- wakeup_count_.store(0, std::memory_order_relaxed);
+ waiter_count_ = 0;
+ wakeup_count_ = 0;
+}
+
+Waiter::~Waiter() {
+ const int err = pthread_mutex_destroy(&mu_);
+ if (err != 0) {
+ ABSL_RAW_LOG(FATAL, "pthread_mutex_destroy failed: %d", err);
+ }
+
+ const int err2 = pthread_cond_destroy(&cv_);
+ if (err2 != 0) {
+ ABSL_RAW_LOG(FATAL, "pthread_cond_destroy failed: %d", err2);
+ }
}
bool Waiter::Wait(KernelTimeout t) {
@@ -218,21 +236,13 @@ bool Waiter::Wait(KernelTimeout t) {
}
PthreadMutexHolder h(&mu_);
- waiter_count_.fetch_add(1, std::memory_order_relaxed);
+ ++waiter_count_;
// Loop until we find a wakeup to consume or timeout.
- while (true) {
- int x = wakeup_count_.load(std::memory_order_relaxed);
- if (x != 0) {
- if (!wakeup_count_.compare_exchange_weak(x, x - 1,
- std::memory_order_acquire,
- std::memory_order_relaxed)) {
- continue; // Raced with someone, retry.
- }
- // Successfully consumed a wakeup, we're done.
- waiter_count_.fetch_sub(1, std::memory_order_relaxed);
- return true;
- }
-
+ // Note that, since the thread ticker is just reset, we don't need to check
+ // whether the thread is idle on the very first pass of the loop.
+ bool first_pass = true;
+ while (wakeup_count_ == 0) {
+ if (!first_pass) MaybeBecomeIdle();
// No wakeups available, time to wait.
if (!t.has_timeout()) {
const int err = pthread_cond_wait(&cv_, &mu_);
@@ -242,46 +252,56 @@ bool Waiter::Wait(KernelTimeout t) {
} else {
const int err = pthread_cond_timedwait(&cv_, &mu_, &abs_timeout);
if (err == ETIMEDOUT) {
- waiter_count_.fetch_sub(1, std::memory_order_relaxed);
+ --waiter_count_;
return false;
}
if (err != 0) {
- ABSL_RAW_LOG(FATAL, "pthread_cond_wait failed: %d", err);
+ ABSL_RAW_LOG(FATAL, "pthread_cond_timedwait failed: %d", err);
}
}
- MaybeBecomeIdle();
+ first_pass = false;
}
+ // Consume a wakeup and we're done.
+ --wakeup_count_;
+ --waiter_count_;
+ return true;
}
void Waiter::Post() {
- wakeup_count_.fetch_add(1, std::memory_order_release);
- Poke();
+ PthreadMutexHolder h(&mu_);
+ ++wakeup_count_;
+ InternalCondVarPoke();
}
void Waiter::Poke() {
- if (waiter_count_.load(std::memory_order_relaxed) == 0) {
- return;
- }
- // Potentially a waker. Take the lock and check again.
PthreadMutexHolder h(&mu_);
- if (waiter_count_.load(std::memory_order_relaxed) == 0) {
- return;
- }
- const int err = pthread_cond_signal(&cv_);
- if (err != 0) {
- ABSL_RAW_LOG(FATAL, "pthread_cond_signal failed: %d", err);
+ InternalCondVarPoke();
+}
+
+void Waiter::InternalCondVarPoke() {
+ if (waiter_count_ != 0) {
+ const int err = pthread_cond_signal(&cv_);
+ if (ABSL_PREDICT_FALSE(err != 0)) {
+ ABSL_RAW_LOG(FATAL, "pthread_cond_signal failed: %d", err);
+ }
}
}
#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_SEM
-void Waiter::Init() {
+Waiter::Waiter() {
if (sem_init(&sem_, 0, 0) != 0) {
ABSL_RAW_LOG(FATAL, "sem_init failed with errno %d\n", errno);
}
wakeups_.store(0, std::memory_order_relaxed);
}
+Waiter::~Waiter() {
+ if (sem_destroy(&sem_) != 0) {
+ ABSL_RAW_LOG(FATAL, "sem_destroy failed with errno %d\n", errno);
+ }
+}
+
bool Waiter::Wait(KernelTimeout t) {
struct timespec abs_timeout;
if (t.has_timeout()) {
@@ -289,9 +309,12 @@ bool Waiter::Wait(KernelTimeout t) {
}
// Loop until we timeout or consume a wakeup.
+ // Note that, since the thread ticker is just reset, we don't need to check
+ // whether the thread is idle on the very first pass of the loop.
+ bool first_pass = true;
while (true) {
int x = wakeups_.load(std::memory_order_relaxed);
- if (x != 0) {
+ while (x != 0) {
if (!wakeups_.compare_exchange_weak(x, x - 1,
std::memory_order_acquire,
std::memory_order_relaxed)) {
@@ -301,6 +324,7 @@ bool Waiter::Wait(KernelTimeout t) {
return true;
}
+ if (!first_pass) MaybeBecomeIdle();
// Nothing to consume, wait (looping on EINTR).
while (true) {
if (!t.has_timeout()) {
@@ -314,13 +338,16 @@ bool Waiter::Wait(KernelTimeout t) {
ABSL_RAW_LOG(FATAL, "sem_timedwait failed: %d", errno);
}
}
- MaybeBecomeIdle();
+ first_pass = false;
}
}
void Waiter::Post() {
- wakeups_.fetch_add(1, std::memory_order_release); // Post a wakeup.
- Poke();
+ // Post a wakeup.
+ if (wakeups_.fetch_add(1, std::memory_order_release) == 0) {
+ // We incremented from 0, need to wake a potential waiter.
+ Poke();
+ }
}
void Waiter::Poke() {
@@ -341,31 +368,29 @@ class Waiter::WinHelper {
return reinterpret_cast<CONDITION_VARIABLE *>(&w->cv_storage_);
}
- static_assert(sizeof(SRWLOCK) == sizeof(Waiter::SRWLockStorage),
- "SRWLockStorage does not have the same size as SRWLOCK");
+ static_assert(sizeof(SRWLOCK) == sizeof(void *),
+ "`mu_storage_` does not have the same size as SRWLOCK");
+ static_assert(alignof(SRWLOCK) == alignof(void *),
+ "`mu_storage_` does not have the same alignment as SRWLOCK");
+
+ static_assert(sizeof(CONDITION_VARIABLE) == sizeof(void *),
+ "`ABSL_CONDITION_VARIABLE_STORAGE` does not have the same size "
+ "as `CONDITION_VARIABLE`");
static_assert(
- alignof(SRWLOCK) == alignof(Waiter::SRWLockStorage),
- "SRWLockStorage does not have the same alignment as SRWLOCK");
-
- static_assert(sizeof(CONDITION_VARIABLE) ==
- sizeof(Waiter::ConditionVariableStorage),
- "ABSL_CONDITION_VARIABLE_STORAGE does not have the same size "
- "as CONDITION_VARIABLE");
- static_assert(alignof(CONDITION_VARIABLE) ==
- alignof(Waiter::ConditionVariableStorage),
- "ConditionVariableStorage does not have the same "
- "alignment as CONDITION_VARIABLE");
-
- // The SRWLOCK and CONDITION_VARIABLE types must be trivially constuctible
+ alignof(CONDITION_VARIABLE) == alignof(void *),
+ "`cv_storage_` does not have the same alignment as `CONDITION_VARIABLE`");
+
+ // The SRWLOCK and CONDITION_VARIABLE types must be trivially constructible
// and destructible because we never call their constructors or destructors.
static_assert(std::is_trivially_constructible<SRWLOCK>::value,
- "The SRWLOCK type must be trivially constructible");
- static_assert(std::is_trivially_constructible<CONDITION_VARIABLE>::value,
- "The CONDITION_VARIABLE type must be trivially constructible");
+ "The `SRWLOCK` type must be trivially constructible");
+ static_assert(
+ std::is_trivially_constructible<CONDITION_VARIABLE>::value,
+ "The `CONDITION_VARIABLE` type must be trivially constructible");
static_assert(std::is_trivially_destructible<SRWLOCK>::value,
- "The SRWLOCK type must be trivially destructible");
+ "The `SRWLOCK` type must be trivially destructible");
static_assert(std::is_trivially_destructible<CONDITION_VARIABLE>::value,
- "The CONDITION_VARIABLE type must be trivially destructible");
+ "The `CONDITION_VARIABLE` type must be trivially destructible");
};
class LockHolder {
@@ -385,36 +410,33 @@ class LockHolder {
SRWLOCK* mu_;
};
-void Waiter::Init() {
+Waiter::Waiter() {
auto *mu = ::new (static_cast<void *>(&mu_storage_)) SRWLOCK;
auto *cv = ::new (static_cast<void *>(&cv_storage_)) CONDITION_VARIABLE;
InitializeSRWLock(mu);
InitializeConditionVariable(cv);
- waiter_count_.store(0, std::memory_order_relaxed);
- wakeup_count_.store(0, std::memory_order_relaxed);
+ waiter_count_ = 0;
+ wakeup_count_ = 0;
}
+// SRW locks and condition variables do not need to be explicitly destroyed.
+// https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-initializesrwlock
+// https://stackoverflow.com/questions/28975958/why-does-windows-have-no-deleteconditionvariable-function-to-go-together-with
+Waiter::~Waiter() = default;
+
bool Waiter::Wait(KernelTimeout t) {
SRWLOCK *mu = WinHelper::GetLock(this);
CONDITION_VARIABLE *cv = WinHelper::GetCond(this);
LockHolder h(mu);
- waiter_count_.fetch_add(1, std::memory_order_relaxed);
+ ++waiter_count_;
// Loop until we find a wakeup to consume or timeout.
- while (true) {
- int x = wakeup_count_.load(std::memory_order_relaxed);
- if (x != 0) {
- if (!wakeup_count_.compare_exchange_weak(x, x - 1,
- std::memory_order_acquire,
- std::memory_order_relaxed)) {
- continue; // Raced with someone, retry.
- }
- // Successfully consumed a wakeup, we're done.
- waiter_count_.fetch_sub(1, std::memory_order_relaxed);
- return true;
- }
-
+ // Note that, since the thread ticker is just reset, we don't need to check
+ // whether the thread is idle on the very first pass of the loop.
+ bool first_pass = true;
+ while (wakeup_count_ == 0) {
+ if (!first_pass) MaybeBecomeIdle();
// No wakeups available, time to wait.
if (!SleepConditionVariableSRW(cv, mu, t.InMillisecondsFromNow(), 0)) {
// GetLastError() returns a Win32 DWORD, but we assign to
@@ -422,32 +444,35 @@ bool Waiter::Wait(KernelTimeout t) {
// initialization guarantees this is not a narrowing conversion.
const unsigned long err{GetLastError()}; // NOLINT(runtime/int)
if (err == ERROR_TIMEOUT) {
- waiter_count_.fetch_sub(1, std::memory_order_relaxed);
+ --waiter_count_;
return false;
} else {
ABSL_RAW_LOG(FATAL, "SleepConditionVariableSRW failed: %lu", err);
}
}
-
- MaybeBecomeIdle();
+ first_pass = false;
}
+ // Consume a wakeup and we're done.
+ --wakeup_count_;
+ --waiter_count_;
+ return true;
}
void Waiter::Post() {
- wakeup_count_.fetch_add(1, std::memory_order_release);
- Poke();
+ LockHolder h(WinHelper::GetLock(this));
+ ++wakeup_count_;
+ InternalCondVarPoke();
}
void Waiter::Poke() {
- if (waiter_count_.load(std::memory_order_relaxed) == 0) {
- return;
- }
- // Potentially a waker. Take the lock and check again.
LockHolder h(WinHelper::GetLock(this));
- if (waiter_count_.load(std::memory_order_relaxed) == 0) {
- return;
+ InternalCondVarPoke();
+}
+
+void Waiter::InternalCondVarPoke() {
+ if (waiter_count_ != 0) {
+ WakeConditionVariable(WinHelper::GetCond(this));
}
- WakeConditionVariable(WinHelper::GetCond(this));
}
#else
@@ -455,5 +480,5 @@ void Waiter::Poke() {
#endif
} // namespace synchronization_internal
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
diff --git a/absl/synchronization/internal/waiter.h b/absl/synchronization/internal/waiter.h
index 06032642..a6e6d4c7 100644
--- a/absl/synchronization/internal/waiter.h
+++ b/absl/synchronization/internal/waiter.h
@@ -18,10 +18,16 @@
#include "absl/base/config.h"
-#ifndef _WIN32
+#ifdef _WIN32
+#include <sdkddkver.h>
+#else
#include <pthread.h>
#endif
+#ifdef __linux__
+#include <linux/futex.h>
+#endif
+
#ifdef ABSL_HAVE_SEMAPHORE_H
#include <semaphore.h>
#endif
@@ -40,9 +46,14 @@
#if defined(ABSL_FORCE_WAITER_MODE)
#define ABSL_WAITER_MODE ABSL_FORCE_WAITER_MODE
-#elif defined(_WIN32)
+#elif defined(_WIN32) && _WIN32_WINNT >= _WIN32_WINNT_VISTA
#define ABSL_WAITER_MODE ABSL_WAITER_MODE_WIN32
-#elif defined(__linux__)
+#elif defined(__BIONIC__)
+// Bionic supports all the futex operations we need even when some of the futex
+// definitions are missing.
+#define ABSL_WAITER_MODE ABSL_WAITER_MODE_FUTEX
+#elif defined(__linux__) && defined(FUTEX_CLOCK_REALTIME)
+// FUTEX_CLOCK_REALTIME requires Linux >= 2.6.28.
#define ABSL_WAITER_MODE ABSL_WAITER_MODE_FUTEX
#elif defined(ABSL_HAVE_SEMAPHORE_H)
#define ABSL_WAITER_MODE ABSL_WAITER_MODE_SEM
@@ -51,20 +62,21 @@
#endif
namespace absl {
-inline namespace lts_2019_08_08 {
+ABSL_NAMESPACE_BEGIN
namespace synchronization_internal {
// Waiter is an OS-specific semaphore.
class Waiter {
public:
- // No constructor, instances use the reserved space in ThreadIdentity.
- // All initialization logic belongs in `Init()`.
- Waiter() = delete;
+ // Prepare any data to track waits.
+ Waiter();
+
+ // Not copyable or movable
Waiter(const Waiter&) = delete;
Waiter& operator=(const Waiter&) = delete;
- // Prepare any data to track waits.
- void Init();
+ // Destroy any data to track waits.
+ ~Waiter();
// Blocks the calling thread until a matching call to `Post()` or
// `t` has passed. Returns `true` if woken (`Post()` called),
@@ -105,10 +117,13 @@ class Waiter {
static_assert(sizeof(int32_t) == sizeof(futex_), "Wrong size for futex");
#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_CONDVAR
+ // REQUIRES: mu_ must be held.
+ void InternalCondVarPoke();
+
pthread_mutex_t mu_;
pthread_cond_t cv_;
- std::atomic<int> waiter_count_;
- std::atomic<int> wakeup_count_; // Unclaimed wakeups, written under lock.
+ int waiter_count_;
+ int wakeup_count_; // Unclaimed wakeups.
#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_SEM
sem_t sem_;
@@ -118,26 +133,19 @@ class Waiter {
std::atomic<int> wakeups_;
#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_WIN32
- // The Windows API has lots of choices for synchronization
- // primivitives. We are using SRWLOCK and CONDITION_VARIABLE
- // because they don't require a destructor to release system
- // resources.
- //
- // However, we can't include Windows.h in our headers, so we use aligned
- // storage buffers to define the storage.
- using SRWLockStorage =
- typename std::aligned_storage<sizeof(void*), alignof(void*)>::type;
- using ConditionVariableStorage =
- typename std::aligned_storage<sizeof(void*), alignof(void*)>::type;
-
// WinHelper - Used to define utilities for accessing the lock and
// condition variable storage once the types are complete.
class WinHelper;
- SRWLockStorage mu_storage_;
- ConditionVariableStorage cv_storage_;
- std::atomic<int> waiter_count_;
- std::atomic<int> wakeup_count_;
+ // REQUIRES: WinHelper::GetLock(this) must be held.
+ void InternalCondVarPoke();
+
+ // We can't include Windows.h in our headers, so we use aligned charachter
+ // buffers to define the storage of SRWLOCK and CONDITION_VARIABLE.
+ alignas(void*) unsigned char mu_storage_[sizeof(void*)];
+ alignas(void*) unsigned char cv_storage_[sizeof(void*)];
+ int waiter_count_;
+ int wakeup_count_;
#else
#error Unknown ABSL_WAITER_MODE
@@ -145,7 +153,7 @@ class Waiter {
};
} // namespace synchronization_internal
-} // inline namespace lts_2019_08_08
+ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_SYNCHRONIZATION_INTERNAL_WAITER_H_
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-19 20:48:24 +0000