From 8e34214922538e43c0fff242b200db922974f275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sun, 4 Feb 2024 13:43:19 +0100 Subject: [PATCH 1/7] refactor the predicate taking variant of '__cxx_atomic_wait' --- libcxx/include/__atomic/atomic_sync.h | 101 +++++++++++++++++--------- libcxx/include/latch | 11 ++- libcxx/include/semaphore | 17 +---- libcxx/src/atomic.cpp | 1 + 4 files changed, 80 insertions(+), 50 deletions(-) diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h index e1994ddde86c1..1eca545afb99f 100644 --- a/libcxx/include/__atomic/atomic_sync.h +++ b/libcxx/include/__atomic/atomic_sync.h @@ -43,15 +43,53 @@ __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); -template +template +struct __libcpp_atomic_wait_poll_impl { + _Atp* __a_; + _Poll __poll_; + memory_order __order_; + + _LIBCPP_AVAILABILITY_SYNC + _LIBCPP_HIDE_FROM_ABI bool operator()() const { + auto __current_val = __cxx_atomic_load(__a_, __order_); + return __poll_(__current_val); + } +}; + +template struct __libcpp_atomic_wait_backoff_impl { _Atp* __a_; - _BackoffTest __backoff_test_; + _Poll __poll_; + memory_order __order_; + + _LIBCPP_AVAILABILITY_SYNC + _LIBCPP_HIDE_FROM_ABI bool + __poll_or_get_monitor(__cxx_atomic_contention_t const volatile*, __cxx_contention_t& __monitor) const { + // In case the atomic can be waited on directly, the monitor value is just + // the value of the atomic. + // `__poll_` takes the current value of the atomic as an in-out argument + // to potentially modify it. After it returns, `__monitor` has a value + // which can be safely waited on by `std::__libcpp_atomic_wait` without any + // ABA style issues. + __monitor = __cxx_atomic_load(__a_, __order_); + return __poll_(__monitor); + } + + _LIBCPP_AVAILABILITY_SYNC + _LIBCPP_HIDE_FROM_ABI bool __poll_or_get_monitor(void const volatile*, __cxx_contention_t& __monitor) const { + // In case we must wait on an atomic from the pool, the monitor comes from + // `std::__libcpp_atomic_monitor`. + // Only then we may read from `__a_`. This is the "event count" pattern. + __monitor = std::__libcpp_atomic_monitor(__a_); + auto __current_val = __cxx_atomic_load(__a_, __order_); + return __poll_(__current_val); + } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const { if (__elapsed > chrono::microseconds(64)) { - auto __monitor = std::__libcpp_atomic_monitor(__a_); - if (__backoff_test_(__monitor)) + __cxx_contention_t __monitor; + if (__poll_or_get_monitor(__a_, __monitor)) return true; std::__libcpp_atomic_wait(__a_, __monitor); } else if (__elapsed > chrono::microseconds(4)) @@ -62,26 +100,20 @@ struct __libcpp_atomic_wait_backoff_impl { } }; -template -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool -__cxx_atomic_wait(_Atp* __a, _Poll&& __poll, _BackoffTest&& __backoff_test) { - __libcpp_atomic_wait_backoff_impl<_Atp, __decay_t<_BackoffTest> > __backoff_fn = {__a, __backoff_test}; - return std::__libcpp_thread_poll_with_backoff(__poll, __backoff_fn); -} - -template -struct __libcpp_atomic_wait_poll_as_backoff_test { - _Poll __poll_; - - _LIBCPP_AVAILABILITY_SYNC - _LIBCPP_HIDE_FROM_ABI bool operator()(__cxx_contention_t&) const { return __poll_(); } -}; - +// The semantics of this function are similar to `atomic`'s +// `.wait(T old, std::memory_order order)`, but instead of having a hardcoded +// predicate (is the loaded value unequal to `old`?), the predicate function is +// specified as an argument. The loaded value is given as an in-out argument to +// the predicate. If the predicate function returns `true`, +// `_cxx_atomic_wait_fn` will return. If the predicate function returns +// `false`, it must set the argument to its current understanding of the atomic +// value. The predicate function must not return `false` spuriously. template -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_wait(_Atp* __a, _Poll&& __poll) { - __libcpp_atomic_wait_backoff_impl<_Atp, __libcpp_atomic_wait_poll_as_backoff_test<_Poll&> > __backoff_fn = { - __a, {__poll}}; - return std::__libcpp_thread_poll_with_backoff(__poll, __backoff_fn); +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void +__cxx_atomic_wait_fn(_Atp* __a, _Poll&& __poll, memory_order __order) { + __libcpp_atomic_wait_poll_impl<_Atp, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order}; + __libcpp_atomic_wait_backoff_impl<_Atp, __decay_t<_Poll> > __backoff_fn = {__a, __poll, __order}; + (void)std::__libcpp_thread_poll_with_backoff(__poll_fn, __backoff_fn); } #else // _LIBCPP_HAS_NO_THREADS @@ -90,9 +122,10 @@ template _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) {} template _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) {} -template -_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_wait(_Atp*, _Fn&& __test_fn) { - return std::__libcpp_thread_poll_with_backoff(__test_fn, __spinning_backoff_policy()); +template +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_wait_fn(_Atp*, _Poll&& __poll, memory_order __order) { + __libcpp_atomic_wait_poll_impl<_Atp, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order}; + (void)std::__libcpp_thread_poll_with_backoff(__poll_fn, __spinning_backoff_policy()); } #endif // _LIBCPP_HAS_NO_THREADS @@ -102,21 +135,19 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp c return std::memcmp(std::addressof(__lhs), std::addressof(__rhs), sizeof(_Tp)) == 0; } -template -struct __cxx_atomic_wait_test_fn_impl { - _Atp* __a; +template +struct __cxx_atomic_wait_poll_fn_impl { _Tp __val; - memory_order __order; - _LIBCPP_HIDE_FROM_ABI bool operator()() const { - return !std::__cxx_nonatomic_compare_equal(std::__cxx_atomic_load(__a, __order), __val); + _LIBCPP_HIDE_FROM_ABI bool operator()(_Tp& __current_val) const { + return !std::__cxx_nonatomic_compare_equal(__current_val, __val); } }; template -_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) { - __cxx_atomic_wait_test_fn_impl<_Atp, _Tp> __test_fn = {__a, __val, __order}; - return std::__cxx_atomic_wait(__a, __test_fn); + __cxx_atomic_wait_poll_fn_impl<_Tp> __poll_fn = {__val}; + std::__cxx_atomic_wait_fn(__a, __poll_fn, __order); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/latch b/libcxx/include/latch index ad7b35579913b..45c1b5c91e6cd 100644 --- a/libcxx/include/latch +++ b/libcxx/include/latch @@ -97,9 +97,13 @@ public: if (__old == __update) __a_.notify_all(); } - inline _LIBCPP_HIDE_FROM_ABI bool try_wait() const noexcept { return 0 == __a_.load(memory_order_acquire); } + inline _LIBCPP_HIDE_FROM_ABI bool try_wait() const noexcept { + auto __value = __a_.load(memory_order_acquire); + return try_wait_impl(__value); + } inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait() const { - __cxx_atomic_wait(&__a_.__a_, [this]() -> bool { return try_wait(); }); + __cxx_atomic_wait_fn( + &__a_.__a_, [this](ptrdiff_t& __value) -> bool { return try_wait_impl(__value); }, memory_order_acquire); } inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_wait(ptrdiff_t __update = 1) { _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__update >= 0, "latch::arrive_and_wait called with a negative value"); @@ -108,6 +112,9 @@ public: count_down(__update); wait(); } + +private: + inline _LIBCPP_HIDE_FROM_ABI bool try_wait_impl(ptrdiff_t& __value) const noexcept { return __value == 0; } }; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/semaphore b/libcxx/include/semaphore index 5235d720bf6fe..27e5951e7d982 100644 --- a/libcxx/include/semaphore +++ b/libcxx/include/semaphore @@ -100,17 +100,8 @@ public: } } _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void acquire() { - auto const __poll_fn = [this]() -> bool { - auto __old = __a_.load(memory_order_relaxed); - return (__old != 0) && __a_.compare_exchange_strong(__old, __old - 1, memory_order_acquire, memory_order_relaxed); - }; - auto const __backoff_test = [this](__cxx_contention_t& __monitor) -> bool { - ptrdiff_t __old = __monitor; - bool __r = __try_acquire_impl(__old); - __monitor = static_cast<__cxx_contention_t>(__old); - return __r; - }; - __cxx_atomic_wait(&__a_.__a_, __poll_fn, __backoff_test); + __cxx_atomic_wait_fn( + &__a_.__a_, [this](ptrdiff_t& __old) { return __try_acquire_impl(__old); }, memory_order_relaxed); } template _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool @@ -121,7 +112,7 @@ public: return std::__libcpp_thread_poll_with_backoff(__poll_fn, __libcpp_timed_backoff_policy(), __rel_time); } _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool try_acquire() { - auto __old = __a_.load(memory_order_acquire); + auto __old = __a_.load(memory_order_relaxed); return __try_acquire_impl(__old); } @@ -130,7 +121,7 @@ private: while (true) { if (__old == 0) return false; - if (__a_.compare_exchange_strong(__old, __old - 1, memory_order_acquire, memory_order_relaxed)) + if (__a_.compare_exchange_weak(__old, __old - 1, memory_order_acquire, memory_order_relaxed)) return true; } } diff --git a/libcxx/src/atomic.cpp b/libcxx/src/atomic.cpp index 2f0389ae6974a..2b67685c8a0a1 100644 --- a/libcxx/src/atomic.cpp +++ b/libcxx/src/atomic.cpp @@ -178,6 +178,7 @@ _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile* __location) { __libcpp_contention_notify(&__libcpp_contention_state(__location)->__contention_state, __location, false); } +// This function is never used, but still exported for ABI compatibility. _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile* __location) { return __libcpp_contention_monitor_for_wait(&__libcpp_contention_state(__location)->__contention_state, __location); From ce27d09d95b3d6f5ec7d08f4522fe75def06aa64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sun, 4 Feb 2024 14:39:47 +0100 Subject: [PATCH 2/7] remove unneeded include of '__atomic/contention_t.h' --- libcxx/include/semaphore | 1 - 1 file changed, 1 deletion(-) diff --git a/libcxx/include/semaphore b/libcxx/include/semaphore index 27e5951e7d982..ae5af24e99e32 100644 --- a/libcxx/include/semaphore +++ b/libcxx/include/semaphore @@ -54,7 +54,6 @@ using binary_semaphore = counting_semaphore<1>; #include <__assert> // all public C++ headers provide the assertion handler #include <__atomic/atomic_base.h> #include <__atomic/atomic_sync.h> -#include <__atomic/contention_t.h> #include <__atomic/memory_order.h> #include <__availability> #include <__chrono/time_point.h> From 6f39049ec50093d22db3c99432276346d3a8cec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sun, 4 Feb 2024 15:01:14 +0100 Subject: [PATCH 3/7] try to fix CI error --- libcxx/include/__atomic/atomic_sync.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h index 1eca545afb99f..78bdd88cf0b8f 100644 --- a/libcxx/include/__atomic/atomic_sync.h +++ b/libcxx/include/__atomic/atomic_sync.h @@ -51,7 +51,7 @@ struct __libcpp_atomic_wait_poll_impl { _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool operator()() const { - auto __current_val = __cxx_atomic_load(__a_, __order_); + auto __current_val = std::__cxx_atomic_load(__a_, __order_); return __poll_(__current_val); } }; @@ -71,7 +71,7 @@ struct __libcpp_atomic_wait_backoff_impl { // to potentially modify it. After it returns, `__monitor` has a value // which can be safely waited on by `std::__libcpp_atomic_wait` without any // ABA style issues. - __monitor = __cxx_atomic_load(__a_, __order_); + __monitor = std::__cxx_atomic_load(__a_, __order_); return __poll_(__monitor); } @@ -81,7 +81,7 @@ struct __libcpp_atomic_wait_backoff_impl { // `std::__libcpp_atomic_monitor`. // Only then we may read from `__a_`. This is the "event count" pattern. __monitor = std::__libcpp_atomic_monitor(__a_); - auto __current_val = __cxx_atomic_load(__a_, __order_); + auto __current_val = std::__cxx_atomic_load(__a_, __order_); return __poll_(__current_val); } From 08660014469b993eac4eef1dee6dfc93360443c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sun, 4 Feb 2024 17:01:38 +0100 Subject: [PATCH 4/7] CI --- libcxx/include/__atomic/atomic_sync.h | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h index 78bdd88cf0b8f..8e17e3a58a110 100644 --- a/libcxx/include/__atomic/atomic_sync.h +++ b/libcxx/include/__atomic/atomic_sync.h @@ -27,6 +27,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD +template +struct __libcpp_atomic_wait_poll_impl { + _Atp* __a_; + _Poll __poll_; + memory_order __order_; + + _LIBCPP_AVAILABILITY_SYNC + _LIBCPP_HIDE_FROM_ABI bool operator()() const { + auto __current_val = std::__cxx_atomic_load(__a_, __order_); + return __poll_(__current_val); + } +}; + #ifndef _LIBCPP_HAS_NO_THREADS _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*); @@ -43,19 +56,6 @@ __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); -template -struct __libcpp_atomic_wait_poll_impl { - _Atp* __a_; - _Poll __poll_; - memory_order __order_; - - _LIBCPP_AVAILABILITY_SYNC - _LIBCPP_HIDE_FROM_ABI bool operator()() const { - auto __current_val = std::__cxx_atomic_load(__a_, __order_); - return __poll_(__current_val); - } -}; - template struct __libcpp_atomic_wait_backoff_impl { _Atp* __a_; From 55411621379b30a10d90a42724bb0b66668f5dbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sun, 4 Feb 2024 18:34:10 +0100 Subject: [PATCH 5/7] CI --- libcxx/include/__atomic/atomic_sync.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h index 8e17e3a58a110..47407fa20d382 100644 --- a/libcxx/include/__atomic/atomic_sync.h +++ b/libcxx/include/__atomic/atomic_sync.h @@ -123,7 +123,7 @@ _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const template _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) {} template -_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_wait_fn(_Atp*, _Poll&& __poll, memory_order __order) { +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_wait_fn(_Atp* __a, _Poll&& __poll, memory_order __order) { __libcpp_atomic_wait_poll_impl<_Atp, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order}; (void)std::__libcpp_thread_poll_with_backoff(__poll_fn, __spinning_backoff_policy()); } From f151e17f36a96d439bd2ca4ab1f08d902a165f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sun, 18 Feb 2024 12:26:14 +0100 Subject: [PATCH 6/7] rename '_cxx_atomic_wait_fn' to '__cxx_atomic_wait_unless' --- libcxx/include/__atomic/atomic_sync.h | 8 ++++---- libcxx/include/latch | 2 +- libcxx/include/semaphore | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h index 47407fa20d382..93b051debf308 100644 --- a/libcxx/include/__atomic/atomic_sync.h +++ b/libcxx/include/__atomic/atomic_sync.h @@ -105,12 +105,12 @@ struct __libcpp_atomic_wait_backoff_impl { // predicate (is the loaded value unequal to `old`?), the predicate function is // specified as an argument. The loaded value is given as an in-out argument to // the predicate. If the predicate function returns `true`, -// `_cxx_atomic_wait_fn` will return. If the predicate function returns +// `_cxx_atomic_wait_unless` will return. If the predicate function returns // `false`, it must set the argument to its current understanding of the atomic // value. The predicate function must not return `false` spuriously. template _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void -__cxx_atomic_wait_fn(_Atp* __a, _Poll&& __poll, memory_order __order) { +__cxx_atomic_wait_unless(_Atp* __a, _Poll&& __poll, memory_order __order) { __libcpp_atomic_wait_poll_impl<_Atp, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order}; __libcpp_atomic_wait_backoff_impl<_Atp, __decay_t<_Poll> > __backoff_fn = {__a, __poll, __order}; (void)std::__libcpp_thread_poll_with_backoff(__poll_fn, __backoff_fn); @@ -123,7 +123,7 @@ _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const template _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) {} template -_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_wait_fn(_Atp* __a, _Poll&& __poll, memory_order __order) { +_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_wait_unless(_Atp* __a, _Poll&& __poll, memory_order __order) { __libcpp_atomic_wait_poll_impl<_Atp, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order}; (void)std::__libcpp_thread_poll_with_backoff(__poll_fn, __spinning_backoff_policy()); } @@ -147,7 +147,7 @@ template _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) { __cxx_atomic_wait_poll_fn_impl<_Tp> __poll_fn = {__val}; - std::__cxx_atomic_wait_fn(__a, __poll_fn, __order); + std::__cxx_atomic_wait_unless(__a, __poll_fn, __order); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/latch b/libcxx/include/latch index 45c1b5c91e6cd..dd389d296f5c1 100644 --- a/libcxx/include/latch +++ b/libcxx/include/latch @@ -102,7 +102,7 @@ public: return try_wait_impl(__value); } inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait() const { - __cxx_atomic_wait_fn( + __cxx_atomic_wait_unless( &__a_.__a_, [this](ptrdiff_t& __value) -> bool { return try_wait_impl(__value); }, memory_order_acquire); } inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_wait(ptrdiff_t __update = 1) { diff --git a/libcxx/include/semaphore b/libcxx/include/semaphore index ae5af24e99e32..448b5fbd8c58c 100644 --- a/libcxx/include/semaphore +++ b/libcxx/include/semaphore @@ -99,7 +99,7 @@ public: } } _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void acquire() { - __cxx_atomic_wait_fn( + __cxx_atomic_wait_unless( &__a_.__a_, [this](ptrdiff_t& __old) { return __try_acquire_impl(__old); }, memory_order_relaxed); } template From 2daeb53709423b7102238411452a575d442722ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kokem=C3=BCller?= Date: Sun, 18 Feb 2024 12:46:43 +0100 Subject: [PATCH 7/7] rename '__cxx_atomic_wait_poll_fn_impl' to '__atomic_compare_unequal_to' --- libcxx/include/__atomic/atomic_sync.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h index 93b051debf308..d07b35878493b 100644 --- a/libcxx/include/__atomic/atomic_sync.h +++ b/libcxx/include/__atomic/atomic_sync.h @@ -136,7 +136,7 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp c } template -struct __cxx_atomic_wait_poll_fn_impl { +struct __atomic_compare_unequal_to { _Tp __val; _LIBCPP_HIDE_FROM_ABI bool operator()(_Tp& __current_val) const { return !std::__cxx_nonatomic_compare_equal(__current_val, __val); @@ -146,7 +146,7 @@ struct __cxx_atomic_wait_poll_fn_impl { template _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) { - __cxx_atomic_wait_poll_fn_impl<_Tp> __poll_fn = {__val}; + __atomic_compare_unequal_to<_Tp> __poll_fn = {__val}; std::__cxx_atomic_wait_unless(__a, __poll_fn, __order); }