diff --git a/libcxx/include/__atomic/atomic_base.h b/libcxx/include/__atomic/atomic_base.h index 3ad3b562c5980..6ca01a7f1bf9b 100644 --- a/libcxx/include/__atomic/atomic_base.h +++ b/libcxx/include/__atomic/atomic_base.h @@ -104,24 +104,20 @@ struct __atomic_base // false _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT { - std::__cxx_atomic_wait(std::addressof(__a_), __v, __m); + std::__atomic_wait(*this, __v, __m); } _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT { - std::__cxx_atomic_wait(std::addressof(__a_), __v, __m); + std::__atomic_wait(*this, __v, __m); } _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { - std::__cxx_atomic_notify_one(std::addressof(__a_)); - } - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { - std::__cxx_atomic_notify_one(std::addressof(__a_)); + std::__atomic_notify_one(*this); } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); } _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { - std::__cxx_atomic_notify_all(std::addressof(__a_)); - } - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { - std::__cxx_atomic_notify_all(std::addressof(__a_)); + std::__atomic_notify_all(*this); } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); } #if _LIBCPP_STD_VER >= 20 _LIBCPP_HIDE_FROM_ABI constexpr __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {} @@ -200,6 +196,32 @@ struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> { _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; } }; +// Here we need _IsIntegral because the default template argument is not enough +// e.g __atomic_base is __atomic_base, which inherits from +// __atomic_base and the caller of the wait function is +// __atomic_base. So specializing __atomic_base<_Tp> does not work +template +struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > { + static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) { + return __a.load(__order); + } + + static _LIBCPP_HIDE_FROM_ABI _Tp + __atomic_load(const volatile __atomic_base<_Tp, _IsIntegral>& __this, memory_order __order) { + return __this.load(__order); + } + + static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_Tp>* + __atomic_contention_address(const __atomic_base<_Tp, _IsIntegral>& __a) { + return std::addressof(__a.__a_); + } + + static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_Tp>* + __atomic_contention_address(const volatile __atomic_base<_Tp, _IsIntegral>& __this) { + return std::addressof(__this.__a_); + } +}; + _LIBCPP_END_NAMESPACE_STD #endif // _LIBCPP___ATOMIC_ATOMIC_BASE_H diff --git a/libcxx/include/__atomic/atomic_flag.h b/libcxx/include/__atomic/atomic_flag.h index 18a864523de06..084366237c16e 100644 --- a/libcxx/include/__atomic/atomic_flag.h +++ b/libcxx/include/__atomic/atomic_flag.h @@ -16,6 +16,7 @@ #include <__availability> #include <__chrono/duration.h> #include <__config> +#include <__memory/addressof.h> #include <__thread/support.h> #include @@ -50,20 +51,20 @@ struct atomic_flag { _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT { - __cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m); + std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m); } _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT { - __cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m); + std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m); } _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { - __cxx_atomic_notify_one(&__a_); + std::__atomic_notify_one(*this); } - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { __cxx_atomic_notify_one(&__a_); } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); } _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { - __cxx_atomic_notify_all(&__a_); + std::__atomic_notify_all(*this); } - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { __cxx_atomic_notify_all(&__a_); } + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); } #if _LIBCPP_STD_VER >= 20 _LIBCPP_HIDE_FROM_ABI constexpr atomic_flag() _NOEXCEPT : __a_(false) {} @@ -78,6 +79,28 @@ struct atomic_flag { atomic_flag& operator=(const atomic_flag&) volatile = delete; }; +template <> +struct __atomic_waitable_traits { + static _LIBCPP_HIDE_FROM_ABI _LIBCPP_ATOMIC_FLAG_TYPE __atomic_load(const atomic_flag& __a, memory_order __order) { + return std::__cxx_atomic_load(&__a.__a_, __order); + } + + static _LIBCPP_HIDE_FROM_ABI _LIBCPP_ATOMIC_FLAG_TYPE + __atomic_load(const volatile atomic_flag& __a, memory_order __order) { + return std::__cxx_atomic_load(&__a.__a_, __order); + } + + static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE>* + __atomic_contention_address(const atomic_flag& __a) { + return std::addressof(__a.__a_); + } + + static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE>* + __atomic_contention_address(const volatile atomic_flag& __a) { + return std::addressof(__a.__a_); + } +}; + inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT { return __o->test(); } inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const atomic_flag* __o) _NOEXCEPT { return __o->test(); } diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h index 2997bad83344b..e583dca38c4c7 100644 --- a/libcxx/include/__atomic/atomic_sync.h +++ b/libcxx/include/__atomic/atomic_sync.h @@ -18,7 +18,11 @@ #include <__memory/addressof.h> #include <__thread/poll_with_backoff.h> #include <__thread/support.h> +#include <__type_traits/conjunction.h> #include <__type_traits/decay.h> +#include <__type_traits/invoke.h> +#include <__type_traits/void_t.h> +#include <__utility/declval.h> #include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -27,15 +31,40 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template -struct __libcpp_atomic_wait_poll_impl { - _Atp* __a_; +// The customisation points to enable the following functions: +// - __atomic_wait +// - __atomic_wait_unless +// - __atomic_notify_one +// - __atomic_notify_all +// Note that std::atomic::wait was back-ported to C++03 +// The below implementations look ugly to support C++03 +template +struct __atomic_waitable_traits { + template + static void __atomic_load(_AtomicWaitable&&, memory_order) = delete; + + template + static void __atomic_contention_address(_AtomicWaitable&&) = delete; +}; + +template +struct __atomic_waitable : false_type {}; + +template +struct __atomic_waitable< _Tp, + __void_t >::__atomic_load( + std::declval(), std::declval())), + decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_contention_address( + std::declval()))> > : true_type {}; + +template +struct __atomic_wait_poll_impl { + const _AtomicWaitable& __a_; _Poll __poll_; memory_order __order_; - _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool operator()() const { - auto __current_val = std::__cxx_atomic_load(__a_, __order_); + auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a_, __order_); return __poll_(__current_val); } }; @@ -56,42 +85,45 @@ __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*); _LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t); -template -struct __libcpp_atomic_wait_backoff_impl { - _Atp* __a_; +template +struct __atomic_wait_backoff_impl { + const _AtomicWaitable& __a_; _Poll __poll_; memory_order __order_; + using __waitable_traits = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >; + _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool - __poll_or_get_monitor(__cxx_atomic_contention_t const volatile*, __cxx_contention_t& __monitor) const { - // In case the atomic can be waited on directly, the monitor value is just - // the value of the atomic. + __update_monitor_val_and_poll(__cxx_atomic_contention_t const volatile*, __cxx_contention_t& __monitor_val) const { + // In case the contention type happens to be __cxx_atomic_contention_t, i.e. __cxx_atomic_impl, + // the platform wait is directly monitoring the atomic value itself. // `__poll_` takes the current value of the atomic as an in-out argument // to potentially modify it. After it returns, `__monitor` has a value // which can be safely waited on by `std::__libcpp_atomic_wait` without any // ABA style issues. - __monitor = std::__cxx_atomic_load(__a_, __order_); - return __poll_(__monitor); + __monitor_val = __waitable_traits::__atomic_load(__a_, __order_); + return __poll_(__monitor_val); } _LIBCPP_AVAILABILITY_SYNC - _LIBCPP_HIDE_FROM_ABI bool __poll_or_get_monitor(void const volatile*, __cxx_contention_t& __monitor) const { - // In case we must wait on an atomic from the pool, the monitor comes from - // `std::__libcpp_atomic_monitor`. - // Only then we may read from `__a_`. This is the "event count" pattern. - __monitor = std::__libcpp_atomic_monitor(__a_); - auto __current_val = std::__cxx_atomic_load(__a_, __order_); + _LIBCPP_HIDE_FROM_ABI bool + __update_monitor_val_and_poll(void const volatile* __contention_address, __cxx_contention_t& __monitor_val) const { + // In case the contention type is anything else, platform wait is monitoring a __cxx_atomic_contention_t + // from the global pool, the monitor comes from __libcpp_atomic_monitor + __monitor_val = std::__libcpp_atomic_monitor(__contention_address); + auto __current_val = __waitable_traits::__atomic_load(__a_, __order_); return __poll_(__current_val); } _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const { if (__elapsed > chrono::microseconds(64)) { - __cxx_contention_t __monitor; - if (__poll_or_get_monitor(__a_, __monitor)) + auto __contention_address = __waitable_traits::__atomic_contention_address(__a_); + __cxx_contention_t __monitor_val; + if (__update_monitor_val_and_poll(__contention_address, __monitor_val)) return true; - std::__libcpp_atomic_wait(__a_, __monitor); + std::__libcpp_atomic_wait(__contention_address, __monitor_val); } else if (__elapsed > chrono::microseconds(4)) __libcpp_thread_yield(); else { @@ -105,29 +137,44 @@ struct __libcpp_atomic_wait_backoff_impl { // predicate (is the loaded value unequal to `old`?), the predicate function is // specified as an argument. The loaded value is given as an in-out argument to // the predicate. If the predicate function returns `true`, -// `_cxx_atomic_wait_unless` will return. If the predicate function returns +// `__atomic_wait_unless` will return. If the predicate function returns // `false`, it must set the argument to its current understanding of the atomic // value. The predicate function must not return `false` spuriously. -template +template _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void -__cxx_atomic_wait_unless(_Atp* __a, _Poll&& __poll, memory_order __order) { - __libcpp_atomic_wait_poll_impl<_Atp, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order}; - __libcpp_atomic_wait_backoff_impl<_Atp, __decay_t<_Poll> > __backoff_fn = {__a, __poll, __order}; - (void)std::__libcpp_thread_poll_with_backoff(__poll_fn, __backoff_fn); +__atomic_wait_unless(const _AtomicWaitable& __a, _Poll&& __poll, memory_order __order) { + static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + __atomic_wait_poll_impl<_AtomicWaitable, __decay_t<_Poll> > __poll_impl = {__a, __poll, __order}; + __atomic_wait_backoff_impl<_AtomicWaitable, __decay_t<_Poll> > __backoff_fn = {__a, __poll, __order}; + std::__libcpp_thread_poll_with_backoff(__poll_impl, __backoff_fn); +} + +template +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) { + static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + std::__cxx_atomic_notify_one(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a)); +} + +template +_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) { + static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + std::__cxx_atomic_notify_all(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a)); } #else // _LIBCPP_HAS_NO_THREADS -template -_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) {} -template -_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) {} -template -_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_wait_unless(_Atp* __a, _Poll&& __poll, memory_order __order) { - __libcpp_atomic_wait_poll_impl<_Atp, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order}; - (void)std::__libcpp_thread_poll_with_backoff(__poll_fn, __spinning_backoff_policy()); +template +_LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, _Poll&& __poll, memory_order __order) { + __atomic_wait_poll_impl<_AtomicWaitable, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order}; + std::__libcpp_thread_poll_with_backoff(__poll_fn, __spinning_backoff_policy()); } +template +_LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable&) {} + +template +_LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable&) {} + #endif // _LIBCPP_HAS_NO_THREADS template @@ -143,11 +190,12 @@ struct __atomic_compare_unequal_to { } }; -template +template _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void -__cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) { - __atomic_compare_unequal_to<_Tp> __poll_fn = {__val}; - std::__cxx_atomic_wait_unless(__a, __poll_fn, __order); +__atomic_wait(_AtomicWaitable& __a, _Up __val, memory_order __order) { + static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + __atomic_compare_unequal_to<_Up> __nonatomic_equal = {__val}; + std::__atomic_wait_unless(__a, __nonatomic_equal, __order); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/latch b/libcxx/include/latch index 3fe201b63d138..3cc7258381143 100644 --- a/libcxx/include/latch +++ b/libcxx/include/latch @@ -102,8 +102,8 @@ public: return try_wait_impl(__value); } inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait() const { - __cxx_atomic_wait_unless( - &__a_.__a_, [this](ptrdiff_t& __value) -> bool { return try_wait_impl(__value); }, memory_order_acquire); + std::__atomic_wait_unless( + __a_, [this](ptrdiff_t& __value) -> bool { return try_wait_impl(__value); }, memory_order_acquire); } inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_wait(ptrdiff_t __update = 1) { _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__update >= 0, "latch::arrive_and_wait called with a negative value"); @@ -114,7 +114,7 @@ public: } private: - inline _LIBCPP_HIDE_FROM_ABI bool try_wait_impl(ptrdiff_t& __value) const noexcept { return __value == 0; } + _LIBCPP_HIDE_FROM_ABI bool try_wait_impl(ptrdiff_t& __value) const noexcept { return __value == 0; } }; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/semaphore b/libcxx/include/semaphore index 2dfdae9aa148c..1375ec3f7c04b 100644 --- a/libcxx/include/semaphore +++ b/libcxx/include/semaphore @@ -99,8 +99,8 @@ public: } } _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void acquire() { - __cxx_atomic_wait_unless( - &__a_.__a_, [this](ptrdiff_t& __old) { return __try_acquire_impl(__old); }, memory_order_relaxed); + std::__atomic_wait_unless( + __a_, [this](ptrdiff_t& __old) { return __try_acquire_impl(__old); }, memory_order_relaxed); } template _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool