Skip to content

[libc++] refactor cxx_atomic_wait to make it reusable for atomic_ref #81427

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 2, 2024
Merged

[libc++] refactor cxx_atomic_wait to make it reusable for atomic_ref #81427

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c3e4d3a
[libc++] refactor `cxx_atomic_wait` to make it reusable for atomic_ref
huixie90 Feb 11, 2024
805e6c3
address feedback
huixie90 Mar 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 32 additions & 10 deletions libcxx/include/__atomic/atomic_base.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,24 +104,20 @@ struct __atomic_base // false

_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const
volatile _NOEXCEPT {
std::__cxx_atomic_wait(std::addressof(__a_), __v, __m);
std::__atomic_wait(*this, __v, __m);
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT {
std::__cxx_atomic_wait(std::addressof(__a_), __v, __m);
std::__atomic_wait(*this, __v, __m);
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT {
std::__cxx_atomic_notify_one(std::addressof(__a_));
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT {
std::__cxx_atomic_notify_one(std::addressof(__a_));
std::__atomic_notify_one(*this);
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); }
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT {
std::__cxx_atomic_notify_all(std::addressof(__a_));
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT {
std::__cxx_atomic_notify_all(std::addressof(__a_));
std::__atomic_notify_all(*this);
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); }

#if _LIBCPP_STD_VER >= 20
_LIBCPP_HIDE_FROM_ABI constexpr __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {}
Expand Down Expand Up @@ -200,6 +196,32 @@ struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> {
_LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; }
};

// Here we need _IsIntegral because the default template argument is not enough
// e.g __atomic_base<int> is __atomic_base<int, true>, which inherits from
// __atomic_base<int, false> and the caller of the wait function is
// __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work
template <class _Tp, bool _IsIntegral>
struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > {
static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) {
return __a.load(__order);
}

static _LIBCPP_HIDE_FROM_ABI _Tp
__atomic_load(const volatile __atomic_base<_Tp, _IsIntegral>& __this, memory_order __order) {
return __this.load(__order);
}

static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_Tp>*
__atomic_contention_address(const __atomic_base<_Tp, _IsIntegral>& __a) {
return std::addressof(__a.__a_);
}

static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_Tp>*
__atomic_contention_address(const volatile __atomic_base<_Tp, _IsIntegral>& __this) {
return std::addressof(__this.__a_);
}
};

_LIBCPP_END_NAMESPACE_STD

#endif // _LIBCPP___ATOMIC_ATOMIC_BASE_H
35 changes: 29 additions & 6 deletions libcxx/include/__atomic/atomic_flag.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include <__availability>
#include <__chrono/duration.h>
#include <__config>
#include <__memory/addressof.h>
#include <__thread/support.h>
#include <cstdint>

Expand Down Expand Up @@ -50,20 +51,20 @@ struct atomic_flag {

_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(bool __v, memory_order __m = memory_order_seq_cst) const
volatile _NOEXCEPT {
__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);
std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT {
__cxx_atomic_wait(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);
std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT {
__cxx_atomic_notify_one(&__a_);
std::__atomic_notify_one(*this);
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { __cxx_atomic_notify_one(&__a_); }
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); }
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT {
__cxx_atomic_notify_all(&__a_);
std::__atomic_notify_all(*this);
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { __cxx_atomic_notify_all(&__a_); }
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); }

#if _LIBCPP_STD_VER >= 20
_LIBCPP_HIDE_FROM_ABI constexpr atomic_flag() _NOEXCEPT : __a_(false) {}
Expand All @@ -78,6 +79,28 @@ struct atomic_flag {
atomic_flag& operator=(const atomic_flag&) volatile = delete;
};

template <>
struct __atomic_waitable_traits<atomic_flag> {
static _LIBCPP_HIDE_FROM_ABI _LIBCPP_ATOMIC_FLAG_TYPE __atomic_load(const atomic_flag& __a, memory_order __order) {
return std::__cxx_atomic_load(&__a.__a_, __order);
}

static _LIBCPP_HIDE_FROM_ABI _LIBCPP_ATOMIC_FLAG_TYPE
__atomic_load(const volatile atomic_flag& __a, memory_order __order) {
return std::__cxx_atomic_load(&__a.__a_, __order);
}

static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE>*
__atomic_contention_address(const atomic_flag& __a) {
return std::addressof(__a.__a_);
}

static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_LIBCPP_ATOMIC_FLAG_TYPE>*
__atomic_contention_address(const volatile atomic_flag& __a) {
return std::addressof(__a.__a_);
}
};

inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT { return __o->test(); }

inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const atomic_flag* __o) _NOEXCEPT { return __o->test(); }
Expand Down
128 changes: 88 additions & 40 deletions libcxx/include/__atomic/atomic_sync.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,11 @@
#include <__memory/addressof.h>
#include <__thread/poll_with_backoff.h>
#include <__thread/support.h>
#include <__type_traits/conjunction.h>
#include <__type_traits/decay.h>
#include <__type_traits/invoke.h>
#include <__type_traits/void_t.h>
#include <__utility/declval.h>
#include <cstring>

#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
Expand All @@ -27,15 +31,40 @@

_LIBCPP_BEGIN_NAMESPACE_STD

template <class _Atp, class _Poll>
struct __libcpp_atomic_wait_poll_impl {
_Atp* __a_;
// The customisation points to enable the following functions:
// - __atomic_wait
// - __atomic_wait_unless
// - __atomic_notify_one
// - __atomic_notify_all
// Note that std::atomic<T>::wait was back-ported to C++03
// The below implementations look ugly to support C++03
template <class _Tp, class = void>
struct __atomic_waitable_traits {
template <class _AtomicWaitable>
static void __atomic_load(_AtomicWaitable&&, memory_order) = delete;

template <class _AtomicWaitable>
static void __atomic_contention_address(_AtomicWaitable&&) = delete;
};

template <class _Tp, class = void>
struct __atomic_waitable : false_type {};

template <class _Tp>
struct __atomic_waitable< _Tp,
__void_t<decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_load(
std::declval<const _Tp&>(), std::declval<memory_order>())),
decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_contention_address(
std::declval<const _Tp&>()))> > : true_type {};

template <class _AtomicWaitable, class _Poll>
struct __atomic_wait_poll_impl {
const _AtomicWaitable& __a_;
_Poll __poll_;
memory_order __order_;

_LIBCPP_AVAILABILITY_SYNC
_LIBCPP_HIDE_FROM_ABI bool operator()() const {
auto __current_val = std::__cxx_atomic_load(__a_, __order_);
auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a_, __order_);
return __poll_(__current_val);
}
};
Expand All @@ -56,42 +85,45 @@ __libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*);
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
__libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t);

template <class _Atp, class _Poll>
struct __libcpp_atomic_wait_backoff_impl {
_Atp* __a_;
template <class _AtomicWaitable, class _Poll>
struct __atomic_wait_backoff_impl {
const _AtomicWaitable& __a_;
_Poll __poll_;
memory_order __order_;

using __waitable_traits = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;

_LIBCPP_AVAILABILITY_SYNC
_LIBCPP_HIDE_FROM_ABI bool
__poll_or_get_monitor(__cxx_atomic_contention_t const volatile*, __cxx_contention_t& __monitor) const {
// In case the atomic can be waited on directly, the monitor value is just
// the value of the atomic.
__update_monitor_val_and_poll(__cxx_atomic_contention_t const volatile*, __cxx_contention_t& __monitor_val) const {
// In case the contention type happens to be __cxx_atomic_contention_t, i.e. __cxx_atomic_impl<int64_t>,
// the platform wait is directly monitoring the atomic value itself.
// `__poll_` takes the current value of the atomic as an in-out argument
// to potentially modify it. After it returns, `__monitor` has a value
// which can be safely waited on by `std::__libcpp_atomic_wait` without any
// ABA style issues.
__monitor = std::__cxx_atomic_load(__a_, __order_);
return __poll_(__monitor);
__monitor_val = __waitable_traits::__atomic_load(__a_, __order_);
return __poll_(__monitor_val);
}

_LIBCPP_AVAILABILITY_SYNC
_LIBCPP_HIDE_FROM_ABI bool __poll_or_get_monitor(void const volatile*, __cxx_contention_t& __monitor) const {
// In case we must wait on an atomic from the pool, the monitor comes from
// `std::__libcpp_atomic_monitor`.
// Only then we may read from `__a_`. This is the "event count" pattern.
__monitor = std::__libcpp_atomic_monitor(__a_);
auto __current_val = std::__cxx_atomic_load(__a_, __order_);
_LIBCPP_HIDE_FROM_ABI bool
__update_monitor_val_and_poll(void const volatile* __contention_address, __cxx_contention_t& __monitor_val) const {
// In case the contention type is anything else, platform wait is monitoring a __cxx_atomic_contention_t
// from the global pool, the monitor comes from __libcpp_atomic_monitor
__monitor_val = std::__libcpp_atomic_monitor(__contention_address);
auto __current_val = __waitable_traits::__atomic_load(__a_, __order_);
return __poll_(__current_val);
}

_LIBCPP_AVAILABILITY_SYNC
_LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const {
if (__elapsed > chrono::microseconds(64)) {
__cxx_contention_t __monitor;
if (__poll_or_get_monitor(__a_, __monitor))
auto __contention_address = __waitable_traits::__atomic_contention_address(__a_);
__cxx_contention_t __monitor_val;
if (__update_monitor_val_and_poll(__contention_address, __monitor_val))
return true;
std::__libcpp_atomic_wait(__a_, __monitor);
std::__libcpp_atomic_wait(__contention_address, __monitor_val);
} else if (__elapsed > chrono::microseconds(4))
__libcpp_thread_yield();
else {
Expand All @@ -105,29 +137,44 @@ struct __libcpp_atomic_wait_backoff_impl {
// predicate (is the loaded value unequal to `old`?), the predicate function is
// specified as an argument. The loaded value is given as an in-out argument to
// the predicate. If the predicate function returns `true`,
// `_cxx_atomic_wait_unless` will return. If the predicate function returns
// `__atomic_wait_unless` will return. If the predicate function returns
// `false`, it must set the argument to its current understanding of the atomic
// value. The predicate function must not return `false` spuriously.
template <class _Atp, class _Poll>
template <class _AtomicWaitable, class _Poll>
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
__cxx_atomic_wait_unless(_Atp* __a, _Poll&& __poll, memory_order __order) {
__libcpp_atomic_wait_poll_impl<_Atp, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order};
__libcpp_atomic_wait_backoff_impl<_Atp, __decay_t<_Poll> > __backoff_fn = {__a, __poll, __order};
(void)std::__libcpp_thread_poll_with_backoff(__poll_fn, __backoff_fn);
__atomic_wait_unless(const _AtomicWaitable& __a, _Poll&& __poll, memory_order __order) {
static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
__atomic_wait_poll_impl<_AtomicWaitable, __decay_t<_Poll> > __poll_impl = {__a, __poll, __order};
__atomic_wait_backoff_impl<_AtomicWaitable, __decay_t<_Poll> > __backoff_fn = {__a, __poll, __order};
std::__libcpp_thread_poll_with_backoff(__poll_impl, __backoff_fn);
}

template <class _AtomicWaitable>
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) {
static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
std::__cxx_atomic_notify_one(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
}

template <class _AtomicWaitable>
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) {
static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
std::__cxx_atomic_notify_all(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
}

#else // _LIBCPP_HAS_NO_THREADS

template <class _Tp>
_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp> const volatile*) {}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp> const volatile*) {}
template <class _Atp, class _Poll>
_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_wait_unless(_Atp* __a, _Poll&& __poll, memory_order __order) {
__libcpp_atomic_wait_poll_impl<_Atp, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order};
(void)std::__libcpp_thread_poll_with_backoff(__poll_fn, __spinning_backoff_policy());
template <class _AtomicWaitable, class _Poll>
_LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, _Poll&& __poll, memory_order __order) {
__atomic_wait_poll_impl<_AtomicWaitable, __decay_t<_Poll> > __poll_fn = {__a, __poll, __order};
std::__libcpp_thread_poll_with_backoff(__poll_fn, __spinning_backoff_policy());
}

template <class _AtomicWaitable>
_LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable&) {}

template <class _AtomicWaitable>
_LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable&) {}

#endif // _LIBCPP_HAS_NO_THREADS

template <typename _Tp>
Expand All @@ -143,11 +190,12 @@ struct __atomic_compare_unequal_to {
}
};

template <class _Atp, class _Tp>
template <class _AtomicWaitable, class _Up>
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
__cxx_atomic_wait(_Atp* __a, _Tp const __val, memory_order __order) {
__atomic_compare_unequal_to<_Tp> __poll_fn = {__val};
std::__cxx_atomic_wait_unless(__a, __poll_fn, __order);
__atomic_wait(_AtomicWaitable& __a, _Up __val, memory_order __order) {
static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
__atomic_compare_unequal_to<_Up> __nonatomic_equal = {__val};
std::__atomic_wait_unless(__a, __nonatomic_equal, __order);
}

_LIBCPP_END_NAMESPACE_STD
Expand Down
6 changes: 3 additions & 3 deletions libcxx/include/latch
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,8 @@ public:
return try_wait_impl(__value);
}
inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait() const {
__cxx_atomic_wait_unless(
&__a_.__a_, [this](ptrdiff_t& __value) -> bool { return try_wait_impl(__value); }, memory_order_acquire);
std::__atomic_wait_unless(
__a_, [this](ptrdiff_t& __value) -> bool { return try_wait_impl(__value); }, memory_order_acquire);
}
inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_wait(ptrdiff_t __update = 1) {
_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__update >= 0, "latch::arrive_and_wait called with a negative value");
Expand All @@ -114,7 +114,7 @@ public:
}

private:
inline _LIBCPP_HIDE_FROM_ABI bool try_wait_impl(ptrdiff_t& __value) const noexcept { return __value == 0; }
_LIBCPP_HIDE_FROM_ABI bool try_wait_impl(ptrdiff_t& __value) const noexcept { return __value == 0; }
};

_LIBCPP_END_NAMESPACE_STD
Expand Down
4 changes: 2 additions & 2 deletions libcxx/include/semaphore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ public:
}
}
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void acquire() {
__cxx_atomic_wait_unless(
&__a_.__a_, [this](ptrdiff_t& __old) { return __try_acquire_impl(__old); }, memory_order_relaxed);
std::__atomic_wait_unless(
__a_, [this](ptrdiff_t& __old) { return __try_acquire_impl(__old); }, memory_order_relaxed);
}
template <class _Rep, class _Period>
_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool
Expand Down