@@ -79,8 +79,6 @@ struct __empty_completion {
79
79
inline _LIBCPP_HIDE_FROM_ABI void operator ()() noexcept {}
80
80
};
81
81
82
- # ifndef _LIBCPP_HAS_NO_TREE_BARRIER
83
-
84
82
/*
85
83
86
84
The default implementation of __barrier_base is a classic tree barrier.
@@ -153,109 +151,6 @@ public:
153
151
}
154
152
};
155
153
156
- # else
157
-
158
- /*
159
-
160
- The alternative implementation of __barrier_base is a central barrier.
161
-
162
- Two versions of this algorithm are provided:
163
- 1. A fairly straightforward implementation of the litterature for the
164
- general case where the completion function is not empty.
165
- 2. An optimized implementation that exploits 2's complement arithmetic
166
- and well-defined overflow in atomic arithmetic, to handle the phase
167
- roll-over for free.
168
-
169
- */
170
-
171
- template <class _CompletionF >
172
- class __barrier_base {
173
- atomic<ptrdiff_t > __expected;
174
- atomic<ptrdiff_t > __arrived;
175
- _CompletionF __completion;
176
- atomic<bool > __phase;
177
-
178
- public:
179
- using arrival_token = bool ;
180
-
181
- static constexpr ptrdiff_t max () noexcept { return numeric_limits<ptrdiff_t >::max (); }
182
-
183
- _LIBCPP_HIDE_FROM_ABI __barrier_base (ptrdiff_t __expected, _CompletionF __completion = _CompletionF())
184
- : __expected(__expected), __arrived(__expected), __completion(std::move(__completion)), __phase(false ) {}
185
- [[nodiscard]] _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI arrival_token arrive (ptrdiff_t update) {
186
- auto const __old_phase = __phase.load (memory_order_relaxed);
187
- auto const __result = __arrived.fetch_sub (update, memory_order_acq_rel) - update;
188
- auto const new_expected = __expected.load (memory_order_relaxed);
189
-
190
- _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN (
191
- update <= new_expected, " update is greater than the expected count for the current barrier phase" );
192
-
193
- if (0 == __result) {
194
- __completion ();
195
- __arrived.store (new_expected, memory_order_relaxed);
196
- __phase.store (!__old_phase, memory_order_release);
197
- __phase.notify_all ();
198
- }
199
- return __old_phase;
200
- }
201
- _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait (arrival_token&& __old_phase) const {
202
- __phase.wait (__old_phase, memory_order_acquire);
203
- }
204
- _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_drop () {
205
- __expected.fetch_sub (1 , memory_order_relaxed);
206
- (void )arrive (1 );
207
- }
208
- };
209
-
210
- template <>
211
- class __barrier_base <__empty_completion> {
212
- static constexpr uint64_t __expected_unit = 1ull ;
213
- static constexpr uint64_t __arrived_unit = 1ull << 32 ;
214
- static constexpr uint64_t __expected_mask = __arrived_unit - 1 ;
215
- static constexpr uint64_t __phase_bit = 1ull << 63 ;
216
- static constexpr uint64_t __arrived_mask = (__phase_bit - 1 ) & ~__expected_mask;
217
-
218
- atomic<uint64_t > __phase_arrived_expected;
219
-
220
- static _LIBCPP_HIDE_FROM_ABI constexpr uint64_t __init (ptrdiff_t __count) _NOEXCEPT {
221
- return ((uint64_t (1u << 31 ) - __count) << 32 ) | (uint64_t (1u << 31 ) - __count);
222
- }
223
-
224
- public:
225
- using arrival_token = uint64_t ;
226
-
227
- static constexpr ptrdiff_t max () noexcept { return ptrdiff_t (1u << 31 ) - 1 ; }
228
-
229
- _LIBCPP_HIDE_FROM_ABI explicit inline __barrier_base (ptrdiff_t __count, __empty_completion = __empty_completion())
230
- : __phase_arrived_expected(__init(__count)) {}
231
- [[nodiscard]] inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI arrival_token arrive (ptrdiff_t update) {
232
- auto const __inc = __arrived_unit * update;
233
- auto const __old = __phase_arrived_expected.fetch_add (__inc, memory_order_acq_rel);
234
-
235
- _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN (
236
- update <= __old, " update is greater than the expected count for the current barrier phase" );
237
-
238
- if ((__old ^ (__old + __inc)) & __phase_bit) {
239
- __phase_arrived_expected.fetch_add ((__old & __expected_mask) << 32 , memory_order_relaxed);
240
- __phase_arrived_expected.notify_all ();
241
- }
242
- return __old & __phase_bit;
243
- }
244
- inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait (arrival_token&& __phase) const {
245
- auto const __test_fn = [=]() -> bool {
246
- uint64_t const __current = __phase_arrived_expected.load (memory_order_acquire);
247
- return ((__current & __phase_bit) != __phase);
248
- };
249
- __libcpp_thread_poll_with_backoff (__test_fn, __libcpp_timed_backoff_policy ());
250
- }
251
- inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_drop () {
252
- __phase_arrived_expected.fetch_add (__expected_unit, memory_order_relaxed);
253
- (void )arrive (1 );
254
- }
255
- };
256
-
257
- # endif // !_LIBCPP_HAS_NO_TREE_BARRIER
258
-
259
154
template <class _CompletionF = __empty_completion>
260
155
class barrier {
261
156
__barrier_base<_CompletionF> __b_;
0 commit comments