1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef _LIBCPP___ATOMIC_ATOMIC_H
10#define _LIBCPP___ATOMIC_ATOMIC_H
11
12#include <__atomic/atomic_sync.h>
13#include <__atomic/atomic_waitable_traits.h>
14#include <__atomic/check_memory_order.h>
15#include <__atomic/floating_point_helper.h>
16#include <__atomic/is_always_lock_free.h>
17#include <__atomic/memory_order.h>
18#include <__atomic/support.h>
19#include <__config>
20#include <__cstddef/ptrdiff_t.h>
21#include <__memory/addressof.h>
22#include <__type_traits/enable_if.h>
23#include <__type_traits/is_floating_point.h>
24#include <__type_traits/is_function.h>
25#include <__type_traits/is_integral.h>
26#include <__type_traits/is_nothrow_constructible.h>
27#include <__type_traits/is_same.h>
28#include <__type_traits/is_trivially_copyable.h>
29#include <__type_traits/remove_const.h>
30#include <__type_traits/remove_pointer.h>
31#include <__type_traits/remove_volatile.h>
32#include <__utility/forward.h>
33#include <cstring>
34
35#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
36# pragma GCC system_header
37#endif
38
39_LIBCPP_BEGIN_NAMESPACE_STD
40
41template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value>
42struct __atomic_base // false
43{
44 mutable __cxx_atomic_impl<_Tp> __a_;
45
46 using value_type = _Tp;
47
48#if _LIBCPP_STD_VER >= 17
49 static constexpr bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value;
50#endif
51
52 [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const volatile _NOEXCEPT {
53 return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>));
54 }
55 [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT {
56 return static_cast<__atomic_base const volatile*>(this)->is_lock_free();
57 }
58 _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
59 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
60 std::__cxx_atomic_store(std::addressof(__a_), __d, __m);
61 }
62 _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT
63 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
64 std::__cxx_atomic_store(std::addressof(__a_), __d, __m);
65 }
66 [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
67 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
68 return std::__cxx_atomic_load(std::addressof(__a_), __m);
69 }
70 [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
71 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
72 return std::__cxx_atomic_load(std::addressof(__a_), __m);
73 }
74 _LIBCPP_HIDE_FROM_ABI operator _Tp() const volatile _NOEXCEPT { return load(); }
75 _LIBCPP_HIDE_FROM_ABI operator _Tp() const _NOEXCEPT { return load(); }
76 _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
77 return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m);
78 }
79 _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
80 return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m);
81 }
82 _LIBCPP_HIDE_FROM_ABI bool
83 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT
84 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
85 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
86 }
87 _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT
88 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
89 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
90 }
91 _LIBCPP_HIDE_FROM_ABI bool
92 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT
93 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
94 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
95 }
96 _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT
97 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
98 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f);
99 }
100 _LIBCPP_HIDE_FROM_ABI bool
101 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
102 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
103 }
104 _LIBCPP_HIDE_FROM_ABI bool
105 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
106 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
107 }
108 _LIBCPP_HIDE_FROM_ABI bool
109 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
110 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
111 }
112 _LIBCPP_HIDE_FROM_ABI bool
113 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
114 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m);
115 }
116
117#if _LIBCPP_STD_VER >= 20
118 _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT {
119 std::__atomic_wait(*this, __v, __m);
120 }
121 _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT {
122 std::__atomic_wait(*this, __v, __m);
123 }
124 _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { std::__atomic_notify_one(*this); }
125 _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); }
126 _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { std::__atomic_notify_all(*this); }
127 _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); }
128#endif // _LIBCPP_STD_VER >= 20
129
130#if _LIBCPP_STD_VER >= 20
131 _LIBCPP_HIDE_FROM_ABI constexpr __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {}
132#else
133 _LIBCPP_HIDE_FROM_ABI __atomic_base() _NOEXCEPT = default;
134#endif
135
136 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {}
137
138 __atomic_base(const __atomic_base&) = delete;
139};
140
141// atomic<Integral>
142
143template <class _Tp>
144struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> {
145 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp, false>;
146
147 using difference_type = typename __base::value_type;
148
149 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __atomic_base() _NOEXCEPT = default;
150
151 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {}
152
153 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
154 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
155 }
156 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
157 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
158 }
159 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
160 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
161 }
162 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
163 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
164 }
165 _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
166 return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m);
167 }
168 _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
169 return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m);
170 }
171 _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
172 return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m);
173 }
174 _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
175 return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m);
176 }
177 _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
178 return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m);
179 }
180 _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
181 return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m);
182 }
183
184 _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) volatile _NOEXCEPT { return fetch_add(_Tp(1)); }
185 _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) _NOEXCEPT { return fetch_add(_Tp(1)); }
186 _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) volatile _NOEXCEPT { return fetch_sub(_Tp(1)); }
187 _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) _NOEXCEPT { return fetch_sub(_Tp(1)); }
188 _LIBCPP_HIDE_FROM_ABI _Tp operator++() volatile _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); }
189 _LIBCPP_HIDE_FROM_ABI _Tp operator++() _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); }
190 _LIBCPP_HIDE_FROM_ABI _Tp operator--() volatile _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); }
191 _LIBCPP_HIDE_FROM_ABI _Tp operator--() _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); }
192 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; }
193 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) _NOEXCEPT { return fetch_add(__op) + __op; }
194 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; }
195 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) _NOEXCEPT { return fetch_sub(__op) - __op; }
196 _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) volatile _NOEXCEPT { return fetch_and(__op) & __op; }
197 _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) _NOEXCEPT { return fetch_and(__op) & __op; }
198 _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) volatile _NOEXCEPT { return fetch_or(__op) | __op; }
199 _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) _NOEXCEPT { return fetch_or(__op) | __op; }
200 _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) volatile _NOEXCEPT { return fetch_xor(__op) ^ __op; }
201 _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; }
202};
203
204#if _LIBCPP_STD_VER >= 20
205// Here we need _IsIntegral because the default template argument is not enough
206// e.g __atomic_base<int> is __atomic_base<int, true>, which inherits from
207// __atomic_base<int, false> and the caller of the wait function is
208// __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work
209template <class _Tp, bool _IsIntegral>
210struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > {
211 using __value_type _LIBCPP_NODEBUG = _Tp;
212
213 static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) {
214 return __a.load(__order);
215 }
216
217 static _LIBCPP_HIDE_FROM_ABI _Tp
218 __atomic_load(const volatile __atomic_base<_Tp, _IsIntegral>& __this, memory_order __order) {
219 return __this.load(__order);
220 }
221
222 static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_Tp>*
223 __atomic_contention_address(const __atomic_base<_Tp, _IsIntegral>& __a) {
224 return std::addressof(__a.__a_);
225 }
226
227 static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_Tp>*
228 __atomic_contention_address(const volatile __atomic_base<_Tp, _IsIntegral>& __this) {
229 return std::addressof(__this.__a_);
230 }
231};
232
233#endif // _LIBCPP_STD_VER >= 20
234
235template <typename _Tp>
236struct __check_atomic_mandates {
237 using type _LIBCPP_NODEBUG = _Tp;
238 static_assert(is_trivially_copyable<_Tp>::value, "std::atomic<T> requires that 'T' be a trivially copyable type");
239};
240
241template <class _Tp>
242struct atomic : public __atomic_base<typename __check_atomic_mandates<_Tp>::type> {
243 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>;
244
245#if _LIBCPP_STD_VER >= 20
246 _LIBCPP_HIDE_FROM_ABI atomic() = default;
247#else
248 _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default;
249#endif
250
251 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {}
252
253 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile _NOEXCEPT {
254 __base::store(__d);
255 return __d;
256 }
257 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) _NOEXCEPT {
258 __base::store(__d);
259 return __d;
260 }
261
262 atomic& operator=(const atomic&) = delete;
263 atomic& operator=(const atomic&) volatile = delete;
264};
265
266// atomic<T*>
267
268template <class _Tp>
269struct atomic<_Tp*> : public __atomic_base<_Tp*> {
270 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp*>;
271
272 using difference_type = ptrdiff_t;
273
274 _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default;
275
276 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {}
277
278 _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) volatile _NOEXCEPT {
279 __base::store(__d);
280 return __d;
281 }
282 _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) _NOEXCEPT {
283 __base::store(__d);
284 return __d;
285 }
286
287 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
288 // __atomic_fetch_add accepts function pointers, guard against them.
289 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
290 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
291 }
292
293 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
294 // __atomic_fetch_add accepts function pointers, guard against them.
295 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
296 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);
297 }
298
299 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {
300 // __atomic_fetch_add accepts function pointers, guard against them.
301 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
302 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
303 }
304
305 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {
306 // __atomic_fetch_add accepts function pointers, guard against them.
307 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");
308 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);
309 }
310
311 _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) volatile _NOEXCEPT { return fetch_add(1); }
312 _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) _NOEXCEPT { return fetch_add(1); }
313 _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) volatile _NOEXCEPT { return fetch_sub(1); }
314 _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) _NOEXCEPT { return fetch_sub(1); }
315 _LIBCPP_HIDE_FROM_ABI _Tp* operator++() volatile _NOEXCEPT { return fetch_add(1) + 1; }
316 _LIBCPP_HIDE_FROM_ABI _Tp* operator++() _NOEXCEPT { return fetch_add(1) + 1; }
317 _LIBCPP_HIDE_FROM_ABI _Tp* operator--() volatile _NOEXCEPT { return fetch_sub(1) - 1; }
318 _LIBCPP_HIDE_FROM_ABI _Tp* operator--() _NOEXCEPT { return fetch_sub(1) - 1; }
319 _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; }
320 _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT { return fetch_add(__op) + __op; }
321 _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; }
322 _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT { return fetch_sub(__op) - __op; }
323
324 atomic& operator=(const atomic&) = delete;
325 atomic& operator=(const atomic&) volatile = delete;
326};
327
328#if _LIBCPP_STD_VER >= 20
329template <class _Tp>
330struct __atomic_waitable_traits<atomic<_Tp> > : __atomic_waitable_traits<__atomic_base<_Tp> > {};
331
332template <class _Tp>
333 requires is_floating_point_v<_Tp>
334struct atomic<_Tp> : __atomic_base<_Tp> {
335private:
336 template <class _This, class _Operation, class _BuiltinOp>
337 _LIBCPP_HIDE_FROM_ABI static _Tp
338 __rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) {
339 if constexpr (std::__has_rmw_builtin<_Tp>()) {
340 return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m);
341 } else {
342 _Tp __old = __self.load(memory_order_relaxed);
343 _Tp __new = __operation(__old, __operand);
344 while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) {
345# ifdef _LIBCPP_COMPILER_CLANG_BASED
346 if constexpr (std::__is_fp80_long_double<_Tp>()) {
347 // https://llvm.org/PR47978
348 // clang bug: __old is not updated on failure for atomic<long double>::compare_exchange_weak
349 // Note __old = __self.load(memory_order_relaxed) will not work
350 std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), std::addressof(__old), memory_order_relaxed);
351 }
352# endif
353 __new = __operation(__old, __operand);
354 }
355 return __old;
356 }
357 }
358
359 template <class _This>
360 _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) {
361 auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
362 return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order);
363 };
364 auto __plus = [](auto __a, auto __b) { return __a + __b; };
365 return __rmw_op(std::forward<_This>(__self), __operand, __m, __plus, __builtin_op);
366 }
367
368 template <class _This>
369 _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) {
370 auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
371 return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order);
372 };
373 auto __minus = [](auto __a, auto __b) { return __a - __b; };
374 return __rmw_op(std::forward<_This>(__self), __operand, __m, __minus, __builtin_op);
375 }
376
377public:
378 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>;
379 using value_type = _Tp;
380 using difference_type = value_type;
381
382 _LIBCPP_HIDE_FROM_ABI constexpr atomic() noexcept = default;
383 _LIBCPP_HIDE_FROM_ABI constexpr atomic(_Tp __d) noexcept : __base(__d) {}
384
385 atomic(const atomic&) = delete;
386 atomic& operator=(const atomic&) = delete;
387 atomic& operator=(const atomic&) volatile = delete;
388
389 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile noexcept
390 requires __base::is_always_lock_free
391 {
392 __base::store(__d);
393 return __d;
394 }
395 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) noexcept {
396 __base::store(__d);
397 return __d;
398 }
399
400 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
401 requires __base::is_always_lock_free
402 {
403 return __fetch_add(*this, __op, __m);
404 }
405
406 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
407 return __fetch_add(*this, __op, __m);
408 }
409
410 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
411 requires __base::is_always_lock_free
412 {
413 return __fetch_sub(*this, __op, __m);
414 }
415
416 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
417 return __fetch_sub(*this, __op, __m);
418 }
419
420 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile noexcept
421 requires __base::is_always_lock_free
422 {
423 return fetch_add(__op) + __op;
424 }
425
426 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; }
427
428 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile noexcept
429 requires __base::is_always_lock_free
430 {
431 return fetch_sub(__op) - __op;
432 }
433
434 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; }
435};
436
437#endif // _LIBCPP_STD_VER >= 20
438
439// atomic_is_lock_free
440
441template <class _Tp>
442[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT {
443 return __o->is_lock_free();
444}
445
446template <class _Tp>
447[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT {
448 return __o->is_lock_free();
449}
450
451// atomic_init
452
453template <class _Tp>
454_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void
455atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
456 std::__cxx_atomic_init(std::addressof(__o->__a_), __d);
457}
458
459template <class _Tp>
460_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void
461atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
462 std::__cxx_atomic_init(std::addressof(__o->__a_), __d);
463}
464
465// atomic_store
466
467template <class _Tp>
468_LIBCPP_HIDE_FROM_ABI void atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
469 __o->store(__d);
470}
471
472template <class _Tp>
473_LIBCPP_HIDE_FROM_ABI void atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
474 __o->store(__d);
475}
476
477// atomic_store_explicit
478
479template <class _Tp>
480_LIBCPP_HIDE_FROM_ABI void
481atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
482 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
483 __o->store(__d, __m);
484}
485
486template <class _Tp>
487_LIBCPP_HIDE_FROM_ABI void
488atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT
489 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
490 __o->store(__d, __m);
491}
492
493// atomic_load
494
495template <class _Tp>
496[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT {
497 return __o->load();
498}
499
500template <class _Tp>
501[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const atomic<_Tp>* __o) _NOEXCEPT {
502 return __o->load();
503}
504
505// atomic_load_explicit
506
507template <class _Tp>
508[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp
509atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
510 return __o->load(__m);
511}
512
513template <class _Tp>
514[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
515 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
516 return __o->load(__m);
517}
518
519// atomic_exchange
520
521template <class _Tp>
522_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
523 return __o->exchange(__d);
524}
525
526template <class _Tp>
527_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
528 return __o->exchange(__d);
529}
530
531// atomic_exchange_explicit
532
533template <class _Tp>
534_LIBCPP_HIDE_FROM_ABI _Tp
535atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT {
536 return __o->exchange(__d, __m);
537}
538
539template <class _Tp>
540_LIBCPP_HIDE_FROM_ABI _Tp
541atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT {
542 return __o->exchange(__d, __m);
543}
544
545// atomic_compare_exchange_weak
546
547template <class _Tp>
548_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak(
549 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
550 return __o->compare_exchange_weak(*__e, __d);
551}
552
553template <class _Tp>
554_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak(
555 atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
556 return __o->compare_exchange_weak(*__e, __d);
557}
558
559// atomic_compare_exchange_strong
560
561template <class _Tp>
562_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong(
563 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
564 return __o->compare_exchange_strong(*__e, __d);
565}
566
567template <class _Tp>
568_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong(
569 atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {
570 return __o->compare_exchange_strong(*__e, __d);
571}
572
573// atomic_compare_exchange_weak_explicit
574
575template <class _Tp>
576_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
577 volatile atomic<_Tp>* __o,
578 typename atomic<_Tp>::value_type* __e,
579 typename atomic<_Tp>::value_type __d,
580 memory_order __s,
581 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
582 return __o->compare_exchange_weak(*__e, __d, __s, __f);
583}
584
585template <class _Tp>
586_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
587 atomic<_Tp>* __o,
588 typename atomic<_Tp>::value_type* __e,
589 typename atomic<_Tp>::value_type __d,
590 memory_order __s,
591 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
592 return __o->compare_exchange_weak(*__e, __d, __s, __f);
593}
594
595// atomic_compare_exchange_strong_explicit
596
597template <class _Tp>
598_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
599 volatile atomic<_Tp>* __o,
600 typename atomic<_Tp>::value_type* __e,
601 typename atomic<_Tp>::value_type __d,
602 memory_order __s,
603 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
604 return __o->compare_exchange_strong(*__e, __d, __s, __f);
605}
606
607template <class _Tp>
608_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
609 atomic<_Tp>* __o,
610 typename atomic<_Tp>::value_type* __e,
611 typename atomic<_Tp>::value_type __d,
612 memory_order __s,
613 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {
614 return __o->compare_exchange_strong(*__e, __d, __s, __f);
615}
616
617#if _LIBCPP_STD_VER >= 20
618
619// atomic_wait
620
621template <class _Tp>
622_LIBCPP_HIDE_FROM_ABI void
623atomic_wait(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT {
624 return __o->wait(__v);
625}
626
627template <class _Tp>
628_LIBCPP_HIDE_FROM_ABI void atomic_wait(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT {
629 return __o->wait(__v);
630}
631
632// atomic_wait_explicit
633
634template <class _Tp>
635_LIBCPP_HIDE_FROM_ABI void
636atomic_wait_explicit(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT
637 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
638 return __o->wait(__v, __m);
639}
640
641template <class _Tp>
642_LIBCPP_HIDE_FROM_ABI void
643atomic_wait_explicit(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT
644 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
645 return __o->wait(__v, __m);
646}
647
648// atomic_notify_one
649
650template <class _Tp>
651_LIBCPP_HIDE_FROM_ABI void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT {
652 __o->notify_one();
653}
654template <class _Tp>
655_LIBCPP_HIDE_FROM_ABI void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT {
656 __o->notify_one();
657}
658
659// atomic_notify_all
660
661template <class _Tp>
662_LIBCPP_HIDE_FROM_ABI void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT {
663 __o->notify_all();
664}
665template <class _Tp>
666_LIBCPP_HIDE_FROM_ABI void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT {
667 __o->notify_all();
668}
669
670#endif // _LIBCPP_STD_VER >= 20
671
672// atomic_fetch_add
673
674template <class _Tp>
675_LIBCPP_HIDE_FROM_ABI _Tp
676atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
677 return __o->fetch_add(__op);
678}
679
680template <class _Tp>
681_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
682 return __o->fetch_add(__op);
683}
684
685// atomic_fetch_add_explicit
686
687template <class _Tp>
688_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add_explicit(
689 volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
690 return __o->fetch_add(__op, __m);
691}
692
693template <class _Tp>
694_LIBCPP_HIDE_FROM_ABI _Tp
695atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
696 return __o->fetch_add(__op, __m);
697}
698
699// atomic_fetch_sub
700
701template <class _Tp>
702_LIBCPP_HIDE_FROM_ABI _Tp
703atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
704 return __o->fetch_sub(__op);
705}
706
707template <class _Tp>
708_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {
709 return __o->fetch_sub(__op);
710}
711
712// atomic_fetch_sub_explicit
713
714template <class _Tp>
715_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub_explicit(
716 volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
717 return __o->fetch_sub(__op, __m);
718}
719
720template <class _Tp>
721_LIBCPP_HIDE_FROM_ABI _Tp
722atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {
723 return __o->fetch_sub(__op, __m);
724}
725
726// atomic_fetch_and
727
728template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
729_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
730 return __o->fetch_and(__op);
731}
732
733template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
734_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
735 return __o->fetch_and(__op);
736}
737
738// atomic_fetch_and_explicit
739
740template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
741_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and_explicit(
742 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
743 return __o->fetch_and(__op, __m);
744}
745
746template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
747_LIBCPP_HIDE_FROM_ABI _Tp
748atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
749 return __o->fetch_and(__op, __m);
750}
751
752// atomic_fetch_or
753
754template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
755_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
756 return __o->fetch_or(__op);
757}
758
759template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
760_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
761 return __o->fetch_or(__op);
762}
763
764// atomic_fetch_or_explicit
765
766template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
767_LIBCPP_HIDE_FROM_ABI _Tp
768atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
769 return __o->fetch_or(__op, __m);
770}
771
772template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
773_LIBCPP_HIDE_FROM_ABI _Tp
774atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
775 return __o->fetch_or(__op, __m);
776}
777
778// atomic_fetch_xor
779
780template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
781_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
782 return __o->fetch_xor(__op);
783}
784
785template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
786_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {
787 return __o->fetch_xor(__op);
788}
789
790// atomic_fetch_xor_explicit
791
792template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
793_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor_explicit(
794 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
795 return __o->fetch_xor(__op, __m);
796}
797
798template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>
799_LIBCPP_HIDE_FROM_ABI _Tp
800atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {
801 return __o->fetch_xor(__op, __m);
802}
803
804_LIBCPP_END_NAMESPACE_STD
805
806#endif // _LIBCPP___ATOMIC_ATOMIC_H