PAPI 7.1.0.0
Loading...
Searching...
No Matches
generalize-arithm.h
Go to the documentation of this file.
1/*
2 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to deal
6 * in the Software without restriction, including without limitation the rights
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 * SOFTWARE.
21 */
22
23/* char_compare_and_swap (based on fetch_compare_and_swap) */
24#if defined(AO_HAVE_char_fetch_compare_and_swap_full) \
25 && !defined(AO_HAVE_char_compare_and_swap_full)
26 AO_INLINE int
27 AO_char_compare_and_swap_full(volatile unsignedchar *addr, unsignedchar old_val,
28 unsignedchar new_val)
29 {
30 return AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
31 == old_val;
32 }
33# define AO_HAVE_char_compare_and_swap_full
34#endif
35
36#if defined(AO_HAVE_char_fetch_compare_and_swap_acquire) \
37 && !defined(AO_HAVE_char_compare_and_swap_acquire)
38 AO_INLINE int
39 AO_char_compare_and_swap_acquire(volatile unsignedchar *addr, unsignedchar old_val,
40 unsignedchar new_val)
41 {
42 return AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val)
43 == old_val;
44 }
45# define AO_HAVE_char_compare_and_swap_acquire
46#endif
47
48#if defined(AO_HAVE_char_fetch_compare_and_swap_release) \
49 && !defined(AO_HAVE_char_compare_and_swap_release)
50 AO_INLINE int
51 AO_char_compare_and_swap_release(volatile unsignedchar *addr, unsignedchar old_val,
52 unsignedchar new_val)
53 {
54 return AO_char_fetch_compare_and_swap_release(addr, old_val, new_val)
55 == old_val;
56 }
57# define AO_HAVE_char_compare_and_swap_release
58#endif
59
60#if defined(AO_HAVE_char_fetch_compare_and_swap_write) \
61 && !defined(AO_HAVE_char_compare_and_swap_write)
62 AO_INLINE int
63 AO_char_compare_and_swap_write(volatile unsignedchar *addr, unsignedchar old_val,
64 unsignedchar new_val)
65 {
66 return AO_char_fetch_compare_and_swap_write(addr, old_val, new_val)
67 == old_val;
68 }
69# define AO_HAVE_char_compare_and_swap_write
70#endif
71
72#if defined(AO_HAVE_char_fetch_compare_and_swap_read) \
73 && !defined(AO_HAVE_char_compare_and_swap_read)
74 AO_INLINE int
75 AO_char_compare_and_swap_read(volatile unsignedchar *addr, unsignedchar old_val,
76 unsignedchar new_val)
77 {
78 return AO_char_fetch_compare_and_swap_read(addr, old_val, new_val)
79 == old_val;
80 }
81# define AO_HAVE_char_compare_and_swap_read
82#endif
83
84#if defined(AO_HAVE_char_fetch_compare_and_swap) \
85 && !defined(AO_HAVE_char_compare_and_swap)
86 AO_INLINE int
87 AO_char_compare_and_swap(volatile unsignedchar *addr, unsignedchar old_val,
88 unsignedchar new_val)
89 {
90 return AO_char_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
91 }
92# define AO_HAVE_char_compare_and_swap
93#endif
94
95#if defined(AO_HAVE_char_fetch_compare_and_swap_release_write) \
96 && !defined(AO_HAVE_char_compare_and_swap_release_write)
97 AO_INLINE int
98 AO_char_compare_and_swap_release_write(volatile unsignedchar *addr,
99 unsignedchar old_val, unsignedchar new_val)
100 {
101 return AO_char_fetch_compare_and_swap_release_write(addr, old_val,
102 new_val) == old_val;
103 }
104# define AO_HAVE_char_compare_and_swap_release_write
105#endif
106
107#if defined(AO_HAVE_char_fetch_compare_and_swap_acquire_read) \
108 && !defined(AO_HAVE_char_compare_and_swap_acquire_read)
109 AO_INLINE int
110 AO_char_compare_and_swap_acquire_read(volatile unsignedchar *addr,
111 unsignedchar old_val, unsignedchar new_val)
112 {
113 return AO_char_fetch_compare_and_swap_acquire_read(addr, old_val,
114 new_val) == old_val;
115 }
116# define AO_HAVE_char_compare_and_swap_acquire_read
117#endif
118
119#if defined(AO_HAVE_char_fetch_compare_and_swap_dd_acquire_read) \
120 && !defined(AO_HAVE_char_compare_and_swap_dd_acquire_read)
121 AO_INLINE int
122 AO_char_compare_and_swap_dd_acquire_read(volatile unsignedchar *addr,
123 unsignedchar old_val, unsignedchar new_val)
124 {
125 return AO_char_fetch_compare_and_swap_dd_acquire_read(addr, old_val,
126 new_val) == old_val;
127 }
128# define AO_HAVE_char_compare_and_swap_dd_acquire_read
129#endif
130
131/* char_fetch_and_add */
132/* We first try to implement fetch_and_add variants in terms of the */
133/* corresponding compare_and_swap variants to minimize adding barriers. */
134#if defined(AO_HAVE_char_compare_and_swap_full) \
135 && !defined(AO_HAVE_char_fetch_and_add_full)
136 AO_ATTR_NO_SANITIZE_THREAD
137 AO_INLINE unsignedchar
138 AO_char_fetch_and_add_full(volatile unsignedchar *addr, unsignedchar incr)
139 {
140 unsignedchar old;
141
142 do
143 {
144 old = *(unsignedchar *)addr;
145 }
146 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
147 old + incr)));
148 return old;
149 }
150# define AO_HAVE_char_fetch_and_add_full
151#endif
152
153#if defined(AO_HAVE_char_compare_and_swap_acquire) \
154 && !defined(AO_HAVE_char_fetch_and_add_acquire)
155 AO_ATTR_NO_SANITIZE_THREAD
156 AO_INLINE unsignedchar
157 AO_char_fetch_and_add_acquire(volatile unsignedchar *addr, unsignedchar incr)
158 {
159 unsignedchar old;
160
161 do
162 {
163 old = *(unsignedchar *)addr;
164 }
165 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_acquire(addr, old,
166 old + incr)));
167 return old;
168 }
169# define AO_HAVE_char_fetch_and_add_acquire
170#endif
171
172#if defined(AO_HAVE_char_compare_and_swap_release) \
173 && !defined(AO_HAVE_char_fetch_and_add_release)
174 AO_ATTR_NO_SANITIZE_THREAD
175 AO_INLINE unsignedchar
176 AO_char_fetch_and_add_release(volatile unsignedchar *addr, unsignedchar incr)
177 {
178 unsignedchar old;
179
180 do
181 {
182 old = *(unsignedchar *)addr;
183 }
184 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_release(addr, old,
185 old + incr)));
186 return old;
187 }
188# define AO_HAVE_char_fetch_and_add_release
189#endif
190
191#if defined(AO_HAVE_char_compare_and_swap) \
192 && !defined(AO_HAVE_char_fetch_and_add)
193 AO_ATTR_NO_SANITIZE_THREAD
194 AO_INLINE unsignedchar
195 AO_char_fetch_and_add(volatile unsignedchar *addr, unsignedchar incr)
196 {
197 unsignedchar old;
198
199 do
200 {
201 old = *(unsignedchar *)addr;
202 }
203 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap(addr, old,
204 old + incr)));
205 return old;
206 }
207# define AO_HAVE_char_fetch_and_add
208#endif
209
210#if defined(AO_HAVE_char_fetch_and_add_full)
211# if !defined(AO_HAVE_char_fetch_and_add_release)
212# define AO_char_fetch_and_add_release(addr, val) \
213 AO_char_fetch_and_add_full(addr, val)
214# define AO_HAVE_char_fetch_and_add_release
215# endif
216# if !defined(AO_HAVE_char_fetch_and_add_acquire)
217# define AO_char_fetch_and_add_acquire(addr, val) \
218 AO_char_fetch_and_add_full(addr, val)
219# define AO_HAVE_char_fetch_and_add_acquire
220# endif
221# if !defined(AO_HAVE_char_fetch_and_add_write)
222# define AO_char_fetch_and_add_write(addr, val) \
223 AO_char_fetch_and_add_full(addr, val)
224# define AO_HAVE_char_fetch_and_add_write
225# endif
226# if !defined(AO_HAVE_char_fetch_and_add_read)
227# define AO_char_fetch_and_add_read(addr, val) \
228 AO_char_fetch_and_add_full(addr, val)
229# define AO_HAVE_char_fetch_and_add_read
230# endif
231#endif /* AO_HAVE_char_fetch_and_add_full */
232
233#if defined(AO_HAVE_char_fetch_and_add) && defined(AO_HAVE_nop_full) \
234 && !defined(AO_HAVE_char_fetch_and_add_acquire)
235 AO_INLINE unsignedchar
236 AO_char_fetch_and_add_acquire(volatile unsignedchar *addr, unsignedchar incr)
237 {
238 unsignedchar result = AO_char_fetch_and_add(addr, incr);
239 AO_nop_full();
240 return result;
241 }
242# define AO_HAVE_char_fetch_and_add_acquire
243#endif
244#if defined(AO_HAVE_char_fetch_and_add) && defined(AO_HAVE_nop_full) \
245 && !defined(AO_HAVE_char_fetch_and_add_release)
246# define AO_char_fetch_and_add_release(addr, incr) \
247 (AO_nop_full(), AO_char_fetch_and_add(addr, incr))
248# define AO_HAVE_char_fetch_and_add_release
249#endif
250
251#if !defined(AO_HAVE_char_fetch_and_add) \
252 && defined(AO_HAVE_char_fetch_and_add_release)
253# define AO_char_fetch_and_add(addr, val) \
254 AO_char_fetch_and_add_release(addr, val)
255# define AO_HAVE_char_fetch_and_add
256#endif
257#if !defined(AO_HAVE_char_fetch_and_add) \
258 && defined(AO_HAVE_char_fetch_and_add_acquire)
259# define AO_char_fetch_and_add(addr, val) \
260 AO_char_fetch_and_add_acquire(addr, val)
261# define AO_HAVE_char_fetch_and_add
262#endif
263#if !defined(AO_HAVE_char_fetch_and_add) \
264 && defined(AO_HAVE_char_fetch_and_add_write)
265# define AO_char_fetch_and_add(addr, val) \
266 AO_char_fetch_and_add_write(addr, val)
267# define AO_HAVE_char_fetch_and_add
268#endif
269#if !defined(AO_HAVE_char_fetch_and_add) \
270 && defined(AO_HAVE_char_fetch_and_add_read)
271# define AO_char_fetch_and_add(addr, val) \
272 AO_char_fetch_and_add_read(addr, val)
273# define AO_HAVE_char_fetch_and_add
274#endif
275
276#if defined(AO_HAVE_char_fetch_and_add_acquire) \
277 && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_char_fetch_and_add_full)
278# define AO_char_fetch_and_add_full(addr, val) \
279 (AO_nop_full(), AO_char_fetch_and_add_acquire(addr, val))
280# define AO_HAVE_char_fetch_and_add_full
281#endif
282
283#if !defined(AO_HAVE_char_fetch_and_add_release_write) \
284 && defined(AO_HAVE_char_fetch_and_add_write)
285# define AO_char_fetch_and_add_release_write(addr, val) \
286 AO_char_fetch_and_add_write(addr, val)
287# define AO_HAVE_char_fetch_and_add_release_write
288#endif
289#if !defined(AO_HAVE_char_fetch_and_add_release_write) \
290 && defined(AO_HAVE_char_fetch_and_add_release)
291# define AO_char_fetch_and_add_release_write(addr, val) \
292 AO_char_fetch_and_add_release(addr, val)
293# define AO_HAVE_char_fetch_and_add_release_write
294#endif
295
296#if !defined(AO_HAVE_char_fetch_and_add_acquire_read) \
297 && defined(AO_HAVE_char_fetch_and_add_read)
298# define AO_char_fetch_and_add_acquire_read(addr, val) \
299 AO_char_fetch_and_add_read(addr, val)
300# define AO_HAVE_char_fetch_and_add_acquire_read
301#endif
302#if !defined(AO_HAVE_char_fetch_and_add_acquire_read) \
303 && defined(AO_HAVE_char_fetch_and_add_acquire)
304# define AO_char_fetch_and_add_acquire_read(addr, val) \
305 AO_char_fetch_and_add_acquire(addr, val)
306# define AO_HAVE_char_fetch_and_add_acquire_read
307#endif
308
309#ifdef AO_NO_DD_ORDERING
310# if defined(AO_HAVE_char_fetch_and_add_acquire_read)
311# define AO_char_fetch_and_add_dd_acquire_read(addr, val) \
312 AO_char_fetch_and_add_acquire_read(addr, val)
313# define AO_HAVE_char_fetch_and_add_dd_acquire_read
314# endif
315#else
316# if defined(AO_HAVE_char_fetch_and_add)
317# define AO_char_fetch_and_add_dd_acquire_read(addr, val) \
318 AO_char_fetch_and_add(addr, val)
319# define AO_HAVE_char_fetch_and_add_dd_acquire_read
320# endif
321#endif /* !AO_NO_DD_ORDERING */
322
323/* char_fetch_and_add1 */
324#if defined(AO_HAVE_char_fetch_and_add_full) \
325 && !defined(AO_HAVE_char_fetch_and_add1_full)
326# define AO_char_fetch_and_add1_full(addr) \
327 AO_char_fetch_and_add_full(addr, 1)
328# define AO_HAVE_char_fetch_and_add1_full
329#endif
330#if defined(AO_HAVE_char_fetch_and_add_release) \
331 && !defined(AO_HAVE_char_fetch_and_add1_release)
332# define AO_char_fetch_and_add1_release(addr) \
333 AO_char_fetch_and_add_release(addr, 1)
334# define AO_HAVE_char_fetch_and_add1_release
335#endif
336#if defined(AO_HAVE_char_fetch_and_add_acquire) \
337 && !defined(AO_HAVE_char_fetch_and_add1_acquire)
338# define AO_char_fetch_and_add1_acquire(addr) \
339 AO_char_fetch_and_add_acquire(addr, 1)
340# define AO_HAVE_char_fetch_and_add1_acquire
341#endif
342#if defined(AO_HAVE_char_fetch_and_add_write) \
343 && !defined(AO_HAVE_char_fetch_and_add1_write)
344# define AO_char_fetch_and_add1_write(addr) \
345 AO_char_fetch_and_add_write(addr, 1)
346# define AO_HAVE_char_fetch_and_add1_write
347#endif
348#if defined(AO_HAVE_char_fetch_and_add_read) \
349 && !defined(AO_HAVE_char_fetch_and_add1_read)
350# define AO_char_fetch_and_add1_read(addr) \
351 AO_char_fetch_and_add_read(addr, 1)
352# define AO_HAVE_char_fetch_and_add1_read
353#endif
354#if defined(AO_HAVE_char_fetch_and_add_release_write) \
355 && !defined(AO_HAVE_char_fetch_and_add1_release_write)
356# define AO_char_fetch_and_add1_release_write(addr) \
357 AO_char_fetch_and_add_release_write(addr, 1)
358# define AO_HAVE_char_fetch_and_add1_release_write
359#endif
360#if defined(AO_HAVE_char_fetch_and_add_acquire_read) \
361 && !defined(AO_HAVE_char_fetch_and_add1_acquire_read)
362# define AO_char_fetch_and_add1_acquire_read(addr) \
363 AO_char_fetch_and_add_acquire_read(addr, 1)
364# define AO_HAVE_char_fetch_and_add1_acquire_read
365#endif
366#if defined(AO_HAVE_char_fetch_and_add) \
367 && !defined(AO_HAVE_char_fetch_and_add1)
368# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add(addr, 1)
369# define AO_HAVE_char_fetch_and_add1
370#endif
371
372#if defined(AO_HAVE_char_fetch_and_add1_full)
373# if !defined(AO_HAVE_char_fetch_and_add1_release)
374# define AO_char_fetch_and_add1_release(addr) \
375 AO_char_fetch_and_add1_full(addr)
376# define AO_HAVE_char_fetch_and_add1_release
377# endif
378# if !defined(AO_HAVE_char_fetch_and_add1_acquire)
379# define AO_char_fetch_and_add1_acquire(addr) \
380 AO_char_fetch_and_add1_full(addr)
381# define AO_HAVE_char_fetch_and_add1_acquire
382# endif
383# if !defined(AO_HAVE_char_fetch_and_add1_write)
384# define AO_char_fetch_and_add1_write(addr) \
385 AO_char_fetch_and_add1_full(addr)
386# define AO_HAVE_char_fetch_and_add1_write
387# endif
388# if !defined(AO_HAVE_char_fetch_and_add1_read)
389# define AO_char_fetch_and_add1_read(addr) \
390 AO_char_fetch_and_add1_full(addr)
391# define AO_HAVE_char_fetch_and_add1_read
392# endif
393#endif /* AO_HAVE_char_fetch_and_add1_full */
394
395#if !defined(AO_HAVE_char_fetch_and_add1) \
396 && defined(AO_HAVE_char_fetch_and_add1_release)
397# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_release(addr)
398# define AO_HAVE_char_fetch_and_add1
399#endif
400#if !defined(AO_HAVE_char_fetch_and_add1) \
401 && defined(AO_HAVE_char_fetch_and_add1_acquire)
402# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_acquire(addr)
403# define AO_HAVE_char_fetch_and_add1
404#endif
405#if !defined(AO_HAVE_char_fetch_and_add1) \
406 && defined(AO_HAVE_char_fetch_and_add1_write)
407# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_write(addr)
408# define AO_HAVE_char_fetch_and_add1
409#endif
410#if !defined(AO_HAVE_char_fetch_and_add1) \
411 && defined(AO_HAVE_char_fetch_and_add1_read)
412# define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_read(addr)
413# define AO_HAVE_char_fetch_and_add1
414#endif
415
416#if defined(AO_HAVE_char_fetch_and_add1_acquire) \
417 && defined(AO_HAVE_nop_full) \
418 && !defined(AO_HAVE_char_fetch_and_add1_full)
419# define AO_char_fetch_and_add1_full(addr) \
420 (AO_nop_full(), AO_char_fetch_and_add1_acquire(addr))
421# define AO_HAVE_char_fetch_and_add1_full
422#endif
423
424#if !defined(AO_HAVE_char_fetch_and_add1_release_write) \
425 && defined(AO_HAVE_char_fetch_and_add1_write)
426# define AO_char_fetch_and_add1_release_write(addr) \
427 AO_char_fetch_and_add1_write(addr)
428# define AO_HAVE_char_fetch_and_add1_release_write
429#endif
430#if !defined(AO_HAVE_char_fetch_and_add1_release_write) \
431 && defined(AO_HAVE_char_fetch_and_add1_release)
432# define AO_char_fetch_and_add1_release_write(addr) \
433 AO_char_fetch_and_add1_release(addr)
434# define AO_HAVE_char_fetch_and_add1_release_write
435#endif
436#if !defined(AO_HAVE_char_fetch_and_add1_acquire_read) \
437 && defined(AO_HAVE_char_fetch_and_add1_read)
438# define AO_char_fetch_and_add1_acquire_read(addr) \
439 AO_char_fetch_and_add1_read(addr)
440# define AO_HAVE_char_fetch_and_add1_acquire_read
441#endif
442#if !defined(AO_HAVE_char_fetch_and_add1_acquire_read) \
443 && defined(AO_HAVE_char_fetch_and_add1_acquire)
444# define AO_char_fetch_and_add1_acquire_read(addr) \
445 AO_char_fetch_and_add1_acquire(addr)
446# define AO_HAVE_char_fetch_and_add1_acquire_read
447#endif
448
449#ifdef AO_NO_DD_ORDERING
450# if defined(AO_HAVE_char_fetch_and_add1_acquire_read)
451# define AO_char_fetch_and_add1_dd_acquire_read(addr) \
452 AO_char_fetch_and_add1_acquire_read(addr)
453# define AO_HAVE_char_fetch_and_add1_dd_acquire_read
454# endif
455#else
456# if defined(AO_HAVE_char_fetch_and_add1)
457# define AO_char_fetch_and_add1_dd_acquire_read(addr) \
458 AO_char_fetch_and_add1(addr)
459# define AO_HAVE_char_fetch_and_add1_dd_acquire_read
460# endif
461#endif /* !AO_NO_DD_ORDERING */
462
463/* char_fetch_and_sub1 */
464#if defined(AO_HAVE_char_fetch_and_add_full) \
465 && !defined(AO_HAVE_char_fetch_and_sub1_full)
466# define AO_char_fetch_and_sub1_full(addr) \
467 AO_char_fetch_and_add_full(addr, (unsignedchar)(-1))
468# define AO_HAVE_char_fetch_and_sub1_full
469#endif
470#if defined(AO_HAVE_char_fetch_and_add_release) \
471 && !defined(AO_HAVE_char_fetch_and_sub1_release)
472# define AO_char_fetch_and_sub1_release(addr) \
473 AO_char_fetch_and_add_release(addr, (unsignedchar)(-1))
474# define AO_HAVE_char_fetch_and_sub1_release
475#endif
476#if defined(AO_HAVE_char_fetch_and_add_acquire) \
477 && !defined(AO_HAVE_char_fetch_and_sub1_acquire)
478# define AO_char_fetch_and_sub1_acquire(addr) \
479 AO_char_fetch_and_add_acquire(addr, (unsignedchar)(-1))
480# define AO_HAVE_char_fetch_and_sub1_acquire
481#endif
482#if defined(AO_HAVE_char_fetch_and_add_write) \
483 && !defined(AO_HAVE_char_fetch_and_sub1_write)
484# define AO_char_fetch_and_sub1_write(addr) \
485 AO_char_fetch_and_add_write(addr, (unsignedchar)(-1))
486# define AO_HAVE_char_fetch_and_sub1_write
487#endif
488#if defined(AO_HAVE_char_fetch_and_add_read) \
489 && !defined(AO_HAVE_char_fetch_and_sub1_read)
490# define AO_char_fetch_and_sub1_read(addr) \
491 AO_char_fetch_and_add_read(addr, (unsignedchar)(-1))
492# define AO_HAVE_char_fetch_and_sub1_read
493#endif
494#if defined(AO_HAVE_char_fetch_and_add_release_write) \
495 && !defined(AO_HAVE_char_fetch_and_sub1_release_write)
496# define AO_char_fetch_and_sub1_release_write(addr) \
497 AO_char_fetch_and_add_release_write(addr, (unsignedchar)(-1))
498# define AO_HAVE_char_fetch_and_sub1_release_write
499#endif
500#if defined(AO_HAVE_char_fetch_and_add_acquire_read) \
501 && !defined(AO_HAVE_char_fetch_and_sub1_acquire_read)
502# define AO_char_fetch_and_sub1_acquire_read(addr) \
503 AO_char_fetch_and_add_acquire_read(addr, (unsignedchar)(-1))
504# define AO_HAVE_char_fetch_and_sub1_acquire_read
505#endif
506#if defined(AO_HAVE_char_fetch_and_add) \
507 && !defined(AO_HAVE_char_fetch_and_sub1)
508# define AO_char_fetch_and_sub1(addr) \
509 AO_char_fetch_and_add(addr, (unsignedchar)(-1))
510# define AO_HAVE_char_fetch_and_sub1
511#endif
512
513#if defined(AO_HAVE_char_fetch_and_sub1_full)
514# if !defined(AO_HAVE_char_fetch_and_sub1_release)
515# define AO_char_fetch_and_sub1_release(addr) \
516 AO_char_fetch_and_sub1_full(addr)
517# define AO_HAVE_char_fetch_and_sub1_release
518# endif
519# if !defined(AO_HAVE_char_fetch_and_sub1_acquire)
520# define AO_char_fetch_and_sub1_acquire(addr) \
521 AO_char_fetch_and_sub1_full(addr)
522# define AO_HAVE_char_fetch_and_sub1_acquire
523# endif
524# if !defined(AO_HAVE_char_fetch_and_sub1_write)
525# define AO_char_fetch_and_sub1_write(addr) \
526 AO_char_fetch_and_sub1_full(addr)
527# define AO_HAVE_char_fetch_and_sub1_write
528# endif
529# if !defined(AO_HAVE_char_fetch_and_sub1_read)
530# define AO_char_fetch_and_sub1_read(addr) \
531 AO_char_fetch_and_sub1_full(addr)
532# define AO_HAVE_char_fetch_and_sub1_read
533# endif
534#endif /* AO_HAVE_char_fetch_and_sub1_full */
535
536#if !defined(AO_HAVE_char_fetch_and_sub1) \
537 && defined(AO_HAVE_char_fetch_and_sub1_release)
538# define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_release(addr)
539# define AO_HAVE_char_fetch_and_sub1
540#endif
541#if !defined(AO_HAVE_char_fetch_and_sub1) \
542 && defined(AO_HAVE_char_fetch_and_sub1_acquire)
543# define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_acquire(addr)
544# define AO_HAVE_char_fetch_and_sub1
545#endif
546#if !defined(AO_HAVE_char_fetch_and_sub1) \
547 && defined(AO_HAVE_char_fetch_and_sub1_write)
548# define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_write(addr)
549# define AO_HAVE_char_fetch_and_sub1
550#endif
551#if !defined(AO_HAVE_char_fetch_and_sub1) \
552 && defined(AO_HAVE_char_fetch_and_sub1_read)
553# define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_read(addr)
554# define AO_HAVE_char_fetch_and_sub1
555#endif
556
557#if defined(AO_HAVE_char_fetch_and_sub1_acquire) \
558 && defined(AO_HAVE_nop_full) \
559 && !defined(AO_HAVE_char_fetch_and_sub1_full)
560# define AO_char_fetch_and_sub1_full(addr) \
561 (AO_nop_full(), AO_char_fetch_and_sub1_acquire(addr))
562# define AO_HAVE_char_fetch_and_sub1_full
563#endif
564
565#if !defined(AO_HAVE_char_fetch_and_sub1_release_write) \
566 && defined(AO_HAVE_char_fetch_and_sub1_write)
567# define AO_char_fetch_and_sub1_release_write(addr) \
568 AO_char_fetch_and_sub1_write(addr)
569# define AO_HAVE_char_fetch_and_sub1_release_write
570#endif
571#if !defined(AO_HAVE_char_fetch_and_sub1_release_write) \
572 && defined(AO_HAVE_char_fetch_and_sub1_release)
573# define AO_char_fetch_and_sub1_release_write(addr) \
574 AO_char_fetch_and_sub1_release(addr)
575# define AO_HAVE_char_fetch_and_sub1_release_write
576#endif
577#if !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) \
578 && defined(AO_HAVE_char_fetch_and_sub1_read)
579# define AO_char_fetch_and_sub1_acquire_read(addr) \
580 AO_char_fetch_and_sub1_read(addr)
581# define AO_HAVE_char_fetch_and_sub1_acquire_read
582#endif
583#if !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) \
584 && defined(AO_HAVE_char_fetch_and_sub1_acquire)
585# define AO_char_fetch_and_sub1_acquire_read(addr) \
586 AO_char_fetch_and_sub1_acquire(addr)
587# define AO_HAVE_char_fetch_and_sub1_acquire_read
588#endif
589
590#ifdef AO_NO_DD_ORDERING
591# if defined(AO_HAVE_char_fetch_and_sub1_acquire_read)
592# define AO_char_fetch_and_sub1_dd_acquire_read(addr) \
593 AO_char_fetch_and_sub1_acquire_read(addr)
594# define AO_HAVE_char_fetch_and_sub1_dd_acquire_read
595# endif
596#else
597# if defined(AO_HAVE_char_fetch_and_sub1)
598# define AO_char_fetch_and_sub1_dd_acquire_read(addr) \
599 AO_char_fetch_and_sub1(addr)
600# define AO_HAVE_char_fetch_and_sub1_dd_acquire_read
601# endif
602#endif /* !AO_NO_DD_ORDERING */
603
604/* char_and */
605#if defined(AO_HAVE_char_compare_and_swap_full) \
606 && !defined(AO_HAVE_char_and_full)
607 AO_ATTR_NO_SANITIZE_THREAD
608 AO_INLINE void
609 AO_char_and_full(volatile unsignedchar *addr, unsignedchar value)
610 {
611 unsignedchar old;
612
613 do
614 {
615 old = *(unsignedchar *)addr;
616 }
617 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
618 old & value)));
619 }
620# define AO_HAVE_char_and_full
621#endif
622
623#if defined(AO_HAVE_char_and_full)
624# if !defined(AO_HAVE_char_and_release)
625# define AO_char_and_release(addr, val) AO_char_and_full(addr, val)
626# define AO_HAVE_char_and_release
627# endif
628# if !defined(AO_HAVE_char_and_acquire)
629# define AO_char_and_acquire(addr, val) AO_char_and_full(addr, val)
630# define AO_HAVE_char_and_acquire
631# endif
632# if !defined(AO_HAVE_char_and_write)
633# define AO_char_and_write(addr, val) AO_char_and_full(addr, val)
634# define AO_HAVE_char_and_write
635# endif
636# if !defined(AO_HAVE_char_and_read)
637# define AO_char_and_read(addr, val) AO_char_and_full(addr, val)
638# define AO_HAVE_char_and_read
639# endif
640#endif /* AO_HAVE_char_and_full */
641
642#if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_release)
643# define AO_char_and(addr, val) AO_char_and_release(addr, val)
644# define AO_HAVE_char_and
645#endif
646#if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_acquire)
647# define AO_char_and(addr, val) AO_char_and_acquire(addr, val)
648# define AO_HAVE_char_and
649#endif
650#if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_write)
651# define AO_char_and(addr, val) AO_char_and_write(addr, val)
652# define AO_HAVE_char_and
653#endif
654#if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_read)
655# define AO_char_and(addr, val) AO_char_and_read(addr, val)
656# define AO_HAVE_char_and
657#endif
658
659#if defined(AO_HAVE_char_and_acquire) && defined(AO_HAVE_nop_full) \
660 && !defined(AO_HAVE_char_and_full)
661# define AO_char_and_full(addr, val) \
662 (AO_nop_full(), AO_char_and_acquire(addr, val))
663# define AO_HAVE_char_and_full
664#endif
665
666#if !defined(AO_HAVE_char_and_release_write) \
667 && defined(AO_HAVE_char_and_write)
668# define AO_char_and_release_write(addr, val) AO_char_and_write(addr, val)
669# define AO_HAVE_char_and_release_write
670#endif
671#if !defined(AO_HAVE_char_and_release_write) \
672 && defined(AO_HAVE_char_and_release)
673# define AO_char_and_release_write(addr, val) AO_char_and_release(addr, val)
674# define AO_HAVE_char_and_release_write
675#endif
676#if !defined(AO_HAVE_char_and_acquire_read) \
677 && defined(AO_HAVE_char_and_read)
678# define AO_char_and_acquire_read(addr, val) AO_char_and_read(addr, val)
679# define AO_HAVE_char_and_acquire_read
680#endif
681#if !defined(AO_HAVE_char_and_acquire_read) \
682 && defined(AO_HAVE_char_and_acquire)
683# define AO_char_and_acquire_read(addr, val) AO_char_and_acquire(addr, val)
684# define AO_HAVE_char_and_acquire_read
685#endif
686
687/* char_or */
688#if defined(AO_HAVE_char_compare_and_swap_full) \
689 && !defined(AO_HAVE_char_or_full)
690 AO_ATTR_NO_SANITIZE_THREAD
691 AO_INLINE void
692 AO_char_or_full(volatile unsignedchar *addr, unsignedchar value)
693 {
694 unsignedchar old;
695
696 do
697 {
698 old = *(unsignedchar *)addr;
699 }
700 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
701 old | value)));
702 }
703# define AO_HAVE_char_or_full
704#endif
705
706#if defined(AO_HAVE_char_or_full)
707# if !defined(AO_HAVE_char_or_release)
708# define AO_char_or_release(addr, val) AO_char_or_full(addr, val)
709# define AO_HAVE_char_or_release
710# endif
711# if !defined(AO_HAVE_char_or_acquire)
712# define AO_char_or_acquire(addr, val) AO_char_or_full(addr, val)
713# define AO_HAVE_char_or_acquire
714# endif
715# if !defined(AO_HAVE_char_or_write)
716# define AO_char_or_write(addr, val) AO_char_or_full(addr, val)
717# define AO_HAVE_char_or_write
718# endif
719# if !defined(AO_HAVE_char_or_read)
720# define AO_char_or_read(addr, val) AO_char_or_full(addr, val)
721# define AO_HAVE_char_or_read
722# endif
723#endif /* AO_HAVE_char_or_full */
724
725#if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_release)
726# define AO_char_or(addr, val) AO_char_or_release(addr, val)
727# define AO_HAVE_char_or
728#endif
729#if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_acquire)
730# define AO_char_or(addr, val) AO_char_or_acquire(addr, val)
731# define AO_HAVE_char_or
732#endif
733#if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_write)
734# define AO_char_or(addr, val) AO_char_or_write(addr, val)
735# define AO_HAVE_char_or
736#endif
737#if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_read)
738# define AO_char_or(addr, val) AO_char_or_read(addr, val)
739# define AO_HAVE_char_or
740#endif
741
742#if defined(AO_HAVE_char_or_acquire) && defined(AO_HAVE_nop_full) \
743 && !defined(AO_HAVE_char_or_full)
744# define AO_char_or_full(addr, val) \
745 (AO_nop_full(), AO_char_or_acquire(addr, val))
746# define AO_HAVE_char_or_full
747#endif
748
749#if !defined(AO_HAVE_char_or_release_write) \
750 && defined(AO_HAVE_char_or_write)
751# define AO_char_or_release_write(addr, val) AO_char_or_write(addr, val)
752# define AO_HAVE_char_or_release_write
753#endif
754#if !defined(AO_HAVE_char_or_release_write) \
755 && defined(AO_HAVE_char_or_release)
756# define AO_char_or_release_write(addr, val) AO_char_or_release(addr, val)
757# define AO_HAVE_char_or_release_write
758#endif
759#if !defined(AO_HAVE_char_or_acquire_read) && defined(AO_HAVE_char_or_read)
760# define AO_char_or_acquire_read(addr, val) AO_char_or_read(addr, val)
761# define AO_HAVE_char_or_acquire_read
762#endif
763#if !defined(AO_HAVE_char_or_acquire_read) \
764 && defined(AO_HAVE_char_or_acquire)
765# define AO_char_or_acquire_read(addr, val) AO_char_or_acquire(addr, val)
766# define AO_HAVE_char_or_acquire_read
767#endif
768
769/* char_xor */
770#if defined(AO_HAVE_char_compare_and_swap_full) \
771 && !defined(AO_HAVE_char_xor_full)
772 AO_ATTR_NO_SANITIZE_THREAD
773 AO_INLINE void
774 AO_char_xor_full(volatile unsignedchar *addr, unsignedchar value)
775 {
776 unsignedchar old;
777
778 do
779 {
780 old = *(unsignedchar *)addr;
781 }
782 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old,
783 old ^ value)));
784 }
785# define AO_HAVE_char_xor_full
786#endif
787
788#if defined(AO_HAVE_char_xor_full)
789# if !defined(AO_HAVE_char_xor_release)
790# define AO_char_xor_release(addr, val) AO_char_xor_full(addr, val)
791# define AO_HAVE_char_xor_release
792# endif
793# if !defined(AO_HAVE_char_xor_acquire)
794# define AO_char_xor_acquire(addr, val) AO_char_xor_full(addr, val)
795# define AO_HAVE_char_xor_acquire
796# endif
797# if !defined(AO_HAVE_char_xor_write)
798# define AO_char_xor_write(addr, val) AO_char_xor_full(addr, val)
799# define AO_HAVE_char_xor_write
800# endif
801# if !defined(AO_HAVE_char_xor_read)
802# define AO_char_xor_read(addr, val) AO_char_xor_full(addr, val)
803# define AO_HAVE_char_xor_read
804# endif
805#endif /* AO_HAVE_char_xor_full */
806
807#if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_release)
808# define AO_char_xor(addr, val) AO_char_xor_release(addr, val)
809# define AO_HAVE_char_xor
810#endif
811#if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_acquire)
812# define AO_char_xor(addr, val) AO_char_xor_acquire(addr, val)
813# define AO_HAVE_char_xor
814#endif
815#if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_write)
816# define AO_char_xor(addr, val) AO_char_xor_write(addr, val)
817# define AO_HAVE_char_xor
818#endif
819#if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_read)
820# define AO_char_xor(addr, val) AO_char_xor_read(addr, val)
821# define AO_HAVE_char_xor
822#endif
823
824#if defined(AO_HAVE_char_xor_acquire) && defined(AO_HAVE_nop_full) \
825 && !defined(AO_HAVE_char_xor_full)
826# define AO_char_xor_full(addr, val) \
827 (AO_nop_full(), AO_char_xor_acquire(addr, val))
828# define AO_HAVE_char_xor_full
829#endif
830
831#if !defined(AO_HAVE_char_xor_release_write) \
832 && defined(AO_HAVE_char_xor_write)
833# define AO_char_xor_release_write(addr, val) AO_char_xor_write(addr, val)
834# define AO_HAVE_char_xor_release_write
835#endif
836#if !defined(AO_HAVE_char_xor_release_write) \
837 && defined(AO_HAVE_char_xor_release)
838# define AO_char_xor_release_write(addr, val) AO_char_xor_release(addr, val)
839# define AO_HAVE_char_xor_release_write
840#endif
841#if !defined(AO_HAVE_char_xor_acquire_read) \
842 && defined(AO_HAVE_char_xor_read)
843# define AO_char_xor_acquire_read(addr, val) AO_char_xor_read(addr, val)
844# define AO_HAVE_char_xor_acquire_read
845#endif
846#if !defined(AO_HAVE_char_xor_acquire_read) \
847 && defined(AO_HAVE_char_xor_acquire)
848# define AO_char_xor_acquire_read(addr, val) AO_char_xor_acquire(addr, val)
849# define AO_HAVE_char_xor_acquire_read
850#endif
851
852/* char_and/or/xor_dd_acquire_read are meaningless. */
853/*
854 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
855 *
856 * Permission is hereby granted, free of charge, to any person obtaining a copy
857 * of this software and associated documentation files (the "Software"), to deal
858 * in the Software without restriction, including without limitation the rights
859 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
860 * copies of the Software, and to permit persons to whom the Software is
861 * furnished to do so, subject to the following conditions:
862 *
863 * The above copyright notice and this permission notice shall be included in
864 * all copies or substantial portions of the Software.
865 *
866 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
867 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
868 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
869 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
870 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
871 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
872 * SOFTWARE.
873 */
874
875/* short_compare_and_swap (based on fetch_compare_and_swap) */
876#if defined(AO_HAVE_short_fetch_compare_and_swap_full) \
877 && !defined(AO_HAVE_short_compare_and_swap_full)
878 AO_INLINE int
879 AO_short_compare_and_swap_full(volatile unsignedshort *addr, unsignedshort old_val,
880 unsignedshort new_val)
881 {
882 return AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
883 == old_val;
884 }
885# define AO_HAVE_short_compare_and_swap_full
886#endif
887
888#if defined(AO_HAVE_short_fetch_compare_and_swap_acquire) \
889 && !defined(AO_HAVE_short_compare_and_swap_acquire)
890 AO_INLINE int
891 AO_short_compare_and_swap_acquire(volatile unsignedshort *addr, unsignedshort old_val,
892 unsignedshort new_val)
893 {
894 return AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val)
895 == old_val;
896 }
897# define AO_HAVE_short_compare_and_swap_acquire
898#endif
899
900#if defined(AO_HAVE_short_fetch_compare_and_swap_release) \
901 && !defined(AO_HAVE_short_compare_and_swap_release)
902 AO_INLINE int
903 AO_short_compare_and_swap_release(volatile unsignedshort *addr, unsignedshort old_val,
904 unsignedshort new_val)
905 {
906 return AO_short_fetch_compare_and_swap_release(addr, old_val, new_val)
907 == old_val;
908 }
909# define AO_HAVE_short_compare_and_swap_release
910#endif
911
912#if defined(AO_HAVE_short_fetch_compare_and_swap_write) \
913 && !defined(AO_HAVE_short_compare_and_swap_write)
914 AO_INLINE int
915 AO_short_compare_and_swap_write(volatile unsignedshort *addr, unsignedshort old_val,
916 unsignedshort new_val)
917 {
918 return AO_short_fetch_compare_and_swap_write(addr, old_val, new_val)
919 == old_val;
920 }
921# define AO_HAVE_short_compare_and_swap_write
922#endif
923
924#if defined(AO_HAVE_short_fetch_compare_and_swap_read) \
925 && !defined(AO_HAVE_short_compare_and_swap_read)
926 AO_INLINE int
927 AO_short_compare_and_swap_read(volatile unsignedshort *addr, unsignedshort old_val,
928 unsignedshort new_val)
929 {
930 return AO_short_fetch_compare_and_swap_read(addr, old_val, new_val)
931 == old_val;
932 }
933# define AO_HAVE_short_compare_and_swap_read
934#endif
935
936#if defined(AO_HAVE_short_fetch_compare_and_swap) \
937 && !defined(AO_HAVE_short_compare_and_swap)
938 AO_INLINE int
939 AO_short_compare_and_swap(volatile unsignedshort *addr, unsignedshort old_val,
940 unsignedshort new_val)
941 {
942 return AO_short_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
943 }
944# define AO_HAVE_short_compare_and_swap
945#endif
946
947#if defined(AO_HAVE_short_fetch_compare_and_swap_release_write) \
948 && !defined(AO_HAVE_short_compare_and_swap_release_write)
949 AO_INLINE int
950 AO_short_compare_and_swap_release_write(volatile unsignedshort *addr,
951 unsignedshort old_val, unsignedshort new_val)
952 {
953 return AO_short_fetch_compare_and_swap_release_write(addr, old_val,
954 new_val) == old_val;
955 }
956# define AO_HAVE_short_compare_and_swap_release_write
957#endif
958
959#if defined(AO_HAVE_short_fetch_compare_and_swap_acquire_read) \
960 && !defined(AO_HAVE_short_compare_and_swap_acquire_read)
961 AO_INLINE int
962 AO_short_compare_and_swap_acquire_read(volatile unsignedshort *addr,
963 unsignedshort old_val, unsignedshort new_val)
964 {
965 return AO_short_fetch_compare_and_swap_acquire_read(addr, old_val,
966 new_val) == old_val;
967 }
968# define AO_HAVE_short_compare_and_swap_acquire_read
969#endif
970
971#if defined(AO_HAVE_short_fetch_compare_and_swap_dd_acquire_read) \
972 && !defined(AO_HAVE_short_compare_and_swap_dd_acquire_read)
973 AO_INLINE int
974 AO_short_compare_and_swap_dd_acquire_read(volatile unsignedshort *addr,
975 unsignedshort old_val, unsignedshort new_val)
976 {
977 return AO_short_fetch_compare_and_swap_dd_acquire_read(addr, old_val,
978 new_val) == old_val;
979 }
980# define AO_HAVE_short_compare_and_swap_dd_acquire_read
981#endif
982
983/* short_fetch_and_add */
984/* We first try to implement fetch_and_add variants in terms of the */
985/* corresponding compare_and_swap variants to minimize adding barriers. */
986#if defined(AO_HAVE_short_compare_and_swap_full) \
987 && !defined(AO_HAVE_short_fetch_and_add_full)
988 AO_ATTR_NO_SANITIZE_THREAD
989 AO_INLINE unsignedshort
990 AO_short_fetch_and_add_full(volatile unsignedshort *addr, unsignedshort incr)
991 {
992 unsignedshort old;
993
994 do
995 {
996 old = *(unsignedshort *)addr;
997 }
998 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
999 old + incr)));
1000 return old;
1001 }
1002# define AO_HAVE_short_fetch_and_add_full
1003#endif
1004
1005#if defined(AO_HAVE_short_compare_and_swap_acquire) \
1006 && !defined(AO_HAVE_short_fetch_and_add_acquire)
1007 AO_ATTR_NO_SANITIZE_THREAD
1008 AO_INLINE unsignedshort
1009 AO_short_fetch_and_add_acquire(volatile unsignedshort *addr, unsignedshort incr)
1010 {
1011 unsignedshort old;
1012
1013 do
1014 {
1015 old = *(unsignedshort *)addr;
1016 }
1017 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_acquire(addr, old,
1018 old + incr)));
1019 return old;
1020 }
1021# define AO_HAVE_short_fetch_and_add_acquire
1022#endif
1023
1024#if defined(AO_HAVE_short_compare_and_swap_release) \
1025 && !defined(AO_HAVE_short_fetch_and_add_release)
1026 AO_ATTR_NO_SANITIZE_THREAD
1027 AO_INLINE unsignedshort
1028 AO_short_fetch_and_add_release(volatile unsignedshort *addr, unsignedshort incr)
1029 {
1030 unsignedshort old;
1031
1032 do
1033 {
1034 old = *(unsignedshort *)addr;
1035 }
1036 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_release(addr, old,
1037 old + incr)));
1038 return old;
1039 }
1040# define AO_HAVE_short_fetch_and_add_release
1041#endif
1042
1043#if defined(AO_HAVE_short_compare_and_swap) \
1044 && !defined(AO_HAVE_short_fetch_and_add)
1045 AO_ATTR_NO_SANITIZE_THREAD
1046 AO_INLINE unsignedshort
1047 AO_short_fetch_and_add(volatile unsignedshort *addr, unsignedshort incr)
1048 {
1049 unsignedshort old;
1050
1051 do
1052 {
1053 old = *(unsignedshort *)addr;
1054 }
1055 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap(addr, old,
1056 old + incr)));
1057 return old;
1058 }
1059# define AO_HAVE_short_fetch_and_add
1060#endif
1061
1062#if defined(AO_HAVE_short_fetch_and_add_full)
1063# if !defined(AO_HAVE_short_fetch_and_add_release)
1064# define AO_short_fetch_and_add_release(addr, val) \
1065 AO_short_fetch_and_add_full(addr, val)
1066# define AO_HAVE_short_fetch_and_add_release
1067# endif
1068# if !defined(AO_HAVE_short_fetch_and_add_acquire)
1069# define AO_short_fetch_and_add_acquire(addr, val) \
1070 AO_short_fetch_and_add_full(addr, val)
1071# define AO_HAVE_short_fetch_and_add_acquire
1072# endif
1073# if !defined(AO_HAVE_short_fetch_and_add_write)
1074# define AO_short_fetch_and_add_write(addr, val) \
1075 AO_short_fetch_and_add_full(addr, val)
1076# define AO_HAVE_short_fetch_and_add_write
1077# endif
1078# if !defined(AO_HAVE_short_fetch_and_add_read)
1079# define AO_short_fetch_and_add_read(addr, val) \
1080 AO_short_fetch_and_add_full(addr, val)
1081# define AO_HAVE_short_fetch_and_add_read
1082# endif
1083#endif /* AO_HAVE_short_fetch_and_add_full */
1084
1085#if defined(AO_HAVE_short_fetch_and_add) && defined(AO_HAVE_nop_full) \
1086 && !defined(AO_HAVE_short_fetch_and_add_acquire)
1087 AO_INLINE unsignedshort
1088 AO_short_fetch_and_add_acquire(volatile unsignedshort *addr, unsignedshort incr)
1089 {
1090 unsignedshort result = AO_short_fetch_and_add(addr, incr);
1091 AO_nop_full();
1092 return result;
1093 }
1094# define AO_HAVE_short_fetch_and_add_acquire
1095#endif
1096#if defined(AO_HAVE_short_fetch_and_add) && defined(AO_HAVE_nop_full) \
1097 && !defined(AO_HAVE_short_fetch_and_add_release)
1098# define AO_short_fetch_and_add_release(addr, incr) \
1099 (AO_nop_full(), AO_short_fetch_and_add(addr, incr))
1100# define AO_HAVE_short_fetch_and_add_release
1101#endif
1102
1103#if !defined(AO_HAVE_short_fetch_and_add) \
1104 && defined(AO_HAVE_short_fetch_and_add_release)
1105# define AO_short_fetch_and_add(addr, val) \
1106 AO_short_fetch_and_add_release(addr, val)
1107# define AO_HAVE_short_fetch_and_add
1108#endif
1109#if !defined(AO_HAVE_short_fetch_and_add) \
1110 && defined(AO_HAVE_short_fetch_and_add_acquire)
1111# define AO_short_fetch_and_add(addr, val) \
1112 AO_short_fetch_and_add_acquire(addr, val)
1113# define AO_HAVE_short_fetch_and_add
1114#endif
1115#if !defined(AO_HAVE_short_fetch_and_add) \
1116 && defined(AO_HAVE_short_fetch_and_add_write)
1117# define AO_short_fetch_and_add(addr, val) \
1118 AO_short_fetch_and_add_write(addr, val)
1119# define AO_HAVE_short_fetch_and_add
1120#endif
1121#if !defined(AO_HAVE_short_fetch_and_add) \
1122 && defined(AO_HAVE_short_fetch_and_add_read)
1123# define AO_short_fetch_and_add(addr, val) \
1124 AO_short_fetch_and_add_read(addr, val)
1125# define AO_HAVE_short_fetch_and_add
1126#endif
1127
1128#if defined(AO_HAVE_short_fetch_and_add_acquire) \
1129 && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_short_fetch_and_add_full)
1130# define AO_short_fetch_and_add_full(addr, val) \
1131 (AO_nop_full(), AO_short_fetch_and_add_acquire(addr, val))
1132# define AO_HAVE_short_fetch_and_add_full
1133#endif
1134
1135#if !defined(AO_HAVE_short_fetch_and_add_release_write) \
1136 && defined(AO_HAVE_short_fetch_and_add_write)
1137# define AO_short_fetch_and_add_release_write(addr, val) \
1138 AO_short_fetch_and_add_write(addr, val)
1139# define AO_HAVE_short_fetch_and_add_release_write
1140#endif
1141#if !defined(AO_HAVE_short_fetch_and_add_release_write) \
1142 && defined(AO_HAVE_short_fetch_and_add_release)
1143# define AO_short_fetch_and_add_release_write(addr, val) \
1144 AO_short_fetch_and_add_release(addr, val)
1145# define AO_HAVE_short_fetch_and_add_release_write
1146#endif
1147
1148#if !defined(AO_HAVE_short_fetch_and_add_acquire_read) \
1149 && defined(AO_HAVE_short_fetch_and_add_read)
1150# define AO_short_fetch_and_add_acquire_read(addr, val) \
1151 AO_short_fetch_and_add_read(addr, val)
1152# define AO_HAVE_short_fetch_and_add_acquire_read
1153#endif
1154#if !defined(AO_HAVE_short_fetch_and_add_acquire_read) \
1155 && defined(AO_HAVE_short_fetch_and_add_acquire)
1156# define AO_short_fetch_and_add_acquire_read(addr, val) \
1157 AO_short_fetch_and_add_acquire(addr, val)
1158# define AO_HAVE_short_fetch_and_add_acquire_read
1159#endif
1160
1161#ifdef AO_NO_DD_ORDERING
1162# if defined(AO_HAVE_short_fetch_and_add_acquire_read)
1163# define AO_short_fetch_and_add_dd_acquire_read(addr, val) \
1164 AO_short_fetch_and_add_acquire_read(addr, val)
1165# define AO_HAVE_short_fetch_and_add_dd_acquire_read
1166# endif
1167#else
1168# if defined(AO_HAVE_short_fetch_and_add)
1169# define AO_short_fetch_and_add_dd_acquire_read(addr, val) \
1170 AO_short_fetch_and_add(addr, val)
1171# define AO_HAVE_short_fetch_and_add_dd_acquire_read
1172# endif
1173#endif /* !AO_NO_DD_ORDERING */
1174
1175/* short_fetch_and_add1 */
1176#if defined(AO_HAVE_short_fetch_and_add_full) \
1177 && !defined(AO_HAVE_short_fetch_and_add1_full)
1178# define AO_short_fetch_and_add1_full(addr) \
1179 AO_short_fetch_and_add_full(addr, 1)
1180# define AO_HAVE_short_fetch_and_add1_full
1181#endif
1182#if defined(AO_HAVE_short_fetch_and_add_release) \
1183 && !defined(AO_HAVE_short_fetch_and_add1_release)
1184# define AO_short_fetch_and_add1_release(addr) \
1185 AO_short_fetch_and_add_release(addr, 1)
1186# define AO_HAVE_short_fetch_and_add1_release
1187#endif
1188#if defined(AO_HAVE_short_fetch_and_add_acquire) \
1189 && !defined(AO_HAVE_short_fetch_and_add1_acquire)
1190# define AO_short_fetch_and_add1_acquire(addr) \
1191 AO_short_fetch_and_add_acquire(addr, 1)
1192# define AO_HAVE_short_fetch_and_add1_acquire
1193#endif
1194#if defined(AO_HAVE_short_fetch_and_add_write) \
1195 && !defined(AO_HAVE_short_fetch_and_add1_write)
1196# define AO_short_fetch_and_add1_write(addr) \
1197 AO_short_fetch_and_add_write(addr, 1)
1198# define AO_HAVE_short_fetch_and_add1_write
1199#endif
1200#if defined(AO_HAVE_short_fetch_and_add_read) \
1201 && !defined(AO_HAVE_short_fetch_and_add1_read)
1202# define AO_short_fetch_and_add1_read(addr) \
1203 AO_short_fetch_and_add_read(addr, 1)
1204# define AO_HAVE_short_fetch_and_add1_read
1205#endif
1206#if defined(AO_HAVE_short_fetch_and_add_release_write) \
1207 && !defined(AO_HAVE_short_fetch_and_add1_release_write)
1208# define AO_short_fetch_and_add1_release_write(addr) \
1209 AO_short_fetch_and_add_release_write(addr, 1)
1210# define AO_HAVE_short_fetch_and_add1_release_write
1211#endif
1212#if defined(AO_HAVE_short_fetch_and_add_acquire_read) \
1213 && !defined(AO_HAVE_short_fetch_and_add1_acquire_read)
1214# define AO_short_fetch_and_add1_acquire_read(addr) \
1215 AO_short_fetch_and_add_acquire_read(addr, 1)
1216# define AO_HAVE_short_fetch_and_add1_acquire_read
1217#endif
1218#if defined(AO_HAVE_short_fetch_and_add) \
1219 && !defined(AO_HAVE_short_fetch_and_add1)
1220# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add(addr, 1)
1221# define AO_HAVE_short_fetch_and_add1
1222#endif
1223
1224#if defined(AO_HAVE_short_fetch_and_add1_full)
1225# if !defined(AO_HAVE_short_fetch_and_add1_release)
1226# define AO_short_fetch_and_add1_release(addr) \
1227 AO_short_fetch_and_add1_full(addr)
1228# define AO_HAVE_short_fetch_and_add1_release
1229# endif
1230# if !defined(AO_HAVE_short_fetch_and_add1_acquire)
1231# define AO_short_fetch_and_add1_acquire(addr) \
1232 AO_short_fetch_and_add1_full(addr)
1233# define AO_HAVE_short_fetch_and_add1_acquire
1234# endif
1235# if !defined(AO_HAVE_short_fetch_and_add1_write)
1236# define AO_short_fetch_and_add1_write(addr) \
1237 AO_short_fetch_and_add1_full(addr)
1238# define AO_HAVE_short_fetch_and_add1_write
1239# endif
1240# if !defined(AO_HAVE_short_fetch_and_add1_read)
1241# define AO_short_fetch_and_add1_read(addr) \
1242 AO_short_fetch_and_add1_full(addr)
1243# define AO_HAVE_short_fetch_and_add1_read
1244# endif
1245#endif /* AO_HAVE_short_fetch_and_add1_full */
1246
1247#if !defined(AO_HAVE_short_fetch_and_add1) \
1248 && defined(AO_HAVE_short_fetch_and_add1_release)
1249# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_release(addr)
1250# define AO_HAVE_short_fetch_and_add1
1251#endif
1252#if !defined(AO_HAVE_short_fetch_and_add1) \
1253 && defined(AO_HAVE_short_fetch_and_add1_acquire)
1254# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_acquire(addr)
1255# define AO_HAVE_short_fetch_and_add1
1256#endif
1257#if !defined(AO_HAVE_short_fetch_and_add1) \
1258 && defined(AO_HAVE_short_fetch_and_add1_write)
1259# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_write(addr)
1260# define AO_HAVE_short_fetch_and_add1
1261#endif
1262#if !defined(AO_HAVE_short_fetch_and_add1) \
1263 && defined(AO_HAVE_short_fetch_and_add1_read)
1264# define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_read(addr)
1265# define AO_HAVE_short_fetch_and_add1
1266#endif
1267
1268#if defined(AO_HAVE_short_fetch_and_add1_acquire) \
1269 && defined(AO_HAVE_nop_full) \
1270 && !defined(AO_HAVE_short_fetch_and_add1_full)
1271# define AO_short_fetch_and_add1_full(addr) \
1272 (AO_nop_full(), AO_short_fetch_and_add1_acquire(addr))
1273# define AO_HAVE_short_fetch_and_add1_full
1274#endif
1275
1276#if !defined(AO_HAVE_short_fetch_and_add1_release_write) \
1277 && defined(AO_HAVE_short_fetch_and_add1_write)
1278# define AO_short_fetch_and_add1_release_write(addr) \
1279 AO_short_fetch_and_add1_write(addr)
1280# define AO_HAVE_short_fetch_and_add1_release_write
1281#endif
1282#if !defined(AO_HAVE_short_fetch_and_add1_release_write) \
1283 && defined(AO_HAVE_short_fetch_and_add1_release)
1284# define AO_short_fetch_and_add1_release_write(addr) \
1285 AO_short_fetch_and_add1_release(addr)
1286# define AO_HAVE_short_fetch_and_add1_release_write
1287#endif
1288#if !defined(AO_HAVE_short_fetch_and_add1_acquire_read) \
1289 && defined(AO_HAVE_short_fetch_and_add1_read)
1290# define AO_short_fetch_and_add1_acquire_read(addr) \
1291 AO_short_fetch_and_add1_read(addr)
1292# define AO_HAVE_short_fetch_and_add1_acquire_read
1293#endif
1294#if !defined(AO_HAVE_short_fetch_and_add1_acquire_read) \
1295 && defined(AO_HAVE_short_fetch_and_add1_acquire)
1296# define AO_short_fetch_and_add1_acquire_read(addr) \
1297 AO_short_fetch_and_add1_acquire(addr)
1298# define AO_HAVE_short_fetch_and_add1_acquire_read
1299#endif
1300
1301#ifdef AO_NO_DD_ORDERING
1302# if defined(AO_HAVE_short_fetch_and_add1_acquire_read)
1303# define AO_short_fetch_and_add1_dd_acquire_read(addr) \
1304 AO_short_fetch_and_add1_acquire_read(addr)
1305# define AO_HAVE_short_fetch_and_add1_dd_acquire_read
1306# endif
1307#else
1308# if defined(AO_HAVE_short_fetch_and_add1)
1309# define AO_short_fetch_and_add1_dd_acquire_read(addr) \
1310 AO_short_fetch_and_add1(addr)
1311# define AO_HAVE_short_fetch_and_add1_dd_acquire_read
1312# endif
1313#endif /* !AO_NO_DD_ORDERING */
1314
1315/* short_fetch_and_sub1 */
1316#if defined(AO_HAVE_short_fetch_and_add_full) \
1317 && !defined(AO_HAVE_short_fetch_and_sub1_full)
1318# define AO_short_fetch_and_sub1_full(addr) \
1319 AO_short_fetch_and_add_full(addr, (unsignedshort)(-1))
1320# define AO_HAVE_short_fetch_and_sub1_full
1321#endif
1322#if defined(AO_HAVE_short_fetch_and_add_release) \
1323 && !defined(AO_HAVE_short_fetch_and_sub1_release)
1324# define AO_short_fetch_and_sub1_release(addr) \
1325 AO_short_fetch_and_add_release(addr, (unsignedshort)(-1))
1326# define AO_HAVE_short_fetch_and_sub1_release
1327#endif
1328#if defined(AO_HAVE_short_fetch_and_add_acquire) \
1329 && !defined(AO_HAVE_short_fetch_and_sub1_acquire)
1330# define AO_short_fetch_and_sub1_acquire(addr) \
1331 AO_short_fetch_and_add_acquire(addr, (unsignedshort)(-1))
1332# define AO_HAVE_short_fetch_and_sub1_acquire
1333#endif
1334#if defined(AO_HAVE_short_fetch_and_add_write) \
1335 && !defined(AO_HAVE_short_fetch_and_sub1_write)
1336# define AO_short_fetch_and_sub1_write(addr) \
1337 AO_short_fetch_and_add_write(addr, (unsignedshort)(-1))
1338# define AO_HAVE_short_fetch_and_sub1_write
1339#endif
1340#if defined(AO_HAVE_short_fetch_and_add_read) \
1341 && !defined(AO_HAVE_short_fetch_and_sub1_read)
1342# define AO_short_fetch_and_sub1_read(addr) \
1343 AO_short_fetch_and_add_read(addr, (unsignedshort)(-1))
1344# define AO_HAVE_short_fetch_and_sub1_read
1345#endif
1346#if defined(AO_HAVE_short_fetch_and_add_release_write) \
1347 && !defined(AO_HAVE_short_fetch_and_sub1_release_write)
1348# define AO_short_fetch_and_sub1_release_write(addr) \
1349 AO_short_fetch_and_add_release_write(addr, (unsignedshort)(-1))
1350# define AO_HAVE_short_fetch_and_sub1_release_write
1351#endif
1352#if defined(AO_HAVE_short_fetch_and_add_acquire_read) \
1353 && !defined(AO_HAVE_short_fetch_and_sub1_acquire_read)
1354# define AO_short_fetch_and_sub1_acquire_read(addr) \
1355 AO_short_fetch_and_add_acquire_read(addr, (unsignedshort)(-1))
1356# define AO_HAVE_short_fetch_and_sub1_acquire_read
1357#endif
1358#if defined(AO_HAVE_short_fetch_and_add) \
1359 && !defined(AO_HAVE_short_fetch_and_sub1)
1360# define AO_short_fetch_and_sub1(addr) \
1361 AO_short_fetch_and_add(addr, (unsignedshort)(-1))
1362# define AO_HAVE_short_fetch_and_sub1
1363#endif
1364
1365#if defined(AO_HAVE_short_fetch_and_sub1_full)
1366# if !defined(AO_HAVE_short_fetch_and_sub1_release)
1367# define AO_short_fetch_and_sub1_release(addr) \
1368 AO_short_fetch_and_sub1_full(addr)
1369# define AO_HAVE_short_fetch_and_sub1_release
1370# endif
1371# if !defined(AO_HAVE_short_fetch_and_sub1_acquire)
1372# define AO_short_fetch_and_sub1_acquire(addr) \
1373 AO_short_fetch_and_sub1_full(addr)
1374# define AO_HAVE_short_fetch_and_sub1_acquire
1375# endif
1376# if !defined(AO_HAVE_short_fetch_and_sub1_write)
1377# define AO_short_fetch_and_sub1_write(addr) \
1378 AO_short_fetch_and_sub1_full(addr)
1379# define AO_HAVE_short_fetch_and_sub1_write
1380# endif
1381# if !defined(AO_HAVE_short_fetch_and_sub1_read)
1382# define AO_short_fetch_and_sub1_read(addr) \
1383 AO_short_fetch_and_sub1_full(addr)
1384# define AO_HAVE_short_fetch_and_sub1_read
1385# endif
1386#endif /* AO_HAVE_short_fetch_and_sub1_full */
1387
1388#if !defined(AO_HAVE_short_fetch_and_sub1) \
1389 && defined(AO_HAVE_short_fetch_and_sub1_release)
1390# define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_release(addr)
1391# define AO_HAVE_short_fetch_and_sub1
1392#endif
1393#if !defined(AO_HAVE_short_fetch_and_sub1) \
1394 && defined(AO_HAVE_short_fetch_and_sub1_acquire)
1395# define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_acquire(addr)
1396# define AO_HAVE_short_fetch_and_sub1
1397#endif
1398#if !defined(AO_HAVE_short_fetch_and_sub1) \
1399 && defined(AO_HAVE_short_fetch_and_sub1_write)
1400# define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_write(addr)
1401# define AO_HAVE_short_fetch_and_sub1
1402#endif
1403#if !defined(AO_HAVE_short_fetch_and_sub1) \
1404 && defined(AO_HAVE_short_fetch_and_sub1_read)
1405# define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_read(addr)
1406# define AO_HAVE_short_fetch_and_sub1
1407#endif
1408
1409#if defined(AO_HAVE_short_fetch_and_sub1_acquire) \
1410 && defined(AO_HAVE_nop_full) \
1411 && !defined(AO_HAVE_short_fetch_and_sub1_full)
1412# define AO_short_fetch_and_sub1_full(addr) \
1413 (AO_nop_full(), AO_short_fetch_and_sub1_acquire(addr))
1414# define AO_HAVE_short_fetch_and_sub1_full
1415#endif
1416
1417#if !defined(AO_HAVE_short_fetch_and_sub1_release_write) \
1418 && defined(AO_HAVE_short_fetch_and_sub1_write)
1419# define AO_short_fetch_and_sub1_release_write(addr) \
1420 AO_short_fetch_and_sub1_write(addr)
1421# define AO_HAVE_short_fetch_and_sub1_release_write
1422#endif
1423#if !defined(AO_HAVE_short_fetch_and_sub1_release_write) \
1424 && defined(AO_HAVE_short_fetch_and_sub1_release)
1425# define AO_short_fetch_and_sub1_release_write(addr) \
1426 AO_short_fetch_and_sub1_release(addr)
1427# define AO_HAVE_short_fetch_and_sub1_release_write
1428#endif
1429#if !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) \
1430 && defined(AO_HAVE_short_fetch_and_sub1_read)
1431# define AO_short_fetch_and_sub1_acquire_read(addr) \
1432 AO_short_fetch_and_sub1_read(addr)
1433# define AO_HAVE_short_fetch_and_sub1_acquire_read
1434#endif
1435#if !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) \
1436 && defined(AO_HAVE_short_fetch_and_sub1_acquire)
1437# define AO_short_fetch_and_sub1_acquire_read(addr) \
1438 AO_short_fetch_and_sub1_acquire(addr)
1439# define AO_HAVE_short_fetch_and_sub1_acquire_read
1440#endif
1441
1442#ifdef AO_NO_DD_ORDERING
1443# if defined(AO_HAVE_short_fetch_and_sub1_acquire_read)
1444# define AO_short_fetch_and_sub1_dd_acquire_read(addr) \
1445 AO_short_fetch_and_sub1_acquire_read(addr)
1446# define AO_HAVE_short_fetch_and_sub1_dd_acquire_read
1447# endif
1448#else
1449# if defined(AO_HAVE_short_fetch_and_sub1)
1450# define AO_short_fetch_and_sub1_dd_acquire_read(addr) \
1451 AO_short_fetch_and_sub1(addr)
1452# define AO_HAVE_short_fetch_and_sub1_dd_acquire_read
1453# endif
1454#endif /* !AO_NO_DD_ORDERING */
1455
1456/* short_and */
1457#if defined(AO_HAVE_short_compare_and_swap_full) \
1458 && !defined(AO_HAVE_short_and_full)
1459 AO_ATTR_NO_SANITIZE_THREAD
1460 AO_INLINE void
1461 AO_short_and_full(volatile unsignedshort *addr, unsignedshort value)
1462 {
1463 unsignedshort old;
1464
1465 do
1466 {
1467 old = *(unsignedshort *)addr;
1468 }
1469 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
1470 old & value)));
1471 }
1472# define AO_HAVE_short_and_full
1473#endif
1474
1475#if defined(AO_HAVE_short_and_full)
1476# if !defined(AO_HAVE_short_and_release)
1477# define AO_short_and_release(addr, val) AO_short_and_full(addr, val)
1478# define AO_HAVE_short_and_release
1479# endif
1480# if !defined(AO_HAVE_short_and_acquire)
1481# define AO_short_and_acquire(addr, val) AO_short_and_full(addr, val)
1482# define AO_HAVE_short_and_acquire
1483# endif
1484# if !defined(AO_HAVE_short_and_write)
1485# define AO_short_and_write(addr, val) AO_short_and_full(addr, val)
1486# define AO_HAVE_short_and_write
1487# endif
1488# if !defined(AO_HAVE_short_and_read)
1489# define AO_short_and_read(addr, val) AO_short_and_full(addr, val)
1490# define AO_HAVE_short_and_read
1491# endif
1492#endif /* AO_HAVE_short_and_full */
1493
1494#if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_release)
1495# define AO_short_and(addr, val) AO_short_and_release(addr, val)
1496# define AO_HAVE_short_and
1497#endif
1498#if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_acquire)
1499# define AO_short_and(addr, val) AO_short_and_acquire(addr, val)
1500# define AO_HAVE_short_and
1501#endif
1502#if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_write)
1503# define AO_short_and(addr, val) AO_short_and_write(addr, val)
1504# define AO_HAVE_short_and
1505#endif
1506#if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_read)
1507# define AO_short_and(addr, val) AO_short_and_read(addr, val)
1508# define AO_HAVE_short_and
1509#endif
1510
1511#if defined(AO_HAVE_short_and_acquire) && defined(AO_HAVE_nop_full) \
1512 && !defined(AO_HAVE_short_and_full)
1513# define AO_short_and_full(addr, val) \
1514 (AO_nop_full(), AO_short_and_acquire(addr, val))
1515# define AO_HAVE_short_and_full
1516#endif
1517
1518#if !defined(AO_HAVE_short_and_release_write) \
1519 && defined(AO_HAVE_short_and_write)
1520# define AO_short_and_release_write(addr, val) AO_short_and_write(addr, val)
1521# define AO_HAVE_short_and_release_write
1522#endif
1523#if !defined(AO_HAVE_short_and_release_write) \
1524 && defined(AO_HAVE_short_and_release)
1525# define AO_short_and_release_write(addr, val) AO_short_and_release(addr, val)
1526# define AO_HAVE_short_and_release_write
1527#endif
1528#if !defined(AO_HAVE_short_and_acquire_read) \
1529 && defined(AO_HAVE_short_and_read)
1530# define AO_short_and_acquire_read(addr, val) AO_short_and_read(addr, val)
1531# define AO_HAVE_short_and_acquire_read
1532#endif
1533#if !defined(AO_HAVE_short_and_acquire_read) \
1534 && defined(AO_HAVE_short_and_acquire)
1535# define AO_short_and_acquire_read(addr, val) AO_short_and_acquire(addr, val)
1536# define AO_HAVE_short_and_acquire_read
1537#endif
1538
1539/* short_or */
1540#if defined(AO_HAVE_short_compare_and_swap_full) \
1541 && !defined(AO_HAVE_short_or_full)
1542 AO_ATTR_NO_SANITIZE_THREAD
1543 AO_INLINE void
1544 AO_short_or_full(volatile unsignedshort *addr, unsignedshort value)
1545 {
1546 unsignedshort old;
1547
1548 do
1549 {
1550 old = *(unsignedshort *)addr;
1551 }
1552 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
1553 old | value)));
1554 }
1555# define AO_HAVE_short_or_full
1556#endif
1557
1558#if defined(AO_HAVE_short_or_full)
1559# if !defined(AO_HAVE_short_or_release)
1560# define AO_short_or_release(addr, val) AO_short_or_full(addr, val)
1561# define AO_HAVE_short_or_release
1562# endif
1563# if !defined(AO_HAVE_short_or_acquire)
1564# define AO_short_or_acquire(addr, val) AO_short_or_full(addr, val)
1565# define AO_HAVE_short_or_acquire
1566# endif
1567# if !defined(AO_HAVE_short_or_write)
1568# define AO_short_or_write(addr, val) AO_short_or_full(addr, val)
1569# define AO_HAVE_short_or_write
1570# endif
1571# if !defined(AO_HAVE_short_or_read)
1572# define AO_short_or_read(addr, val) AO_short_or_full(addr, val)
1573# define AO_HAVE_short_or_read
1574# endif
1575#endif /* AO_HAVE_short_or_full */
1576
1577#if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_release)
1578# define AO_short_or(addr, val) AO_short_or_release(addr, val)
1579# define AO_HAVE_short_or
1580#endif
1581#if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_acquire)
1582# define AO_short_or(addr, val) AO_short_or_acquire(addr, val)
1583# define AO_HAVE_short_or
1584#endif
1585#if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_write)
1586# define AO_short_or(addr, val) AO_short_or_write(addr, val)
1587# define AO_HAVE_short_or
1588#endif
1589#if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_read)
1590# define AO_short_or(addr, val) AO_short_or_read(addr, val)
1591# define AO_HAVE_short_or
1592#endif
1593
1594#if defined(AO_HAVE_short_or_acquire) && defined(AO_HAVE_nop_full) \
1595 && !defined(AO_HAVE_short_or_full)
1596# define AO_short_or_full(addr, val) \
1597 (AO_nop_full(), AO_short_or_acquire(addr, val))
1598# define AO_HAVE_short_or_full
1599#endif
1600
1601#if !defined(AO_HAVE_short_or_release_write) \
1602 && defined(AO_HAVE_short_or_write)
1603# define AO_short_or_release_write(addr, val) AO_short_or_write(addr, val)
1604# define AO_HAVE_short_or_release_write
1605#endif
1606#if !defined(AO_HAVE_short_or_release_write) \
1607 && defined(AO_HAVE_short_or_release)
1608# define AO_short_or_release_write(addr, val) AO_short_or_release(addr, val)
1609# define AO_HAVE_short_or_release_write
1610#endif
1611#if !defined(AO_HAVE_short_or_acquire_read) && defined(AO_HAVE_short_or_read)
1612# define AO_short_or_acquire_read(addr, val) AO_short_or_read(addr, val)
1613# define AO_HAVE_short_or_acquire_read
1614#endif
1615#if !defined(AO_HAVE_short_or_acquire_read) \
1616 && defined(AO_HAVE_short_or_acquire)
1617# define AO_short_or_acquire_read(addr, val) AO_short_or_acquire(addr, val)
1618# define AO_HAVE_short_or_acquire_read
1619#endif
1620
1621/* short_xor */
1622#if defined(AO_HAVE_short_compare_and_swap_full) \
1623 && !defined(AO_HAVE_short_xor_full)
1624 AO_ATTR_NO_SANITIZE_THREAD
1625 AO_INLINE void
1626 AO_short_xor_full(volatile unsignedshort *addr, unsignedshort value)
1627 {
1628 unsignedshort old;
1629
1630 do
1631 {
1632 old = *(unsignedshort *)addr;
1633 }
1634 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old,
1635 old ^ value)));
1636 }
1637# define AO_HAVE_short_xor_full
1638#endif
1639
1640#if defined(AO_HAVE_short_xor_full)
1641# if !defined(AO_HAVE_short_xor_release)
1642# define AO_short_xor_release(addr, val) AO_short_xor_full(addr, val)
1643# define AO_HAVE_short_xor_release
1644# endif
1645# if !defined(AO_HAVE_short_xor_acquire)
1646# define AO_short_xor_acquire(addr, val) AO_short_xor_full(addr, val)
1647# define AO_HAVE_short_xor_acquire
1648# endif
1649# if !defined(AO_HAVE_short_xor_write)
1650# define AO_short_xor_write(addr, val) AO_short_xor_full(addr, val)
1651# define AO_HAVE_short_xor_write
1652# endif
1653# if !defined(AO_HAVE_short_xor_read)
1654# define AO_short_xor_read(addr, val) AO_short_xor_full(addr, val)
1655# define AO_HAVE_short_xor_read
1656# endif
1657#endif /* AO_HAVE_short_xor_full */
1658
1659#if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_release)
1660# define AO_short_xor(addr, val) AO_short_xor_release(addr, val)
1661# define AO_HAVE_short_xor
1662#endif
1663#if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_acquire)
1664# define AO_short_xor(addr, val) AO_short_xor_acquire(addr, val)
1665# define AO_HAVE_short_xor
1666#endif
1667#if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_write)
1668# define AO_short_xor(addr, val) AO_short_xor_write(addr, val)
1669# define AO_HAVE_short_xor
1670#endif
1671#if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_read)
1672# define AO_short_xor(addr, val) AO_short_xor_read(addr, val)
1673# define AO_HAVE_short_xor
1674#endif
1675
1676#if defined(AO_HAVE_short_xor_acquire) && defined(AO_HAVE_nop_full) \
1677 && !defined(AO_HAVE_short_xor_full)
1678# define AO_short_xor_full(addr, val) \
1679 (AO_nop_full(), AO_short_xor_acquire(addr, val))
1680# define AO_HAVE_short_xor_full
1681#endif
1682
1683#if !defined(AO_HAVE_short_xor_release_write) \
1684 && defined(AO_HAVE_short_xor_write)
1685# define AO_short_xor_release_write(addr, val) AO_short_xor_write(addr, val)
1686# define AO_HAVE_short_xor_release_write
1687#endif
1688#if !defined(AO_HAVE_short_xor_release_write) \
1689 && defined(AO_HAVE_short_xor_release)
1690# define AO_short_xor_release_write(addr, val) AO_short_xor_release(addr, val)
1691# define AO_HAVE_short_xor_release_write
1692#endif
1693#if !defined(AO_HAVE_short_xor_acquire_read) \
1694 && defined(AO_HAVE_short_xor_read)
1695# define AO_short_xor_acquire_read(addr, val) AO_short_xor_read(addr, val)
1696# define AO_HAVE_short_xor_acquire_read
1697#endif
1698#if !defined(AO_HAVE_short_xor_acquire_read) \
1699 && defined(AO_HAVE_short_xor_acquire)
1700# define AO_short_xor_acquire_read(addr, val) AO_short_xor_acquire(addr, val)
1701# define AO_HAVE_short_xor_acquire_read
1702#endif
1703
1704/* short_and/or/xor_dd_acquire_read are meaningless. */
1705/*
1706 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
1707 *
1708 * Permission is hereby granted, free of charge, to any person obtaining a copy
1709 * of this software and associated documentation files (the "Software"), to deal
1710 * in the Software without restriction, including without limitation the rights
1711 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
1712 * copies of the Software, and to permit persons to whom the Software is
1713 * furnished to do so, subject to the following conditions:
1714 *
1715 * The above copyright notice and this permission notice shall be included in
1716 * all copies or substantial portions of the Software.
1717 *
1718 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1719 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1720 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
1721 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
1722 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
1723 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
1724 * SOFTWARE.
1725 */
1726
1727/* int_compare_and_swap (based on fetch_compare_and_swap) */
1728#if defined(AO_HAVE_int_fetch_compare_and_swap_full) \
1729 && !defined(AO_HAVE_int_compare_and_swap_full)
1730 AO_INLINE int
1731 AO_int_compare_and_swap_full(volatile unsigned *addr, unsigned old_val,
1732 unsigned new_val)
1733 {
1734 return AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1735 == old_val;
1736 }
1737# define AO_HAVE_int_compare_and_swap_full
1738#endif
1739
1740#if defined(AO_HAVE_int_fetch_compare_and_swap_acquire) \
1741 && !defined(AO_HAVE_int_compare_and_swap_acquire)
1742 AO_INLINE int
1743 AO_int_compare_and_swap_acquire(volatile unsigned *addr, unsigned old_val,
1744 unsigned new_val)
1745 {
1746 return AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1747 == old_val;
1748 }
1749# define AO_HAVE_int_compare_and_swap_acquire
1750#endif
1751
1752#if defined(AO_HAVE_int_fetch_compare_and_swap_release) \
1753 && !defined(AO_HAVE_int_compare_and_swap_release)
1754 AO_INLINE int
1755 AO_int_compare_and_swap_release(volatile unsigned *addr, unsigned old_val,
1756 unsigned new_val)
1757 {
1758 return AO_int_fetch_compare_and_swap_release(addr, old_val, new_val)
1759 == old_val;
1760 }
1761# define AO_HAVE_int_compare_and_swap_release
1762#endif
1763
1764#if defined(AO_HAVE_int_fetch_compare_and_swap_write) \
1765 && !defined(AO_HAVE_int_compare_and_swap_write)
1766 AO_INLINE int
1767 AO_int_compare_and_swap_write(volatile unsigned *addr, unsigned old_val,
1768 unsigned new_val)
1769 {
1770 return AO_int_fetch_compare_and_swap_write(addr, old_val, new_val)
1771 == old_val;
1772 }
1773# define AO_HAVE_int_compare_and_swap_write
1774#endif
1775
1776#if defined(AO_HAVE_int_fetch_compare_and_swap_read) \
1777 && !defined(AO_HAVE_int_compare_and_swap_read)
1778 AO_INLINE int
1779 AO_int_compare_and_swap_read(volatile unsigned *addr, unsigned old_val,
1780 unsigned new_val)
1781 {
1782 return AO_int_fetch_compare_and_swap_read(addr, old_val, new_val)
1783 == old_val;
1784 }
1785# define AO_HAVE_int_compare_and_swap_read
1786#endif
1787
1788#if defined(AO_HAVE_int_fetch_compare_and_swap) \
1789 && !defined(AO_HAVE_int_compare_and_swap)
1790 AO_INLINE int
1791 AO_int_compare_and_swap(volatile unsigned *addr, unsigned old_val,
1792 unsigned new_val)
1793 {
1794 return AO_int_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
1795 }
1796# define AO_HAVE_int_compare_and_swap
1797#endif
1798
1799#if defined(AO_HAVE_int_fetch_compare_and_swap_release_write) \
1800 && !defined(AO_HAVE_int_compare_and_swap_release_write)
1801 AO_INLINE int
1802 AO_int_compare_and_swap_release_write(volatile unsigned *addr,
1803 unsigned old_val, unsigned new_val)
1804 {
1805 return AO_int_fetch_compare_and_swap_release_write(addr, old_val,
1806 new_val) == old_val;
1807 }
1808# define AO_HAVE_int_compare_and_swap_release_write
1809#endif
1810
1811#if defined(AO_HAVE_int_fetch_compare_and_swap_acquire_read) \
1812 && !defined(AO_HAVE_int_compare_and_swap_acquire_read)
1813 AO_INLINE int
1814 AO_int_compare_and_swap_acquire_read(volatile unsigned *addr,
1815 unsigned old_val, unsigned new_val)
1816 {
1817 return AO_int_fetch_compare_and_swap_acquire_read(addr, old_val,
1818 new_val) == old_val;
1819 }
1820# define AO_HAVE_int_compare_and_swap_acquire_read
1821#endif
1822
1823#if defined(AO_HAVE_int_fetch_compare_and_swap_dd_acquire_read) \
1824 && !defined(AO_HAVE_int_compare_and_swap_dd_acquire_read)
1825 AO_INLINE int
1826 AO_int_compare_and_swap_dd_acquire_read(volatile unsigned *addr,
1827 unsigned old_val, unsigned new_val)
1828 {
1829 return AO_int_fetch_compare_and_swap_dd_acquire_read(addr, old_val,
1830 new_val) == old_val;
1831 }
1832# define AO_HAVE_int_compare_and_swap_dd_acquire_read
1833#endif
1834
1835/* int_fetch_and_add */
1836/* We first try to implement fetch_and_add variants in terms of the */
1837/* corresponding compare_and_swap variants to minimize adding barriers. */
1838#if defined(AO_HAVE_int_compare_and_swap_full) \
1839 && !defined(AO_HAVE_int_fetch_and_add_full)
1840 AO_ATTR_NO_SANITIZE_THREAD
1841 AO_INLINE unsigned
1842 AO_int_fetch_and_add_full(volatile unsigned *addr, unsigned incr)
1843 {
1844 unsigned old;
1845
1846 do
1847 {
1848 old = *(unsigned *)addr;
1849 }
1850 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
1851 old + incr)));
1852 return old;
1853 }
1854# define AO_HAVE_int_fetch_and_add_full
1855#endif
1856
1857#if defined(AO_HAVE_int_compare_and_swap_acquire) \
1858 && !defined(AO_HAVE_int_fetch_and_add_acquire)
1859 AO_ATTR_NO_SANITIZE_THREAD
1860 AO_INLINE unsigned
1861 AO_int_fetch_and_add_acquire(volatile unsigned *addr, unsigned incr)
1862 {
1863 unsigned old;
1864
1865 do
1866 {
1867 old = *(unsigned *)addr;
1868 }
1869 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_acquire(addr, old,
1870 old + incr)));
1871 return old;
1872 }
1873# define AO_HAVE_int_fetch_and_add_acquire
1874#endif
1875
1876#if defined(AO_HAVE_int_compare_and_swap_release) \
1877 && !defined(AO_HAVE_int_fetch_and_add_release)
1878 AO_ATTR_NO_SANITIZE_THREAD
1879 AO_INLINE unsigned
1880 AO_int_fetch_and_add_release(volatile unsigned *addr, unsigned incr)
1881 {
1882 unsigned old;
1883
1884 do
1885 {
1886 old = *(unsigned *)addr;
1887 }
1888 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_release(addr, old,
1889 old + incr)));
1890 return old;
1891 }
1892# define AO_HAVE_int_fetch_and_add_release
1893#endif
1894
1895#if defined(AO_HAVE_int_compare_and_swap) \
1896 && !defined(AO_HAVE_int_fetch_and_add)
1897 AO_ATTR_NO_SANITIZE_THREAD
1898 AO_INLINE unsigned
1899 AO_int_fetch_and_add(volatile unsigned *addr, unsigned incr)
1900 {
1901 unsigned old;
1902
1903 do
1904 {
1905 old = *(unsigned *)addr;
1906 }
1907 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap(addr, old,
1908 old + incr)));
1909 return old;
1910 }
1911# define AO_HAVE_int_fetch_and_add
1912#endif
1913
1914#if defined(AO_HAVE_int_fetch_and_add_full)
1915# if !defined(AO_HAVE_int_fetch_and_add_release)
1916# define AO_int_fetch_and_add_release(addr, val) \
1917 AO_int_fetch_and_add_full(addr, val)
1918# define AO_HAVE_int_fetch_and_add_release
1919# endif
1920# if !defined(AO_HAVE_int_fetch_and_add_acquire)
1921# define AO_int_fetch_and_add_acquire(addr, val) \
1922 AO_int_fetch_and_add_full(addr, val)
1923# define AO_HAVE_int_fetch_and_add_acquire
1924# endif
1925# if !defined(AO_HAVE_int_fetch_and_add_write)
1926# define AO_int_fetch_and_add_write(addr, val) \
1927 AO_int_fetch_and_add_full(addr, val)
1928# define AO_HAVE_int_fetch_and_add_write
1929# endif
1930# if !defined(AO_HAVE_int_fetch_and_add_read)
1931# define AO_int_fetch_and_add_read(addr, val) \
1932 AO_int_fetch_and_add_full(addr, val)
1933# define AO_HAVE_int_fetch_and_add_read
1934# endif
1935#endif /* AO_HAVE_int_fetch_and_add_full */
1936
1937#if defined(AO_HAVE_int_fetch_and_add) && defined(AO_HAVE_nop_full) \
1938 && !defined(AO_HAVE_int_fetch_and_add_acquire)
1939 AO_INLINE unsigned
1940 AO_int_fetch_and_add_acquire(volatile unsigned *addr, unsigned incr)
1941 {
1942 unsigned result = AO_int_fetch_and_add(addr, incr);
1943 AO_nop_full();
1944 return result;
1945 }
1946# define AO_HAVE_int_fetch_and_add_acquire
1947#endif
1948#if defined(AO_HAVE_int_fetch_and_add) && defined(AO_HAVE_nop_full) \
1949 && !defined(AO_HAVE_int_fetch_and_add_release)
1950# define AO_int_fetch_and_add_release(addr, incr) \
1951 (AO_nop_full(), AO_int_fetch_and_add(addr, incr))
1952# define AO_HAVE_int_fetch_and_add_release
1953#endif
1954
1955#if !defined(AO_HAVE_int_fetch_and_add) \
1956 && defined(AO_HAVE_int_fetch_and_add_release)
1957# define AO_int_fetch_and_add(addr, val) \
1958 AO_int_fetch_and_add_release(addr, val)
1959# define AO_HAVE_int_fetch_and_add
1960#endif
1961#if !defined(AO_HAVE_int_fetch_and_add) \
1962 && defined(AO_HAVE_int_fetch_and_add_acquire)
1963# define AO_int_fetch_and_add(addr, val) \
1964 AO_int_fetch_and_add_acquire(addr, val)
1965# define AO_HAVE_int_fetch_and_add
1966#endif
1967#if !defined(AO_HAVE_int_fetch_and_add) \
1968 && defined(AO_HAVE_int_fetch_and_add_write)
1969# define AO_int_fetch_and_add(addr, val) \
1970 AO_int_fetch_and_add_write(addr, val)
1971# define AO_HAVE_int_fetch_and_add
1972#endif
1973#if !defined(AO_HAVE_int_fetch_and_add) \
1974 && defined(AO_HAVE_int_fetch_and_add_read)
1975# define AO_int_fetch_and_add(addr, val) \
1976 AO_int_fetch_and_add_read(addr, val)
1977# define AO_HAVE_int_fetch_and_add
1978#endif
1979
1980#if defined(AO_HAVE_int_fetch_and_add_acquire) \
1981 && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_int_fetch_and_add_full)
1982# define AO_int_fetch_and_add_full(addr, val) \
1983 (AO_nop_full(), AO_int_fetch_and_add_acquire(addr, val))
1984# define AO_HAVE_int_fetch_and_add_full
1985#endif
1986
1987#if !defined(AO_HAVE_int_fetch_and_add_release_write) \
1988 && defined(AO_HAVE_int_fetch_and_add_write)
1989# define AO_int_fetch_and_add_release_write(addr, val) \
1990 AO_int_fetch_and_add_write(addr, val)
1991# define AO_HAVE_int_fetch_and_add_release_write
1992#endif
1993#if !defined(AO_HAVE_int_fetch_and_add_release_write) \
1994 && defined(AO_HAVE_int_fetch_and_add_release)
1995# define AO_int_fetch_and_add_release_write(addr, val) \
1996 AO_int_fetch_and_add_release(addr, val)
1997# define AO_HAVE_int_fetch_and_add_release_write
1998#endif
1999
2000#if !defined(AO_HAVE_int_fetch_and_add_acquire_read) \
2001 && defined(AO_HAVE_int_fetch_and_add_read)
2002# define AO_int_fetch_and_add_acquire_read(addr, val) \
2003 AO_int_fetch_and_add_read(addr, val)
2004# define AO_HAVE_int_fetch_and_add_acquire_read
2005#endif
2006#if !defined(AO_HAVE_int_fetch_and_add_acquire_read) \
2007 && defined(AO_HAVE_int_fetch_and_add_acquire)
2008# define AO_int_fetch_and_add_acquire_read(addr, val) \
2009 AO_int_fetch_and_add_acquire(addr, val)
2010# define AO_HAVE_int_fetch_and_add_acquire_read
2011#endif
2012
2013#ifdef AO_NO_DD_ORDERING
2014# if defined(AO_HAVE_int_fetch_and_add_acquire_read)
2015# define AO_int_fetch_and_add_dd_acquire_read(addr, val) \
2016 AO_int_fetch_and_add_acquire_read(addr, val)
2017# define AO_HAVE_int_fetch_and_add_dd_acquire_read
2018# endif
2019#else
2020# if defined(AO_HAVE_int_fetch_and_add)
2021# define AO_int_fetch_and_add_dd_acquire_read(addr, val) \
2022 AO_int_fetch_and_add(addr, val)
2023# define AO_HAVE_int_fetch_and_add_dd_acquire_read
2024# endif
2025#endif /* !AO_NO_DD_ORDERING */
2026
2027/* int_fetch_and_add1 */
2028#if defined(AO_HAVE_int_fetch_and_add_full) \
2029 && !defined(AO_HAVE_int_fetch_and_add1_full)
2030# define AO_int_fetch_and_add1_full(addr) \
2031 AO_int_fetch_and_add_full(addr, 1)
2032# define AO_HAVE_int_fetch_and_add1_full
2033#endif
2034#if defined(AO_HAVE_int_fetch_and_add_release) \
2035 && !defined(AO_HAVE_int_fetch_and_add1_release)
2036# define AO_int_fetch_and_add1_release(addr) \
2037 AO_int_fetch_and_add_release(addr, 1)
2038# define AO_HAVE_int_fetch_and_add1_release
2039#endif
2040#if defined(AO_HAVE_int_fetch_and_add_acquire) \
2041 && !defined(AO_HAVE_int_fetch_and_add1_acquire)
2042# define AO_int_fetch_and_add1_acquire(addr) \
2043 AO_int_fetch_and_add_acquire(addr, 1)
2044# define AO_HAVE_int_fetch_and_add1_acquire
2045#endif
2046#if defined(AO_HAVE_int_fetch_and_add_write) \
2047 && !defined(AO_HAVE_int_fetch_and_add1_write)
2048# define AO_int_fetch_and_add1_write(addr) \
2049 AO_int_fetch_and_add_write(addr, 1)
2050# define AO_HAVE_int_fetch_and_add1_write
2051#endif
2052#if defined(AO_HAVE_int_fetch_and_add_read) \
2053 && !defined(AO_HAVE_int_fetch_and_add1_read)
2054# define AO_int_fetch_and_add1_read(addr) \
2055 AO_int_fetch_and_add_read(addr, 1)
2056# define AO_HAVE_int_fetch_and_add1_read
2057#endif
2058#if defined(AO_HAVE_int_fetch_and_add_release_write) \
2059 && !defined(AO_HAVE_int_fetch_and_add1_release_write)
2060# define AO_int_fetch_and_add1_release_write(addr) \
2061 AO_int_fetch_and_add_release_write(addr, 1)
2062# define AO_HAVE_int_fetch_and_add1_release_write
2063#endif
2064#if defined(AO_HAVE_int_fetch_and_add_acquire_read) \
2065 && !defined(AO_HAVE_int_fetch_and_add1_acquire_read)
2066# define AO_int_fetch_and_add1_acquire_read(addr) \
2067 AO_int_fetch_and_add_acquire_read(addr, 1)
2068# define AO_HAVE_int_fetch_and_add1_acquire_read
2069#endif
2070#if defined(AO_HAVE_int_fetch_and_add) \
2071 && !defined(AO_HAVE_int_fetch_and_add1)
2072# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add(addr, 1)
2073# define AO_HAVE_int_fetch_and_add1
2074#endif
2075
2076#if defined(AO_HAVE_int_fetch_and_add1_full)
2077# if !defined(AO_HAVE_int_fetch_and_add1_release)
2078# define AO_int_fetch_and_add1_release(addr) \
2079 AO_int_fetch_and_add1_full(addr)
2080# define AO_HAVE_int_fetch_and_add1_release
2081# endif
2082# if !defined(AO_HAVE_int_fetch_and_add1_acquire)
2083# define AO_int_fetch_and_add1_acquire(addr) \
2084 AO_int_fetch_and_add1_full(addr)
2085# define AO_HAVE_int_fetch_and_add1_acquire
2086# endif
2087# if !defined(AO_HAVE_int_fetch_and_add1_write)
2088# define AO_int_fetch_and_add1_write(addr) \
2089 AO_int_fetch_and_add1_full(addr)
2090# define AO_HAVE_int_fetch_and_add1_write
2091# endif
2092# if !defined(AO_HAVE_int_fetch_and_add1_read)
2093# define AO_int_fetch_and_add1_read(addr) \
2094 AO_int_fetch_and_add1_full(addr)
2095# define AO_HAVE_int_fetch_and_add1_read
2096# endif
2097#endif /* AO_HAVE_int_fetch_and_add1_full */
2098
2099#if !defined(AO_HAVE_int_fetch_and_add1) \
2100 && defined(AO_HAVE_int_fetch_and_add1_release)
2101# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_release(addr)
2102# define AO_HAVE_int_fetch_and_add1
2103#endif
2104#if !defined(AO_HAVE_int_fetch_and_add1) \
2105 && defined(AO_HAVE_int_fetch_and_add1_acquire)
2106# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_acquire(addr)
2107# define AO_HAVE_int_fetch_and_add1
2108#endif
2109#if !defined(AO_HAVE_int_fetch_and_add1) \
2110 && defined(AO_HAVE_int_fetch_and_add1_write)
2111# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_write(addr)
2112# define AO_HAVE_int_fetch_and_add1
2113#endif
2114#if !defined(AO_HAVE_int_fetch_and_add1) \
2115 && defined(AO_HAVE_int_fetch_and_add1_read)
2116# define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_read(addr)
2117# define AO_HAVE_int_fetch_and_add1
2118#endif
2119
2120#if defined(AO_HAVE_int_fetch_and_add1_acquire) \
2121 && defined(AO_HAVE_nop_full) \
2122 && !defined(AO_HAVE_int_fetch_and_add1_full)
2123# define AO_int_fetch_and_add1_full(addr) \
2124 (AO_nop_full(), AO_int_fetch_and_add1_acquire(addr))
2125# define AO_HAVE_int_fetch_and_add1_full
2126#endif
2127
2128#if !defined(AO_HAVE_int_fetch_and_add1_release_write) \
2129 && defined(AO_HAVE_int_fetch_and_add1_write)
2130# define AO_int_fetch_and_add1_release_write(addr) \
2131 AO_int_fetch_and_add1_write(addr)
2132# define AO_HAVE_int_fetch_and_add1_release_write
2133#endif
2134#if !defined(AO_HAVE_int_fetch_and_add1_release_write) \
2135 && defined(AO_HAVE_int_fetch_and_add1_release)
2136# define AO_int_fetch_and_add1_release_write(addr) \
2137 AO_int_fetch_and_add1_release(addr)
2138# define AO_HAVE_int_fetch_and_add1_release_write
2139#endif
2140#if !defined(AO_HAVE_int_fetch_and_add1_acquire_read) \
2141 && defined(AO_HAVE_int_fetch_and_add1_read)
2142# define AO_int_fetch_and_add1_acquire_read(addr) \
2143 AO_int_fetch_and_add1_read(addr)
2144# define AO_HAVE_int_fetch_and_add1_acquire_read
2145#endif
2146#if !defined(AO_HAVE_int_fetch_and_add1_acquire_read) \
2147 && defined(AO_HAVE_int_fetch_and_add1_acquire)
2148# define AO_int_fetch_and_add1_acquire_read(addr) \
2149 AO_int_fetch_and_add1_acquire(addr)
2150# define AO_HAVE_int_fetch_and_add1_acquire_read
2151#endif
2152
2153#ifdef AO_NO_DD_ORDERING
2154# if defined(AO_HAVE_int_fetch_and_add1_acquire_read)
2155# define AO_int_fetch_and_add1_dd_acquire_read(addr) \
2156 AO_int_fetch_and_add1_acquire_read(addr)
2157# define AO_HAVE_int_fetch_and_add1_dd_acquire_read
2158# endif
2159#else
2160# if defined(AO_HAVE_int_fetch_and_add1)
2161# define AO_int_fetch_and_add1_dd_acquire_read(addr) \
2162 AO_int_fetch_and_add1(addr)
2163# define AO_HAVE_int_fetch_and_add1_dd_acquire_read
2164# endif
2165#endif /* !AO_NO_DD_ORDERING */
2166
2167/* int_fetch_and_sub1 */
2168#if defined(AO_HAVE_int_fetch_and_add_full) \
2169 && !defined(AO_HAVE_int_fetch_and_sub1_full)
2170# define AO_int_fetch_and_sub1_full(addr) \
2171 AO_int_fetch_and_add_full(addr, (unsigned)(-1))
2172# define AO_HAVE_int_fetch_and_sub1_full
2173#endif
2174#if defined(AO_HAVE_int_fetch_and_add_release) \
2175 && !defined(AO_HAVE_int_fetch_and_sub1_release)
2176# define AO_int_fetch_and_sub1_release(addr) \
2177 AO_int_fetch_and_add_release(addr, (unsigned)(-1))
2178# define AO_HAVE_int_fetch_and_sub1_release
2179#endif
2180#if defined(AO_HAVE_int_fetch_and_add_acquire) \
2181 && !defined(AO_HAVE_int_fetch_and_sub1_acquire)
2182# define AO_int_fetch_and_sub1_acquire(addr) \
2183 AO_int_fetch_and_add_acquire(addr, (unsigned)(-1))
2184# define AO_HAVE_int_fetch_and_sub1_acquire
2185#endif
2186#if defined(AO_HAVE_int_fetch_and_add_write) \
2187 && !defined(AO_HAVE_int_fetch_and_sub1_write)
2188# define AO_int_fetch_and_sub1_write(addr) \
2189 AO_int_fetch_and_add_write(addr, (unsigned)(-1))
2190# define AO_HAVE_int_fetch_and_sub1_write
2191#endif
2192#if defined(AO_HAVE_int_fetch_and_add_read) \
2193 && !defined(AO_HAVE_int_fetch_and_sub1_read)
2194# define AO_int_fetch_and_sub1_read(addr) \
2195 AO_int_fetch_and_add_read(addr, (unsigned)(-1))
2196# define AO_HAVE_int_fetch_and_sub1_read
2197#endif
2198#if defined(AO_HAVE_int_fetch_and_add_release_write) \
2199 && !defined(AO_HAVE_int_fetch_and_sub1_release_write)
2200# define AO_int_fetch_and_sub1_release_write(addr) \
2201 AO_int_fetch_and_add_release_write(addr, (unsigned)(-1))
2202# define AO_HAVE_int_fetch_and_sub1_release_write
2203#endif
2204#if defined(AO_HAVE_int_fetch_and_add_acquire_read) \
2205 && !defined(AO_HAVE_int_fetch_and_sub1_acquire_read)
2206# define AO_int_fetch_and_sub1_acquire_read(addr) \
2207 AO_int_fetch_and_add_acquire_read(addr, (unsigned)(-1))
2208# define AO_HAVE_int_fetch_and_sub1_acquire_read
2209#endif
2210#if defined(AO_HAVE_int_fetch_and_add) \
2211 && !defined(AO_HAVE_int_fetch_and_sub1)
2212# define AO_int_fetch_and_sub1(addr) \
2213 AO_int_fetch_and_add(addr, (unsigned)(-1))
2214# define AO_HAVE_int_fetch_and_sub1
2215#endif
2216
2217#if defined(AO_HAVE_int_fetch_and_sub1_full)
2218# if !defined(AO_HAVE_int_fetch_and_sub1_release)
2219# define AO_int_fetch_and_sub1_release(addr) \
2220 AO_int_fetch_and_sub1_full(addr)
2221# define AO_HAVE_int_fetch_and_sub1_release
2222# endif
2223# if !defined(AO_HAVE_int_fetch_and_sub1_acquire)
2224# define AO_int_fetch_and_sub1_acquire(addr) \
2225 AO_int_fetch_and_sub1_full(addr)
2226# define AO_HAVE_int_fetch_and_sub1_acquire
2227# endif
2228# if !defined(AO_HAVE_int_fetch_and_sub1_write)
2229# define AO_int_fetch_and_sub1_write(addr) \
2230 AO_int_fetch_and_sub1_full(addr)
2231# define AO_HAVE_int_fetch_and_sub1_write
2232# endif
2233# if !defined(AO_HAVE_int_fetch_and_sub1_read)
2234# define AO_int_fetch_and_sub1_read(addr) \
2235 AO_int_fetch_and_sub1_full(addr)
2236# define AO_HAVE_int_fetch_and_sub1_read
2237# endif
2238#endif /* AO_HAVE_int_fetch_and_sub1_full */
2239
2240#if !defined(AO_HAVE_int_fetch_and_sub1) \
2241 && defined(AO_HAVE_int_fetch_and_sub1_release)
2242# define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_release(addr)
2243# define AO_HAVE_int_fetch_and_sub1
2244#endif
2245#if !defined(AO_HAVE_int_fetch_and_sub1) \
2246 && defined(AO_HAVE_int_fetch_and_sub1_acquire)
2247# define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_acquire(addr)
2248# define AO_HAVE_int_fetch_and_sub1
2249#endif
2250#if !defined(AO_HAVE_int_fetch_and_sub1) \
2251 && defined(AO_HAVE_int_fetch_and_sub1_write)
2252# define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_write(addr)
2253# define AO_HAVE_int_fetch_and_sub1
2254#endif
2255#if !defined(AO_HAVE_int_fetch_and_sub1) \
2256 && defined(AO_HAVE_int_fetch_and_sub1_read)
2257# define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_read(addr)
2258# define AO_HAVE_int_fetch_and_sub1
2259#endif
2260
2261#if defined(AO_HAVE_int_fetch_and_sub1_acquire) \
2262 && defined(AO_HAVE_nop_full) \
2263 && !defined(AO_HAVE_int_fetch_and_sub1_full)
2264# define AO_int_fetch_and_sub1_full(addr) \
2265 (AO_nop_full(), AO_int_fetch_and_sub1_acquire(addr))
2266# define AO_HAVE_int_fetch_and_sub1_full
2267#endif
2268
2269#if !defined(AO_HAVE_int_fetch_and_sub1_release_write) \
2270 && defined(AO_HAVE_int_fetch_and_sub1_write)
2271# define AO_int_fetch_and_sub1_release_write(addr) \
2272 AO_int_fetch_and_sub1_write(addr)
2273# define AO_HAVE_int_fetch_and_sub1_release_write
2274#endif
2275#if !defined(AO_HAVE_int_fetch_and_sub1_release_write) \
2276 && defined(AO_HAVE_int_fetch_and_sub1_release)
2277# define AO_int_fetch_and_sub1_release_write(addr) \
2278 AO_int_fetch_and_sub1_release(addr)
2279# define AO_HAVE_int_fetch_and_sub1_release_write
2280#endif
2281#if !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) \
2282 && defined(AO_HAVE_int_fetch_and_sub1_read)
2283# define AO_int_fetch_and_sub1_acquire_read(addr) \
2284 AO_int_fetch_and_sub1_read(addr)
2285# define AO_HAVE_int_fetch_and_sub1_acquire_read
2286#endif
2287#if !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) \
2288 && defined(AO_HAVE_int_fetch_and_sub1_acquire)
2289# define AO_int_fetch_and_sub1_acquire_read(addr) \
2290 AO_int_fetch_and_sub1_acquire(addr)
2291# define AO_HAVE_int_fetch_and_sub1_acquire_read
2292#endif
2293
2294#ifdef AO_NO_DD_ORDERING
2295# if defined(AO_HAVE_int_fetch_and_sub1_acquire_read)
2296# define AO_int_fetch_and_sub1_dd_acquire_read(addr) \
2297 AO_int_fetch_and_sub1_acquire_read(addr)
2298# define AO_HAVE_int_fetch_and_sub1_dd_acquire_read
2299# endif
2300#else
2301# if defined(AO_HAVE_int_fetch_and_sub1)
2302# define AO_int_fetch_and_sub1_dd_acquire_read(addr) \
2303 AO_int_fetch_and_sub1(addr)
2304# define AO_HAVE_int_fetch_and_sub1_dd_acquire_read
2305# endif
2306#endif /* !AO_NO_DD_ORDERING */
2307
2308/* int_and */
2309#if defined(AO_HAVE_int_compare_and_swap_full) \
2310 && !defined(AO_HAVE_int_and_full)
2311 AO_ATTR_NO_SANITIZE_THREAD
2312 AO_INLINE void
2313 AO_int_and_full(volatile unsigned *addr, unsigned value)
2314 {
2315 unsigned old;
2316
2317 do
2318 {
2319 old = *(unsigned *)addr;
2320 }
2321 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
2322 old & value)));
2323 }
2324# define AO_HAVE_int_and_full
2325#endif
2326
2327#if defined(AO_HAVE_int_and_full)
2328# if !defined(AO_HAVE_int_and_release)
2329# define AO_int_and_release(addr, val) AO_int_and_full(addr, val)
2330# define AO_HAVE_int_and_release
2331# endif
2332# if !defined(AO_HAVE_int_and_acquire)
2333# define AO_int_and_acquire(addr, val) AO_int_and_full(addr, val)
2334# define AO_HAVE_int_and_acquire
2335# endif
2336# if !defined(AO_HAVE_int_and_write)
2337# define AO_int_and_write(addr, val) AO_int_and_full(addr, val)
2338# define AO_HAVE_int_and_write
2339# endif
2340# if !defined(AO_HAVE_int_and_read)
2341# define AO_int_and_read(addr, val) AO_int_and_full(addr, val)
2342# define AO_HAVE_int_and_read
2343# endif
2344#endif /* AO_HAVE_int_and_full */
2345
2346#if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_release)
2347# define AO_int_and(addr, val) AO_int_and_release(addr, val)
2348# define AO_HAVE_int_and
2349#endif
2350#if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_acquire)
2351# define AO_int_and(addr, val) AO_int_and_acquire(addr, val)
2352# define AO_HAVE_int_and
2353#endif
2354#if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_write)
2355# define AO_int_and(addr, val) AO_int_and_write(addr, val)
2356# define AO_HAVE_int_and
2357#endif
2358#if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_read)
2359# define AO_int_and(addr, val) AO_int_and_read(addr, val)
2360# define AO_HAVE_int_and
2361#endif
2362
2363#if defined(AO_HAVE_int_and_acquire) && defined(AO_HAVE_nop_full) \
2364 && !defined(AO_HAVE_int_and_full)
2365# define AO_int_and_full(addr, val) \
2366 (AO_nop_full(), AO_int_and_acquire(addr, val))
2367# define AO_HAVE_int_and_full
2368#endif
2369
2370#if !defined(AO_HAVE_int_and_release_write) \
2371 && defined(AO_HAVE_int_and_write)
2372# define AO_int_and_release_write(addr, val) AO_int_and_write(addr, val)
2373# define AO_HAVE_int_and_release_write
2374#endif
2375#if !defined(AO_HAVE_int_and_release_write) \
2376 && defined(AO_HAVE_int_and_release)
2377# define AO_int_and_release_write(addr, val) AO_int_and_release(addr, val)
2378# define AO_HAVE_int_and_release_write
2379#endif
2380#if !defined(AO_HAVE_int_and_acquire_read) \
2381 && defined(AO_HAVE_int_and_read)
2382# define AO_int_and_acquire_read(addr, val) AO_int_and_read(addr, val)
2383# define AO_HAVE_int_and_acquire_read
2384#endif
2385#if !defined(AO_HAVE_int_and_acquire_read) \
2386 && defined(AO_HAVE_int_and_acquire)
2387# define AO_int_and_acquire_read(addr, val) AO_int_and_acquire(addr, val)
2388# define AO_HAVE_int_and_acquire_read
2389#endif
2390
2391/* int_or */
2392#if defined(AO_HAVE_int_compare_and_swap_full) \
2393 && !defined(AO_HAVE_int_or_full)
2394 AO_ATTR_NO_SANITIZE_THREAD
2395 AO_INLINE void
2396 AO_int_or_full(volatile unsigned *addr, unsigned value)
2397 {
2398 unsigned old;
2399
2400 do
2401 {
2402 old = *(unsigned *)addr;
2403 }
2404 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
2405 old | value)));
2406 }
2407# define AO_HAVE_int_or_full
2408#endif
2409
2410#if defined(AO_HAVE_int_or_full)
2411# if !defined(AO_HAVE_int_or_release)
2412# define AO_int_or_release(addr, val) AO_int_or_full(addr, val)
2413# define AO_HAVE_int_or_release
2414# endif
2415# if !defined(AO_HAVE_int_or_acquire)
2416# define AO_int_or_acquire(addr, val) AO_int_or_full(addr, val)
2417# define AO_HAVE_int_or_acquire
2418# endif
2419# if !defined(AO_HAVE_int_or_write)
2420# define AO_int_or_write(addr, val) AO_int_or_full(addr, val)
2421# define AO_HAVE_int_or_write
2422# endif
2423# if !defined(AO_HAVE_int_or_read)
2424# define AO_int_or_read(addr, val) AO_int_or_full(addr, val)
2425# define AO_HAVE_int_or_read
2426# endif
2427#endif /* AO_HAVE_int_or_full */
2428
2429#if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_release)
2430# define AO_int_or(addr, val) AO_int_or_release(addr, val)
2431# define AO_HAVE_int_or
2432#endif
2433#if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_acquire)
2434# define AO_int_or(addr, val) AO_int_or_acquire(addr, val)
2435# define AO_HAVE_int_or
2436#endif
2437#if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_write)
2438# define AO_int_or(addr, val) AO_int_or_write(addr, val)
2439# define AO_HAVE_int_or
2440#endif
2441#if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_read)
2442# define AO_int_or(addr, val) AO_int_or_read(addr, val)
2443# define AO_HAVE_int_or
2444#endif
2445
2446#if defined(AO_HAVE_int_or_acquire) && defined(AO_HAVE_nop_full) \
2447 && !defined(AO_HAVE_int_or_full)
2448# define AO_int_or_full(addr, val) \
2449 (AO_nop_full(), AO_int_or_acquire(addr, val))
2450# define AO_HAVE_int_or_full
2451#endif
2452
2453#if !defined(AO_HAVE_int_or_release_write) \
2454 && defined(AO_HAVE_int_or_write)
2455# define AO_int_or_release_write(addr, val) AO_int_or_write(addr, val)
2456# define AO_HAVE_int_or_release_write
2457#endif
2458#if !defined(AO_HAVE_int_or_release_write) \
2459 && defined(AO_HAVE_int_or_release)
2460# define AO_int_or_release_write(addr, val) AO_int_or_release(addr, val)
2461# define AO_HAVE_int_or_release_write
2462#endif
2463#if !defined(AO_HAVE_int_or_acquire_read) && defined(AO_HAVE_int_or_read)
2464# define AO_int_or_acquire_read(addr, val) AO_int_or_read(addr, val)
2465# define AO_HAVE_int_or_acquire_read
2466#endif
2467#if !defined(AO_HAVE_int_or_acquire_read) \
2468 && defined(AO_HAVE_int_or_acquire)
2469# define AO_int_or_acquire_read(addr, val) AO_int_or_acquire(addr, val)
2470# define AO_HAVE_int_or_acquire_read
2471#endif
2472
2473/* int_xor */
2474#if defined(AO_HAVE_int_compare_and_swap_full) \
2475 && !defined(AO_HAVE_int_xor_full)
2476 AO_ATTR_NO_SANITIZE_THREAD
2477 AO_INLINE void
2478 AO_int_xor_full(volatile unsigned *addr, unsigned value)
2479 {
2480 unsigned old;
2481
2482 do
2483 {
2484 old = *(unsigned *)addr;
2485 }
2486 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old,
2487 old ^ value)));
2488 }
2489# define AO_HAVE_int_xor_full
2490#endif
2491
2492#if defined(AO_HAVE_int_xor_full)
2493# if !defined(AO_HAVE_int_xor_release)
2494# define AO_int_xor_release(addr, val) AO_int_xor_full(addr, val)
2495# define AO_HAVE_int_xor_release
2496# endif
2497# if !defined(AO_HAVE_int_xor_acquire)
2498# define AO_int_xor_acquire(addr, val) AO_int_xor_full(addr, val)
2499# define AO_HAVE_int_xor_acquire
2500# endif
2501# if !defined(AO_HAVE_int_xor_write)
2502# define AO_int_xor_write(addr, val) AO_int_xor_full(addr, val)
2503# define AO_HAVE_int_xor_write
2504# endif
2505# if !defined(AO_HAVE_int_xor_read)
2506# define AO_int_xor_read(addr, val) AO_int_xor_full(addr, val)
2507# define AO_HAVE_int_xor_read
2508# endif
2509#endif /* AO_HAVE_int_xor_full */
2510
2511#if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_release)
2512# define AO_int_xor(addr, val) AO_int_xor_release(addr, val)
2513# define AO_HAVE_int_xor
2514#endif
2515#if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_acquire)
2516# define AO_int_xor(addr, val) AO_int_xor_acquire(addr, val)
2517# define AO_HAVE_int_xor
2518#endif
2519#if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_write)
2520# define AO_int_xor(addr, val) AO_int_xor_write(addr, val)
2521# define AO_HAVE_int_xor
2522#endif
2523#if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_read)
2524# define AO_int_xor(addr, val) AO_int_xor_read(addr, val)
2525# define AO_HAVE_int_xor
2526#endif
2527
2528#if defined(AO_HAVE_int_xor_acquire) && defined(AO_HAVE_nop_full) \
2529 && !defined(AO_HAVE_int_xor_full)
2530# define AO_int_xor_full(addr, val) \
2531 (AO_nop_full(), AO_int_xor_acquire(addr, val))
2532# define AO_HAVE_int_xor_full
2533#endif
2534
2535#if !defined(AO_HAVE_int_xor_release_write) \
2536 && defined(AO_HAVE_int_xor_write)
2537# define AO_int_xor_release_write(addr, val) AO_int_xor_write(addr, val)
2538# define AO_HAVE_int_xor_release_write
2539#endif
2540#if !defined(AO_HAVE_int_xor_release_write) \
2541 && defined(AO_HAVE_int_xor_release)
2542# define AO_int_xor_release_write(addr, val) AO_int_xor_release(addr, val)
2543# define AO_HAVE_int_xor_release_write
2544#endif
2545#if !defined(AO_HAVE_int_xor_acquire_read) \
2546 && defined(AO_HAVE_int_xor_read)
2547# define AO_int_xor_acquire_read(addr, val) AO_int_xor_read(addr, val)
2548# define AO_HAVE_int_xor_acquire_read
2549#endif
2550#if !defined(AO_HAVE_int_xor_acquire_read) \
2551 && defined(AO_HAVE_int_xor_acquire)
2552# define AO_int_xor_acquire_read(addr, val) AO_int_xor_acquire(addr, val)
2553# define AO_HAVE_int_xor_acquire_read
2554#endif
2555
2556/* int_and/or/xor_dd_acquire_read are meaningless. */
2557/*
2558 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
2559 *
2560 * Permission is hereby granted, free of charge, to any person obtaining a copy
2561 * of this software and associated documentation files (the "Software"), to deal
2562 * in the Software without restriction, including without limitation the rights
2563 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
2564 * copies of the Software, and to permit persons to whom the Software is
2565 * furnished to do so, subject to the following conditions:
2566 *
2567 * The above copyright notice and this permission notice shall be included in
2568 * all copies or substantial portions of the Software.
2569 *
2570 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
2571 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
2572 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
2573 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
2574 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
2575 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2576 * SOFTWARE.
2577 */
2578
2579/* compare_and_swap (based on fetch_compare_and_swap) */
2580#if defined(AO_HAVE_fetch_compare_and_swap_full) \
2581 && !defined(AO_HAVE_compare_and_swap_full)
2582 AO_INLINE int
2583 AO_compare_and_swap_full(volatile AO_t *addr, AO_t old_val,
2584 AO_t new_val)
2585 {
2586 return AO_fetch_compare_and_swap_full(addr, old_val, new_val)
2587 == old_val;
2588 }
2589# define AO_HAVE_compare_and_swap_full
2590#endif
2591
2592#if defined(AO_HAVE_fetch_compare_and_swap_acquire) \
2593 && !defined(AO_HAVE_compare_and_swap_acquire)
2594 AO_INLINE int
2595 AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old_val,
2596 AO_t new_val)
2597 {
2598 return AO_fetch_compare_and_swap_acquire(addr, old_val, new_val)
2599 == old_val;
2600 }
2601# define AO_HAVE_compare_and_swap_acquire
2602#endif
2603
2604#if defined(AO_HAVE_fetch_compare_and_swap_release) \
2605 && !defined(AO_HAVE_compare_and_swap_release)
2606 AO_INLINE int
2607 AO_compare_and_swap_release(volatile AO_t *addr, AO_t old_val,
2608 AO_t new_val)
2609 {
2610 return AO_fetch_compare_and_swap_release(addr, old_val, new_val)
2611 == old_val;
2612 }
2613# define AO_HAVE_compare_and_swap_release
2614#endif
2615
2616#if defined(AO_HAVE_fetch_compare_and_swap_write) \
2617 && !defined(AO_HAVE_compare_and_swap_write)
2618 AO_INLINE int
2619 AO_compare_and_swap_write(volatile AO_t *addr, AO_t old_val,
2620 AO_t new_val)
2621 {
2622 return AO_fetch_compare_and_swap_write(addr, old_val, new_val)
2623 == old_val;
2624 }
2625# define AO_HAVE_compare_and_swap_write
2626#endif
2627
2628#if defined(AO_HAVE_fetch_compare_and_swap_read) \
2629 && !defined(AO_HAVE_compare_and_swap_read)
2630 AO_INLINE int
2631 AO_compare_and_swap_read(volatile AO_t *addr, AO_t old_val,
2632 AO_t new_val)
2633 {
2634 return AO_fetch_compare_and_swap_read(addr, old_val, new_val)
2635 == old_val;
2636 }
2637# define AO_HAVE_compare_and_swap_read
2638#endif
2639
2640#if defined(AO_HAVE_fetch_compare_and_swap) \
2641 && !defined(AO_HAVE_compare_and_swap)
2642 AO_INLINE int
2643 AO_compare_and_swap(volatile AO_t *addr, AO_t old_val,
2644 AO_t new_val)
2645 {
2646 return AO_fetch_compare_and_swap(addr, old_val, new_val) == old_val;
2647 }
2648# define AO_HAVE_compare_and_swap
2649#endif
2650
2651#if defined(AO_HAVE_fetch_compare_and_swap_release_write) \
2652 && !defined(AO_HAVE_compare_and_swap_release_write)
2653 AO_INLINE int
2654 AO_compare_and_swap_release_write(volatile AO_t *addr,
2655 AO_t old_val, AO_t new_val)
2656 {
2657 return AO_fetch_compare_and_swap_release_write(addr, old_val,
2658 new_val) == old_val;
2659 }
2660# define AO_HAVE_compare_and_swap_release_write
2661#endif
2662
2663#if defined(AO_HAVE_fetch_compare_and_swap_acquire_read) \
2664 && !defined(AO_HAVE_compare_and_swap_acquire_read)
2665 AO_INLINE int
2666 AO_compare_and_swap_acquire_read(volatile AO_t *addr,
2667 AO_t old_val, AO_t new_val)
2668 {
2669 return AO_fetch_compare_and_swap_acquire_read(addr, old_val,
2670 new_val) == old_val;
2671 }
2672# define AO_HAVE_compare_and_swap_acquire_read
2673#endif
2674
2675#if defined(AO_HAVE_fetch_compare_and_swap_dd_acquire_read) \
2676 && !defined(AO_HAVE_compare_and_swap_dd_acquire_read)
2677 AO_INLINE int
2678 AO_compare_and_swap_dd_acquire_read(volatile AO_t *addr,
2679 AO_t old_val, AO_t new_val)
2680 {
2681 return AO_fetch_compare_and_swap_dd_acquire_read(addr, old_val,
2682 new_val) == old_val;
2683 }
2684# define AO_HAVE_compare_and_swap_dd_acquire_read
2685#endif
2686
2687/* fetch_and_add */
2688/* We first try to implement fetch_and_add variants in terms of the */
2689/* corresponding compare_and_swap variants to minimize adding barriers. */
2690#if defined(AO_HAVE_compare_and_swap_full) \
2691 && !defined(AO_HAVE_fetch_and_add_full)
2692 AO_ATTR_NO_SANITIZE_THREAD
2693 AO_INLINE AO_t
2694 AO_fetch_and_add_full(volatile AO_t *addr, AO_t incr)
2695 {
2696 AO_t old;
2697
2698 do
2699 {
2700 old = *(AO_t *)addr;
2701 }
2702 while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
2703 old + incr)));
2704 return old;
2705 }
2706# define AO_HAVE_fetch_and_add_full
2707#endif
2708
2709#if defined(AO_HAVE_compare_and_swap_acquire) \
2710 && !defined(AO_HAVE_fetch_and_add_acquire)
2711 AO_ATTR_NO_SANITIZE_THREAD
2712 AO_INLINE AO_t
2713 AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr)
2714 {
2715 AO_t old;
2716
2717 do
2718 {
2719 old = *(AO_t *)addr;
2720 }
2721 while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire(addr, old,
2722 old + incr)));
2723 return old;
2724 }
2725# define AO_HAVE_fetch_and_add_acquire
2726#endif
2727
2728#if defined(AO_HAVE_compare_and_swap_release) \
2729 && !defined(AO_HAVE_fetch_and_add_release)
2730 AO_ATTR_NO_SANITIZE_THREAD
2731 AO_INLINE AO_t
2732 AO_fetch_and_add_release(volatile AO_t *addr, AO_t incr)
2733 {
2734 AO_t old;
2735
2736 do
2737 {
2738 old = *(AO_t *)addr;
2739 }
2740 while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(addr, old,
2741 old + incr)));
2742 return old;
2743 }
2744# define AO_HAVE_fetch_and_add_release
2745#endif
2746
2747#if defined(AO_HAVE_compare_and_swap) \
2748 && !defined(AO_HAVE_fetch_and_add)
2749 AO_ATTR_NO_SANITIZE_THREAD
2750 AO_INLINE AO_t
2751 AO_fetch_and_add(volatile AO_t *addr, AO_t incr)
2752 {
2753 AO_t old;
2754
2755 do
2756 {
2757 old = *(AO_t *)addr;
2758 }
2759 while (AO_EXPECT_FALSE(!AO_compare_and_swap(addr, old,
2760 old + incr)));
2761 return old;
2762 }
2763# define AO_HAVE_fetch_and_add
2764#endif
2765
2766#if defined(AO_HAVE_fetch_and_add_full)
2767# if !defined(AO_HAVE_fetch_and_add_release)
2768# define AO_fetch_and_add_release(addr, val) \
2769 AO_fetch_and_add_full(addr, val)
2770# define AO_HAVE_fetch_and_add_release
2771# endif
2772# if !defined(AO_HAVE_fetch_and_add_acquire)
2773# define AO_fetch_and_add_acquire(addr, val) \
2774 AO_fetch_and_add_full(addr, val)
2775# define AO_HAVE_fetch_and_add_acquire
2776# endif
2777# if !defined(AO_HAVE_fetch_and_add_write)
2778# define AO_fetch_and_add_write(addr, val) \
2779 AO_fetch_and_add_full(addr, val)
2780# define AO_HAVE_fetch_and_add_write
2781# endif
2782# if !defined(AO_HAVE_fetch_and_add_read)
2783# define AO_fetch_and_add_read(addr, val) \
2784 AO_fetch_and_add_full(addr, val)
2785# define AO_HAVE_fetch_and_add_read
2786# endif
2787#endif /* AO_HAVE_fetch_and_add_full */
2788
2789#if defined(AO_HAVE_fetch_and_add) && defined(AO_HAVE_nop_full) \
2790 && !defined(AO_HAVE_fetch_and_add_acquire)
2791 AO_INLINE AO_t
2792 AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr)
2793 {
2794 AO_t result = AO_fetch_and_add(addr, incr);
2795 AO_nop_full();
2796 return result;
2797 }
2798# define AO_HAVE_fetch_and_add_acquire
2799#endif
2800#if defined(AO_HAVE_fetch_and_add) && defined(AO_HAVE_nop_full) \
2801 && !defined(AO_HAVE_fetch_and_add_release)
2802# define AO_fetch_and_add_release(addr, incr) \
2803 (AO_nop_full(), AO_fetch_and_add(addr, incr))
2804# define AO_HAVE_fetch_and_add_release
2805#endif
2806
2807#if !defined(AO_HAVE_fetch_and_add) \
2808 && defined(AO_HAVE_fetch_and_add_release)
2809# define AO_fetch_and_add(addr, val) \
2810 AO_fetch_and_add_release(addr, val)
2811# define AO_HAVE_fetch_and_add
2812#endif
2813#if !defined(AO_HAVE_fetch_and_add) \
2814 && defined(AO_HAVE_fetch_and_add_acquire)
2815# define AO_fetch_and_add(addr, val) \
2816 AO_fetch_and_add_acquire(addr, val)
2817# define AO_HAVE_fetch_and_add
2818#endif
2819#if !defined(AO_HAVE_fetch_and_add) \
2820 && defined(AO_HAVE_fetch_and_add_write)
2821# define AO_fetch_and_add(addr, val) \
2822 AO_fetch_and_add_write(addr, val)
2823# define AO_HAVE_fetch_and_add
2824#endif
2825#if !defined(AO_HAVE_fetch_and_add) \
2826 && defined(AO_HAVE_fetch_and_add_read)
2827# define AO_fetch_and_add(addr, val) \
2828 AO_fetch_and_add_read(addr, val)
2829# define AO_HAVE_fetch_and_add
2830#endif
2831
2832#if defined(AO_HAVE_fetch_and_add_acquire) \
2833 && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_fetch_and_add_full)
2834# define AO_fetch_and_add_full(addr, val) \
2835 (AO_nop_full(), AO_fetch_and_add_acquire(addr, val))
2836# define AO_HAVE_fetch_and_add_full
2837#endif
2838
2839#if !defined(AO_HAVE_fetch_and_add_release_write) \
2840 && defined(AO_HAVE_fetch_and_add_write)
2841# define AO_fetch_and_add_release_write(addr, val) \
2842 AO_fetch_and_add_write(addr, val)
2843# define AO_HAVE_fetch_and_add_release_write
2844#endif
2845#if !defined(AO_HAVE_fetch_and_add_release_write) \
2846 && defined(AO_HAVE_fetch_and_add_release)
2847# define AO_fetch_and_add_release_write(addr, val) \
2848 AO_fetch_and_add_release(addr, val)
2849# define AO_HAVE_fetch_and_add_release_write
2850#endif
2851
2852#if !defined(AO_HAVE_fetch_and_add_acquire_read) \
2853 && defined(AO_HAVE_fetch_and_add_read)
2854# define AO_fetch_and_add_acquire_read(addr, val) \
2855 AO_fetch_and_add_read(addr, val)
2856# define AO_HAVE_fetch_and_add_acquire_read
2857#endif
2858#if !defined(AO_HAVE_fetch_and_add_acquire_read) \
2859 && defined(AO_HAVE_fetch_and_add_acquire)
2860# define AO_fetch_and_add_acquire_read(addr, val) \
2861 AO_fetch_and_add_acquire(addr, val)
2862# define AO_HAVE_fetch_and_add_acquire_read
2863#endif
2864
2865#ifdef AO_NO_DD_ORDERING
2866# if defined(AO_HAVE_fetch_and_add_acquire_read)
2867# define AO_fetch_and_add_dd_acquire_read(addr, val) \
2868 AO_fetch_and_add_acquire_read(addr, val)
2869# define AO_HAVE_fetch_and_add_dd_acquire_read
2870# endif
2871#else
2872# if defined(AO_HAVE_fetch_and_add)
2873# define AO_fetch_and_add_dd_acquire_read(addr, val) \
2874 AO_fetch_and_add(addr, val)
2875# define AO_HAVE_fetch_and_add_dd_acquire_read
2876# endif
2877#endif /* !AO_NO_DD_ORDERING */
2878
2879/* fetch_and_add1 */
2880#if defined(AO_HAVE_fetch_and_add_full) \
2881 && !defined(AO_HAVE_fetch_and_add1_full)
2882# define AO_fetch_and_add1_full(addr) \
2883 AO_fetch_and_add_full(addr, 1)
2884# define AO_HAVE_fetch_and_add1_full
2885#endif
2886#if defined(AO_HAVE_fetch_and_add_release) \
2887 && !defined(AO_HAVE_fetch_and_add1_release)
2888# define AO_fetch_and_add1_release(addr) \
2889 AO_fetch_and_add_release(addr, 1)
2890# define AO_HAVE_fetch_and_add1_release
2891#endif
2892#if defined(AO_HAVE_fetch_and_add_acquire) \
2893 && !defined(AO_HAVE_fetch_and_add1_acquire)
2894# define AO_fetch_and_add1_acquire(addr) \
2895 AO_fetch_and_add_acquire(addr, 1)
2896# define AO_HAVE_fetch_and_add1_acquire
2897#endif
2898#if defined(AO_HAVE_fetch_and_add_write) \
2899 && !defined(AO_HAVE_fetch_and_add1_write)
2900# define AO_fetch_and_add1_write(addr) \
2901 AO_fetch_and_add_write(addr, 1)
2902# define AO_HAVE_fetch_and_add1_write
2903#endif
2904#if defined(AO_HAVE_fetch_and_add_read) \
2905 && !defined(AO_HAVE_fetch_and_add1_read)
2906# define AO_fetch_and_add1_read(addr) \
2907 AO_fetch_and_add_read(addr, 1)
2908# define AO_HAVE_fetch_and_add1_read
2909#endif
2910#if defined(AO_HAVE_fetch_and_add_release_write) \
2911 && !defined(AO_HAVE_fetch_and_add1_release_write)
2912# define AO_fetch_and_add1_release_write(addr) \
2913 AO_fetch_and_add_release_write(addr, 1)
2914# define AO_HAVE_fetch_and_add1_release_write
2915#endif
2916#if defined(AO_HAVE_fetch_and_add_acquire_read) \
2917 && !defined(AO_HAVE_fetch_and_add1_acquire_read)
2918# define AO_fetch_and_add1_acquire_read(addr) \
2919 AO_fetch_and_add_acquire_read(addr, 1)
2920# define AO_HAVE_fetch_and_add1_acquire_read
2921#endif
2922#if defined(AO_HAVE_fetch_and_add) \
2923 && !defined(AO_HAVE_fetch_and_add1)
2924# define AO_fetch_and_add1(addr) AO_fetch_and_add(addr, 1)
2925# define AO_HAVE_fetch_and_add1
2926#endif
2927
2928#if defined(AO_HAVE_fetch_and_add1_full)
2929# if !defined(AO_HAVE_fetch_and_add1_release)
2930# define AO_fetch_and_add1_release(addr) \
2931 AO_fetch_and_add1_full(addr)
2932# define AO_HAVE_fetch_and_add1_release
2933# endif
2934# if !defined(AO_HAVE_fetch_and_add1_acquire)
2935# define AO_fetch_and_add1_acquire(addr) \
2936 AO_fetch_and_add1_full(addr)
2937# define AO_HAVE_fetch_and_add1_acquire
2938# endif
2939# if !defined(AO_HAVE_fetch_and_add1_write)
2940# define AO_fetch_and_add1_write(addr) \
2941 AO_fetch_and_add1_full(addr)
2942# define AO_HAVE_fetch_and_add1_write
2943# endif
2944# if !defined(AO_HAVE_fetch_and_add1_read)
2945# define AO_fetch_and_add1_read(addr) \
2946 AO_fetch_and_add1_full(addr)
2947# define AO_HAVE_fetch_and_add1_read
2948# endif
2949#endif /* AO_HAVE_fetch_and_add1_full */
2950
2951#if !defined(AO_HAVE_fetch_and_add1) \
2952 && defined(AO_HAVE_fetch_and_add1_release)
2953# define AO_fetch_and_add1(addr) AO_fetch_and_add1_release(addr)
2954# define AO_HAVE_fetch_and_add1
2955#endif
2956#if !defined(AO_HAVE_fetch_and_add1) \
2957 && defined(AO_HAVE_fetch_and_add1_acquire)
2958# define AO_fetch_and_add1(addr) AO_fetch_and_add1_acquire(addr)
2959# define AO_HAVE_fetch_and_add1
2960#endif
2961#if !defined(AO_HAVE_fetch_and_add1) \
2962 && defined(AO_HAVE_fetch_and_add1_write)
2963# define AO_fetch_and_add1(addr) AO_fetch_and_add1_write(addr)
2964# define AO_HAVE_fetch_and_add1
2965#endif
2966#if !defined(AO_HAVE_fetch_and_add1) \
2967 && defined(AO_HAVE_fetch_and_add1_read)
2968# define AO_fetch_and_add1(addr) AO_fetch_and_add1_read(addr)
2969# define AO_HAVE_fetch_and_add1
2970#endif
2971
2972#if defined(AO_HAVE_fetch_and_add1_acquire) \
2973 && defined(AO_HAVE_nop_full) \
2974 && !defined(AO_HAVE_fetch_and_add1_full)
2975# define AO_fetch_and_add1_full(addr) \
2976 (AO_nop_full(), AO_fetch_and_add1_acquire(addr))
2977# define AO_HAVE_fetch_and_add1_full
2978#endif
2979
2980#if !defined(AO_HAVE_fetch_and_add1_release_write) \
2981 && defined(AO_HAVE_fetch_and_add1_write)
2982# define AO_fetch_and_add1_release_write(addr) \
2983 AO_fetch_and_add1_write(addr)
2984# define AO_HAVE_fetch_and_add1_release_write
2985#endif
2986#if !defined(AO_HAVE_fetch_and_add1_release_write) \
2987 && defined(AO_HAVE_fetch_and_add1_release)
2988# define AO_fetch_and_add1_release_write(addr) \
2989 AO_fetch_and_add1_release(addr)
2990# define AO_HAVE_fetch_and_add1_release_write
2991#endif
2992#if !defined(AO_HAVE_fetch_and_add1_acquire_read) \
2993 && defined(AO_HAVE_fetch_and_add1_read)
2994# define AO_fetch_and_add1_acquire_read(addr) \
2995 AO_fetch_and_add1_read(addr)
2996# define AO_HAVE_fetch_and_add1_acquire_read
2997#endif
2998#if !defined(AO_HAVE_fetch_and_add1_acquire_read) \
2999 && defined(AO_HAVE_fetch_and_add1_acquire)
3000# define AO_fetch_and_add1_acquire_read(addr) \
3001 AO_fetch_and_add1_acquire(addr)
3002# define AO_HAVE_fetch_and_add1_acquire_read
3003#endif
3004
3005#ifdef AO_NO_DD_ORDERING
3006# if defined(AO_HAVE_fetch_and_add1_acquire_read)
3007# define AO_fetch_and_add1_dd_acquire_read(addr) \
3008 AO_fetch_and_add1_acquire_read(addr)
3009# define AO_HAVE_fetch_and_add1_dd_acquire_read
3010# endif
3011#else
3012# if defined(AO_HAVE_fetch_and_add1)
3013# define AO_fetch_and_add1_dd_acquire_read(addr) \
3014 AO_fetch_and_add1(addr)
3015# define AO_HAVE_fetch_and_add1_dd_acquire_read
3016# endif
3017#endif /* !AO_NO_DD_ORDERING */
3018
3019/* fetch_and_sub1 */
3020#if defined(AO_HAVE_fetch_and_add_full) \
3021 && !defined(AO_HAVE_fetch_and_sub1_full)
3022# define AO_fetch_and_sub1_full(addr) \
3023 AO_fetch_and_add_full(addr, (AO_t)(-1))
3024# define AO_HAVE_fetch_and_sub1_full
3025#endif
3026#if defined(AO_HAVE_fetch_and_add_release) \
3027 && !defined(AO_HAVE_fetch_and_sub1_release)
3028# define AO_fetch_and_sub1_release(addr) \
3029 AO_fetch_and_add_release(addr, (AO_t)(-1))
3030# define AO_HAVE_fetch_and_sub1_release
3031#endif
3032#if defined(AO_HAVE_fetch_and_add_acquire) \
3033 && !defined(AO_HAVE_fetch_and_sub1_acquire)
3034# define AO_fetch_and_sub1_acquire(addr) \
3035 AO_fetch_and_add_acquire(addr, (AO_t)(-1))
3036# define AO_HAVE_fetch_and_sub1_acquire
3037#endif
3038#if defined(AO_HAVE_fetch_and_add_write) \
3039 && !defined(AO_HAVE_fetch_and_sub1_write)
3040# define AO_fetch_and_sub1_write(addr) \
3041 AO_fetch_and_add_write(addr, (AO_t)(-1))
3042# define AO_HAVE_fetch_and_sub1_write
3043#endif
3044#if defined(AO_HAVE_fetch_and_add_read) \
3045 && !defined(AO_HAVE_fetch_and_sub1_read)
3046# define AO_fetch_and_sub1_read(addr) \
3047 AO_fetch_and_add_read(addr, (AO_t)(-1))
3048# define AO_HAVE_fetch_and_sub1_read
3049#endif
3050#if defined(AO_HAVE_fetch_and_add_release_write) \
3051 && !defined(AO_HAVE_fetch_and_sub1_release_write)
3052# define AO_fetch_and_sub1_release_write(addr) \
3053 AO_fetch_and_add_release_write(addr, (AO_t)(-1))
3054# define AO_HAVE_fetch_and_sub1_release_write
3055#endif
3056#if defined(AO_HAVE_fetch_and_add_acquire_read) \
3057 && !defined(AO_HAVE_fetch_and_sub1_acquire_read)
3058# define AO_fetch_and_sub1_acquire_read(addr) \
3059 AO_fetch_and_add_acquire_read(addr, (AO_t)(-1))
3060# define AO_HAVE_fetch_and_sub1_acquire_read
3061#endif
3062#if defined(AO_HAVE_fetch_and_add) \
3063 && !defined(AO_HAVE_fetch_and_sub1)
3064# define AO_fetch_and_sub1(addr) \
3065 AO_fetch_and_add(addr, (AO_t)(-1))
3066# define AO_HAVE_fetch_and_sub1
3067#endif
3068
3069#if defined(AO_HAVE_fetch_and_sub1_full)
3070# if !defined(AO_HAVE_fetch_and_sub1_release)
3071# define AO_fetch_and_sub1_release(addr) \
3072 AO_fetch_and_sub1_full(addr)
3073# define AO_HAVE_fetch_and_sub1_release
3074# endif
3075# if !defined(AO_HAVE_fetch_and_sub1_acquire)
3076# define AO_fetch_and_sub1_acquire(addr) \
3077 AO_fetch_and_sub1_full(addr)
3078# define AO_HAVE_fetch_and_sub1_acquire
3079# endif
3080# if !defined(AO_HAVE_fetch_and_sub1_write)
3081# define AO_fetch_and_sub1_write(addr) \
3082 AO_fetch_and_sub1_full(addr)
3083# define AO_HAVE_fetch_and_sub1_write
3084# endif
3085# if !defined(AO_HAVE_fetch_and_sub1_read)
3086# define AO_fetch_and_sub1_read(addr) \
3087 AO_fetch_and_sub1_full(addr)
3088# define AO_HAVE_fetch_and_sub1_read
3089# endif
3090#endif /* AO_HAVE_fetch_and_sub1_full */
3091
3092#if !defined(AO_HAVE_fetch_and_sub1) \
3093 && defined(AO_HAVE_fetch_and_sub1_release)
3094# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_release(addr)
3095# define AO_HAVE_fetch_and_sub1
3096#endif
3097#if !defined(AO_HAVE_fetch_and_sub1) \
3098 && defined(AO_HAVE_fetch_and_sub1_acquire)
3099# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_acquire(addr)
3100# define AO_HAVE_fetch_and_sub1
3101#endif
3102#if !defined(AO_HAVE_fetch_and_sub1) \
3103 && defined(AO_HAVE_fetch_and_sub1_write)
3104# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_write(addr)
3105# define AO_HAVE_fetch_and_sub1
3106#endif
3107#if !defined(AO_HAVE_fetch_and_sub1) \
3108 && defined(AO_HAVE_fetch_and_sub1_read)
3109# define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_read(addr)
3110# define AO_HAVE_fetch_and_sub1
3111#endif
3112
3113#if defined(AO_HAVE_fetch_and_sub1_acquire) \
3114 && defined(AO_HAVE_nop_full) \
3115 && !defined(AO_HAVE_fetch_and_sub1_full)
3116# define AO_fetch_and_sub1_full(addr) \
3117 (AO_nop_full(), AO_fetch_and_sub1_acquire(addr))
3118# define AO_HAVE_fetch_and_sub1_full
3119#endif
3120
3121#if !defined(AO_HAVE_fetch_and_sub1_release_write) \
3122 && defined(AO_HAVE_fetch_and_sub1_write)
3123# define AO_fetch_and_sub1_release_write(addr) \
3124 AO_fetch_and_sub1_write(addr)
3125# define AO_HAVE_fetch_and_sub1_release_write
3126#endif
3127#if !defined(AO_HAVE_fetch_and_sub1_release_write) \
3128 && defined(AO_HAVE_fetch_and_sub1_release)
3129# define AO_fetch_and_sub1_release_write(addr) \
3130 AO_fetch_and_sub1_release(addr)
3131# define AO_HAVE_fetch_and_sub1_release_write
3132#endif
3133#if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \
3134 && defined(AO_HAVE_fetch_and_sub1_read)
3135# define AO_fetch_and_sub1_acquire_read(addr) \
3136 AO_fetch_and_sub1_read(addr)
3137# define AO_HAVE_fetch_and_sub1_acquire_read
3138#endif
3139#if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \
3140 && defined(AO_HAVE_fetch_and_sub1_acquire)
3141# define AO_fetch_and_sub1_acquire_read(addr) \
3142 AO_fetch_and_sub1_acquire(addr)
3143# define AO_HAVE_fetch_and_sub1_acquire_read
3144#endif
3145
3146#ifdef AO_NO_DD_ORDERING
3147# if defined(AO_HAVE_fetch_and_sub1_acquire_read)
3148# define AO_fetch_and_sub1_dd_acquire_read(addr) \
3149 AO_fetch_and_sub1_acquire_read(addr)
3150# define AO_HAVE_fetch_and_sub1_dd_acquire_read
3151# endif
3152#else
3153# if defined(AO_HAVE_fetch_and_sub1)
3154# define AO_fetch_and_sub1_dd_acquire_read(addr) \
3155 AO_fetch_and_sub1(addr)
3156# define AO_HAVE_fetch_and_sub1_dd_acquire_read
3157# endif
3158#endif /* !AO_NO_DD_ORDERING */
3159
3160/* and */
3161#if defined(AO_HAVE_compare_and_swap_full) \
3162 && !defined(AO_HAVE_and_full)
3163 AO_ATTR_NO_SANITIZE_THREAD
3164 AO_INLINE void
3165 AO_and_full(volatile AO_t *addr, AO_t value)
3166 {
3167 AO_t old;
3168
3169 do
3170 {
3171 old = *(AO_t *)addr;
3172 }
3173 while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
3174 old & value)));
3175 }
3176# define AO_HAVE_and_full
3177#endif
3178
3179#if defined(AO_HAVE_and_full)
3180# if !defined(AO_HAVE_and_release)
3181# define AO_and_release(addr, val) AO_and_full(addr, val)
3182# define AO_HAVE_and_release
3183# endif
3184# if !defined(AO_HAVE_and_acquire)
3185# define AO_and_acquire(addr, val) AO_and_full(addr, val)
3186# define AO_HAVE_and_acquire
3187# endif
3188# if !defined(AO_HAVE_and_write)
3189# define AO_and_write(addr, val) AO_and_full(addr, val)
3190# define AO_HAVE_and_write
3191# endif
3192# if !defined(AO_HAVE_and_read)
3193# define AO_and_read(addr, val) AO_and_full(addr, val)
3194# define AO_HAVE_and_read
3195# endif
3196#endif /* AO_HAVE_and_full */
3197
3198#if !defined(AO_HAVE_and) && defined(AO_HAVE_and_release)
3199# define AO_and(addr, val) AO_and_release(addr, val)
3200# define AO_HAVE_and
3201#endif
3202#if !defined(AO_HAVE_and) && defined(AO_HAVE_and_acquire)
3203# define AO_and(addr, val) AO_and_acquire(addr, val)
3204# define AO_HAVE_and
3205#endif
3206#if !defined(AO_HAVE_and) && defined(AO_HAVE_and_write)
3207# define AO_and(addr, val) AO_and_write(addr, val)
3208# define AO_HAVE_and
3209#endif
3210#if !defined(AO_HAVE_and) && defined(AO_HAVE_and_read)
3211# define AO_and(addr, val) AO_and_read(addr, val)
3212# define AO_HAVE_and
3213#endif
3214
3215#if defined(AO_HAVE_and_acquire) && defined(AO_HAVE_nop_full) \
3216 && !defined(AO_HAVE_and_full)
3217# define AO_and_full(addr, val) \
3218 (AO_nop_full(), AO_and_acquire(addr, val))
3219# define AO_HAVE_and_full
3220#endif
3221
3222#if !defined(AO_HAVE_and_release_write) \
3223 && defined(AO_HAVE_and_write)
3224# define AO_and_release_write(addr, val) AO_and_write(addr, val)
3225# define AO_HAVE_and_release_write
3226#endif
3227#if !defined(AO_HAVE_and_release_write) \
3228 && defined(AO_HAVE_and_release)
3229# define AO_and_release_write(addr, val) AO_and_release(addr, val)
3230# define AO_HAVE_and_release_write
3231#endif
3232#if !defined(AO_HAVE_and_acquire_read) \
3233 && defined(AO_HAVE_and_read)
3234# define AO_and_acquire_read(addr, val) AO_and_read(addr, val)
3235# define AO_HAVE_and_acquire_read
3236#endif
3237#if !defined(AO_HAVE_and_acquire_read) \
3238 && defined(AO_HAVE_and_acquire)
3239# define AO_and_acquire_read(addr, val) AO_and_acquire(addr, val)
3240# define AO_HAVE_and_acquire_read
3241#endif
3242
3243/* or */
3244#if defined(AO_HAVE_compare_and_swap_full) \
3245 && !defined(AO_HAVE_or_full)
3246 AO_ATTR_NO_SANITIZE_THREAD
3247 AO_INLINE void
3248 AO_or_full(volatile AO_t *addr, AO_t value)
3249 {
3250 AO_t old;
3251
3252 do
3253 {
3254 old = *(AO_t *)addr;
3255 }
3256 while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
3257 old | value)));
3258 }
3259# define AO_HAVE_or_full
3260#endif
3261
3262#if defined(AO_HAVE_or_full)
3263# if !defined(AO_HAVE_or_release)
3264# define AO_or_release(addr, val) AO_or_full(addr, val)
3265# define AO_HAVE_or_release
3266# endif
3267# if !defined(AO_HAVE_or_acquire)
3268# define AO_or_acquire(addr, val) AO_or_full(addr, val)
3269# define AO_HAVE_or_acquire
3270# endif
3271# if !defined(AO_HAVE_or_write)
3272# define AO_or_write(addr, val) AO_or_full(addr, val)
3273# define AO_HAVE_or_write
3274# endif
3275# if !defined(AO_HAVE_or_read)
3276# define AO_or_read(addr, val) AO_or_full(addr, val)
3277# define AO_HAVE_or_read
3278# endif
3279#endif /* AO_HAVE_or_full */
3280
3281#if !defined(AO_HAVE_or) && defined(AO_HAVE_or_release)
3282# define AO_or(addr, val) AO_or_release(addr, val)
3283# define AO_HAVE_or
3284#endif
3285#if !defined(AO_HAVE_or) && defined(AO_HAVE_or_acquire)
3286# define AO_or(addr, val) AO_or_acquire(addr, val)
3287# define AO_HAVE_or
3288#endif
3289#if !defined(AO_HAVE_or) && defined(AO_HAVE_or_write)
3290# define AO_or(addr, val) AO_or_write(addr, val)
3291# define AO_HAVE_or
3292#endif
3293#if !defined(AO_HAVE_or) && defined(AO_HAVE_or_read)
3294# define AO_or(addr, val) AO_or_read(addr, val)
3295# define AO_HAVE_or
3296#endif
3297
3298#if defined(AO_HAVE_or_acquire) && defined(AO_HAVE_nop_full) \
3299 && !defined(AO_HAVE_or_full)
3300# define AO_or_full(addr, val) \
3301 (AO_nop_full(), AO_or_acquire(addr, val))
3302# define AO_HAVE_or_full
3303#endif
3304
3305#if !defined(AO_HAVE_or_release_write) \
3306 && defined(AO_HAVE_or_write)
3307# define AO_or_release_write(addr, val) AO_or_write(addr, val)
3308# define AO_HAVE_or_release_write
3309#endif
3310#if !defined(AO_HAVE_or_release_write) \
3311 && defined(AO_HAVE_or_release)
3312# define AO_or_release_write(addr, val) AO_or_release(addr, val)
3313# define AO_HAVE_or_release_write
3314#endif
3315#if !defined(AO_HAVE_or_acquire_read) && defined(AO_HAVE_or_read)
3316# define AO_or_acquire_read(addr, val) AO_or_read(addr, val)
3317# define AO_HAVE_or_acquire_read
3318#endif
3319#if !defined(AO_HAVE_or_acquire_read) \
3320 && defined(AO_HAVE_or_acquire)
3321# define AO_or_acquire_read(addr, val) AO_or_acquire(addr, val)
3322# define AO_HAVE_or_acquire_read
3323#endif
3324
3325/* xor */
3326#if defined(AO_HAVE_compare_and_swap_full) \
3327 && !defined(AO_HAVE_xor_full)
3328 AO_ATTR_NO_SANITIZE_THREAD
3329 AO_INLINE void
3330 AO_xor_full(volatile AO_t *addr, AO_t value)
3331 {
3332 AO_t old;
3333
3334 do
3335 {
3336 old = *(AO_t *)addr;
3337 }
3338 while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old,
3339 old ^ value)));
3340 }
3341# define AO_HAVE_xor_full
3342#endif
3343
3344#if defined(AO_HAVE_xor_full)
3345# if !defined(AO_HAVE_xor_release)
3346# define AO_xor_release(addr, val) AO_xor_full(addr, val)
3347# define AO_HAVE_xor_release
3348# endif
3349# if !defined(AO_HAVE_xor_acquire)
3350# define AO_xor_acquire(addr, val) AO_xor_full(addr, val)
3351# define AO_HAVE_xor_acquire
3352# endif
3353# if !defined(AO_HAVE_xor_write)
3354# define AO_xor_write(addr, val) AO_xor_full(addr, val)
3355# define AO_HAVE_xor_write
3356# endif
3357# if !defined(AO_HAVE_xor_read)
3358# define AO_xor_read(addr, val) AO_xor_full(addr, val)
3359# define AO_HAVE_xor_read
3360# endif
3361#endif /* AO_HAVE_xor_full */
3362
3363#if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_release)
3364# define AO_xor(addr, val) AO_xor_release(addr, val)
3365# define AO_HAVE_xor
3366#endif
3367#if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_acquire)
3368# define AO_xor(addr, val) AO_xor_acquire(addr, val)
3369# define AO_HAVE_xor
3370#endif
3371#if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_write)
3372# define AO_xor(addr, val) AO_xor_write(addr, val)
3373# define AO_HAVE_xor
3374#endif
3375#if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_read)
3376# define AO_xor(addr, val) AO_xor_read(addr, val)
3377# define AO_HAVE_xor
3378#endif
3379
3380#if defined(AO_HAVE_xor_acquire) && defined(AO_HAVE_nop_full) \
3381 && !defined(AO_HAVE_xor_full)
3382# define AO_xor_full(addr, val) \
3383 (AO_nop_full(), AO_xor_acquire(addr, val))
3384# define AO_HAVE_xor_full
3385#endif
3386
3387#if !defined(AO_HAVE_xor_release_write) \
3388 && defined(AO_HAVE_xor_write)
3389# define AO_xor_release_write(addr, val) AO_xor_write(addr, val)
3390# define AO_HAVE_xor_release_write
3391#endif
3392#if !defined(AO_HAVE_xor_release_write) \
3393 && defined(AO_HAVE_xor_release)
3394# define AO_xor_release_write(addr, val) AO_xor_release(addr, val)
3395# define AO_HAVE_xor_release_write
3396#endif
3397#if !defined(AO_HAVE_xor_acquire_read) \
3398 && defined(AO_HAVE_xor_read)
3399# define AO_xor_acquire_read(addr, val) AO_xor_read(addr, val)
3400# define AO_HAVE_xor_acquire_read
3401#endif
3402#if !defined(AO_HAVE_xor_acquire_read) \
3403 && defined(AO_HAVE_xor_acquire)
3404# define AO_xor_acquire_read(addr, val) AO_xor_acquire(addr, val)
3405# define AO_HAVE_xor_acquire_read
3406#endif
3407
3408/* and/or/xor_dd_acquire_read are meaningless. */
result
volatile int result
Definition: Lib_With_Created_Counter.c:21
AO_compare_and_swap
AO_INLINE int AO_compare_and_swap(volatile AO_t *addr, AO_t old, AO_t new_val)
Definition: alpha.h:46
AO_nop_full
AO_INLINE void AO_nop_full(void)
Definition: alpha.h:27
AO_t
#define AO_t
Definition: atomic_ops.h:156
AO_ATTR_NO_SANITIZE_THREAD
#define AO_ATTR_NO_SANITIZE_THREAD
Definition: atomic_ops.h:226
AO_INLINE
#define AO_INLINE
Definition: atomic_ops.h:186
AO_EXPECT_FALSE
#define AO_EXPECT_FALSE(expr)
Definition: atomic_ops.h:193
AO_compare_and_swap_full
AO_INLINE int AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val)
Definition: avr32.h:49
AO_fetch_compare_and_swap_full
#define AO_fetch_compare_and_swap_full(addr, old, newval)
Definition: emul_cas.h:61
AO_short_fetch_and_add
AO_INLINE unsigned short AO_short_fetch_and_add(volatile unsigned short *p, unsigned short incr)
Definition: gcc/arm.h:502
AO_char_fetch_and_add
AO_INLINE unsigned char AO_char_fetch_and_add(volatile unsigned char *p, unsigned char incr)
Definition: gcc/arm.h:478
AO_fetch_and_add
AO_INLINE AO_t AO_fetch_and_add(volatile AO_t *p, AO_t incr)
Definition: gcc/arm.h:338
AO_fetch_compare_and_swap
AO_INLINE AO_t AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val)
Definition: gcc/arm.h:559
AO_short_fetch_compare_and_swap_acquire
AO_INLINE unsigned short AO_short_fetch_compare_and_swap_acquire(volatile unsigned short *addr, unsigned short old, unsigned short new_val)
Definition: gcc/ia64.h:173
AO_char_fetch_compare_and_swap_acquire
AO_INLINE unsigned char AO_char_fetch_compare_and_swap_acquire(volatile unsigned char *addr, unsigned char old, unsigned char new_val)
Definition: gcc/ia64.h:145
AO_int_fetch_compare_and_swap_acquire
AO_INLINE unsigned int AO_int_fetch_compare_and_swap_acquire(volatile unsigned int *addr, unsigned int old, unsigned int new_val)
Definition: gcc/ia64.h:254
AO_int_fetch_compare_and_swap_release
AO_INLINE unsigned int AO_int_fetch_compare_and_swap_release(volatile unsigned int *addr, unsigned int old, unsigned int new_val)
Definition: gcc/ia64.h:267
AO_fetch_compare_and_swap_release
AO_INLINE AO_t AO_fetch_compare_and_swap_release(volatile AO_t *addr, AO_t old, AO_t new_val)
Definition: gcc/ia64.h:130
AO_short_fetch_compare_and_swap_release
AO_INLINE unsigned short AO_short_fetch_compare_and_swap_release(volatile unsigned short *addr, unsigned short old, unsigned short new_val)
Definition: gcc/ia64.h:187
AO_char_fetch_compare_and_swap_release
AO_INLINE unsigned char AO_char_fetch_compare_and_swap_release(volatile unsigned char *addr, unsigned char old, unsigned char new_val)
Definition: gcc/ia64.h:159
AO_fetch_compare_and_swap_acquire
AO_INLINE AO_t AO_fetch_compare_and_swap_acquire(volatile AO_t *addr, AO_t old, AO_t new_val)
Definition: gcc/ia64.h:115
AO_compare_and_swap_acquire
AO_INLINE int AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old, AO_t new_val)
Definition: gcc/powerpc.h:213
AO_compare_and_swap_release
AO_INLINE int AO_compare_and_swap_release(volatile AO_t *addr, AO_t old, AO_t new_val)
Definition: gcc/powerpc.h:222
AO_short_fetch_compare_and_swap_full
AO_INLINE unsigned short AO_short_fetch_compare_and_swap_full(volatile unsigned short *addr, unsigned short old_val, unsigned short new_val)
Definition: gcc/x86.h:343
AO_char_fetch_compare_and_swap_full
AO_INLINE unsigned char AO_char_fetch_compare_and_swap_full(volatile unsigned char *addr, unsigned char old_val, unsigned char new_val)
Definition: gcc/x86.h:323
AO_fetch_and_add_release
AO_INLINE AO_t AO_fetch_and_add_release(volatile AO_t *addr, AO_t incr)
Definition: generic-arithm.h:615
AO_int_fetch_and_add_full
AO_INLINE unsigned AO_int_fetch_and_add_full(volatile unsigned *addr, unsigned incr)
Definition: generic-arithm.h:777
AO_char_fetch_and_add_full
AO_INLINE unsigned char AO_char_fetch_and_add_full(volatile unsignedchar *addr, unsignedchar incr)
Definition: generic-arithm.h:669
AO_short_xor_full
AO_INLINE void AO_short_xor_full(volatile unsignedshort *addr, unsignedshort value)
Definition: generic-arithm.h:749
AO_char_fetch_and_add_release
AO_INLINE unsigned char AO_char_fetch_and_add_release(volatile unsignedchar *addr, unsignedchar incr)
Definition: generic-arithm.h:453
AO_fetch_and_add_acquire
AO_INLINE AO_t AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr)
Definition: generic-arithm.h:399
AO_char_and_full
AO_INLINE void AO_char_and_full(volatile unsignedchar *addr, unsignedchar value)
Definition: generic-arithm.h:677
AO_short_fetch_and_add_full
AO_INLINE unsigned short AO_short_fetch_and_add_full(volatile unsignedshort *addr, unsignedshort incr)
Definition: generic-arithm.h:723
AO_short_fetch_and_add_release
AO_INLINE unsigned short AO_short_fetch_and_add_release(volatile unsignedshort *addr, unsignedshort incr)
Definition: generic-arithm.h:507
AO_char_or_full
AO_INLINE void AO_char_or_full(volatile unsignedchar *addr, unsignedchar value)
Definition: generic-arithm.h:686
AO_short_and_full
AO_INLINE void AO_short_and_full(volatile unsignedshort *addr, unsignedshort value)
Definition: generic-arithm.h:731
AO_char_xor_full
AO_INLINE void AO_char_xor_full(volatile unsignedchar *addr, unsignedchar value)
Definition: generic-arithm.h:695
AO_short_or_full
AO_INLINE void AO_short_or_full(volatile unsignedshort *addr, unsignedshort value)
Definition: generic-arithm.h:740
AO_short_fetch_and_add_acquire
AO_INLINE unsigned short AO_short_fetch_and_add_acquire(volatile unsignedshort *addr, unsignedshort incr)
Definition: generic-arithm.h:291
AO_int_and_full
AO_INLINE void AO_int_and_full(volatile unsigned *addr, unsigned value)
Definition: generic-arithm.h:785
AO_fetch_and_add_full
AO_INLINE AO_t AO_fetch_and_add_full(volatile AO_t *addr, AO_t incr)
Definition: generic-arithm.h:831
AO_or_full
AO_INLINE void AO_or_full(volatile AO_t *addr, AO_t value)
Definition: generic-arithm.h:848
AO_and_full
AO_INLINE void AO_and_full(volatile AO_t *addr, AO_t value)
Definition: generic-arithm.h:839
AO_int_xor_full
AO_INLINE void AO_int_xor_full(volatile unsigned *addr, unsigned value)
Definition: generic-arithm.h:803
AO_int_or_full
AO_INLINE void AO_int_or_full(volatile unsigned *addr, unsigned value)
Definition: generic-arithm.h:794
AO_int_fetch_and_add
AO_INLINE unsigned AO_int_fetch_and_add(volatile unsigned *addr, unsigned incr)
Definition: generic-arithm.h:129
AO_xor_full
AO_INLINE void AO_xor_full(volatile AO_t *addr, AO_t value)
Definition: generic-arithm.h:857
AO_int_fetch_and_add_acquire
AO_INLINE unsigned AO_int_fetch_and_add_acquire(volatile unsigned *addr, unsigned incr)
Definition: generic-arithm.h:345
AO_char_fetch_and_add_acquire
AO_INLINE unsigned char AO_char_fetch_and_add_acquire(volatile unsignedchar *addr, unsignedchar incr)
Definition: generic-arithm.h:237
AO_int_fetch_and_add_release
AO_INLINE unsigned AO_int_fetch_and_add_release(volatile unsigned *addr, unsigned incr)
Definition: generic-arithm.h:561
AO_int_fetch_compare_and_swap_full
AO_INLINE unsigned AO_int_fetch_compare_and_swap_full(volatile unsigned *addr, unsigned old_val, unsigned new_val)
Definition: generic_pthread.h:357