PAPI 7.1.0.0
Loading...
Searching...
No Matches
generalize-small.h
Go to the documentation of this file.
1/*
2 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to deal
6 * in the Software without restriction, including without limitation the rights
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 * SOFTWARE.
21 */
22
23/* char_fetch_compare_and_swap */
24#if defined(AO_HAVE_char_fetch_compare_and_swap) \
25 && defined(AO_HAVE_nop_full) \
26 && !defined(AO_HAVE_char_fetch_compare_and_swap_acquire)
27 AO_INLINE unsignedchar
28 AO_char_fetch_compare_and_swap_acquire(volatile unsignedchar *addr,
29 unsignedchar old_val, unsignedchar new_val)
30 {
31 unsignedchar result = AO_char_fetch_compare_and_swap(addr, old_val, new_val);
32 AO_nop_full();
33 return result;
34 }
35# define AO_HAVE_char_fetch_compare_and_swap_acquire
36#endif
37#if defined(AO_HAVE_char_fetch_compare_and_swap) \
38 && defined(AO_HAVE_nop_full) \
39 && !defined(AO_HAVE_char_fetch_compare_and_swap_release)
40# define AO_char_fetch_compare_and_swap_release(addr, old_val, new_val) \
41 (AO_nop_full(), \
42 AO_char_fetch_compare_and_swap(addr, old_val, new_val))
43# define AO_HAVE_char_fetch_compare_and_swap_release
44#endif
45#if defined(AO_HAVE_char_fetch_compare_and_swap_full)
46# if !defined(AO_HAVE_char_fetch_compare_and_swap_release)
47# define AO_char_fetch_compare_and_swap_release(addr, old_val, new_val) \
48 AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
49# define AO_HAVE_char_fetch_compare_and_swap_release
50# endif
51# if !defined(AO_HAVE_char_fetch_compare_and_swap_acquire)
52# define AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
53 AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
54# define AO_HAVE_char_fetch_compare_and_swap_acquire
55# endif
56# if !defined(AO_HAVE_char_fetch_compare_and_swap_write)
57# define AO_char_fetch_compare_and_swap_write(addr, old_val, new_val) \
58 AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
59# define AO_HAVE_char_fetch_compare_and_swap_write
60# endif
61# if !defined(AO_HAVE_char_fetch_compare_and_swap_read)
62# define AO_char_fetch_compare_and_swap_read(addr, old_val, new_val) \
63 AO_char_fetch_compare_and_swap_full(addr, old_val, new_val)
64# define AO_HAVE_char_fetch_compare_and_swap_read
65# endif
66#endif /* AO_HAVE_char_fetch_compare_and_swap_full */
67
68#if !defined(AO_HAVE_char_fetch_compare_and_swap) \
69 && defined(AO_HAVE_char_fetch_compare_and_swap_release)
70# define AO_char_fetch_compare_and_swap(addr, old_val, new_val) \
71 AO_char_fetch_compare_and_swap_release(addr, old_val, new_val)
72# define AO_HAVE_char_fetch_compare_and_swap
73#endif
74#if !defined(AO_HAVE_char_fetch_compare_and_swap) \
75 && defined(AO_HAVE_char_fetch_compare_and_swap_acquire)
76# define AO_char_fetch_compare_and_swap(addr, old_val, new_val) \
77 AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val)
78# define AO_HAVE_char_fetch_compare_and_swap
79#endif
80#if !defined(AO_HAVE_char_fetch_compare_and_swap) \
81 && defined(AO_HAVE_char_fetch_compare_and_swap_write)
82# define AO_char_fetch_compare_and_swap(addr, old_val, new_val) \
83 AO_char_fetch_compare_and_swap_write(addr, old_val, new_val)
84# define AO_HAVE_char_fetch_compare_and_swap
85#endif
86#if !defined(AO_HAVE_char_fetch_compare_and_swap) \
87 && defined(AO_HAVE_char_fetch_compare_and_swap_read)
88# define AO_char_fetch_compare_and_swap(addr, old_val, new_val) \
89 AO_char_fetch_compare_and_swap_read(addr, old_val, new_val)
90# define AO_HAVE_char_fetch_compare_and_swap
91#endif
92
93#if defined(AO_HAVE_char_fetch_compare_and_swap_acquire) \
94 && defined(AO_HAVE_nop_full) \
95 && !defined(AO_HAVE_char_fetch_compare_and_swap_full)
96# define AO_char_fetch_compare_and_swap_full(addr, old_val, new_val) \
97 (AO_nop_full(), \
98 AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val))
99# define AO_HAVE_char_fetch_compare_and_swap_full
100#endif
101
102#if !defined(AO_HAVE_char_fetch_compare_and_swap_release_write) \
103 && defined(AO_HAVE_char_fetch_compare_and_swap_write)
104# define AO_char_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
105 AO_char_fetch_compare_and_swap_write(addr, old_val, new_val)
106# define AO_HAVE_char_fetch_compare_and_swap_release_write
107#endif
108#if !defined(AO_HAVE_char_fetch_compare_and_swap_release_write) \
109 && defined(AO_HAVE_char_fetch_compare_and_swap_release)
110# define AO_char_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
111 AO_char_fetch_compare_and_swap_release(addr, old_val, new_val)
112# define AO_HAVE_char_fetch_compare_and_swap_release_write
113#endif
114#if !defined(AO_HAVE_char_fetch_compare_and_swap_acquire_read) \
115 && defined(AO_HAVE_char_fetch_compare_and_swap_read)
116# define AO_char_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
117 AO_char_fetch_compare_and_swap_read(addr, old_val, new_val)
118# define AO_HAVE_char_fetch_compare_and_swap_acquire_read
119#endif
120#if !defined(AO_HAVE_char_fetch_compare_and_swap_acquire_read) \
121 && defined(AO_HAVE_char_fetch_compare_and_swap_acquire)
122# define AO_char_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
123 AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val)
124# define AO_HAVE_char_fetch_compare_and_swap_acquire_read
125#endif
126
127#ifdef AO_NO_DD_ORDERING
128# if defined(AO_HAVE_char_fetch_compare_and_swap_acquire_read)
129# define AO_char_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
130 AO_char_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
131# define AO_HAVE_char_fetch_compare_and_swap_dd_acquire_read
132# endif
133#else
134# if defined(AO_HAVE_char_fetch_compare_and_swap)
135# define AO_char_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
136 AO_char_fetch_compare_and_swap(addr, old_val, new_val)
137# define AO_HAVE_char_fetch_compare_and_swap_dd_acquire_read
138# endif
139#endif /* !AO_NO_DD_ORDERING */
140
141/* char_compare_and_swap */
142#if defined(AO_HAVE_char_compare_and_swap) && defined(AO_HAVE_nop_full) \
143 && !defined(AO_HAVE_char_compare_and_swap_acquire)
144 AO_INLINE int
145 AO_char_compare_and_swap_acquire(volatile unsignedchar *addr, unsignedchar old,
146 unsignedchar new_val)
147 {
148 int result = AO_char_compare_and_swap(addr, old, new_val);
149 AO_nop_full();
150 return result;
151 }
152# define AO_HAVE_char_compare_and_swap_acquire
153#endif
154#if defined(AO_HAVE_char_compare_and_swap) && defined(AO_HAVE_nop_full) \
155 && !defined(AO_HAVE_char_compare_and_swap_release)
156# define AO_char_compare_and_swap_release(addr, old, new_val) \
157 (AO_nop_full(), AO_char_compare_and_swap(addr, old, new_val))
158# define AO_HAVE_char_compare_and_swap_release
159#endif
160#if defined(AO_HAVE_char_compare_and_swap_full)
161# if !defined(AO_HAVE_char_compare_and_swap_release)
162# define AO_char_compare_and_swap_release(addr, old, new_val) \
163 AO_char_compare_and_swap_full(addr, old, new_val)
164# define AO_HAVE_char_compare_and_swap_release
165# endif
166# if !defined(AO_HAVE_char_compare_and_swap_acquire)
167# define AO_char_compare_and_swap_acquire(addr, old, new_val) \
168 AO_char_compare_and_swap_full(addr, old, new_val)
169# define AO_HAVE_char_compare_and_swap_acquire
170# endif
171# if !defined(AO_HAVE_char_compare_and_swap_write)
172# define AO_char_compare_and_swap_write(addr, old, new_val) \
173 AO_char_compare_and_swap_full(addr, old, new_val)
174# define AO_HAVE_char_compare_and_swap_write
175# endif
176# if !defined(AO_HAVE_char_compare_and_swap_read)
177# define AO_char_compare_and_swap_read(addr, old, new_val) \
178 AO_char_compare_and_swap_full(addr, old, new_val)
179# define AO_HAVE_char_compare_and_swap_read
180# endif
181#endif /* AO_HAVE_char_compare_and_swap_full */
182
183#if !defined(AO_HAVE_char_compare_and_swap) \
184 && defined(AO_HAVE_char_compare_and_swap_release)
185# define AO_char_compare_and_swap(addr, old, new_val) \
186 AO_char_compare_and_swap_release(addr, old, new_val)
187# define AO_HAVE_char_compare_and_swap
188#endif
189#if !defined(AO_HAVE_char_compare_and_swap) \
190 && defined(AO_HAVE_char_compare_and_swap_acquire)
191# define AO_char_compare_and_swap(addr, old, new_val) \
192 AO_char_compare_and_swap_acquire(addr, old, new_val)
193# define AO_HAVE_char_compare_and_swap
194#endif
195#if !defined(AO_HAVE_char_compare_and_swap) \
196 && defined(AO_HAVE_char_compare_and_swap_write)
197# define AO_char_compare_and_swap(addr, old, new_val) \
198 AO_char_compare_and_swap_write(addr, old, new_val)
199# define AO_HAVE_char_compare_and_swap
200#endif
201#if !defined(AO_HAVE_char_compare_and_swap) \
202 && defined(AO_HAVE_char_compare_and_swap_read)
203# define AO_char_compare_and_swap(addr, old, new_val) \
204 AO_char_compare_and_swap_read(addr, old, new_val)
205# define AO_HAVE_char_compare_and_swap
206#endif
207
208#if defined(AO_HAVE_char_compare_and_swap_acquire) \
209 && defined(AO_HAVE_nop_full) \
210 && !defined(AO_HAVE_char_compare_and_swap_full)
211# define AO_char_compare_and_swap_full(addr, old, new_val) \
212 (AO_nop_full(), \
213 AO_char_compare_and_swap_acquire(addr, old, new_val))
214# define AO_HAVE_char_compare_and_swap_full
215#endif
216
217#if !defined(AO_HAVE_char_compare_and_swap_release_write) \
218 && defined(AO_HAVE_char_compare_and_swap_write)
219# define AO_char_compare_and_swap_release_write(addr, old, new_val) \
220 AO_char_compare_and_swap_write(addr, old, new_val)
221# define AO_HAVE_char_compare_and_swap_release_write
222#endif
223#if !defined(AO_HAVE_char_compare_and_swap_release_write) \
224 && defined(AO_HAVE_char_compare_and_swap_release)
225# define AO_char_compare_and_swap_release_write(addr, old, new_val) \
226 AO_char_compare_and_swap_release(addr, old, new_val)
227# define AO_HAVE_char_compare_and_swap_release_write
228#endif
229#if !defined(AO_HAVE_char_compare_and_swap_acquire_read) \
230 && defined(AO_HAVE_char_compare_and_swap_read)
231# define AO_char_compare_and_swap_acquire_read(addr, old, new_val) \
232 AO_char_compare_and_swap_read(addr, old, new_val)
233# define AO_HAVE_char_compare_and_swap_acquire_read
234#endif
235#if !defined(AO_HAVE_char_compare_and_swap_acquire_read) \
236 && defined(AO_HAVE_char_compare_and_swap_acquire)
237# define AO_char_compare_and_swap_acquire_read(addr, old, new_val) \
238 AO_char_compare_and_swap_acquire(addr, old, new_val)
239# define AO_HAVE_char_compare_and_swap_acquire_read
240#endif
241
242#ifdef AO_NO_DD_ORDERING
243# if defined(AO_HAVE_char_compare_and_swap_acquire_read)
244# define AO_char_compare_and_swap_dd_acquire_read(addr, old, new_val) \
245 AO_char_compare_and_swap_acquire_read(addr, old, new_val)
246# define AO_HAVE_char_compare_and_swap_dd_acquire_read
247# endif
248#else
249# if defined(AO_HAVE_char_compare_and_swap)
250# define AO_char_compare_and_swap_dd_acquire_read(addr, old, new_val) \
251 AO_char_compare_and_swap(addr, old, new_val)
252# define AO_HAVE_char_compare_and_swap_dd_acquire_read
253# endif
254#endif /* !AO_NO_DD_ORDERING */
255
256/* char_load */
257#if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_acquire)
258# define AO_char_load_acquire(addr) AO_char_load_full(addr)
259# define AO_HAVE_char_load_acquire
260#endif
261
262#if defined(AO_HAVE_char_load_acquire) && !defined(AO_HAVE_char_load)
263# define AO_char_load(addr) AO_char_load_acquire(addr)
264# define AO_HAVE_char_load
265#endif
266
267#if defined(AO_HAVE_char_load_full) && !defined(AO_HAVE_char_load_read)
268# define AO_char_load_read(addr) AO_char_load_full(addr)
269# define AO_HAVE_char_load_read
270#endif
271
272#if !defined(AO_HAVE_char_load_acquire_read) \
273 && defined(AO_HAVE_char_load_acquire)
274# define AO_char_load_acquire_read(addr) AO_char_load_acquire(addr)
275# define AO_HAVE_char_load_acquire_read
276#endif
277
278#if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_full) \
279 && !defined(AO_HAVE_char_load_acquire)
280 AO_INLINE unsignedchar
281 AO_char_load_acquire(const volatile unsignedchar *addr)
282 {
283 unsignedchar result = AO_char_load(addr);
284
285 /* Acquire barrier would be useless, since the load could be delayed */
286 /* beyond it. */
287 AO_nop_full();
288 return result;
289 }
290# define AO_HAVE_char_load_acquire
291#endif
292
293#if defined(AO_HAVE_char_load) && defined(AO_HAVE_nop_read) \
294 && !defined(AO_HAVE_char_load_read)
295 AO_INLINE unsignedchar
296 AO_char_load_read(const volatile unsignedchar *addr)
297 {
298 unsignedchar result = AO_char_load(addr);
299
300 AO_nop_read();
301 return result;
302 }
303# define AO_HAVE_char_load_read
304#endif
305
306#if defined(AO_HAVE_char_load_acquire) && defined(AO_HAVE_nop_full) \
307 && !defined(AO_HAVE_char_load_full)
308# define AO_char_load_full(addr) (AO_nop_full(), AO_char_load_acquire(addr))
309# define AO_HAVE_char_load_full
310#endif
311
312#if defined(AO_HAVE_char_compare_and_swap_read) \
313 && !defined(AO_HAVE_char_load_read)
314# define AO_char_CAS_BASED_LOAD_READ
315 AO_ATTR_NO_SANITIZE_THREAD
316 AO_INLINE unsignedchar
317 AO_char_load_read(const volatile unsignedchar *addr)
318 {
319 unsignedchar result;
320
321 do {
322 result = *(const unsignedchar *)addr;
323 } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_read(
324 (volatile unsignedchar *)addr,
325 result, result)));
326 return result;
327 }
328# define AO_HAVE_char_load_read
329#endif
330
331#if !defined(AO_HAVE_char_load_acquire_read) \
332 && defined(AO_HAVE_char_load_read)
333# define AO_char_load_acquire_read(addr) AO_char_load_read(addr)
334# define AO_HAVE_char_load_acquire_read
335#endif
336
337#if defined(AO_HAVE_char_load_acquire_read) && !defined(AO_HAVE_char_load) \
338 && (!defined(AO_char_CAS_BASED_LOAD_READ) \
339 || !defined(AO_HAVE_char_compare_and_swap))
340# define AO_char_load(addr) AO_char_load_acquire_read(addr)
341# define AO_HAVE_char_load
342#endif
343
344#if defined(AO_HAVE_char_compare_and_swap_full) \
345 && !defined(AO_HAVE_char_load_full)
346 AO_ATTR_NO_SANITIZE_THREAD
347 AO_INLINE unsignedchar
348 AO_char_load_full(const volatile unsignedchar *addr)
349 {
350 unsignedchar result;
351
352 do {
353 result = *(const unsignedchar *)addr;
354 } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(
355 (volatile unsignedchar *)addr,
356 result, result)));
357 return result;
358 }
359# define AO_HAVE_char_load_full
360#endif
361
362#if defined(AO_HAVE_char_compare_and_swap_acquire) \
363 && !defined(AO_HAVE_char_load_acquire)
364 AO_ATTR_NO_SANITIZE_THREAD
365 AO_INLINE unsignedchar
366 AO_char_load_acquire(const volatile unsignedchar *addr)
367 {
368 unsignedchar result;
369
370 do {
371 result = *(const unsignedchar *)addr;
372 } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_acquire(
373 (volatile unsignedchar *)addr,
374 result, result)));
375 return result;
376 }
377# define AO_HAVE_char_load_acquire
378#endif
379
380#if defined(AO_HAVE_char_compare_and_swap) && !defined(AO_HAVE_char_load)
381 AO_ATTR_NO_SANITIZE_THREAD
382 AO_INLINE unsignedchar
383 AO_char_load(const volatile unsignedchar *addr)
384 {
385 unsignedchar result;
386
387 do {
388 result = *(const unsignedchar *)addr;
389 } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap(
390 (volatile unsignedchar *)addr,
391 result, result)));
392 return result;
393 }
394# define AO_HAVE_char_load
395#endif
396
397#ifdef AO_NO_DD_ORDERING
398# if defined(AO_HAVE_char_load_acquire_read)
399# define AO_char_load_dd_acquire_read(addr) \
400 AO_char_load_acquire_read(addr)
401# define AO_HAVE_char_load_dd_acquire_read
402# endif
403#else
404# if defined(AO_HAVE_char_load)
405# define AO_char_load_dd_acquire_read(addr) AO_char_load(addr)
406# define AO_HAVE_char_load_dd_acquire_read
407# endif
408#endif /* !AO_NO_DD_ORDERING */
409
410/* char_store */
411#if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_release)
412# define AO_char_store_release(addr, val) AO_char_store_full(addr, val)
413# define AO_HAVE_char_store_release
414#endif
415
416#if defined(AO_HAVE_char_store_release) && !defined(AO_HAVE_char_store)
417# define AO_char_store(addr, val) AO_char_store_release(addr, val)
418# define AO_HAVE_char_store
419#endif
420
421#if defined(AO_HAVE_char_store_full) && !defined(AO_HAVE_char_store_write)
422# define AO_char_store_write(addr, val) AO_char_store_full(addr, val)
423# define AO_HAVE_char_store_write
424#endif
425
426#if defined(AO_HAVE_char_store_release) \
427 && !defined(AO_HAVE_char_store_release_write)
428# define AO_char_store_release_write(addr, val) \
429 AO_char_store_release(addr, val)
430# define AO_HAVE_char_store_release_write
431#endif
432
433#if defined(AO_HAVE_char_store_write) && !defined(AO_HAVE_char_store)
434# define AO_char_store(addr, val) AO_char_store_write(addr, val)
435# define AO_HAVE_char_store
436#endif
437
438#if defined(AO_HAVE_char_store) && defined(AO_HAVE_nop_full) \
439 && !defined(AO_HAVE_char_store_release)
440# define AO_char_store_release(addr, val) \
441 (AO_nop_full(), AO_char_store(addr, val))
442# define AO_HAVE_char_store_release
443#endif
444
445#if defined(AO_HAVE_char_store) && defined(AO_HAVE_nop_write) \
446 && !defined(AO_HAVE_char_store_write)
447# define AO_char_store_write(addr, val) \
448 (AO_nop_write(), AO_char_store(addr, val))
449# define AO_HAVE_char_store_write
450#endif
451
452#if defined(AO_HAVE_char_compare_and_swap_write) \
453 && !defined(AO_HAVE_char_store_write)
454 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
455 AO_INLINE void
456 AO_char_store_write(volatile unsignedchar *addr, unsignedchar new_val)
457 {
458 unsignedchar old_val;
459
460 do {
461 old_val = *(unsignedchar *)addr;
462 } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_write(addr, old_val,
463 new_val)));
464 }
465# define AO_HAVE_char_store_write
466#endif
467
468#if defined(AO_HAVE_char_store_write) \
469 && !defined(AO_HAVE_char_store_release_write)
470# define AO_char_store_release_write(addr, val) \
471 AO_char_store_write(addr, val)
472# define AO_HAVE_char_store_release_write
473#endif
474
475#if defined(AO_HAVE_char_store_release) && defined(AO_HAVE_nop_full) \
476 && !defined(AO_HAVE_char_store_full)
477# define AO_char_store_full(addr, val) \
478 (AO_char_store_release(addr, val), \
479 AO_nop_full())
480# define AO_HAVE_char_store_full
481#endif
482
483#if defined(AO_HAVE_char_compare_and_swap) && !defined(AO_HAVE_char_store)
484 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
485 AO_INLINE void
486 AO_char_store(volatile unsignedchar *addr, unsignedchar new_val)
487 {
488 unsignedchar old_val;
489
490 do {
491 old_val = *(unsignedchar *)addr;
492 } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap(addr,
493 old_val, new_val)));
494 }
495# define AO_HAVE_char_store
496#endif
497
498#if defined(AO_HAVE_char_compare_and_swap_release) \
499 && !defined(AO_HAVE_char_store_release)
500 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
501 AO_INLINE void
502 AO_char_store_release(volatile unsignedchar *addr, unsignedchar new_val)
503 {
504 unsignedchar old_val;
505
506 do {
507 old_val = *(unsignedchar *)addr;
508 } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_release(addr, old_val,
509 new_val)));
510 }
511# define AO_HAVE_char_store_release
512#endif
513
514#if defined(AO_HAVE_char_compare_and_swap_full) \
515 && !defined(AO_HAVE_char_store_full)
516 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
517 AO_INLINE void
518 AO_char_store_full(volatile unsignedchar *addr, unsignedchar new_val)
519 {
520 unsignedchar old_val;
521
522 do {
523 old_val = *(unsignedchar *)addr;
524 } while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old_val,
525 new_val)));
526 }
527# define AO_HAVE_char_store_full
528#endif
529/*
530 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
531 *
532 * Permission is hereby granted, free of charge, to any person obtaining a copy
533 * of this software and associated documentation files (the "Software"), to deal
534 * in the Software without restriction, including without limitation the rights
535 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
536 * copies of the Software, and to permit persons to whom the Software is
537 * furnished to do so, subject to the following conditions:
538 *
539 * The above copyright notice and this permission notice shall be included in
540 * all copies or substantial portions of the Software.
541 *
542 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
543 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
544 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
545 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
546 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
547 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
548 * SOFTWARE.
549 */
550
551/* short_fetch_compare_and_swap */
552#if defined(AO_HAVE_short_fetch_compare_and_swap) \
553 && defined(AO_HAVE_nop_full) \
554 && !defined(AO_HAVE_short_fetch_compare_and_swap_acquire)
555 AO_INLINE unsignedshort
556 AO_short_fetch_compare_and_swap_acquire(volatile unsignedshort *addr,
557 unsignedshort old_val, unsignedshort new_val)
558 {
559 unsignedshort result = AO_short_fetch_compare_and_swap(addr, old_val, new_val);
560 AO_nop_full();
561 return result;
562 }
563# define AO_HAVE_short_fetch_compare_and_swap_acquire
564#endif
565#if defined(AO_HAVE_short_fetch_compare_and_swap) \
566 && defined(AO_HAVE_nop_full) \
567 && !defined(AO_HAVE_short_fetch_compare_and_swap_release)
568# define AO_short_fetch_compare_and_swap_release(addr, old_val, new_val) \
569 (AO_nop_full(), \
570 AO_short_fetch_compare_and_swap(addr, old_val, new_val))
571# define AO_HAVE_short_fetch_compare_and_swap_release
572#endif
573#if defined(AO_HAVE_short_fetch_compare_and_swap_full)
574# if !defined(AO_HAVE_short_fetch_compare_and_swap_release)
575# define AO_short_fetch_compare_and_swap_release(addr, old_val, new_val) \
576 AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
577# define AO_HAVE_short_fetch_compare_and_swap_release
578# endif
579# if !defined(AO_HAVE_short_fetch_compare_and_swap_acquire)
580# define AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
581 AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
582# define AO_HAVE_short_fetch_compare_and_swap_acquire
583# endif
584# if !defined(AO_HAVE_short_fetch_compare_and_swap_write)
585# define AO_short_fetch_compare_and_swap_write(addr, old_val, new_val) \
586 AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
587# define AO_HAVE_short_fetch_compare_and_swap_write
588# endif
589# if !defined(AO_HAVE_short_fetch_compare_and_swap_read)
590# define AO_short_fetch_compare_and_swap_read(addr, old_val, new_val) \
591 AO_short_fetch_compare_and_swap_full(addr, old_val, new_val)
592# define AO_HAVE_short_fetch_compare_and_swap_read
593# endif
594#endif /* AO_HAVE_short_fetch_compare_and_swap_full */
595
596#if !defined(AO_HAVE_short_fetch_compare_and_swap) \
597 && defined(AO_HAVE_short_fetch_compare_and_swap_release)
598# define AO_short_fetch_compare_and_swap(addr, old_val, new_val) \
599 AO_short_fetch_compare_and_swap_release(addr, old_val, new_val)
600# define AO_HAVE_short_fetch_compare_and_swap
601#endif
602#if !defined(AO_HAVE_short_fetch_compare_and_swap) \
603 && defined(AO_HAVE_short_fetch_compare_and_swap_acquire)
604# define AO_short_fetch_compare_and_swap(addr, old_val, new_val) \
605 AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val)
606# define AO_HAVE_short_fetch_compare_and_swap
607#endif
608#if !defined(AO_HAVE_short_fetch_compare_and_swap) \
609 && defined(AO_HAVE_short_fetch_compare_and_swap_write)
610# define AO_short_fetch_compare_and_swap(addr, old_val, new_val) \
611 AO_short_fetch_compare_and_swap_write(addr, old_val, new_val)
612# define AO_HAVE_short_fetch_compare_and_swap
613#endif
614#if !defined(AO_HAVE_short_fetch_compare_and_swap) \
615 && defined(AO_HAVE_short_fetch_compare_and_swap_read)
616# define AO_short_fetch_compare_and_swap(addr, old_val, new_val) \
617 AO_short_fetch_compare_and_swap_read(addr, old_val, new_val)
618# define AO_HAVE_short_fetch_compare_and_swap
619#endif
620
621#if defined(AO_HAVE_short_fetch_compare_and_swap_acquire) \
622 && defined(AO_HAVE_nop_full) \
623 && !defined(AO_HAVE_short_fetch_compare_and_swap_full)
624# define AO_short_fetch_compare_and_swap_full(addr, old_val, new_val) \
625 (AO_nop_full(), \
626 AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val))
627# define AO_HAVE_short_fetch_compare_and_swap_full
628#endif
629
630#if !defined(AO_HAVE_short_fetch_compare_and_swap_release_write) \
631 && defined(AO_HAVE_short_fetch_compare_and_swap_write)
632# define AO_short_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
633 AO_short_fetch_compare_and_swap_write(addr, old_val, new_val)
634# define AO_HAVE_short_fetch_compare_and_swap_release_write
635#endif
636#if !defined(AO_HAVE_short_fetch_compare_and_swap_release_write) \
637 && defined(AO_HAVE_short_fetch_compare_and_swap_release)
638# define AO_short_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
639 AO_short_fetch_compare_and_swap_release(addr, old_val, new_val)
640# define AO_HAVE_short_fetch_compare_and_swap_release_write
641#endif
642#if !defined(AO_HAVE_short_fetch_compare_and_swap_acquire_read) \
643 && defined(AO_HAVE_short_fetch_compare_and_swap_read)
644# define AO_short_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
645 AO_short_fetch_compare_and_swap_read(addr, old_val, new_val)
646# define AO_HAVE_short_fetch_compare_and_swap_acquire_read
647#endif
648#if !defined(AO_HAVE_short_fetch_compare_and_swap_acquire_read) \
649 && defined(AO_HAVE_short_fetch_compare_and_swap_acquire)
650# define AO_short_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
651 AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val)
652# define AO_HAVE_short_fetch_compare_and_swap_acquire_read
653#endif
654
655#ifdef AO_NO_DD_ORDERING
656# if defined(AO_HAVE_short_fetch_compare_and_swap_acquire_read)
657# define AO_short_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
658 AO_short_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
659# define AO_HAVE_short_fetch_compare_and_swap_dd_acquire_read
660# endif
661#else
662# if defined(AO_HAVE_short_fetch_compare_and_swap)
663# define AO_short_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
664 AO_short_fetch_compare_and_swap(addr, old_val, new_val)
665# define AO_HAVE_short_fetch_compare_and_swap_dd_acquire_read
666# endif
667#endif /* !AO_NO_DD_ORDERING */
668
669/* short_compare_and_swap */
670#if defined(AO_HAVE_short_compare_and_swap) && defined(AO_HAVE_nop_full) \
671 && !defined(AO_HAVE_short_compare_and_swap_acquire)
672 AO_INLINE int
673 AO_short_compare_and_swap_acquire(volatile unsignedshort *addr, unsignedshort old,
674 unsignedshort new_val)
675 {
676 int result = AO_short_compare_and_swap(addr, old, new_val);
677 AO_nop_full();
678 return result;
679 }
680# define AO_HAVE_short_compare_and_swap_acquire
681#endif
682#if defined(AO_HAVE_short_compare_and_swap) && defined(AO_HAVE_nop_full) \
683 && !defined(AO_HAVE_short_compare_and_swap_release)
684# define AO_short_compare_and_swap_release(addr, old, new_val) \
685 (AO_nop_full(), AO_short_compare_and_swap(addr, old, new_val))
686# define AO_HAVE_short_compare_and_swap_release
687#endif
688#if defined(AO_HAVE_short_compare_and_swap_full)
689# if !defined(AO_HAVE_short_compare_and_swap_release)
690# define AO_short_compare_and_swap_release(addr, old, new_val) \
691 AO_short_compare_and_swap_full(addr, old, new_val)
692# define AO_HAVE_short_compare_and_swap_release
693# endif
694# if !defined(AO_HAVE_short_compare_and_swap_acquire)
695# define AO_short_compare_and_swap_acquire(addr, old, new_val) \
696 AO_short_compare_and_swap_full(addr, old, new_val)
697# define AO_HAVE_short_compare_and_swap_acquire
698# endif
699# if !defined(AO_HAVE_short_compare_and_swap_write)
700# define AO_short_compare_and_swap_write(addr, old, new_val) \
701 AO_short_compare_and_swap_full(addr, old, new_val)
702# define AO_HAVE_short_compare_and_swap_write
703# endif
704# if !defined(AO_HAVE_short_compare_and_swap_read)
705# define AO_short_compare_and_swap_read(addr, old, new_val) \
706 AO_short_compare_and_swap_full(addr, old, new_val)
707# define AO_HAVE_short_compare_and_swap_read
708# endif
709#endif /* AO_HAVE_short_compare_and_swap_full */
710
711#if !defined(AO_HAVE_short_compare_and_swap) \
712 && defined(AO_HAVE_short_compare_and_swap_release)
713# define AO_short_compare_and_swap(addr, old, new_val) \
714 AO_short_compare_and_swap_release(addr, old, new_val)
715# define AO_HAVE_short_compare_and_swap
716#endif
717#if !defined(AO_HAVE_short_compare_and_swap) \
718 && defined(AO_HAVE_short_compare_and_swap_acquire)
719# define AO_short_compare_and_swap(addr, old, new_val) \
720 AO_short_compare_and_swap_acquire(addr, old, new_val)
721# define AO_HAVE_short_compare_and_swap
722#endif
723#if !defined(AO_HAVE_short_compare_and_swap) \
724 && defined(AO_HAVE_short_compare_and_swap_write)
725# define AO_short_compare_and_swap(addr, old, new_val) \
726 AO_short_compare_and_swap_write(addr, old, new_val)
727# define AO_HAVE_short_compare_and_swap
728#endif
729#if !defined(AO_HAVE_short_compare_and_swap) \
730 && defined(AO_HAVE_short_compare_and_swap_read)
731# define AO_short_compare_and_swap(addr, old, new_val) \
732 AO_short_compare_and_swap_read(addr, old, new_val)
733# define AO_HAVE_short_compare_and_swap
734#endif
735
736#if defined(AO_HAVE_short_compare_and_swap_acquire) \
737 && defined(AO_HAVE_nop_full) \
738 && !defined(AO_HAVE_short_compare_and_swap_full)
739# define AO_short_compare_and_swap_full(addr, old, new_val) \
740 (AO_nop_full(), \
741 AO_short_compare_and_swap_acquire(addr, old, new_val))
742# define AO_HAVE_short_compare_and_swap_full
743#endif
744
745#if !defined(AO_HAVE_short_compare_and_swap_release_write) \
746 && defined(AO_HAVE_short_compare_and_swap_write)
747# define AO_short_compare_and_swap_release_write(addr, old, new_val) \
748 AO_short_compare_and_swap_write(addr, old, new_val)
749# define AO_HAVE_short_compare_and_swap_release_write
750#endif
751#if !defined(AO_HAVE_short_compare_and_swap_release_write) \
752 && defined(AO_HAVE_short_compare_and_swap_release)
753# define AO_short_compare_and_swap_release_write(addr, old, new_val) \
754 AO_short_compare_and_swap_release(addr, old, new_val)
755# define AO_HAVE_short_compare_and_swap_release_write
756#endif
757#if !defined(AO_HAVE_short_compare_and_swap_acquire_read) \
758 && defined(AO_HAVE_short_compare_and_swap_read)
759# define AO_short_compare_and_swap_acquire_read(addr, old, new_val) \
760 AO_short_compare_and_swap_read(addr, old, new_val)
761# define AO_HAVE_short_compare_and_swap_acquire_read
762#endif
763#if !defined(AO_HAVE_short_compare_and_swap_acquire_read) \
764 && defined(AO_HAVE_short_compare_and_swap_acquire)
765# define AO_short_compare_and_swap_acquire_read(addr, old, new_val) \
766 AO_short_compare_and_swap_acquire(addr, old, new_val)
767# define AO_HAVE_short_compare_and_swap_acquire_read
768#endif
769
770#ifdef AO_NO_DD_ORDERING
771# if defined(AO_HAVE_short_compare_and_swap_acquire_read)
772# define AO_short_compare_and_swap_dd_acquire_read(addr, old, new_val) \
773 AO_short_compare_and_swap_acquire_read(addr, old, new_val)
774# define AO_HAVE_short_compare_and_swap_dd_acquire_read
775# endif
776#else
777# if defined(AO_HAVE_short_compare_and_swap)
778# define AO_short_compare_and_swap_dd_acquire_read(addr, old, new_val) \
779 AO_short_compare_and_swap(addr, old, new_val)
780# define AO_HAVE_short_compare_and_swap_dd_acquire_read
781# endif
782#endif /* !AO_NO_DD_ORDERING */
783
784/* short_load */
785#if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_acquire)
786# define AO_short_load_acquire(addr) AO_short_load_full(addr)
787# define AO_HAVE_short_load_acquire
788#endif
789
790#if defined(AO_HAVE_short_load_acquire) && !defined(AO_HAVE_short_load)
791# define AO_short_load(addr) AO_short_load_acquire(addr)
792# define AO_HAVE_short_load
793#endif
794
795#if defined(AO_HAVE_short_load_full) && !defined(AO_HAVE_short_load_read)
796# define AO_short_load_read(addr) AO_short_load_full(addr)
797# define AO_HAVE_short_load_read
798#endif
799
800#if !defined(AO_HAVE_short_load_acquire_read) \
801 && defined(AO_HAVE_short_load_acquire)
802# define AO_short_load_acquire_read(addr) AO_short_load_acquire(addr)
803# define AO_HAVE_short_load_acquire_read
804#endif
805
806#if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_full) \
807 && !defined(AO_HAVE_short_load_acquire)
808 AO_INLINE unsignedshort
809 AO_short_load_acquire(const volatile unsignedshort *addr)
810 {
811 unsignedshort result = AO_short_load(addr);
812
813 /* Acquire barrier would be useless, since the load could be delayed */
814 /* beyond it. */
815 AO_nop_full();
816 return result;
817 }
818# define AO_HAVE_short_load_acquire
819#endif
820
821#if defined(AO_HAVE_short_load) && defined(AO_HAVE_nop_read) \
822 && !defined(AO_HAVE_short_load_read)
823 AO_INLINE unsignedshort
824 AO_short_load_read(const volatile unsignedshort *addr)
825 {
826 unsignedshort result = AO_short_load(addr);
827
828 AO_nop_read();
829 return result;
830 }
831# define AO_HAVE_short_load_read
832#endif
833
834#if defined(AO_HAVE_short_load_acquire) && defined(AO_HAVE_nop_full) \
835 && !defined(AO_HAVE_short_load_full)
836# define AO_short_load_full(addr) (AO_nop_full(), AO_short_load_acquire(addr))
837# define AO_HAVE_short_load_full
838#endif
839
840#if defined(AO_HAVE_short_compare_and_swap_read) \
841 && !defined(AO_HAVE_short_load_read)
842# define AO_short_CAS_BASED_LOAD_READ
843 AO_ATTR_NO_SANITIZE_THREAD
844 AO_INLINE unsignedshort
845 AO_short_load_read(const volatile unsignedshort *addr)
846 {
847 unsignedshort result;
848
849 do {
850 result = *(const unsignedshort *)addr;
851 } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_read(
852 (volatile unsignedshort *)addr,
853 result, result)));
854 return result;
855 }
856# define AO_HAVE_short_load_read
857#endif
858
859#if !defined(AO_HAVE_short_load_acquire_read) \
860 && defined(AO_HAVE_short_load_read)
861# define AO_short_load_acquire_read(addr) AO_short_load_read(addr)
862# define AO_HAVE_short_load_acquire_read
863#endif
864
865#if defined(AO_HAVE_short_load_acquire_read) && !defined(AO_HAVE_short_load) \
866 && (!defined(AO_short_CAS_BASED_LOAD_READ) \
867 || !defined(AO_HAVE_short_compare_and_swap))
868# define AO_short_load(addr) AO_short_load_acquire_read(addr)
869# define AO_HAVE_short_load
870#endif
871
872#if defined(AO_HAVE_short_compare_and_swap_full) \
873 && !defined(AO_HAVE_short_load_full)
874 AO_ATTR_NO_SANITIZE_THREAD
875 AO_INLINE unsignedshort
876 AO_short_load_full(const volatile unsignedshort *addr)
877 {
878 unsignedshort result;
879
880 do {
881 result = *(const unsignedshort *)addr;
882 } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(
883 (volatile unsignedshort *)addr,
884 result, result)));
885 return result;
886 }
887# define AO_HAVE_short_load_full
888#endif
889
890#if defined(AO_HAVE_short_compare_and_swap_acquire) \
891 && !defined(AO_HAVE_short_load_acquire)
892 AO_ATTR_NO_SANITIZE_THREAD
893 AO_INLINE unsignedshort
894 AO_short_load_acquire(const volatile unsignedshort *addr)
895 {
896 unsignedshort result;
897
898 do {
899 result = *(const unsignedshort *)addr;
900 } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_acquire(
901 (volatile unsignedshort *)addr,
902 result, result)));
903 return result;
904 }
905# define AO_HAVE_short_load_acquire
906#endif
907
908#if defined(AO_HAVE_short_compare_and_swap) && !defined(AO_HAVE_short_load)
909 AO_ATTR_NO_SANITIZE_THREAD
910 AO_INLINE unsignedshort
911 AO_short_load(const volatile unsignedshort *addr)
912 {
913 unsignedshort result;
914
915 do {
916 result = *(const unsignedshort *)addr;
917 } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap(
918 (volatile unsignedshort *)addr,
919 result, result)));
920 return result;
921 }
922# define AO_HAVE_short_load
923#endif
924
925#ifdef AO_NO_DD_ORDERING
926# if defined(AO_HAVE_short_load_acquire_read)
927# define AO_short_load_dd_acquire_read(addr) \
928 AO_short_load_acquire_read(addr)
929# define AO_HAVE_short_load_dd_acquire_read
930# endif
931#else
932# if defined(AO_HAVE_short_load)
933# define AO_short_load_dd_acquire_read(addr) AO_short_load(addr)
934# define AO_HAVE_short_load_dd_acquire_read
935# endif
936#endif /* !AO_NO_DD_ORDERING */
937
938/* short_store */
939#if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_release)
940# define AO_short_store_release(addr, val) AO_short_store_full(addr, val)
941# define AO_HAVE_short_store_release
942#endif
943
944#if defined(AO_HAVE_short_store_release) && !defined(AO_HAVE_short_store)
945# define AO_short_store(addr, val) AO_short_store_release(addr, val)
946# define AO_HAVE_short_store
947#endif
948
949#if defined(AO_HAVE_short_store_full) && !defined(AO_HAVE_short_store_write)
950# define AO_short_store_write(addr, val) AO_short_store_full(addr, val)
951# define AO_HAVE_short_store_write
952#endif
953
954#if defined(AO_HAVE_short_store_release) \
955 && !defined(AO_HAVE_short_store_release_write)
956# define AO_short_store_release_write(addr, val) \
957 AO_short_store_release(addr, val)
958# define AO_HAVE_short_store_release_write
959#endif
960
961#if defined(AO_HAVE_short_store_write) && !defined(AO_HAVE_short_store)
962# define AO_short_store(addr, val) AO_short_store_write(addr, val)
963# define AO_HAVE_short_store
964#endif
965
966#if defined(AO_HAVE_short_store) && defined(AO_HAVE_nop_full) \
967 && !defined(AO_HAVE_short_store_release)
968# define AO_short_store_release(addr, val) \
969 (AO_nop_full(), AO_short_store(addr, val))
970# define AO_HAVE_short_store_release
971#endif
972
973#if defined(AO_HAVE_short_store) && defined(AO_HAVE_nop_write) \
974 && !defined(AO_HAVE_short_store_write)
975# define AO_short_store_write(addr, val) \
976 (AO_nop_write(), AO_short_store(addr, val))
977# define AO_HAVE_short_store_write
978#endif
979
980#if defined(AO_HAVE_short_compare_and_swap_write) \
981 && !defined(AO_HAVE_short_store_write)
982 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
983 AO_INLINE void
984 AO_short_store_write(volatile unsignedshort *addr, unsignedshort new_val)
985 {
986 unsignedshort old_val;
987
988 do {
989 old_val = *(unsignedshort *)addr;
990 } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_write(addr, old_val,
991 new_val)));
992 }
993# define AO_HAVE_short_store_write
994#endif
995
996#if defined(AO_HAVE_short_store_write) \
997 && !defined(AO_HAVE_short_store_release_write)
998# define AO_short_store_release_write(addr, val) \
999 AO_short_store_write(addr, val)
1000# define AO_HAVE_short_store_release_write
1001#endif
1002
1003#if defined(AO_HAVE_short_store_release) && defined(AO_HAVE_nop_full) \
1004 && !defined(AO_HAVE_short_store_full)
1005# define AO_short_store_full(addr, val) \
1006 (AO_short_store_release(addr, val), \
1007 AO_nop_full())
1008# define AO_HAVE_short_store_full
1009#endif
1010
1011#if defined(AO_HAVE_short_compare_and_swap) && !defined(AO_HAVE_short_store)
1012 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1013 AO_INLINE void
1014 AO_short_store(volatile unsignedshort *addr, unsignedshort new_val)
1015 {
1016 unsignedshort old_val;
1017
1018 do {
1019 old_val = *(unsignedshort *)addr;
1020 } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap(addr,
1021 old_val, new_val)));
1022 }
1023# define AO_HAVE_short_store
1024#endif
1025
1026#if defined(AO_HAVE_short_compare_and_swap_release) \
1027 && !defined(AO_HAVE_short_store_release)
1028 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1029 AO_INLINE void
1030 AO_short_store_release(volatile unsignedshort *addr, unsignedshort new_val)
1031 {
1032 unsignedshort old_val;
1033
1034 do {
1035 old_val = *(unsignedshort *)addr;
1036 } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_release(addr, old_val,
1037 new_val)));
1038 }
1039# define AO_HAVE_short_store_release
1040#endif
1041
1042#if defined(AO_HAVE_short_compare_and_swap_full) \
1043 && !defined(AO_HAVE_short_store_full)
1044 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1045 AO_INLINE void
1046 AO_short_store_full(volatile unsignedshort *addr, unsignedshort new_val)
1047 {
1048 unsignedshort old_val;
1049
1050 do {
1051 old_val = *(unsignedshort *)addr;
1052 } while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old_val,
1053 new_val)));
1054 }
1055# define AO_HAVE_short_store_full
1056#endif
1057/*
1058 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
1059 *
1060 * Permission is hereby granted, free of charge, to any person obtaining a copy
1061 * of this software and associated documentation files (the "Software"), to deal
1062 * in the Software without restriction, including without limitation the rights
1063 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
1064 * copies of the Software, and to permit persons to whom the Software is
1065 * furnished to do so, subject to the following conditions:
1066 *
1067 * The above copyright notice and this permission notice shall be included in
1068 * all copies or substantial portions of the Software.
1069 *
1070 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1071 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1072 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
1073 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
1074 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
1075 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
1076 * SOFTWARE.
1077 */
1078
1079/* int_fetch_compare_and_swap */
1080#if defined(AO_HAVE_int_fetch_compare_and_swap) \
1081 && defined(AO_HAVE_nop_full) \
1082 && !defined(AO_HAVE_int_fetch_compare_and_swap_acquire)
1083 AO_INLINE unsigned
1084 AO_int_fetch_compare_and_swap_acquire(volatile unsigned *addr,
1085 unsigned old_val, unsigned new_val)
1086 {
1087 unsigned result = AO_int_fetch_compare_and_swap(addr, old_val, new_val);
1088 AO_nop_full();
1089 return result;
1090 }
1091# define AO_HAVE_int_fetch_compare_and_swap_acquire
1092#endif
1093#if defined(AO_HAVE_int_fetch_compare_and_swap) \
1094 && defined(AO_HAVE_nop_full) \
1095 && !defined(AO_HAVE_int_fetch_compare_and_swap_release)
1096# define AO_int_fetch_compare_and_swap_release(addr, old_val, new_val) \
1097 (AO_nop_full(), \
1098 AO_int_fetch_compare_and_swap(addr, old_val, new_val))
1099# define AO_HAVE_int_fetch_compare_and_swap_release
1100#endif
1101#if defined(AO_HAVE_int_fetch_compare_and_swap_full)
1102# if !defined(AO_HAVE_int_fetch_compare_and_swap_release)
1103# define AO_int_fetch_compare_and_swap_release(addr, old_val, new_val) \
1104 AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1105# define AO_HAVE_int_fetch_compare_and_swap_release
1106# endif
1107# if !defined(AO_HAVE_int_fetch_compare_and_swap_acquire)
1108# define AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
1109 AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1110# define AO_HAVE_int_fetch_compare_and_swap_acquire
1111# endif
1112# if !defined(AO_HAVE_int_fetch_compare_and_swap_write)
1113# define AO_int_fetch_compare_and_swap_write(addr, old_val, new_val) \
1114 AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1115# define AO_HAVE_int_fetch_compare_and_swap_write
1116# endif
1117# if !defined(AO_HAVE_int_fetch_compare_and_swap_read)
1118# define AO_int_fetch_compare_and_swap_read(addr, old_val, new_val) \
1119 AO_int_fetch_compare_and_swap_full(addr, old_val, new_val)
1120# define AO_HAVE_int_fetch_compare_and_swap_read
1121# endif
1122#endif /* AO_HAVE_int_fetch_compare_and_swap_full */
1123
1124#if !defined(AO_HAVE_int_fetch_compare_and_swap) \
1125 && defined(AO_HAVE_int_fetch_compare_and_swap_release)
1126# define AO_int_fetch_compare_and_swap(addr, old_val, new_val) \
1127 AO_int_fetch_compare_and_swap_release(addr, old_val, new_val)
1128# define AO_HAVE_int_fetch_compare_and_swap
1129#endif
1130#if !defined(AO_HAVE_int_fetch_compare_and_swap) \
1131 && defined(AO_HAVE_int_fetch_compare_and_swap_acquire)
1132# define AO_int_fetch_compare_and_swap(addr, old_val, new_val) \
1133 AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1134# define AO_HAVE_int_fetch_compare_and_swap
1135#endif
1136#if !defined(AO_HAVE_int_fetch_compare_and_swap) \
1137 && defined(AO_HAVE_int_fetch_compare_and_swap_write)
1138# define AO_int_fetch_compare_and_swap(addr, old_val, new_val) \
1139 AO_int_fetch_compare_and_swap_write(addr, old_val, new_val)
1140# define AO_HAVE_int_fetch_compare_and_swap
1141#endif
1142#if !defined(AO_HAVE_int_fetch_compare_and_swap) \
1143 && defined(AO_HAVE_int_fetch_compare_and_swap_read)
1144# define AO_int_fetch_compare_and_swap(addr, old_val, new_val) \
1145 AO_int_fetch_compare_and_swap_read(addr, old_val, new_val)
1146# define AO_HAVE_int_fetch_compare_and_swap
1147#endif
1148
1149#if defined(AO_HAVE_int_fetch_compare_and_swap_acquire) \
1150 && defined(AO_HAVE_nop_full) \
1151 && !defined(AO_HAVE_int_fetch_compare_and_swap_full)
1152# define AO_int_fetch_compare_and_swap_full(addr, old_val, new_val) \
1153 (AO_nop_full(), \
1154 AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val))
1155# define AO_HAVE_int_fetch_compare_and_swap_full
1156#endif
1157
1158#if !defined(AO_HAVE_int_fetch_compare_and_swap_release_write) \
1159 && defined(AO_HAVE_int_fetch_compare_and_swap_write)
1160# define AO_int_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
1161 AO_int_fetch_compare_and_swap_write(addr, old_val, new_val)
1162# define AO_HAVE_int_fetch_compare_and_swap_release_write
1163#endif
1164#if !defined(AO_HAVE_int_fetch_compare_and_swap_release_write) \
1165 && defined(AO_HAVE_int_fetch_compare_and_swap_release)
1166# define AO_int_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
1167 AO_int_fetch_compare_and_swap_release(addr, old_val, new_val)
1168# define AO_HAVE_int_fetch_compare_and_swap_release_write
1169#endif
1170#if !defined(AO_HAVE_int_fetch_compare_and_swap_acquire_read) \
1171 && defined(AO_HAVE_int_fetch_compare_and_swap_read)
1172# define AO_int_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
1173 AO_int_fetch_compare_and_swap_read(addr, old_val, new_val)
1174# define AO_HAVE_int_fetch_compare_and_swap_acquire_read
1175#endif
1176#if !defined(AO_HAVE_int_fetch_compare_and_swap_acquire_read) \
1177 && defined(AO_HAVE_int_fetch_compare_and_swap_acquire)
1178# define AO_int_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
1179 AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1180# define AO_HAVE_int_fetch_compare_and_swap_acquire_read
1181#endif
1182
1183#ifdef AO_NO_DD_ORDERING
1184# if defined(AO_HAVE_int_fetch_compare_and_swap_acquire_read)
1185# define AO_int_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
1186 AO_int_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
1187# define AO_HAVE_int_fetch_compare_and_swap_dd_acquire_read
1188# endif
1189#else
1190# if defined(AO_HAVE_int_fetch_compare_and_swap)
1191# define AO_int_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
1192 AO_int_fetch_compare_and_swap(addr, old_val, new_val)
1193# define AO_HAVE_int_fetch_compare_and_swap_dd_acquire_read
1194# endif
1195#endif /* !AO_NO_DD_ORDERING */
1196
1197/* int_compare_and_swap */
1198#if defined(AO_HAVE_int_compare_and_swap) && defined(AO_HAVE_nop_full) \
1199 && !defined(AO_HAVE_int_compare_and_swap_acquire)
1200 AO_INLINE int
1201 AO_int_compare_and_swap_acquire(volatile unsigned *addr, unsigned old,
1202 unsigned new_val)
1203 {
1204 int result = AO_int_compare_and_swap(addr, old, new_val);
1205 AO_nop_full();
1206 return result;
1207 }
1208# define AO_HAVE_int_compare_and_swap_acquire
1209#endif
1210#if defined(AO_HAVE_int_compare_and_swap) && defined(AO_HAVE_nop_full) \
1211 && !defined(AO_HAVE_int_compare_and_swap_release)
1212# define AO_int_compare_and_swap_release(addr, old, new_val) \
1213 (AO_nop_full(), AO_int_compare_and_swap(addr, old, new_val))
1214# define AO_HAVE_int_compare_and_swap_release
1215#endif
1216#if defined(AO_HAVE_int_compare_and_swap_full)
1217# if !defined(AO_HAVE_int_compare_and_swap_release)
1218# define AO_int_compare_and_swap_release(addr, old, new_val) \
1219 AO_int_compare_and_swap_full(addr, old, new_val)
1220# define AO_HAVE_int_compare_and_swap_release
1221# endif
1222# if !defined(AO_HAVE_int_compare_and_swap_acquire)
1223# define AO_int_compare_and_swap_acquire(addr, old, new_val) \
1224 AO_int_compare_and_swap_full(addr, old, new_val)
1225# define AO_HAVE_int_compare_and_swap_acquire
1226# endif
1227# if !defined(AO_HAVE_int_compare_and_swap_write)
1228# define AO_int_compare_and_swap_write(addr, old, new_val) \
1229 AO_int_compare_and_swap_full(addr, old, new_val)
1230# define AO_HAVE_int_compare_and_swap_write
1231# endif
1232# if !defined(AO_HAVE_int_compare_and_swap_read)
1233# define AO_int_compare_and_swap_read(addr, old, new_val) \
1234 AO_int_compare_and_swap_full(addr, old, new_val)
1235# define AO_HAVE_int_compare_and_swap_read
1236# endif
1237#endif /* AO_HAVE_int_compare_and_swap_full */
1238
1239#if !defined(AO_HAVE_int_compare_and_swap) \
1240 && defined(AO_HAVE_int_compare_and_swap_release)
1241# define AO_int_compare_and_swap(addr, old, new_val) \
1242 AO_int_compare_and_swap_release(addr, old, new_val)
1243# define AO_HAVE_int_compare_and_swap
1244#endif
1245#if !defined(AO_HAVE_int_compare_and_swap) \
1246 && defined(AO_HAVE_int_compare_and_swap_acquire)
1247# define AO_int_compare_and_swap(addr, old, new_val) \
1248 AO_int_compare_and_swap_acquire(addr, old, new_val)
1249# define AO_HAVE_int_compare_and_swap
1250#endif
1251#if !defined(AO_HAVE_int_compare_and_swap) \
1252 && defined(AO_HAVE_int_compare_and_swap_write)
1253# define AO_int_compare_and_swap(addr, old, new_val) \
1254 AO_int_compare_and_swap_write(addr, old, new_val)
1255# define AO_HAVE_int_compare_and_swap
1256#endif
1257#if !defined(AO_HAVE_int_compare_and_swap) \
1258 && defined(AO_HAVE_int_compare_and_swap_read)
1259# define AO_int_compare_and_swap(addr, old, new_val) \
1260 AO_int_compare_and_swap_read(addr, old, new_val)
1261# define AO_HAVE_int_compare_and_swap
1262#endif
1263
1264#if defined(AO_HAVE_int_compare_and_swap_acquire) \
1265 && defined(AO_HAVE_nop_full) \
1266 && !defined(AO_HAVE_int_compare_and_swap_full)
1267# define AO_int_compare_and_swap_full(addr, old, new_val) \
1268 (AO_nop_full(), \
1269 AO_int_compare_and_swap_acquire(addr, old, new_val))
1270# define AO_HAVE_int_compare_and_swap_full
1271#endif
1272
1273#if !defined(AO_HAVE_int_compare_and_swap_release_write) \
1274 && defined(AO_HAVE_int_compare_and_swap_write)
1275# define AO_int_compare_and_swap_release_write(addr, old, new_val) \
1276 AO_int_compare_and_swap_write(addr, old, new_val)
1277# define AO_HAVE_int_compare_and_swap_release_write
1278#endif
1279#if !defined(AO_HAVE_int_compare_and_swap_release_write) \
1280 && defined(AO_HAVE_int_compare_and_swap_release)
1281# define AO_int_compare_and_swap_release_write(addr, old, new_val) \
1282 AO_int_compare_and_swap_release(addr, old, new_val)
1283# define AO_HAVE_int_compare_and_swap_release_write
1284#endif
1285#if !defined(AO_HAVE_int_compare_and_swap_acquire_read) \
1286 && defined(AO_HAVE_int_compare_and_swap_read)
1287# define AO_int_compare_and_swap_acquire_read(addr, old, new_val) \
1288 AO_int_compare_and_swap_read(addr, old, new_val)
1289# define AO_HAVE_int_compare_and_swap_acquire_read
1290#endif
1291#if !defined(AO_HAVE_int_compare_and_swap_acquire_read) \
1292 && defined(AO_HAVE_int_compare_and_swap_acquire)
1293# define AO_int_compare_and_swap_acquire_read(addr, old, new_val) \
1294 AO_int_compare_and_swap_acquire(addr, old, new_val)
1295# define AO_HAVE_int_compare_and_swap_acquire_read
1296#endif
1297
1298#ifdef AO_NO_DD_ORDERING
1299# if defined(AO_HAVE_int_compare_and_swap_acquire_read)
1300# define AO_int_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1301 AO_int_compare_and_swap_acquire_read(addr, old, new_val)
1302# define AO_HAVE_int_compare_and_swap_dd_acquire_read
1303# endif
1304#else
1305# if defined(AO_HAVE_int_compare_and_swap)
1306# define AO_int_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1307 AO_int_compare_and_swap(addr, old, new_val)
1308# define AO_HAVE_int_compare_and_swap_dd_acquire_read
1309# endif
1310#endif /* !AO_NO_DD_ORDERING */
1311
1312/* int_load */
1313#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_acquire)
1314# define AO_int_load_acquire(addr) AO_int_load_full(addr)
1315# define AO_HAVE_int_load_acquire
1316#endif
1317
1318#if defined(AO_HAVE_int_load_acquire) && !defined(AO_HAVE_int_load)
1319# define AO_int_load(addr) AO_int_load_acquire(addr)
1320# define AO_HAVE_int_load
1321#endif
1322
1323#if defined(AO_HAVE_int_load_full) && !defined(AO_HAVE_int_load_read)
1324# define AO_int_load_read(addr) AO_int_load_full(addr)
1325# define AO_HAVE_int_load_read
1326#endif
1327
1328#if !defined(AO_HAVE_int_load_acquire_read) \
1329 && defined(AO_HAVE_int_load_acquire)
1330# define AO_int_load_acquire_read(addr) AO_int_load_acquire(addr)
1331# define AO_HAVE_int_load_acquire_read
1332#endif
1333
1334#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_full) \
1335 && !defined(AO_HAVE_int_load_acquire)
1336 AO_INLINE unsigned
1337 AO_int_load_acquire(const volatile unsigned *addr)
1338 {
1339 unsigned result = AO_int_load(addr);
1340
1341 /* Acquire barrier would be useless, since the load could be delayed */
1342 /* beyond it. */
1343 AO_nop_full();
1344 return result;
1345 }
1346# define AO_HAVE_int_load_acquire
1347#endif
1348
1349#if defined(AO_HAVE_int_load) && defined(AO_HAVE_nop_read) \
1350 && !defined(AO_HAVE_int_load_read)
1351 AO_INLINE unsigned
1352 AO_int_load_read(const volatile unsigned *addr)
1353 {
1354 unsigned result = AO_int_load(addr);
1355
1356 AO_nop_read();
1357 return result;
1358 }
1359# define AO_HAVE_int_load_read
1360#endif
1361
1362#if defined(AO_HAVE_int_load_acquire) && defined(AO_HAVE_nop_full) \
1363 && !defined(AO_HAVE_int_load_full)
1364# define AO_int_load_full(addr) (AO_nop_full(), AO_int_load_acquire(addr))
1365# define AO_HAVE_int_load_full
1366#endif
1367
1368#if defined(AO_HAVE_int_compare_and_swap_read) \
1369 && !defined(AO_HAVE_int_load_read)
1370# define AO_int_CAS_BASED_LOAD_READ
1371 AO_ATTR_NO_SANITIZE_THREAD
1372 AO_INLINE unsigned
1373 AO_int_load_read(const volatile unsigned *addr)
1374 {
1375 unsigned result;
1376
1377 do {
1378 result = *(const unsigned *)addr;
1379 } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_read(
1380 (volatile unsigned *)addr,
1381 result, result)));
1382 return result;
1383 }
1384# define AO_HAVE_int_load_read
1385#endif
1386
1387#if !defined(AO_HAVE_int_load_acquire_read) \
1388 && defined(AO_HAVE_int_load_read)
1389# define AO_int_load_acquire_read(addr) AO_int_load_read(addr)
1390# define AO_HAVE_int_load_acquire_read
1391#endif
1392
1393#if defined(AO_HAVE_int_load_acquire_read) && !defined(AO_HAVE_int_load) \
1394 && (!defined(AO_int_CAS_BASED_LOAD_READ) \
1395 || !defined(AO_HAVE_int_compare_and_swap))
1396# define AO_int_load(addr) AO_int_load_acquire_read(addr)
1397# define AO_HAVE_int_load
1398#endif
1399
1400#if defined(AO_HAVE_int_compare_and_swap_full) \
1401 && !defined(AO_HAVE_int_load_full)
1402 AO_ATTR_NO_SANITIZE_THREAD
1403 AO_INLINE unsigned
1404 AO_int_load_full(const volatile unsigned *addr)
1405 {
1406 unsigned result;
1407
1408 do {
1409 result = *(const unsigned *)addr;
1410 } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(
1411 (volatile unsigned *)addr,
1412 result, result)));
1413 return result;
1414 }
1415# define AO_HAVE_int_load_full
1416#endif
1417
1418#if defined(AO_HAVE_int_compare_and_swap_acquire) \
1419 && !defined(AO_HAVE_int_load_acquire)
1420 AO_ATTR_NO_SANITIZE_THREAD
1421 AO_INLINE unsigned
1422 AO_int_load_acquire(const volatile unsigned *addr)
1423 {
1424 unsigned result;
1425
1426 do {
1427 result = *(const unsigned *)addr;
1428 } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_acquire(
1429 (volatile unsigned *)addr,
1430 result, result)));
1431 return result;
1432 }
1433# define AO_HAVE_int_load_acquire
1434#endif
1435
1436#if defined(AO_HAVE_int_compare_and_swap) && !defined(AO_HAVE_int_load)
1437 AO_ATTR_NO_SANITIZE_THREAD
1438 AO_INLINE unsigned
1439 AO_int_load(const volatile unsigned *addr)
1440 {
1441 unsigned result;
1442
1443 do {
1444 result = *(const unsigned *)addr;
1445 } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap(
1446 (volatile unsigned *)addr,
1447 result, result)));
1448 return result;
1449 }
1450# define AO_HAVE_int_load
1451#endif
1452
1453#ifdef AO_NO_DD_ORDERING
1454# if defined(AO_HAVE_int_load_acquire_read)
1455# define AO_int_load_dd_acquire_read(addr) \
1456 AO_int_load_acquire_read(addr)
1457# define AO_HAVE_int_load_dd_acquire_read
1458# endif
1459#else
1460# if defined(AO_HAVE_int_load)
1461# define AO_int_load_dd_acquire_read(addr) AO_int_load(addr)
1462# define AO_HAVE_int_load_dd_acquire_read
1463# endif
1464#endif /* !AO_NO_DD_ORDERING */
1465
1466/* int_store */
1467#if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_release)
1468# define AO_int_store_release(addr, val) AO_int_store_full(addr, val)
1469# define AO_HAVE_int_store_release
1470#endif
1471
1472#if defined(AO_HAVE_int_store_release) && !defined(AO_HAVE_int_store)
1473# define AO_int_store(addr, val) AO_int_store_release(addr, val)
1474# define AO_HAVE_int_store
1475#endif
1476
1477#if defined(AO_HAVE_int_store_full) && !defined(AO_HAVE_int_store_write)
1478# define AO_int_store_write(addr, val) AO_int_store_full(addr, val)
1479# define AO_HAVE_int_store_write
1480#endif
1481
1482#if defined(AO_HAVE_int_store_release) \
1483 && !defined(AO_HAVE_int_store_release_write)
1484# define AO_int_store_release_write(addr, val) \
1485 AO_int_store_release(addr, val)
1486# define AO_HAVE_int_store_release_write
1487#endif
1488
1489#if defined(AO_HAVE_int_store_write) && !defined(AO_HAVE_int_store)
1490# define AO_int_store(addr, val) AO_int_store_write(addr, val)
1491# define AO_HAVE_int_store
1492#endif
1493
1494#if defined(AO_HAVE_int_store) && defined(AO_HAVE_nop_full) \
1495 && !defined(AO_HAVE_int_store_release)
1496# define AO_int_store_release(addr, val) \
1497 (AO_nop_full(), AO_int_store(addr, val))
1498# define AO_HAVE_int_store_release
1499#endif
1500
1501#if defined(AO_HAVE_int_store) && defined(AO_HAVE_nop_write) \
1502 && !defined(AO_HAVE_int_store_write)
1503# define AO_int_store_write(addr, val) \
1504 (AO_nop_write(), AO_int_store(addr, val))
1505# define AO_HAVE_int_store_write
1506#endif
1507
1508#if defined(AO_HAVE_int_compare_and_swap_write) \
1509 && !defined(AO_HAVE_int_store_write)
1510 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1511 AO_INLINE void
1512 AO_int_store_write(volatile unsigned *addr, unsigned new_val)
1513 {
1514 unsigned old_val;
1515
1516 do {
1517 old_val = *(unsigned *)addr;
1518 } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_write(addr, old_val,
1519 new_val)));
1520 }
1521# define AO_HAVE_int_store_write
1522#endif
1523
1524#if defined(AO_HAVE_int_store_write) \
1525 && !defined(AO_HAVE_int_store_release_write)
1526# define AO_int_store_release_write(addr, val) \
1527 AO_int_store_write(addr, val)
1528# define AO_HAVE_int_store_release_write
1529#endif
1530
1531#if defined(AO_HAVE_int_store_release) && defined(AO_HAVE_nop_full) \
1532 && !defined(AO_HAVE_int_store_full)
1533# define AO_int_store_full(addr, val) \
1534 (AO_int_store_release(addr, val), \
1535 AO_nop_full())
1536# define AO_HAVE_int_store_full
1537#endif
1538
1539#if defined(AO_HAVE_int_compare_and_swap) && !defined(AO_HAVE_int_store)
1540 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1541 AO_INLINE void
1542 AO_int_store(volatile unsigned *addr, unsigned new_val)
1543 {
1544 unsigned old_val;
1545
1546 do {
1547 old_val = *(unsigned *)addr;
1548 } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap(addr,
1549 old_val, new_val)));
1550 }
1551# define AO_HAVE_int_store
1552#endif
1553
1554#if defined(AO_HAVE_int_compare_and_swap_release) \
1555 && !defined(AO_HAVE_int_store_release)
1556 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1557 AO_INLINE void
1558 AO_int_store_release(volatile unsigned *addr, unsigned new_val)
1559 {
1560 unsigned old_val;
1561
1562 do {
1563 old_val = *(unsigned *)addr;
1564 } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_release(addr, old_val,
1565 new_val)));
1566 }
1567# define AO_HAVE_int_store_release
1568#endif
1569
1570#if defined(AO_HAVE_int_compare_and_swap_full) \
1571 && !defined(AO_HAVE_int_store_full)
1572 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
1573 AO_INLINE void
1574 AO_int_store_full(volatile unsigned *addr, unsigned new_val)
1575 {
1576 unsigned old_val;
1577
1578 do {
1579 old_val = *(unsigned *)addr;
1580 } while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old_val,
1581 new_val)));
1582 }
1583# define AO_HAVE_int_store_full
1584#endif
1585/*
1586 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
1587 *
1588 * Permission is hereby granted, free of charge, to any person obtaining a copy
1589 * of this software and associated documentation files (the "Software"), to deal
1590 * in the Software without restriction, including without limitation the rights
1591 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
1592 * copies of the Software, and to permit persons to whom the Software is
1593 * furnished to do so, subject to the following conditions:
1594 *
1595 * The above copyright notice and this permission notice shall be included in
1596 * all copies or substantial portions of the Software.
1597 *
1598 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1599 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1600 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
1601 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
1602 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
1603 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
1604 * SOFTWARE.
1605 */
1606
1607/* fetch_compare_and_swap */
1608#if defined(AO_HAVE_fetch_compare_and_swap) \
1609 && defined(AO_HAVE_nop_full) \
1610 && !defined(AO_HAVE_fetch_compare_and_swap_acquire)
1611 AO_INLINE AO_t
1612 AO_fetch_compare_and_swap_acquire(volatile AO_t *addr,
1613 AO_t old_val, AO_t new_val)
1614 {
1615 AO_t result = AO_fetch_compare_and_swap(addr, old_val, new_val);
1616 AO_nop_full();
1617 return result;
1618 }
1619# define AO_HAVE_fetch_compare_and_swap_acquire
1620#endif
1621#if defined(AO_HAVE_fetch_compare_and_swap) \
1622 && defined(AO_HAVE_nop_full) \
1623 && !defined(AO_HAVE_fetch_compare_and_swap_release)
1624# define AO_fetch_compare_and_swap_release(addr, old_val, new_val) \
1625 (AO_nop_full(), \
1626 AO_fetch_compare_and_swap(addr, old_val, new_val))
1627# define AO_HAVE_fetch_compare_and_swap_release
1628#endif
1629#if defined(AO_HAVE_fetch_compare_and_swap_full)
1630# if !defined(AO_HAVE_fetch_compare_and_swap_release)
1631# define AO_fetch_compare_and_swap_release(addr, old_val, new_val) \
1632 AO_fetch_compare_and_swap_full(addr, old_val, new_val)
1633# define AO_HAVE_fetch_compare_and_swap_release
1634# endif
1635# if !defined(AO_HAVE_fetch_compare_and_swap_acquire)
1636# define AO_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
1637 AO_fetch_compare_and_swap_full(addr, old_val, new_val)
1638# define AO_HAVE_fetch_compare_and_swap_acquire
1639# endif
1640# if !defined(AO_HAVE_fetch_compare_and_swap_write)
1641# define AO_fetch_compare_and_swap_write(addr, old_val, new_val) \
1642 AO_fetch_compare_and_swap_full(addr, old_val, new_val)
1643# define AO_HAVE_fetch_compare_and_swap_write
1644# endif
1645# if !defined(AO_HAVE_fetch_compare_and_swap_read)
1646# define AO_fetch_compare_and_swap_read(addr, old_val, new_val) \
1647 AO_fetch_compare_and_swap_full(addr, old_val, new_val)
1648# define AO_HAVE_fetch_compare_and_swap_read
1649# endif
1650#endif /* AO_HAVE_fetch_compare_and_swap_full */
1651
1652#if !defined(AO_HAVE_fetch_compare_and_swap) \
1653 && defined(AO_HAVE_fetch_compare_and_swap_release)
1654# define AO_fetch_compare_and_swap(addr, old_val, new_val) \
1655 AO_fetch_compare_and_swap_release(addr, old_val, new_val)
1656# define AO_HAVE_fetch_compare_and_swap
1657#endif
1658#if !defined(AO_HAVE_fetch_compare_and_swap) \
1659 && defined(AO_HAVE_fetch_compare_and_swap_acquire)
1660# define AO_fetch_compare_and_swap(addr, old_val, new_val) \
1661 AO_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1662# define AO_HAVE_fetch_compare_and_swap
1663#endif
1664#if !defined(AO_HAVE_fetch_compare_and_swap) \
1665 && defined(AO_HAVE_fetch_compare_and_swap_write)
1666# define AO_fetch_compare_and_swap(addr, old_val, new_val) \
1667 AO_fetch_compare_and_swap_write(addr, old_val, new_val)
1668# define AO_HAVE_fetch_compare_and_swap
1669#endif
1670#if !defined(AO_HAVE_fetch_compare_and_swap) \
1671 && defined(AO_HAVE_fetch_compare_and_swap_read)
1672# define AO_fetch_compare_and_swap(addr, old_val, new_val) \
1673 AO_fetch_compare_and_swap_read(addr, old_val, new_val)
1674# define AO_HAVE_fetch_compare_and_swap
1675#endif
1676
1677#if defined(AO_HAVE_fetch_compare_and_swap_acquire) \
1678 && defined(AO_HAVE_nop_full) \
1679 && !defined(AO_HAVE_fetch_compare_and_swap_full)
1680# define AO_fetch_compare_and_swap_full(addr, old_val, new_val) \
1681 (AO_nop_full(), \
1682 AO_fetch_compare_and_swap_acquire(addr, old_val, new_val))
1683# define AO_HAVE_fetch_compare_and_swap_full
1684#endif
1685
1686#if !defined(AO_HAVE_fetch_compare_and_swap_release_write) \
1687 && defined(AO_HAVE_fetch_compare_and_swap_write)
1688# define AO_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
1689 AO_fetch_compare_and_swap_write(addr, old_val, new_val)
1690# define AO_HAVE_fetch_compare_and_swap_release_write
1691#endif
1692#if !defined(AO_HAVE_fetch_compare_and_swap_release_write) \
1693 && defined(AO_HAVE_fetch_compare_and_swap_release)
1694# define AO_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
1695 AO_fetch_compare_and_swap_release(addr, old_val, new_val)
1696# define AO_HAVE_fetch_compare_and_swap_release_write
1697#endif
1698#if !defined(AO_HAVE_fetch_compare_and_swap_acquire_read) \
1699 && defined(AO_HAVE_fetch_compare_and_swap_read)
1700# define AO_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
1701 AO_fetch_compare_and_swap_read(addr, old_val, new_val)
1702# define AO_HAVE_fetch_compare_and_swap_acquire_read
1703#endif
1704#if !defined(AO_HAVE_fetch_compare_and_swap_acquire_read) \
1705 && defined(AO_HAVE_fetch_compare_and_swap_acquire)
1706# define AO_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
1707 AO_fetch_compare_and_swap_acquire(addr, old_val, new_val)
1708# define AO_HAVE_fetch_compare_and_swap_acquire_read
1709#endif
1710
1711#ifdef AO_NO_DD_ORDERING
1712# if defined(AO_HAVE_fetch_compare_and_swap_acquire_read)
1713# define AO_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
1714 AO_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
1715# define AO_HAVE_fetch_compare_and_swap_dd_acquire_read
1716# endif
1717#else
1718# if defined(AO_HAVE_fetch_compare_and_swap)
1719# define AO_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
1720 AO_fetch_compare_and_swap(addr, old_val, new_val)
1721# define AO_HAVE_fetch_compare_and_swap_dd_acquire_read
1722# endif
1723#endif /* !AO_NO_DD_ORDERING */
1724
1725/* compare_and_swap */
1726#if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full) \
1727 && !defined(AO_HAVE_compare_and_swap_acquire)
1728 AO_INLINE int
1729 AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old,
1730 AO_t new_val)
1731 {
1732 int result = AO_compare_and_swap(addr, old, new_val);
1733 AO_nop_full();
1734 return result;
1735 }
1736# define AO_HAVE_compare_and_swap_acquire
1737#endif
1738#if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full) \
1739 && !defined(AO_HAVE_compare_and_swap_release)
1740# define AO_compare_and_swap_release(addr, old, new_val) \
1741 (AO_nop_full(), AO_compare_and_swap(addr, old, new_val))
1742# define AO_HAVE_compare_and_swap_release
1743#endif
1744#if defined(AO_HAVE_compare_and_swap_full)
1745# if !defined(AO_HAVE_compare_and_swap_release)
1746# define AO_compare_and_swap_release(addr, old, new_val) \
1747 AO_compare_and_swap_full(addr, old, new_val)
1748# define AO_HAVE_compare_and_swap_release
1749# endif
1750# if !defined(AO_HAVE_compare_and_swap_acquire)
1751# define AO_compare_and_swap_acquire(addr, old, new_val) \
1752 AO_compare_and_swap_full(addr, old, new_val)
1753# define AO_HAVE_compare_and_swap_acquire
1754# endif
1755# if !defined(AO_HAVE_compare_and_swap_write)
1756# define AO_compare_and_swap_write(addr, old, new_val) \
1757 AO_compare_and_swap_full(addr, old, new_val)
1758# define AO_HAVE_compare_and_swap_write
1759# endif
1760# if !defined(AO_HAVE_compare_and_swap_read)
1761# define AO_compare_and_swap_read(addr, old, new_val) \
1762 AO_compare_and_swap_full(addr, old, new_val)
1763# define AO_HAVE_compare_and_swap_read
1764# endif
1765#endif /* AO_HAVE_compare_and_swap_full */
1766
1767#if !defined(AO_HAVE_compare_and_swap) \
1768 && defined(AO_HAVE_compare_and_swap_release)
1769# define AO_compare_and_swap(addr, old, new_val) \
1770 AO_compare_and_swap_release(addr, old, new_val)
1771# define AO_HAVE_compare_and_swap
1772#endif
1773#if !defined(AO_HAVE_compare_and_swap) \
1774 && defined(AO_HAVE_compare_and_swap_acquire)
1775# define AO_compare_and_swap(addr, old, new_val) \
1776 AO_compare_and_swap_acquire(addr, old, new_val)
1777# define AO_HAVE_compare_and_swap
1778#endif
1779#if !defined(AO_HAVE_compare_and_swap) \
1780 && defined(AO_HAVE_compare_and_swap_write)
1781# define AO_compare_and_swap(addr, old, new_val) \
1782 AO_compare_and_swap_write(addr, old, new_val)
1783# define AO_HAVE_compare_and_swap
1784#endif
1785#if !defined(AO_HAVE_compare_and_swap) \
1786 && defined(AO_HAVE_compare_and_swap_read)
1787# define AO_compare_and_swap(addr, old, new_val) \
1788 AO_compare_and_swap_read(addr, old, new_val)
1789# define AO_HAVE_compare_and_swap
1790#endif
1791
1792#if defined(AO_HAVE_compare_and_swap_acquire) \
1793 && defined(AO_HAVE_nop_full) \
1794 && !defined(AO_HAVE_compare_and_swap_full)
1795# define AO_compare_and_swap_full(addr, old, new_val) \
1796 (AO_nop_full(), \
1797 AO_compare_and_swap_acquire(addr, old, new_val))
1798# define AO_HAVE_compare_and_swap_full
1799#endif
1800
1801#if !defined(AO_HAVE_compare_and_swap_release_write) \
1802 && defined(AO_HAVE_compare_and_swap_write)
1803# define AO_compare_and_swap_release_write(addr, old, new_val) \
1804 AO_compare_and_swap_write(addr, old, new_val)
1805# define AO_HAVE_compare_and_swap_release_write
1806#endif
1807#if !defined(AO_HAVE_compare_and_swap_release_write) \
1808 && defined(AO_HAVE_compare_and_swap_release)
1809# define AO_compare_and_swap_release_write(addr, old, new_val) \
1810 AO_compare_and_swap_release(addr, old, new_val)
1811# define AO_HAVE_compare_and_swap_release_write
1812#endif
1813#if !defined(AO_HAVE_compare_and_swap_acquire_read) \
1814 && defined(AO_HAVE_compare_and_swap_read)
1815# define AO_compare_and_swap_acquire_read(addr, old, new_val) \
1816 AO_compare_and_swap_read(addr, old, new_val)
1817# define AO_HAVE_compare_and_swap_acquire_read
1818#endif
1819#if !defined(AO_HAVE_compare_and_swap_acquire_read) \
1820 && defined(AO_HAVE_compare_and_swap_acquire)
1821# define AO_compare_and_swap_acquire_read(addr, old, new_val) \
1822 AO_compare_and_swap_acquire(addr, old, new_val)
1823# define AO_HAVE_compare_and_swap_acquire_read
1824#endif
1825
1826#ifdef AO_NO_DD_ORDERING
1827# if defined(AO_HAVE_compare_and_swap_acquire_read)
1828# define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1829 AO_compare_and_swap_acquire_read(addr, old, new_val)
1830# define AO_HAVE_compare_and_swap_dd_acquire_read
1831# endif
1832#else
1833# if defined(AO_HAVE_compare_and_swap)
1834# define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1835 AO_compare_and_swap(addr, old, new_val)
1836# define AO_HAVE_compare_and_swap_dd_acquire_read
1837# endif
1838#endif /* !AO_NO_DD_ORDERING */
1839
1840/* load */
1841#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_acquire)
1842# define AO_load_acquire(addr) AO_load_full(addr)
1843# define AO_HAVE_load_acquire
1844#endif
1845
1846#if defined(AO_HAVE_load_acquire) && !defined(AO_HAVE_load)
1847# define AO_load(addr) AO_load_acquire(addr)
1848# define AO_HAVE_load
1849#endif
1850
1851#if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_read)
1852# define AO_load_read(addr) AO_load_full(addr)
1853# define AO_HAVE_load_read
1854#endif
1855
1856#if !defined(AO_HAVE_load_acquire_read) \
1857 && defined(AO_HAVE_load_acquire)
1858# define AO_load_acquire_read(addr) AO_load_acquire(addr)
1859# define AO_HAVE_load_acquire_read
1860#endif
1861
1862#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_full) \
1863 && !defined(AO_HAVE_load_acquire)
1864 AO_INLINE AO_t
1865 AO_load_acquire(const volatile AO_t *addr)
1866 {
1867 AO_t result = AO_load(addr);
1868
1869 /* Acquire barrier would be useless, since the load could be delayed */
1870 /* beyond it. */
1871 AO_nop_full();
1872 return result;
1873 }
1874# define AO_HAVE_load_acquire
1875#endif
1876
1877#if defined(AO_HAVE_load) && defined(AO_HAVE_nop_read) \
1878 && !defined(AO_HAVE_load_read)
1879 AO_INLINE AO_t
1880 AO_load_read(const volatile AO_t *addr)
1881 {
1882 AO_t result = AO_load(addr);
1883
1884 AO_nop_read();
1885 return result;
1886 }
1887# define AO_HAVE_load_read
1888#endif
1889
1890#if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_nop_full) \
1891 && !defined(AO_HAVE_load_full)
1892# define AO_load_full(addr) (AO_nop_full(), AO_load_acquire(addr))
1893# define AO_HAVE_load_full
1894#endif
1895
1896#if defined(AO_HAVE_compare_and_swap_read) \
1897 && !defined(AO_HAVE_load_read)
1898# define AO_CAS_BASED_LOAD_READ
1899 AO_ATTR_NO_SANITIZE_THREAD
1900 AO_INLINE AO_t
1901 AO_load_read(const volatile AO_t *addr)
1902 {
1903 AO_t result;
1904
1905 do {
1906 result = *(const AO_t *)addr;
1907 } while (AO_EXPECT_FALSE(!AO_compare_and_swap_read(
1908 (volatile AO_t *)addr,
1909 result, result)));
1910 return result;
1911 }
1912# define AO_HAVE_load_read
1913#endif
1914
1915#if !defined(AO_HAVE_load_acquire_read) \
1916 && defined(AO_HAVE_load_read)
1917# define AO_load_acquire_read(addr) AO_load_read(addr)
1918# define AO_HAVE_load_acquire_read
1919#endif
1920
1921#if defined(AO_HAVE_load_acquire_read) && !defined(AO_HAVE_load) \
1922 && (!defined(AO_CAS_BASED_LOAD_READ) \
1923 || !defined(AO_HAVE_compare_and_swap))
1924# define AO_load(addr) AO_load_acquire_read(addr)
1925# define AO_HAVE_load
1926#endif
1927
1928#if defined(AO_HAVE_compare_and_swap_full) \
1929 && !defined(AO_HAVE_load_full)
1930 AO_ATTR_NO_SANITIZE_THREAD
1931 AO_INLINE AO_t
1932 AO_load_full(const volatile AO_t *addr)
1933 {
1934 AO_t result;
1935
1936 do {
1937 result = *(const AO_t *)addr;
1938 } while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(
1939 (volatile AO_t *)addr,
1940 result, result)));
1941 return result;
1942 }
1943# define AO_HAVE_load_full
1944#endif
1945
1946#if defined(AO_HAVE_compare_and_swap_acquire) \
1947 && !defined(AO_HAVE_load_acquire)
1948 AO_ATTR_NO_SANITIZE_THREAD
1949 AO_INLINE AO_t
1950 AO_load_acquire(const volatile AO_t *addr)
1951 {
1952 AO_t result;
1953
1954 do {
1955 result = *(const AO_t *)addr;
1956 } while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire(
1957 (volatile AO_t *)addr,
1958 result, result)));
1959 return result;
1960 }
1961# define AO_HAVE_load_acquire
1962#endif
1963
1964#if defined(AO_HAVE_compare_and_swap) && !defined(AO_HAVE_load)
1965 AO_ATTR_NO_SANITIZE_THREAD
1966 AO_INLINE AO_t
1967 AO_load(const volatile AO_t *addr)
1968 {
1969 AO_t result;
1970
1971 do {
1972 result = *(const AO_t *)addr;
1973 } while (AO_EXPECT_FALSE(!AO_compare_and_swap(
1974 (volatile AO_t *)addr,
1975 result, result)));
1976 return result;
1977 }
1978# define AO_HAVE_load
1979#endif
1980
1981#ifdef AO_NO_DD_ORDERING
1982# if defined(AO_HAVE_load_acquire_read)
1983# define AO_load_dd_acquire_read(addr) \
1984 AO_load_acquire_read(addr)
1985# define AO_HAVE_load_dd_acquire_read
1986# endif
1987#else
1988# if defined(AO_HAVE_load)
1989# define AO_load_dd_acquire_read(addr) AO_load(addr)
1990# define AO_HAVE_load_dd_acquire_read
1991# endif
1992#endif /* !AO_NO_DD_ORDERING */
1993
1994/* store */
1995#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_release)
1996# define AO_store_release(addr, val) AO_store_full(addr, val)
1997# define AO_HAVE_store_release
1998#endif
1999
2000#if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store)
2001# define AO_store(addr, val) AO_store_release(addr, val)
2002# define AO_HAVE_store
2003#endif
2004
2005#if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_write)
2006# define AO_store_write(addr, val) AO_store_full(addr, val)
2007# define AO_HAVE_store_write
2008#endif
2009
2010#if defined(AO_HAVE_store_release) \
2011 && !defined(AO_HAVE_store_release_write)
2012# define AO_store_release_write(addr, val) \
2013 AO_store_release(addr, val)
2014# define AO_HAVE_store_release_write
2015#endif
2016
2017#if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store)
2018# define AO_store(addr, val) AO_store_write(addr, val)
2019# define AO_HAVE_store
2020#endif
2021
2022#if defined(AO_HAVE_store) && defined(AO_HAVE_nop_full) \
2023 && !defined(AO_HAVE_store_release)
2024# define AO_store_release(addr, val) \
2025 (AO_nop_full(), AO_store(addr, val))
2026# define AO_HAVE_store_release
2027#endif
2028
2029#if defined(AO_HAVE_store) && defined(AO_HAVE_nop_write) \
2030 && !defined(AO_HAVE_store_write)
2031# define AO_store_write(addr, val) \
2032 (AO_nop_write(), AO_store(addr, val))
2033# define AO_HAVE_store_write
2034#endif
2035
2036#if defined(AO_HAVE_compare_and_swap_write) \
2037 && !defined(AO_HAVE_store_write)
2038 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2039 AO_INLINE void
2040 AO_store_write(volatile AO_t *addr, AO_t new_val)
2041 {
2042 AO_t old_val;
2043
2044 do {
2045 old_val = *(AO_t *)addr;
2046 } while (AO_EXPECT_FALSE(!AO_compare_and_swap_write(addr, old_val,
2047 new_val)));
2048 }
2049# define AO_HAVE_store_write
2050#endif
2051
2052#if defined(AO_HAVE_store_write) \
2053 && !defined(AO_HAVE_store_release_write)
2054# define AO_store_release_write(addr, val) \
2055 AO_store_write(addr, val)
2056# define AO_HAVE_store_release_write
2057#endif
2058
2059#if defined(AO_HAVE_store_release) && defined(AO_HAVE_nop_full) \
2060 && !defined(AO_HAVE_store_full)
2061# define AO_store_full(addr, val) \
2062 (AO_store_release(addr, val), \
2063 AO_nop_full())
2064# define AO_HAVE_store_full
2065#endif
2066
2067#if defined(AO_HAVE_compare_and_swap) && !defined(AO_HAVE_store)
2068 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2069 AO_INLINE void
2070 AO_store(volatile AO_t *addr, AO_t new_val)
2071 {
2072 AO_t old_val;
2073
2074 do {
2075 old_val = *(AO_t *)addr;
2076 } while (AO_EXPECT_FALSE(!AO_compare_and_swap(addr,
2077 old_val, new_val)));
2078 }
2079# define AO_HAVE_store
2080#endif
2081
2082#if defined(AO_HAVE_compare_and_swap_release) \
2083 && !defined(AO_HAVE_store_release)
2084 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2085 AO_INLINE void
2086 AO_store_release(volatile AO_t *addr, AO_t new_val)
2087 {
2088 AO_t old_val;
2089
2090 do {
2091 old_val = *(AO_t *)addr;
2092 } while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(addr, old_val,
2093 new_val)));
2094 }
2095# define AO_HAVE_store_release
2096#endif
2097
2098#if defined(AO_HAVE_compare_and_swap_full) \
2099 && !defined(AO_HAVE_store_full)
2100 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2101 AO_INLINE void
2102 AO_store_full(volatile AO_t *addr, AO_t new_val)
2103 {
2104 AO_t old_val;
2105
2106 do {
2107 old_val = *(AO_t *)addr;
2108 } while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old_val,
2109 new_val)));
2110 }
2111# define AO_HAVE_store_full
2112#endif
2113/*
2114 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
2115 *
2116 * Permission is hereby granted, free of charge, to any person obtaining a copy
2117 * of this software and associated documentation files (the "Software"), to deal
2118 * in the Software without restriction, including without limitation the rights
2119 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
2120 * copies of the Software, and to permit persons to whom the Software is
2121 * furnished to do so, subject to the following conditions:
2122 *
2123 * The above copyright notice and this permission notice shall be included in
2124 * all copies or substantial portions of the Software.
2125 *
2126 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
2127 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
2128 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
2129 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
2130 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
2131 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2132 * SOFTWARE.
2133 */
2134
2135/* double_fetch_compare_and_swap */
2136#if defined(AO_HAVE_double_fetch_compare_and_swap) \
2137 && defined(AO_HAVE_nop_full) \
2138 && !defined(AO_HAVE_double_fetch_compare_and_swap_acquire)
2139 AO_INLINE AO_double_t
2140 AO_double_fetch_compare_and_swap_acquire(volatile AO_double_t *addr,
2141 AO_double_t old_val, AO_double_t new_val)
2142 {
2143 AO_double_t result = AO_double_fetch_compare_and_swap(addr, old_val, new_val);
2144 AO_nop_full();
2145 return result;
2146 }
2147# define AO_HAVE_double_fetch_compare_and_swap_acquire
2148#endif
2149#if defined(AO_HAVE_double_fetch_compare_and_swap) \
2150 && defined(AO_HAVE_nop_full) \
2151 && !defined(AO_HAVE_double_fetch_compare_and_swap_release)
2152# define AO_double_fetch_compare_and_swap_release(addr, old_val, new_val) \
2153 (AO_nop_full(), \
2154 AO_double_fetch_compare_and_swap(addr, old_val, new_val))
2155# define AO_HAVE_double_fetch_compare_and_swap_release
2156#endif
2157#if defined(AO_HAVE_double_fetch_compare_and_swap_full)
2158# if !defined(AO_HAVE_double_fetch_compare_and_swap_release)
2159# define AO_double_fetch_compare_and_swap_release(addr, old_val, new_val) \
2160 AO_double_fetch_compare_and_swap_full(addr, old_val, new_val)
2161# define AO_HAVE_double_fetch_compare_and_swap_release
2162# endif
2163# if !defined(AO_HAVE_double_fetch_compare_and_swap_acquire)
2164# define AO_double_fetch_compare_and_swap_acquire(addr, old_val, new_val) \
2165 AO_double_fetch_compare_and_swap_full(addr, old_val, new_val)
2166# define AO_HAVE_double_fetch_compare_and_swap_acquire
2167# endif
2168# if !defined(AO_HAVE_double_fetch_compare_and_swap_write)
2169# define AO_double_fetch_compare_and_swap_write(addr, old_val, new_val) \
2170 AO_double_fetch_compare_and_swap_full(addr, old_val, new_val)
2171# define AO_HAVE_double_fetch_compare_and_swap_write
2172# endif
2173# if !defined(AO_HAVE_double_fetch_compare_and_swap_read)
2174# define AO_double_fetch_compare_and_swap_read(addr, old_val, new_val) \
2175 AO_double_fetch_compare_and_swap_full(addr, old_val, new_val)
2176# define AO_HAVE_double_fetch_compare_and_swap_read
2177# endif
2178#endif /* AO_HAVE_double_fetch_compare_and_swap_full */
2179
2180#if !defined(AO_HAVE_double_fetch_compare_and_swap) \
2181 && defined(AO_HAVE_double_fetch_compare_and_swap_release)
2182# define AO_double_fetch_compare_and_swap(addr, old_val, new_val) \
2183 AO_double_fetch_compare_and_swap_release(addr, old_val, new_val)
2184# define AO_HAVE_double_fetch_compare_and_swap
2185#endif
2186#if !defined(AO_HAVE_double_fetch_compare_and_swap) \
2187 && defined(AO_HAVE_double_fetch_compare_and_swap_acquire)
2188# define AO_double_fetch_compare_and_swap(addr, old_val, new_val) \
2189 AO_double_fetch_compare_and_swap_acquire(addr, old_val, new_val)
2190# define AO_HAVE_double_fetch_compare_and_swap
2191#endif
2192#if !defined(AO_HAVE_double_fetch_compare_and_swap) \
2193 && defined(AO_HAVE_double_fetch_compare_and_swap_write)
2194# define AO_double_fetch_compare_and_swap(addr, old_val, new_val) \
2195 AO_double_fetch_compare_and_swap_write(addr, old_val, new_val)
2196# define AO_HAVE_double_fetch_compare_and_swap
2197#endif
2198#if !defined(AO_HAVE_double_fetch_compare_and_swap) \
2199 && defined(AO_HAVE_double_fetch_compare_and_swap_read)
2200# define AO_double_fetch_compare_and_swap(addr, old_val, new_val) \
2201 AO_double_fetch_compare_and_swap_read(addr, old_val, new_val)
2202# define AO_HAVE_double_fetch_compare_and_swap
2203#endif
2204
2205#if defined(AO_HAVE_double_fetch_compare_and_swap_acquire) \
2206 && defined(AO_HAVE_nop_full) \
2207 && !defined(AO_HAVE_double_fetch_compare_and_swap_full)
2208# define AO_double_fetch_compare_and_swap_full(addr, old_val, new_val) \
2209 (AO_nop_full(), \
2210 AO_double_fetch_compare_and_swap_acquire(addr, old_val, new_val))
2211# define AO_HAVE_double_fetch_compare_and_swap_full
2212#endif
2213
2214#if !defined(AO_HAVE_double_fetch_compare_and_swap_release_write) \
2215 && defined(AO_HAVE_double_fetch_compare_and_swap_write)
2216# define AO_double_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
2217 AO_double_fetch_compare_and_swap_write(addr, old_val, new_val)
2218# define AO_HAVE_double_fetch_compare_and_swap_release_write
2219#endif
2220#if !defined(AO_HAVE_double_fetch_compare_and_swap_release_write) \
2221 && defined(AO_HAVE_double_fetch_compare_and_swap_release)
2222# define AO_double_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
2223 AO_double_fetch_compare_and_swap_release(addr, old_val, new_val)
2224# define AO_HAVE_double_fetch_compare_and_swap_release_write
2225#endif
2226#if !defined(AO_HAVE_double_fetch_compare_and_swap_acquire_read) \
2227 && defined(AO_HAVE_double_fetch_compare_and_swap_read)
2228# define AO_double_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
2229 AO_double_fetch_compare_and_swap_read(addr, old_val, new_val)
2230# define AO_HAVE_double_fetch_compare_and_swap_acquire_read
2231#endif
2232#if !defined(AO_HAVE_double_fetch_compare_and_swap_acquire_read) \
2233 && defined(AO_HAVE_double_fetch_compare_and_swap_acquire)
2234# define AO_double_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
2235 AO_double_fetch_compare_and_swap_acquire(addr, old_val, new_val)
2236# define AO_HAVE_double_fetch_compare_and_swap_acquire_read
2237#endif
2238
2239#ifdef AO_NO_DD_ORDERING
2240# if defined(AO_HAVE_double_fetch_compare_and_swap_acquire_read)
2241# define AO_double_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
2242 AO_double_fetch_compare_and_swap_acquire_read(addr, old_val, new_val)
2243# define AO_HAVE_double_fetch_compare_and_swap_dd_acquire_read
2244# endif
2245#else
2246# if defined(AO_HAVE_double_fetch_compare_and_swap)
2247# define AO_double_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
2248 AO_double_fetch_compare_and_swap(addr, old_val, new_val)
2249# define AO_HAVE_double_fetch_compare_and_swap_dd_acquire_read
2250# endif
2251#endif /* !AO_NO_DD_ORDERING */
2252
2253/* double_compare_and_swap */
2254#if defined(AO_HAVE_double_compare_and_swap) && defined(AO_HAVE_nop_full) \
2255 && !defined(AO_HAVE_double_compare_and_swap_acquire)
2256 AO_INLINE int
2257 AO_double_compare_and_swap_acquire(volatile AO_double_t *addr, AO_double_t old,
2258 AO_double_t new_val)
2259 {
2260 int result = AO_double_compare_and_swap(addr, old, new_val);
2261 AO_nop_full();
2262 return result;
2263 }
2264# define AO_HAVE_double_compare_and_swap_acquire
2265#endif
2266#if defined(AO_HAVE_double_compare_and_swap) && defined(AO_HAVE_nop_full) \
2267 && !defined(AO_HAVE_double_compare_and_swap_release)
2268# define AO_double_compare_and_swap_release(addr, old, new_val) \
2269 (AO_nop_full(), AO_double_compare_and_swap(addr, old, new_val))
2270# define AO_HAVE_double_compare_and_swap_release
2271#endif
2272#if defined(AO_HAVE_double_compare_and_swap_full)
2273# if !defined(AO_HAVE_double_compare_and_swap_release)
2274# define AO_double_compare_and_swap_release(addr, old, new_val) \
2275 AO_double_compare_and_swap_full(addr, old, new_val)
2276# define AO_HAVE_double_compare_and_swap_release
2277# endif
2278# if !defined(AO_HAVE_double_compare_and_swap_acquire)
2279# define AO_double_compare_and_swap_acquire(addr, old, new_val) \
2280 AO_double_compare_and_swap_full(addr, old, new_val)
2281# define AO_HAVE_double_compare_and_swap_acquire
2282# endif
2283# if !defined(AO_HAVE_double_compare_and_swap_write)
2284# define AO_double_compare_and_swap_write(addr, old, new_val) \
2285 AO_double_compare_and_swap_full(addr, old, new_val)
2286# define AO_HAVE_double_compare_and_swap_write
2287# endif
2288# if !defined(AO_HAVE_double_compare_and_swap_read)
2289# define AO_double_compare_and_swap_read(addr, old, new_val) \
2290 AO_double_compare_and_swap_full(addr, old, new_val)
2291# define AO_HAVE_double_compare_and_swap_read
2292# endif
2293#endif /* AO_HAVE_double_compare_and_swap_full */
2294
2295#if !defined(AO_HAVE_double_compare_and_swap) \
2296 && defined(AO_HAVE_double_compare_and_swap_release)
2297# define AO_double_compare_and_swap(addr, old, new_val) \
2298 AO_double_compare_and_swap_release(addr, old, new_val)
2299# define AO_HAVE_double_compare_and_swap
2300#endif
2301#if !defined(AO_HAVE_double_compare_and_swap) \
2302 && defined(AO_HAVE_double_compare_and_swap_acquire)
2303# define AO_double_compare_and_swap(addr, old, new_val) \
2304 AO_double_compare_and_swap_acquire(addr, old, new_val)
2305# define AO_HAVE_double_compare_and_swap
2306#endif
2307#if !defined(AO_HAVE_double_compare_and_swap) \
2308 && defined(AO_HAVE_double_compare_and_swap_write)
2309# define AO_double_compare_and_swap(addr, old, new_val) \
2310 AO_double_compare_and_swap_write(addr, old, new_val)
2311# define AO_HAVE_double_compare_and_swap
2312#endif
2313#if !defined(AO_HAVE_double_compare_and_swap) \
2314 && defined(AO_HAVE_double_compare_and_swap_read)
2315# define AO_double_compare_and_swap(addr, old, new_val) \
2316 AO_double_compare_and_swap_read(addr, old, new_val)
2317# define AO_HAVE_double_compare_and_swap
2318#endif
2319
2320#if defined(AO_HAVE_double_compare_and_swap_acquire) \
2321 && defined(AO_HAVE_nop_full) \
2322 && !defined(AO_HAVE_double_compare_and_swap_full)
2323# define AO_double_compare_and_swap_full(addr, old, new_val) \
2324 (AO_nop_full(), \
2325 AO_double_compare_and_swap_acquire(addr, old, new_val))
2326# define AO_HAVE_double_compare_and_swap_full
2327#endif
2328
2329#if !defined(AO_HAVE_double_compare_and_swap_release_write) \
2330 && defined(AO_HAVE_double_compare_and_swap_write)
2331# define AO_double_compare_and_swap_release_write(addr, old, new_val) \
2332 AO_double_compare_and_swap_write(addr, old, new_val)
2333# define AO_HAVE_double_compare_and_swap_release_write
2334#endif
2335#if !defined(AO_HAVE_double_compare_and_swap_release_write) \
2336 && defined(AO_HAVE_double_compare_and_swap_release)
2337# define AO_double_compare_and_swap_release_write(addr, old, new_val) \
2338 AO_double_compare_and_swap_release(addr, old, new_val)
2339# define AO_HAVE_double_compare_and_swap_release_write
2340#endif
2341#if !defined(AO_HAVE_double_compare_and_swap_acquire_read) \
2342 && defined(AO_HAVE_double_compare_and_swap_read)
2343# define AO_double_compare_and_swap_acquire_read(addr, old, new_val) \
2344 AO_double_compare_and_swap_read(addr, old, new_val)
2345# define AO_HAVE_double_compare_and_swap_acquire_read
2346#endif
2347#if !defined(AO_HAVE_double_compare_and_swap_acquire_read) \
2348 && defined(AO_HAVE_double_compare_and_swap_acquire)
2349# define AO_double_compare_and_swap_acquire_read(addr, old, new_val) \
2350 AO_double_compare_and_swap_acquire(addr, old, new_val)
2351# define AO_HAVE_double_compare_and_swap_acquire_read
2352#endif
2353
2354#ifdef AO_NO_DD_ORDERING
2355# if defined(AO_HAVE_double_compare_and_swap_acquire_read)
2356# define AO_double_compare_and_swap_dd_acquire_read(addr, old, new_val) \
2357 AO_double_compare_and_swap_acquire_read(addr, old, new_val)
2358# define AO_HAVE_double_compare_and_swap_dd_acquire_read
2359# endif
2360#else
2361# if defined(AO_HAVE_double_compare_and_swap)
2362# define AO_double_compare_and_swap_dd_acquire_read(addr, old, new_val) \
2363 AO_double_compare_and_swap(addr, old, new_val)
2364# define AO_HAVE_double_compare_and_swap_dd_acquire_read
2365# endif
2366#endif /* !AO_NO_DD_ORDERING */
2367
2368/* double_load */
2369#if defined(AO_HAVE_double_load_full) && !defined(AO_HAVE_double_load_acquire)
2370# define AO_double_load_acquire(addr) AO_double_load_full(addr)
2371# define AO_HAVE_double_load_acquire
2372#endif
2373
2374#if defined(AO_HAVE_double_load_acquire) && !defined(AO_HAVE_double_load)
2375# define AO_double_load(addr) AO_double_load_acquire(addr)
2376# define AO_HAVE_double_load
2377#endif
2378
2379#if defined(AO_HAVE_double_load_full) && !defined(AO_HAVE_double_load_read)
2380# define AO_double_load_read(addr) AO_double_load_full(addr)
2381# define AO_HAVE_double_load_read
2382#endif
2383
2384#if !defined(AO_HAVE_double_load_acquire_read) \
2385 && defined(AO_HAVE_double_load_acquire)
2386# define AO_double_load_acquire_read(addr) AO_double_load_acquire(addr)
2387# define AO_HAVE_double_load_acquire_read
2388#endif
2389
2390#if defined(AO_HAVE_double_load) && defined(AO_HAVE_nop_full) \
2391 && !defined(AO_HAVE_double_load_acquire)
2392 AO_INLINE AO_double_t
2393 AO_double_load_acquire(const volatile AO_double_t *addr)
2394 {
2395 AO_double_t result = AO_double_load(addr);
2396
2397 /* Acquire barrier would be useless, since the load could be delayed */
2398 /* beyond it. */
2399 AO_nop_full();
2400 return result;
2401 }
2402# define AO_HAVE_double_load_acquire
2403#endif
2404
2405#if defined(AO_HAVE_double_load) && defined(AO_HAVE_nop_read) \
2406 && !defined(AO_HAVE_double_load_read)
2407 AO_INLINE AO_double_t
2408 AO_double_load_read(const volatile AO_double_t *addr)
2409 {
2410 AO_double_t result = AO_double_load(addr);
2411
2412 AO_nop_read();
2413 return result;
2414 }
2415# define AO_HAVE_double_load_read
2416#endif
2417
2418#if defined(AO_HAVE_double_load_acquire) && defined(AO_HAVE_nop_full) \
2419 && !defined(AO_HAVE_double_load_full)
2420# define AO_double_load_full(addr) (AO_nop_full(), AO_double_load_acquire(addr))
2421# define AO_HAVE_double_load_full
2422#endif
2423
2424#if defined(AO_HAVE_double_compare_and_swap_read) \
2425 && !defined(AO_HAVE_double_load_read)
2426# define AO_double_CAS_BASED_LOAD_READ
2427 AO_ATTR_NO_SANITIZE_THREAD
2428 AO_INLINE AO_double_t
2429 AO_double_load_read(const volatile AO_double_t *addr)
2430 {
2431 AO_double_t result;
2432
2433 do {
2434 result = *(const AO_double_t *)addr;
2435 } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_read(
2436 (volatile AO_double_t *)addr,
2437 result, result)));
2438 return result;
2439 }
2440# define AO_HAVE_double_load_read
2441#endif
2442
2443#if !defined(AO_HAVE_double_load_acquire_read) \
2444 && defined(AO_HAVE_double_load_read)
2445# define AO_double_load_acquire_read(addr) AO_double_load_read(addr)
2446# define AO_HAVE_double_load_acquire_read
2447#endif
2448
2449#if defined(AO_HAVE_double_load_acquire_read) && !defined(AO_HAVE_double_load) \
2450 && (!defined(AO_double_CAS_BASED_LOAD_READ) \
2451 || !defined(AO_HAVE_double_compare_and_swap))
2452# define AO_double_load(addr) AO_double_load_acquire_read(addr)
2453# define AO_HAVE_double_load
2454#endif
2455
2456#if defined(AO_HAVE_double_compare_and_swap_full) \
2457 && !defined(AO_HAVE_double_load_full)
2458 AO_ATTR_NO_SANITIZE_THREAD
2459 AO_INLINE AO_double_t
2460 AO_double_load_full(const volatile AO_double_t *addr)
2461 {
2462 AO_double_t result;
2463
2464 do {
2465 result = *(const AO_double_t *)addr;
2466 } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_full(
2467 (volatile AO_double_t *)addr,
2468 result, result)));
2469 return result;
2470 }
2471# define AO_HAVE_double_load_full
2472#endif
2473
2474#if defined(AO_HAVE_double_compare_and_swap_acquire) \
2475 && !defined(AO_HAVE_double_load_acquire)
2476 AO_ATTR_NO_SANITIZE_THREAD
2477 AO_INLINE AO_double_t
2478 AO_double_load_acquire(const volatile AO_double_t *addr)
2479 {
2480 AO_double_t result;
2481
2482 do {
2483 result = *(const AO_double_t *)addr;
2484 } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_acquire(
2485 (volatile AO_double_t *)addr,
2486 result, result)));
2487 return result;
2488 }
2489# define AO_HAVE_double_load_acquire
2490#endif
2491
2492#if defined(AO_HAVE_double_compare_and_swap) && !defined(AO_HAVE_double_load)
2493 AO_ATTR_NO_SANITIZE_THREAD
2494 AO_INLINE AO_double_t
2495 AO_double_load(const volatile AO_double_t *addr)
2496 {
2497 AO_double_t result;
2498
2499 do {
2500 result = *(const AO_double_t *)addr;
2501 } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap(
2502 (volatile AO_double_t *)addr,
2503 result, result)));
2504 return result;
2505 }
2506# define AO_HAVE_double_load
2507#endif
2508
2509#ifdef AO_NO_DD_ORDERING
2510# if defined(AO_HAVE_double_load_acquire_read)
2511# define AO_double_load_dd_acquire_read(addr) \
2512 AO_double_load_acquire_read(addr)
2513# define AO_HAVE_double_load_dd_acquire_read
2514# endif
2515#else
2516# if defined(AO_HAVE_double_load)
2517# define AO_double_load_dd_acquire_read(addr) AO_double_load(addr)
2518# define AO_HAVE_double_load_dd_acquire_read
2519# endif
2520#endif /* !AO_NO_DD_ORDERING */
2521
2522/* double_store */
2523#if defined(AO_HAVE_double_store_full) && !defined(AO_HAVE_double_store_release)
2524# define AO_double_store_release(addr, val) AO_double_store_full(addr, val)
2525# define AO_HAVE_double_store_release
2526#endif
2527
2528#if defined(AO_HAVE_double_store_release) && !defined(AO_HAVE_double_store)
2529# define AO_double_store(addr, val) AO_double_store_release(addr, val)
2530# define AO_HAVE_double_store
2531#endif
2532
2533#if defined(AO_HAVE_double_store_full) && !defined(AO_HAVE_double_store_write)
2534# define AO_double_store_write(addr, val) AO_double_store_full(addr, val)
2535# define AO_HAVE_double_store_write
2536#endif
2537
2538#if defined(AO_HAVE_double_store_release) \
2539 && !defined(AO_HAVE_double_store_release_write)
2540# define AO_double_store_release_write(addr, val) \
2541 AO_double_store_release(addr, val)
2542# define AO_HAVE_double_store_release_write
2543#endif
2544
2545#if defined(AO_HAVE_double_store_write) && !defined(AO_HAVE_double_store)
2546# define AO_double_store(addr, val) AO_double_store_write(addr, val)
2547# define AO_HAVE_double_store
2548#endif
2549
2550#if defined(AO_HAVE_double_store) && defined(AO_HAVE_nop_full) \
2551 && !defined(AO_HAVE_double_store_release)
2552# define AO_double_store_release(addr, val) \
2553 (AO_nop_full(), AO_double_store(addr, val))
2554# define AO_HAVE_double_store_release
2555#endif
2556
2557#if defined(AO_HAVE_double_store) && defined(AO_HAVE_nop_write) \
2558 && !defined(AO_HAVE_double_store_write)
2559# define AO_double_store_write(addr, val) \
2560 (AO_nop_write(), AO_double_store(addr, val))
2561# define AO_HAVE_double_store_write
2562#endif
2563
2564#if defined(AO_HAVE_double_compare_and_swap_write) \
2565 && !defined(AO_HAVE_double_store_write)
2566 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2567 AO_INLINE void
2568 AO_double_store_write(volatile AO_double_t *addr, AO_double_t new_val)
2569 {
2570 AO_double_t old_val;
2571
2572 do {
2573 old_val = *(AO_double_t *)addr;
2574 } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_write(addr, old_val,
2575 new_val)));
2576 }
2577# define AO_HAVE_double_store_write
2578#endif
2579
2580#if defined(AO_HAVE_double_store_write) \
2581 && !defined(AO_HAVE_double_store_release_write)
2582# define AO_double_store_release_write(addr, val) \
2583 AO_double_store_write(addr, val)
2584# define AO_HAVE_double_store_release_write
2585#endif
2586
2587#if defined(AO_HAVE_double_store_release) && defined(AO_HAVE_nop_full) \
2588 && !defined(AO_HAVE_double_store_full)
2589# define AO_double_store_full(addr, val) \
2590 (AO_double_store_release(addr, val), \
2591 AO_nop_full())
2592# define AO_HAVE_double_store_full
2593#endif
2594
2595#if defined(AO_HAVE_double_compare_and_swap) && !defined(AO_HAVE_double_store)
2596 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2597 AO_INLINE void
2598 AO_double_store(volatile AO_double_t *addr, AO_double_t new_val)
2599 {
2600 AO_double_t old_val;
2601
2602 do {
2603 old_val = *(AO_double_t *)addr;
2604 } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap(addr,
2605 old_val, new_val)));
2606 }
2607# define AO_HAVE_double_store
2608#endif
2609
2610#if defined(AO_HAVE_double_compare_and_swap_release) \
2611 && !defined(AO_HAVE_double_store_release)
2612 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2613 AO_INLINE void
2614 AO_double_store_release(volatile AO_double_t *addr, AO_double_t new_val)
2615 {
2616 AO_double_t old_val;
2617
2618 do {
2619 old_val = *(AO_double_t *)addr;
2620 } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_release(addr, old_val,
2621 new_val)));
2622 }
2623# define AO_HAVE_double_store_release
2624#endif
2625
2626#if defined(AO_HAVE_double_compare_and_swap_full) \
2627 && !defined(AO_HAVE_double_store_full)
2628 AO_ATTR_NO_SANITIZE_MEMORY AO_ATTR_NO_SANITIZE_THREAD
2629 AO_INLINE void
2630 AO_double_store_full(volatile AO_double_t *addr, AO_double_t new_val)
2631 {
2632 AO_double_t old_val;
2633
2634 do {
2635 old_val = *(AO_double_t *)addr;
2636 } while (AO_EXPECT_FALSE(!AO_double_compare_and_swap_full(addr, old_val,
2637 new_val)));
2638 }
2639# define AO_HAVE_double_store_full
2640#endif
result
volatile int result
Definition: Lib_With_Created_Counter.c:21
AO_double_compare_and_swap
AO_INLINE int AO_double_compare_and_swap(volatile AO_double_t *addr, AO_double_t old_val, AO_double_t new_val)
Definition: aarch64.h:141
AO_double_compare_and_swap_release
AO_INLINE int AO_double_compare_and_swap_release(volatile AO_double_t *addr, AO_double_t old_val, AO_double_t new_val)
Definition: aarch64.h:203
AO_double_store_release
AO_INLINE void AO_double_store_release(volatile AO_double_t *addr, AO_double_t value)
Definition: aarch64.h:118
AO_double_compare_and_swap_acquire
AO_INLINE int AO_double_compare_and_swap_acquire(volatile AO_double_t *addr, AO_double_t old_val, AO_double_t new_val)
Definition: aarch64.h:172
AO_double_load_acquire
AO_INLINE AO_double_t AO_double_load_acquire(const volatile AO_double_t *addr)
Definition: aarch64.h:71
AO_double_compare_and_swap_full
AO_INLINE int AO_double_compare_and_swap_full(volatile AO_double_t *addr, AO_double_t old_val, AO_double_t new_val)
Definition: aarch64.h:234
AO_double_load
AO_INLINE AO_double_t AO_double_load(const volatile AO_double_t *addr)
Definition: aarch64.h:47
AO_double_store
AO_INLINE void AO_double_store(volatile AO_double_t *addr, AO_double_t value)
Definition: aarch64.h:93
AO_compare_and_swap
AO_INLINE int AO_compare_and_swap(volatile AO_t *addr, AO_t old, AO_t new_val)
Definition: alpha.h:46
AO_nop_full
AO_INLINE void AO_nop_full(void)
Definition: alpha.h:27
AO_t
#define AO_t
Definition: atomic_ops.h:156
AO_ATTR_NO_SANITIZE_THREAD
#define AO_ATTR_NO_SANITIZE_THREAD
Definition: atomic_ops.h:226
AO_INLINE
#define AO_INLINE
Definition: atomic_ops.h:186
AO_EXPECT_FALSE
#define AO_EXPECT_FALSE(expr)
Definition: atomic_ops.h:193
AO_ATTR_NO_SANITIZE_MEMORY
#define AO_ATTR_NO_SANITIZE_MEMORY
Definition: atomic_ops.h:216
AO_compare_and_swap_full
AO_INLINE int AO_compare_and_swap_full(volatile AO_t *addr, AO_t old, AO_t new_val)
Definition: avr32.h:49
AO_store_full
#define AO_store_full(addr, val)
Definition: emul_cas.h:82
AO_fetch_compare_and_swap
AO_INLINE AO_t AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val)
Definition: gcc/arm.h:559
AO_short_fetch_compare_and_swap_acquire
AO_INLINE unsigned short AO_short_fetch_compare_and_swap_acquire(volatile unsigned short *addr, unsigned short old, unsigned short new_val)
Definition: gcc/ia64.h:173
AO_char_fetch_compare_and_swap_acquire
AO_INLINE unsigned char AO_char_fetch_compare_and_swap_acquire(volatile unsigned char *addr, unsigned char old, unsigned char new_val)
Definition: gcc/ia64.h:145
AO_int_fetch_compare_and_swap_acquire
AO_INLINE unsigned int AO_int_fetch_compare_and_swap_acquire(volatile unsigned int *addr, unsigned int old, unsigned int new_val)
Definition: gcc/ia64.h:254
AO_fetch_compare_and_swap_acquire
AO_INLINE AO_t AO_fetch_compare_and_swap_acquire(volatile AO_t *addr, AO_t old, AO_t new_val)
Definition: gcc/ia64.h:115
AO_nop_read
#define AO_nop_read()
Definition: gcc/powerpc.h:71
AO_compare_and_swap_acquire
AO_INLINE int AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old, AO_t new_val)
Definition: gcc/powerpc.h:213
AO_compare_and_swap_release
AO_INLINE int AO_compare_and_swap_release(volatile AO_t *addr, AO_t old, AO_t new_val)
Definition: gcc/powerpc.h:222
AO_char_store
AO_INLINE void AO_char_store(volatile unsignedchar *addr, unsignedchar value)
Definition: generic-small.h:51
AO_short_load
AO_INLINE unsigned short AO_short_load(const volatile unsignedshort *addr)
Definition: generic-small.h:179
AO_int_load_acquire
AO_INLINE unsigned AO_int_load_acquire(const volatile unsigned *addr)
Definition: generic-small.h:344
AO_store_release
AO_INLINE void AO_store_release(volatile AO_t *addr, AO_t value)
Definition: generic-small.h:534
AO_int_load
AO_INLINE unsigned AO_int_load(const volatile unsigned *addr)
Definition: generic-small.h:337
AO_char_load_acquire
AO_INLINE unsigned char AO_char_load_acquire(const volatile unsignedchar *addr)
Definition: generic-small.h:28
AO_char_store_release
AO_INLINE void AO_char_store_release(volatile unsignedchar *addr, unsignedchar value)
Definition: generic-small.h:60
AO_int_store_release
AO_INLINE void AO_int_store_release(volatile unsigned *addr, unsigned value)
Definition: generic-small.h:376
AO_short_load_acquire
AO_INLINE unsigned short AO_short_load_acquire(const volatile unsignedshort *addr)
Definition: generic-small.h:186
AO_store
AO_INLINE void AO_store(volatile AO_t *addr, AO_t value)
Definition: generic-small.h:525
AO_short_store
AO_INLINE void AO_short_store(volatile unsignedshort *addr, unsignedshort value)
Definition: generic-small.h:209
AO_load
AO_INLINE AO_t AO_load(const volatile AO_t *addr)
Definition: generic-small.h:495
AO_int_store
AO_INLINE void AO_int_store(volatile unsigned *addr, unsigned value)
Definition: generic-small.h:367
AO_char_load
AO_INLINE unsigned char AO_char_load(const volatile unsignedchar *addr)
Definition: generic-small.h:21
AO_short_store_release
AO_INLINE void AO_short_store_release(volatile unsignedshort *addr, unsignedshort value)
Definition: generic-small.h:218
AO_load_acquire
AO_INLINE AO_t AO_load_acquire(const volatile AO_t *addr)
Definition: generic-small.h:502
AO_double_store_full
AO_INLINE void AO_double_store_full(volatile AO_double_t *addr, AO_double_t value)
Definition: generic_pthread.h:395
AO_short_load_full
AO_INLINE unsigned short AO_short_load_full(const volatile unsigned short *addr)
Definition: generic_pthread.h:97
AO_int_store_full
AO_INLINE void AO_int_store_full(volatile unsigned int *addr, unsigned int val)
Definition: generic_pthread.h:128
AO_int_load_full
AO_INLINE unsigned int AO_int_load_full(const volatile unsigned int *addr)
Definition: generic_pthread.h:117
AO_short_store_full
AO_INLINE void AO_short_store_full(volatile unsigned short *addr, unsigned short val)
Definition: generic_pthread.h:108
AO_char_store_full
AO_INLINE void AO_char_store_full(volatile unsigned char *addr, unsigned char val)
Definition: generic_pthread.h:88
AO_double_load_full
AO_INLINE AO_double_t AO_double_load_full(const volatile AO_double_t *addr)
Definition: generic_pthread.h:382
AO_char_load_full
AO_INLINE unsigned char AO_char_load_full(const volatile unsigned char *addr)
Definition: generic_pthread.h:77
AO_load_full
AO_INLINE AO_t AO_load_full(const volatile AO_t *addr)
Definition: generic_pthread.h:57
AO_double_t
Definition: generic_pthread.h:373