1 // SPDX-License-Identifier: GPL-2.0
2
3 // Generated by scripts/atomic/gen-atomic-instrumented.sh
4 // DO NOT MODIFY THIS FILE DIRECTLY
5
6 /*
7 * This file provides wrappers with KASAN instrumentation for atomic operations.
8 * To use this functionality an arch's atomic.h file needs to define all
9 * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
10 * this file at the end. This file provides atomic_read() that forwards to
11 * arch_atomic_read() for actual atomic operation.
12 * Note: if an arch atomic operation is implemented by means of other atomic
13 * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
14 * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15 * double instrumentation.
16 */
17 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
18 #define _LINUX_ATOMIC_INSTRUMENTED_H
19
20 #include <linux/build_bug.h>
21 #include <linux/compiler.h>
22 #include <linux/instrumented.h>
23
24 static __always_inline int
atomic_read(const atomic_t * v)25 atomic_read(const atomic_t *v)
26 {
27 instrument_atomic_read(v, sizeof(*v));
28 return arch_atomic_read(v);
29 }
30
31 static __always_inline int
atomic_read_acquire(const atomic_t * v)32 atomic_read_acquire(const atomic_t *v)
33 {
34 instrument_atomic_read(v, sizeof(*v));
35 return arch_atomic_read_acquire(v);
36 }
37
38 static __always_inline void
atomic_set(atomic_t * v,int i)39 atomic_set(atomic_t *v, int i)
40 {
41 instrument_atomic_write(v, sizeof(*v));
42 arch_atomic_set(v, i);
43 }
44
45 static __always_inline void
atomic_set_release(atomic_t * v,int i)46 atomic_set_release(atomic_t *v, int i)
47 {
48 kcsan_release();
49 instrument_atomic_write(v, sizeof(*v));
50 arch_atomic_set_release(v, i);
51 }
52
53 static __always_inline void
atomic_add(int i,atomic_t * v)54 atomic_add(int i, atomic_t *v)
55 {
56 instrument_atomic_read_write(v, sizeof(*v));
57 arch_atomic_add(i, v);
58 }
59
60 static __always_inline int
atomic_add_return(int i,atomic_t * v)61 atomic_add_return(int i, atomic_t *v)
62 {
63 kcsan_mb();
64 instrument_atomic_read_write(v, sizeof(*v));
65 return arch_atomic_add_return(i, v);
66 }
67
68 static __always_inline int
atomic_add_return_acquire(int i,atomic_t * v)69 atomic_add_return_acquire(int i, atomic_t *v)
70 {
71 instrument_atomic_read_write(v, sizeof(*v));
72 return arch_atomic_add_return_acquire(i, v);
73 }
74
75 static __always_inline int
atomic_add_return_release(int i,atomic_t * v)76 atomic_add_return_release(int i, atomic_t *v)
77 {
78 kcsan_release();
79 instrument_atomic_read_write(v, sizeof(*v));
80 return arch_atomic_add_return_release(i, v);
81 }
82
83 static __always_inline int
atomic_add_return_relaxed(int i,atomic_t * v)84 atomic_add_return_relaxed(int i, atomic_t *v)
85 {
86 instrument_atomic_read_write(v, sizeof(*v));
87 return arch_atomic_add_return_relaxed(i, v);
88 }
89
90 static __always_inline int
atomic_fetch_add(int i,atomic_t * v)91 atomic_fetch_add(int i, atomic_t *v)
92 {
93 kcsan_mb();
94 instrument_atomic_read_write(v, sizeof(*v));
95 return arch_atomic_fetch_add(i, v);
96 }
97
98 static __always_inline int
atomic_fetch_add_acquire(int i,atomic_t * v)99 atomic_fetch_add_acquire(int i, atomic_t *v)
100 {
101 instrument_atomic_read_write(v, sizeof(*v));
102 return arch_atomic_fetch_add_acquire(i, v);
103 }
104
105 static __always_inline int
atomic_fetch_add_release(int i,atomic_t * v)106 atomic_fetch_add_release(int i, atomic_t *v)
107 {
108 kcsan_release();
109 instrument_atomic_read_write(v, sizeof(*v));
110 return arch_atomic_fetch_add_release(i, v);
111 }
112
113 static __always_inline int
atomic_fetch_add_relaxed(int i,atomic_t * v)114 atomic_fetch_add_relaxed(int i, atomic_t *v)
115 {
116 instrument_atomic_read_write(v, sizeof(*v));
117 return arch_atomic_fetch_add_relaxed(i, v);
118 }
119
120 static __always_inline void
atomic_sub(int i,atomic_t * v)121 atomic_sub(int i, atomic_t *v)
122 {
123 instrument_atomic_read_write(v, sizeof(*v));
124 arch_atomic_sub(i, v);
125 }
126
127 static __always_inline int
atomic_sub_return(int i,atomic_t * v)128 atomic_sub_return(int i, atomic_t *v)
129 {
130 kcsan_mb();
131 instrument_atomic_read_write(v, sizeof(*v));
132 return arch_atomic_sub_return(i, v);
133 }
134
135 static __always_inline int
atomic_sub_return_acquire(int i,atomic_t * v)136 atomic_sub_return_acquire(int i, atomic_t *v)
137 {
138 instrument_atomic_read_write(v, sizeof(*v));
139 return arch_atomic_sub_return_acquire(i, v);
140 }
141
142 static __always_inline int
atomic_sub_return_release(int i,atomic_t * v)143 atomic_sub_return_release(int i, atomic_t *v)
144 {
145 kcsan_release();
146 instrument_atomic_read_write(v, sizeof(*v));
147 return arch_atomic_sub_return_release(i, v);
148 }
149
150 static __always_inline int
atomic_sub_return_relaxed(int i,atomic_t * v)151 atomic_sub_return_relaxed(int i, atomic_t *v)
152 {
153 instrument_atomic_read_write(v, sizeof(*v));
154 return arch_atomic_sub_return_relaxed(i, v);
155 }
156
157 static __always_inline int
atomic_fetch_sub(int i,atomic_t * v)158 atomic_fetch_sub(int i, atomic_t *v)
159 {
160 kcsan_mb();
161 instrument_atomic_read_write(v, sizeof(*v));
162 return arch_atomic_fetch_sub(i, v);
163 }
164
165 static __always_inline int
atomic_fetch_sub_acquire(int i,atomic_t * v)166 atomic_fetch_sub_acquire(int i, atomic_t *v)
167 {
168 instrument_atomic_read_write(v, sizeof(*v));
169 return arch_atomic_fetch_sub_acquire(i, v);
170 }
171
172 static __always_inline int
atomic_fetch_sub_release(int i,atomic_t * v)173 atomic_fetch_sub_release(int i, atomic_t *v)
174 {
175 kcsan_release();
176 instrument_atomic_read_write(v, sizeof(*v));
177 return arch_atomic_fetch_sub_release(i, v);
178 }
179
180 static __always_inline int
atomic_fetch_sub_relaxed(int i,atomic_t * v)181 atomic_fetch_sub_relaxed(int i, atomic_t *v)
182 {
183 instrument_atomic_read_write(v, sizeof(*v));
184 return arch_atomic_fetch_sub_relaxed(i, v);
185 }
186
187 static __always_inline void
atomic_inc(atomic_t * v)188 atomic_inc(atomic_t *v)
189 {
190 instrument_atomic_read_write(v, sizeof(*v));
191 arch_atomic_inc(v);
192 }
193
194 static __always_inline int
atomic_inc_return(atomic_t * v)195 atomic_inc_return(atomic_t *v)
196 {
197 kcsan_mb();
198 instrument_atomic_read_write(v, sizeof(*v));
199 return arch_atomic_inc_return(v);
200 }
201
202 static __always_inline int
atomic_inc_return_acquire(atomic_t * v)203 atomic_inc_return_acquire(atomic_t *v)
204 {
205 instrument_atomic_read_write(v, sizeof(*v));
206 return arch_atomic_inc_return_acquire(v);
207 }
208
209 static __always_inline int
atomic_inc_return_release(atomic_t * v)210 atomic_inc_return_release(atomic_t *v)
211 {
212 kcsan_release();
213 instrument_atomic_read_write(v, sizeof(*v));
214 return arch_atomic_inc_return_release(v);
215 }
216
217 static __always_inline int
atomic_inc_return_relaxed(atomic_t * v)218 atomic_inc_return_relaxed(atomic_t *v)
219 {
220 instrument_atomic_read_write(v, sizeof(*v));
221 return arch_atomic_inc_return_relaxed(v);
222 }
223
224 static __always_inline int
atomic_fetch_inc(atomic_t * v)225 atomic_fetch_inc(atomic_t *v)
226 {
227 kcsan_mb();
228 instrument_atomic_read_write(v, sizeof(*v));
229 return arch_atomic_fetch_inc(v);
230 }
231
232 static __always_inline int
atomic_fetch_inc_acquire(atomic_t * v)233 atomic_fetch_inc_acquire(atomic_t *v)
234 {
235 instrument_atomic_read_write(v, sizeof(*v));
236 return arch_atomic_fetch_inc_acquire(v);
237 }
238
239 static __always_inline int
atomic_fetch_inc_release(atomic_t * v)240 atomic_fetch_inc_release(atomic_t *v)
241 {
242 kcsan_release();
243 instrument_atomic_read_write(v, sizeof(*v));
244 return arch_atomic_fetch_inc_release(v);
245 }
246
247 static __always_inline int
atomic_fetch_inc_relaxed(atomic_t * v)248 atomic_fetch_inc_relaxed(atomic_t *v)
249 {
250 instrument_atomic_read_write(v, sizeof(*v));
251 return arch_atomic_fetch_inc_relaxed(v);
252 }
253
254 static __always_inline void
atomic_dec(atomic_t * v)255 atomic_dec(atomic_t *v)
256 {
257 instrument_atomic_read_write(v, sizeof(*v));
258 arch_atomic_dec(v);
259 }
260
261 static __always_inline int
atomic_dec_return(atomic_t * v)262 atomic_dec_return(atomic_t *v)
263 {
264 kcsan_mb();
265 instrument_atomic_read_write(v, sizeof(*v));
266 return arch_atomic_dec_return(v);
267 }
268
269 static __always_inline int
atomic_dec_return_acquire(atomic_t * v)270 atomic_dec_return_acquire(atomic_t *v)
271 {
272 instrument_atomic_read_write(v, sizeof(*v));
273 return arch_atomic_dec_return_acquire(v);
274 }
275
276 static __always_inline int
atomic_dec_return_release(atomic_t * v)277 atomic_dec_return_release(atomic_t *v)
278 {
279 kcsan_release();
280 instrument_atomic_read_write(v, sizeof(*v));
281 return arch_atomic_dec_return_release(v);
282 }
283
284 static __always_inline int
atomic_dec_return_relaxed(atomic_t * v)285 atomic_dec_return_relaxed(atomic_t *v)
286 {
287 instrument_atomic_read_write(v, sizeof(*v));
288 return arch_atomic_dec_return_relaxed(v);
289 }
290
291 static __always_inline int
atomic_fetch_dec(atomic_t * v)292 atomic_fetch_dec(atomic_t *v)
293 {
294 kcsan_mb();
295 instrument_atomic_read_write(v, sizeof(*v));
296 return arch_atomic_fetch_dec(v);
297 }
298
299 static __always_inline int
atomic_fetch_dec_acquire(atomic_t * v)300 atomic_fetch_dec_acquire(atomic_t *v)
301 {
302 instrument_atomic_read_write(v, sizeof(*v));
303 return arch_atomic_fetch_dec_acquire(v);
304 }
305
306 static __always_inline int
atomic_fetch_dec_release(atomic_t * v)307 atomic_fetch_dec_release(atomic_t *v)
308 {
309 kcsan_release();
310 instrument_atomic_read_write(v, sizeof(*v));
311 return arch_atomic_fetch_dec_release(v);
312 }
313
314 static __always_inline int
atomic_fetch_dec_relaxed(atomic_t * v)315 atomic_fetch_dec_relaxed(atomic_t *v)
316 {
317 instrument_atomic_read_write(v, sizeof(*v));
318 return arch_atomic_fetch_dec_relaxed(v);
319 }
320
321 static __always_inline void
atomic_and(int i,atomic_t * v)322 atomic_and(int i, atomic_t *v)
323 {
324 instrument_atomic_read_write(v, sizeof(*v));
325 arch_atomic_and(i, v);
326 }
327
328 static __always_inline int
atomic_fetch_and(int i,atomic_t * v)329 atomic_fetch_and(int i, atomic_t *v)
330 {
331 kcsan_mb();
332 instrument_atomic_read_write(v, sizeof(*v));
333 return arch_atomic_fetch_and(i, v);
334 }
335
336 static __always_inline int
atomic_fetch_and_acquire(int i,atomic_t * v)337 atomic_fetch_and_acquire(int i, atomic_t *v)
338 {
339 instrument_atomic_read_write(v, sizeof(*v));
340 return arch_atomic_fetch_and_acquire(i, v);
341 }
342
343 static __always_inline int
atomic_fetch_and_release(int i,atomic_t * v)344 atomic_fetch_and_release(int i, atomic_t *v)
345 {
346 kcsan_release();
347 instrument_atomic_read_write(v, sizeof(*v));
348 return arch_atomic_fetch_and_release(i, v);
349 }
350
351 static __always_inline int
atomic_fetch_and_relaxed(int i,atomic_t * v)352 atomic_fetch_and_relaxed(int i, atomic_t *v)
353 {
354 instrument_atomic_read_write(v, sizeof(*v));
355 return arch_atomic_fetch_and_relaxed(i, v);
356 }
357
358 static __always_inline void
atomic_andnot(int i,atomic_t * v)359 atomic_andnot(int i, atomic_t *v)
360 {
361 instrument_atomic_read_write(v, sizeof(*v));
362 arch_atomic_andnot(i, v);
363 }
364
365 static __always_inline int
atomic_fetch_andnot(int i,atomic_t * v)366 atomic_fetch_andnot(int i, atomic_t *v)
367 {
368 kcsan_mb();
369 instrument_atomic_read_write(v, sizeof(*v));
370 return arch_atomic_fetch_andnot(i, v);
371 }
372
373 static __always_inline int
atomic_fetch_andnot_acquire(int i,atomic_t * v)374 atomic_fetch_andnot_acquire(int i, atomic_t *v)
375 {
376 instrument_atomic_read_write(v, sizeof(*v));
377 return arch_atomic_fetch_andnot_acquire(i, v);
378 }
379
380 static __always_inline int
atomic_fetch_andnot_release(int i,atomic_t * v)381 atomic_fetch_andnot_release(int i, atomic_t *v)
382 {
383 kcsan_release();
384 instrument_atomic_read_write(v, sizeof(*v));
385 return arch_atomic_fetch_andnot_release(i, v);
386 }
387
388 static __always_inline int
atomic_fetch_andnot_relaxed(int i,atomic_t * v)389 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
390 {
391 instrument_atomic_read_write(v, sizeof(*v));
392 return arch_atomic_fetch_andnot_relaxed(i, v);
393 }
394
395 static __always_inline void
atomic_or(int i,atomic_t * v)396 atomic_or(int i, atomic_t *v)
397 {
398 instrument_atomic_read_write(v, sizeof(*v));
399 arch_atomic_or(i, v);
400 }
401
402 static __always_inline int
atomic_fetch_or(int i,atomic_t * v)403 atomic_fetch_or(int i, atomic_t *v)
404 {
405 kcsan_mb();
406 instrument_atomic_read_write(v, sizeof(*v));
407 return arch_atomic_fetch_or(i, v);
408 }
409
410 static __always_inline int
atomic_fetch_or_acquire(int i,atomic_t * v)411 atomic_fetch_or_acquire(int i, atomic_t *v)
412 {
413 instrument_atomic_read_write(v, sizeof(*v));
414 return arch_atomic_fetch_or_acquire(i, v);
415 }
416
417 static __always_inline int
atomic_fetch_or_release(int i,atomic_t * v)418 atomic_fetch_or_release(int i, atomic_t *v)
419 {
420 kcsan_release();
421 instrument_atomic_read_write(v, sizeof(*v));
422 return arch_atomic_fetch_or_release(i, v);
423 }
424
425 static __always_inline int
atomic_fetch_or_relaxed(int i,atomic_t * v)426 atomic_fetch_or_relaxed(int i, atomic_t *v)
427 {
428 instrument_atomic_read_write(v, sizeof(*v));
429 return arch_atomic_fetch_or_relaxed(i, v);
430 }
431
432 static __always_inline void
atomic_xor(int i,atomic_t * v)433 atomic_xor(int i, atomic_t *v)
434 {
435 instrument_atomic_read_write(v, sizeof(*v));
436 arch_atomic_xor(i, v);
437 }
438
439 static __always_inline int
atomic_fetch_xor(int i,atomic_t * v)440 atomic_fetch_xor(int i, atomic_t *v)
441 {
442 kcsan_mb();
443 instrument_atomic_read_write(v, sizeof(*v));
444 return arch_atomic_fetch_xor(i, v);
445 }
446
447 static __always_inline int
atomic_fetch_xor_acquire(int i,atomic_t * v)448 atomic_fetch_xor_acquire(int i, atomic_t *v)
449 {
450 instrument_atomic_read_write(v, sizeof(*v));
451 return arch_atomic_fetch_xor_acquire(i, v);
452 }
453
454 static __always_inline int
atomic_fetch_xor_release(int i,atomic_t * v)455 atomic_fetch_xor_release(int i, atomic_t *v)
456 {
457 kcsan_release();
458 instrument_atomic_read_write(v, sizeof(*v));
459 return arch_atomic_fetch_xor_release(i, v);
460 }
461
462 static __always_inline int
atomic_fetch_xor_relaxed(int i,atomic_t * v)463 atomic_fetch_xor_relaxed(int i, atomic_t *v)
464 {
465 instrument_atomic_read_write(v, sizeof(*v));
466 return arch_atomic_fetch_xor_relaxed(i, v);
467 }
468
469 static __always_inline int
atomic_xchg(atomic_t * v,int i)470 atomic_xchg(atomic_t *v, int i)
471 {
472 kcsan_mb();
473 instrument_atomic_read_write(v, sizeof(*v));
474 return arch_atomic_xchg(v, i);
475 }
476
477 static __always_inline int
atomic_xchg_acquire(atomic_t * v,int i)478 atomic_xchg_acquire(atomic_t *v, int i)
479 {
480 instrument_atomic_read_write(v, sizeof(*v));
481 return arch_atomic_xchg_acquire(v, i);
482 }
483
484 static __always_inline int
atomic_xchg_release(atomic_t * v,int i)485 atomic_xchg_release(atomic_t *v, int i)
486 {
487 kcsan_release();
488 instrument_atomic_read_write(v, sizeof(*v));
489 return arch_atomic_xchg_release(v, i);
490 }
491
492 static __always_inline int
atomic_xchg_relaxed(atomic_t * v,int i)493 atomic_xchg_relaxed(atomic_t *v, int i)
494 {
495 instrument_atomic_read_write(v, sizeof(*v));
496 return arch_atomic_xchg_relaxed(v, i);
497 }
498
499 static __always_inline int
atomic_cmpxchg(atomic_t * v,int old,int new)500 atomic_cmpxchg(atomic_t *v, int old, int new)
501 {
502 kcsan_mb();
503 instrument_atomic_read_write(v, sizeof(*v));
504 return arch_atomic_cmpxchg(v, old, new);
505 }
506
507 static __always_inline int
atomic_cmpxchg_acquire(atomic_t * v,int old,int new)508 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
509 {
510 instrument_atomic_read_write(v, sizeof(*v));
511 return arch_atomic_cmpxchg_acquire(v, old, new);
512 }
513
514 static __always_inline int
atomic_cmpxchg_release(atomic_t * v,int old,int new)515 atomic_cmpxchg_release(atomic_t *v, int old, int new)
516 {
517 kcsan_release();
518 instrument_atomic_read_write(v, sizeof(*v));
519 return arch_atomic_cmpxchg_release(v, old, new);
520 }
521
522 static __always_inline int
atomic_cmpxchg_relaxed(atomic_t * v,int old,int new)523 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
524 {
525 instrument_atomic_read_write(v, sizeof(*v));
526 return arch_atomic_cmpxchg_relaxed(v, old, new);
527 }
528
529 static __always_inline bool
atomic_try_cmpxchg(atomic_t * v,int * old,int new)530 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
531 {
532 kcsan_mb();
533 instrument_atomic_read_write(v, sizeof(*v));
534 instrument_atomic_read_write(old, sizeof(*old));
535 return arch_atomic_try_cmpxchg(v, old, new);
536 }
537
538 static __always_inline bool
atomic_try_cmpxchg_acquire(atomic_t * v,int * old,int new)539 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
540 {
541 instrument_atomic_read_write(v, sizeof(*v));
542 instrument_atomic_read_write(old, sizeof(*old));
543 return arch_atomic_try_cmpxchg_acquire(v, old, new);
544 }
545
546 static __always_inline bool
atomic_try_cmpxchg_release(atomic_t * v,int * old,int new)547 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
548 {
549 kcsan_release();
550 instrument_atomic_read_write(v, sizeof(*v));
551 instrument_atomic_read_write(old, sizeof(*old));
552 return arch_atomic_try_cmpxchg_release(v, old, new);
553 }
554
555 static __always_inline bool
atomic_try_cmpxchg_relaxed(atomic_t * v,int * old,int new)556 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
557 {
558 instrument_atomic_read_write(v, sizeof(*v));
559 instrument_atomic_read_write(old, sizeof(*old));
560 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
561 }
562
563 static __always_inline bool
atomic_sub_and_test(int i,atomic_t * v)564 atomic_sub_and_test(int i, atomic_t *v)
565 {
566 kcsan_mb();
567 instrument_atomic_read_write(v, sizeof(*v));
568 return arch_atomic_sub_and_test(i, v);
569 }
570
571 static __always_inline bool
atomic_dec_and_test(atomic_t * v)572 atomic_dec_and_test(atomic_t *v)
573 {
574 kcsan_mb();
575 instrument_atomic_read_write(v, sizeof(*v));
576 return arch_atomic_dec_and_test(v);
577 }
578
579 static __always_inline bool
atomic_inc_and_test(atomic_t * v)580 atomic_inc_and_test(atomic_t *v)
581 {
582 kcsan_mb();
583 instrument_atomic_read_write(v, sizeof(*v));
584 return arch_atomic_inc_and_test(v);
585 }
586
587 static __always_inline bool
atomic_add_negative(int i,atomic_t * v)588 atomic_add_negative(int i, atomic_t *v)
589 {
590 kcsan_mb();
591 instrument_atomic_read_write(v, sizeof(*v));
592 return arch_atomic_add_negative(i, v);
593 }
594
595 static __always_inline int
atomic_fetch_add_unless(atomic_t * v,int a,int u)596 atomic_fetch_add_unless(atomic_t *v, int a, int u)
597 {
598 kcsan_mb();
599 instrument_atomic_read_write(v, sizeof(*v));
600 return arch_atomic_fetch_add_unless(v, a, u);
601 }
602
603 static __always_inline bool
atomic_add_unless(atomic_t * v,int a,int u)604 atomic_add_unless(atomic_t *v, int a, int u)
605 {
606 kcsan_mb();
607 instrument_atomic_read_write(v, sizeof(*v));
608 return arch_atomic_add_unless(v, a, u);
609 }
610
611 static __always_inline bool
atomic_inc_not_zero(atomic_t * v)612 atomic_inc_not_zero(atomic_t *v)
613 {
614 kcsan_mb();
615 instrument_atomic_read_write(v, sizeof(*v));
616 return arch_atomic_inc_not_zero(v);
617 }
618
619 static __always_inline bool
atomic_inc_unless_negative(atomic_t * v)620 atomic_inc_unless_negative(atomic_t *v)
621 {
622 kcsan_mb();
623 instrument_atomic_read_write(v, sizeof(*v));
624 return arch_atomic_inc_unless_negative(v);
625 }
626
627 static __always_inline bool
atomic_dec_unless_positive(atomic_t * v)628 atomic_dec_unless_positive(atomic_t *v)
629 {
630 kcsan_mb();
631 instrument_atomic_read_write(v, sizeof(*v));
632 return arch_atomic_dec_unless_positive(v);
633 }
634
635 static __always_inline int
atomic_dec_if_positive(atomic_t * v)636 atomic_dec_if_positive(atomic_t *v)
637 {
638 kcsan_mb();
639 instrument_atomic_read_write(v, sizeof(*v));
640 return arch_atomic_dec_if_positive(v);
641 }
642
643 static __always_inline s64
atomic64_read(const atomic64_t * v)644 atomic64_read(const atomic64_t *v)
645 {
646 instrument_atomic_read(v, sizeof(*v));
647 return arch_atomic64_read(v);
648 }
649
650 static __always_inline s64
atomic64_read_acquire(const atomic64_t * v)651 atomic64_read_acquire(const atomic64_t *v)
652 {
653 instrument_atomic_read(v, sizeof(*v));
654 return arch_atomic64_read_acquire(v);
655 }
656
657 static __always_inline void
atomic64_set(atomic64_t * v,s64 i)658 atomic64_set(atomic64_t *v, s64 i)
659 {
660 instrument_atomic_write(v, sizeof(*v));
661 arch_atomic64_set(v, i);
662 }
663
664 static __always_inline void
atomic64_set_release(atomic64_t * v,s64 i)665 atomic64_set_release(atomic64_t *v, s64 i)
666 {
667 kcsan_release();
668 instrument_atomic_write(v, sizeof(*v));
669 arch_atomic64_set_release(v, i);
670 }
671
672 static __always_inline void
atomic64_add(s64 i,atomic64_t * v)673 atomic64_add(s64 i, atomic64_t *v)
674 {
675 instrument_atomic_read_write(v, sizeof(*v));
676 arch_atomic64_add(i, v);
677 }
678
679 static __always_inline s64
atomic64_add_return(s64 i,atomic64_t * v)680 atomic64_add_return(s64 i, atomic64_t *v)
681 {
682 kcsan_mb();
683 instrument_atomic_read_write(v, sizeof(*v));
684 return arch_atomic64_add_return(i, v);
685 }
686
687 static __always_inline s64
atomic64_add_return_acquire(s64 i,atomic64_t * v)688 atomic64_add_return_acquire(s64 i, atomic64_t *v)
689 {
690 instrument_atomic_read_write(v, sizeof(*v));
691 return arch_atomic64_add_return_acquire(i, v);
692 }
693
694 static __always_inline s64
atomic64_add_return_release(s64 i,atomic64_t * v)695 atomic64_add_return_release(s64 i, atomic64_t *v)
696 {
697 kcsan_release();
698 instrument_atomic_read_write(v, sizeof(*v));
699 return arch_atomic64_add_return_release(i, v);
700 }
701
702 static __always_inline s64
atomic64_add_return_relaxed(s64 i,atomic64_t * v)703 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
704 {
705 instrument_atomic_read_write(v, sizeof(*v));
706 return arch_atomic64_add_return_relaxed(i, v);
707 }
708
709 static __always_inline s64
atomic64_fetch_add(s64 i,atomic64_t * v)710 atomic64_fetch_add(s64 i, atomic64_t *v)
711 {
712 kcsan_mb();
713 instrument_atomic_read_write(v, sizeof(*v));
714 return arch_atomic64_fetch_add(i, v);
715 }
716
717 static __always_inline s64
atomic64_fetch_add_acquire(s64 i,atomic64_t * v)718 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
719 {
720 instrument_atomic_read_write(v, sizeof(*v));
721 return arch_atomic64_fetch_add_acquire(i, v);
722 }
723
724 static __always_inline s64
atomic64_fetch_add_release(s64 i,atomic64_t * v)725 atomic64_fetch_add_release(s64 i, atomic64_t *v)
726 {
727 kcsan_release();
728 instrument_atomic_read_write(v, sizeof(*v));
729 return arch_atomic64_fetch_add_release(i, v);
730 }
731
732 static __always_inline s64
atomic64_fetch_add_relaxed(s64 i,atomic64_t * v)733 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
734 {
735 instrument_atomic_read_write(v, sizeof(*v));
736 return arch_atomic64_fetch_add_relaxed(i, v);
737 }
738
739 static __always_inline void
atomic64_sub(s64 i,atomic64_t * v)740 atomic64_sub(s64 i, atomic64_t *v)
741 {
742 instrument_atomic_read_write(v, sizeof(*v));
743 arch_atomic64_sub(i, v);
744 }
745
746 static __always_inline s64
atomic64_sub_return(s64 i,atomic64_t * v)747 atomic64_sub_return(s64 i, atomic64_t *v)
748 {
749 kcsan_mb();
750 instrument_atomic_read_write(v, sizeof(*v));
751 return arch_atomic64_sub_return(i, v);
752 }
753
754 static __always_inline s64
atomic64_sub_return_acquire(s64 i,atomic64_t * v)755 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
756 {
757 instrument_atomic_read_write(v, sizeof(*v));
758 return arch_atomic64_sub_return_acquire(i, v);
759 }
760
761 static __always_inline s64
atomic64_sub_return_release(s64 i,atomic64_t * v)762 atomic64_sub_return_release(s64 i, atomic64_t *v)
763 {
764 kcsan_release();
765 instrument_atomic_read_write(v, sizeof(*v));
766 return arch_atomic64_sub_return_release(i, v);
767 }
768
769 static __always_inline s64
atomic64_sub_return_relaxed(s64 i,atomic64_t * v)770 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
771 {
772 instrument_atomic_read_write(v, sizeof(*v));
773 return arch_atomic64_sub_return_relaxed(i, v);
774 }
775
776 static __always_inline s64
atomic64_fetch_sub(s64 i,atomic64_t * v)777 atomic64_fetch_sub(s64 i, atomic64_t *v)
778 {
779 kcsan_mb();
780 instrument_atomic_read_write(v, sizeof(*v));
781 return arch_atomic64_fetch_sub(i, v);
782 }
783
784 static __always_inline s64
atomic64_fetch_sub_acquire(s64 i,atomic64_t * v)785 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
786 {
787 instrument_atomic_read_write(v, sizeof(*v));
788 return arch_atomic64_fetch_sub_acquire(i, v);
789 }
790
791 static __always_inline s64
atomic64_fetch_sub_release(s64 i,atomic64_t * v)792 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
793 {
794 kcsan_release();
795 instrument_atomic_read_write(v, sizeof(*v));
796 return arch_atomic64_fetch_sub_release(i, v);
797 }
798
799 static __always_inline s64
atomic64_fetch_sub_relaxed(s64 i,atomic64_t * v)800 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
801 {
802 instrument_atomic_read_write(v, sizeof(*v));
803 return arch_atomic64_fetch_sub_relaxed(i, v);
804 }
805
806 static __always_inline void
atomic64_inc(atomic64_t * v)807 atomic64_inc(atomic64_t *v)
808 {
809 instrument_atomic_read_write(v, sizeof(*v));
810 arch_atomic64_inc(v);
811 }
812
813 static __always_inline s64
atomic64_inc_return(atomic64_t * v)814 atomic64_inc_return(atomic64_t *v)
815 {
816 kcsan_mb();
817 instrument_atomic_read_write(v, sizeof(*v));
818 return arch_atomic64_inc_return(v);
819 }
820
821 static __always_inline s64
atomic64_inc_return_acquire(atomic64_t * v)822 atomic64_inc_return_acquire(atomic64_t *v)
823 {
824 instrument_atomic_read_write(v, sizeof(*v));
825 return arch_atomic64_inc_return_acquire(v);
826 }
827
828 static __always_inline s64
atomic64_inc_return_release(atomic64_t * v)829 atomic64_inc_return_release(atomic64_t *v)
830 {
831 kcsan_release();
832 instrument_atomic_read_write(v, sizeof(*v));
833 return arch_atomic64_inc_return_release(v);
834 }
835
836 static __always_inline s64
atomic64_inc_return_relaxed(atomic64_t * v)837 atomic64_inc_return_relaxed(atomic64_t *v)
838 {
839 instrument_atomic_read_write(v, sizeof(*v));
840 return arch_atomic64_inc_return_relaxed(v);
841 }
842
843 static __always_inline s64
atomic64_fetch_inc(atomic64_t * v)844 atomic64_fetch_inc(atomic64_t *v)
845 {
846 kcsan_mb();
847 instrument_atomic_read_write(v, sizeof(*v));
848 return arch_atomic64_fetch_inc(v);
849 }
850
851 static __always_inline s64
atomic64_fetch_inc_acquire(atomic64_t * v)852 atomic64_fetch_inc_acquire(atomic64_t *v)
853 {
854 instrument_atomic_read_write(v, sizeof(*v));
855 return arch_atomic64_fetch_inc_acquire(v);
856 }
857
858 static __always_inline s64
atomic64_fetch_inc_release(atomic64_t * v)859 atomic64_fetch_inc_release(atomic64_t *v)
860 {
861 kcsan_release();
862 instrument_atomic_read_write(v, sizeof(*v));
863 return arch_atomic64_fetch_inc_release(v);
864 }
865
866 static __always_inline s64
atomic64_fetch_inc_relaxed(atomic64_t * v)867 atomic64_fetch_inc_relaxed(atomic64_t *v)
868 {
869 instrument_atomic_read_write(v, sizeof(*v));
870 return arch_atomic64_fetch_inc_relaxed(v);
871 }
872
873 static __always_inline void
atomic64_dec(atomic64_t * v)874 atomic64_dec(atomic64_t *v)
875 {
876 instrument_atomic_read_write(v, sizeof(*v));
877 arch_atomic64_dec(v);
878 }
879
880 static __always_inline s64
atomic64_dec_return(atomic64_t * v)881 atomic64_dec_return(atomic64_t *v)
882 {
883 kcsan_mb();
884 instrument_atomic_read_write(v, sizeof(*v));
885 return arch_atomic64_dec_return(v);
886 }
887
888 static __always_inline s64
atomic64_dec_return_acquire(atomic64_t * v)889 atomic64_dec_return_acquire(atomic64_t *v)
890 {
891 instrument_atomic_read_write(v, sizeof(*v));
892 return arch_atomic64_dec_return_acquire(v);
893 }
894
895 static __always_inline s64
atomic64_dec_return_release(atomic64_t * v)896 atomic64_dec_return_release(atomic64_t *v)
897 {
898 kcsan_release();
899 instrument_atomic_read_write(v, sizeof(*v));
900 return arch_atomic64_dec_return_release(v);
901 }
902
903 static __always_inline s64
atomic64_dec_return_relaxed(atomic64_t * v)904 atomic64_dec_return_relaxed(atomic64_t *v)
905 {
906 instrument_atomic_read_write(v, sizeof(*v));
907 return arch_atomic64_dec_return_relaxed(v);
908 }
909
910 static __always_inline s64
atomic64_fetch_dec(atomic64_t * v)911 atomic64_fetch_dec(atomic64_t *v)
912 {
913 kcsan_mb();
914 instrument_atomic_read_write(v, sizeof(*v));
915 return arch_atomic64_fetch_dec(v);
916 }
917
918 static __always_inline s64
atomic64_fetch_dec_acquire(atomic64_t * v)919 atomic64_fetch_dec_acquire(atomic64_t *v)
920 {
921 instrument_atomic_read_write(v, sizeof(*v));
922 return arch_atomic64_fetch_dec_acquire(v);
923 }
924
925 static __always_inline s64
atomic64_fetch_dec_release(atomic64_t * v)926 atomic64_fetch_dec_release(atomic64_t *v)
927 {
928 kcsan_release();
929 instrument_atomic_read_write(v, sizeof(*v));
930 return arch_atomic64_fetch_dec_release(v);
931 }
932
933 static __always_inline s64
atomic64_fetch_dec_relaxed(atomic64_t * v)934 atomic64_fetch_dec_relaxed(atomic64_t *v)
935 {
936 instrument_atomic_read_write(v, sizeof(*v));
937 return arch_atomic64_fetch_dec_relaxed(v);
938 }
939
940 static __always_inline void
atomic64_and(s64 i,atomic64_t * v)941 atomic64_and(s64 i, atomic64_t *v)
942 {
943 instrument_atomic_read_write(v, sizeof(*v));
944 arch_atomic64_and(i, v);
945 }
946
947 static __always_inline s64
atomic64_fetch_and(s64 i,atomic64_t * v)948 atomic64_fetch_and(s64 i, atomic64_t *v)
949 {
950 kcsan_mb();
951 instrument_atomic_read_write(v, sizeof(*v));
952 return arch_atomic64_fetch_and(i, v);
953 }
954
955 static __always_inline s64
atomic64_fetch_and_acquire(s64 i,atomic64_t * v)956 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
957 {
958 instrument_atomic_read_write(v, sizeof(*v));
959 return arch_atomic64_fetch_and_acquire(i, v);
960 }
961
962 static __always_inline s64
atomic64_fetch_and_release(s64 i,atomic64_t * v)963 atomic64_fetch_and_release(s64 i, atomic64_t *v)
964 {
965 kcsan_release();
966 instrument_atomic_read_write(v, sizeof(*v));
967 return arch_atomic64_fetch_and_release(i, v);
968 }
969
970 static __always_inline s64
atomic64_fetch_and_relaxed(s64 i,atomic64_t * v)971 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
972 {
973 instrument_atomic_read_write(v, sizeof(*v));
974 return arch_atomic64_fetch_and_relaxed(i, v);
975 }
976
977 static __always_inline void
atomic64_andnot(s64 i,atomic64_t * v)978 atomic64_andnot(s64 i, atomic64_t *v)
979 {
980 instrument_atomic_read_write(v, sizeof(*v));
981 arch_atomic64_andnot(i, v);
982 }
983
984 static __always_inline s64
atomic64_fetch_andnot(s64 i,atomic64_t * v)985 atomic64_fetch_andnot(s64 i, atomic64_t *v)
986 {
987 kcsan_mb();
988 instrument_atomic_read_write(v, sizeof(*v));
989 return arch_atomic64_fetch_andnot(i, v);
990 }
991
992 static __always_inline s64
atomic64_fetch_andnot_acquire(s64 i,atomic64_t * v)993 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
994 {
995 instrument_atomic_read_write(v, sizeof(*v));
996 return arch_atomic64_fetch_andnot_acquire(i, v);
997 }
998
999 static __always_inline s64
atomic64_fetch_andnot_release(s64 i,atomic64_t * v)1000 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1001 {
1002 kcsan_release();
1003 instrument_atomic_read_write(v, sizeof(*v));
1004 return arch_atomic64_fetch_andnot_release(i, v);
1005 }
1006
1007 static __always_inline s64
atomic64_fetch_andnot_relaxed(s64 i,atomic64_t * v)1008 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1009 {
1010 instrument_atomic_read_write(v, sizeof(*v));
1011 return arch_atomic64_fetch_andnot_relaxed(i, v);
1012 }
1013
1014 static __always_inline void
atomic64_or(s64 i,atomic64_t * v)1015 atomic64_or(s64 i, atomic64_t *v)
1016 {
1017 instrument_atomic_read_write(v, sizeof(*v));
1018 arch_atomic64_or(i, v);
1019 }
1020
1021 static __always_inline s64
atomic64_fetch_or(s64 i,atomic64_t * v)1022 atomic64_fetch_or(s64 i, atomic64_t *v)
1023 {
1024 kcsan_mb();
1025 instrument_atomic_read_write(v, sizeof(*v));
1026 return arch_atomic64_fetch_or(i, v);
1027 }
1028
1029 static __always_inline s64
atomic64_fetch_or_acquire(s64 i,atomic64_t * v)1030 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1031 {
1032 instrument_atomic_read_write(v, sizeof(*v));
1033 return arch_atomic64_fetch_or_acquire(i, v);
1034 }
1035
1036 static __always_inline s64
atomic64_fetch_or_release(s64 i,atomic64_t * v)1037 atomic64_fetch_or_release(s64 i, atomic64_t *v)
1038 {
1039 kcsan_release();
1040 instrument_atomic_read_write(v, sizeof(*v));
1041 return arch_atomic64_fetch_or_release(i, v);
1042 }
1043
1044 static __always_inline s64
atomic64_fetch_or_relaxed(s64 i,atomic64_t * v)1045 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1046 {
1047 instrument_atomic_read_write(v, sizeof(*v));
1048 return arch_atomic64_fetch_or_relaxed(i, v);
1049 }
1050
1051 static __always_inline void
atomic64_xor(s64 i,atomic64_t * v)1052 atomic64_xor(s64 i, atomic64_t *v)
1053 {
1054 instrument_atomic_read_write(v, sizeof(*v));
1055 arch_atomic64_xor(i, v);
1056 }
1057
1058 static __always_inline s64
atomic64_fetch_xor(s64 i,atomic64_t * v)1059 atomic64_fetch_xor(s64 i, atomic64_t *v)
1060 {
1061 kcsan_mb();
1062 instrument_atomic_read_write(v, sizeof(*v));
1063 return arch_atomic64_fetch_xor(i, v);
1064 }
1065
1066 static __always_inline s64
atomic64_fetch_xor_acquire(s64 i,atomic64_t * v)1067 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1068 {
1069 instrument_atomic_read_write(v, sizeof(*v));
1070 return arch_atomic64_fetch_xor_acquire(i, v);
1071 }
1072
1073 static __always_inline s64
atomic64_fetch_xor_release(s64 i,atomic64_t * v)1074 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1075 {
1076 kcsan_release();
1077 instrument_atomic_read_write(v, sizeof(*v));
1078 return arch_atomic64_fetch_xor_release(i, v);
1079 }
1080
1081 static __always_inline s64
atomic64_fetch_xor_relaxed(s64 i,atomic64_t * v)1082 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1083 {
1084 instrument_atomic_read_write(v, sizeof(*v));
1085 return arch_atomic64_fetch_xor_relaxed(i, v);
1086 }
1087
1088 static __always_inline s64
atomic64_xchg(atomic64_t * v,s64 i)1089 atomic64_xchg(atomic64_t *v, s64 i)
1090 {
1091 kcsan_mb();
1092 instrument_atomic_read_write(v, sizeof(*v));
1093 return arch_atomic64_xchg(v, i);
1094 }
1095
1096 static __always_inline s64
atomic64_xchg_acquire(atomic64_t * v,s64 i)1097 atomic64_xchg_acquire(atomic64_t *v, s64 i)
1098 {
1099 instrument_atomic_read_write(v, sizeof(*v));
1100 return arch_atomic64_xchg_acquire(v, i);
1101 }
1102
1103 static __always_inline s64
atomic64_xchg_release(atomic64_t * v,s64 i)1104 atomic64_xchg_release(atomic64_t *v, s64 i)
1105 {
1106 kcsan_release();
1107 instrument_atomic_read_write(v, sizeof(*v));
1108 return arch_atomic64_xchg_release(v, i);
1109 }
1110
1111 static __always_inline s64
atomic64_xchg_relaxed(atomic64_t * v,s64 i)1112 atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1113 {
1114 instrument_atomic_read_write(v, sizeof(*v));
1115 return arch_atomic64_xchg_relaxed(v, i);
1116 }
1117
1118 static __always_inline s64
atomic64_cmpxchg(atomic64_t * v,s64 old,s64 new)1119 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1120 {
1121 kcsan_mb();
1122 instrument_atomic_read_write(v, sizeof(*v));
1123 return arch_atomic64_cmpxchg(v, old, new);
1124 }
1125
1126 static __always_inline s64
atomic64_cmpxchg_acquire(atomic64_t * v,s64 old,s64 new)1127 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1128 {
1129 instrument_atomic_read_write(v, sizeof(*v));
1130 return arch_atomic64_cmpxchg_acquire(v, old, new);
1131 }
1132
1133 static __always_inline s64
atomic64_cmpxchg_release(atomic64_t * v,s64 old,s64 new)1134 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1135 {
1136 kcsan_release();
1137 instrument_atomic_read_write(v, sizeof(*v));
1138 return arch_atomic64_cmpxchg_release(v, old, new);
1139 }
1140
1141 static __always_inline s64
atomic64_cmpxchg_relaxed(atomic64_t * v,s64 old,s64 new)1142 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1143 {
1144 instrument_atomic_read_write(v, sizeof(*v));
1145 return arch_atomic64_cmpxchg_relaxed(v, old, new);
1146 }
1147
1148 static __always_inline bool
atomic64_try_cmpxchg(atomic64_t * v,s64 * old,s64 new)1149 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1150 {
1151 kcsan_mb();
1152 instrument_atomic_read_write(v, sizeof(*v));
1153 instrument_atomic_read_write(old, sizeof(*old));
1154 return arch_atomic64_try_cmpxchg(v, old, new);
1155 }
1156
1157 static __always_inline bool
atomic64_try_cmpxchg_acquire(atomic64_t * v,s64 * old,s64 new)1158 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1159 {
1160 instrument_atomic_read_write(v, sizeof(*v));
1161 instrument_atomic_read_write(old, sizeof(*old));
1162 return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1163 }
1164
1165 static __always_inline bool
atomic64_try_cmpxchg_release(atomic64_t * v,s64 * old,s64 new)1166 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1167 {
1168 kcsan_release();
1169 instrument_atomic_read_write(v, sizeof(*v));
1170 instrument_atomic_read_write(old, sizeof(*old));
1171 return arch_atomic64_try_cmpxchg_release(v, old, new);
1172 }
1173
1174 static __always_inline bool
atomic64_try_cmpxchg_relaxed(atomic64_t * v,s64 * old,s64 new)1175 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1176 {
1177 instrument_atomic_read_write(v, sizeof(*v));
1178 instrument_atomic_read_write(old, sizeof(*old));
1179 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1180 }
1181
1182 static __always_inline bool
atomic64_sub_and_test(s64 i,atomic64_t * v)1183 atomic64_sub_and_test(s64 i, atomic64_t *v)
1184 {
1185 kcsan_mb();
1186 instrument_atomic_read_write(v, sizeof(*v));
1187 return arch_atomic64_sub_and_test(i, v);
1188 }
1189
1190 static __always_inline bool
atomic64_dec_and_test(atomic64_t * v)1191 atomic64_dec_and_test(atomic64_t *v)
1192 {
1193 kcsan_mb();
1194 instrument_atomic_read_write(v, sizeof(*v));
1195 return arch_atomic64_dec_and_test(v);
1196 }
1197
1198 static __always_inline bool
atomic64_inc_and_test(atomic64_t * v)1199 atomic64_inc_and_test(atomic64_t *v)
1200 {
1201 kcsan_mb();
1202 instrument_atomic_read_write(v, sizeof(*v));
1203 return arch_atomic64_inc_and_test(v);
1204 }
1205
1206 static __always_inline bool
atomic64_add_negative(s64 i,atomic64_t * v)1207 atomic64_add_negative(s64 i, atomic64_t *v)
1208 {
1209 kcsan_mb();
1210 instrument_atomic_read_write(v, sizeof(*v));
1211 return arch_atomic64_add_negative(i, v);
1212 }
1213
1214 static __always_inline s64
atomic64_fetch_add_unless(atomic64_t * v,s64 a,s64 u)1215 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1216 {
1217 kcsan_mb();
1218 instrument_atomic_read_write(v, sizeof(*v));
1219 return arch_atomic64_fetch_add_unless(v, a, u);
1220 }
1221
1222 static __always_inline bool
atomic64_add_unless(atomic64_t * v,s64 a,s64 u)1223 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1224 {
1225 kcsan_mb();
1226 instrument_atomic_read_write(v, sizeof(*v));
1227 return arch_atomic64_add_unless(v, a, u);
1228 }
1229
1230 static __always_inline bool
atomic64_inc_not_zero(atomic64_t * v)1231 atomic64_inc_not_zero(atomic64_t *v)
1232 {
1233 kcsan_mb();
1234 instrument_atomic_read_write(v, sizeof(*v));
1235 return arch_atomic64_inc_not_zero(v);
1236 }
1237
1238 static __always_inline bool
atomic64_inc_unless_negative(atomic64_t * v)1239 atomic64_inc_unless_negative(atomic64_t *v)
1240 {
1241 kcsan_mb();
1242 instrument_atomic_read_write(v, sizeof(*v));
1243 return arch_atomic64_inc_unless_negative(v);
1244 }
1245
1246 static __always_inline bool
atomic64_dec_unless_positive(atomic64_t * v)1247 atomic64_dec_unless_positive(atomic64_t *v)
1248 {
1249 kcsan_mb();
1250 instrument_atomic_read_write(v, sizeof(*v));
1251 return arch_atomic64_dec_unless_positive(v);
1252 }
1253
1254 static __always_inline s64
atomic64_dec_if_positive(atomic64_t * v)1255 atomic64_dec_if_positive(atomic64_t *v)
1256 {
1257 kcsan_mb();
1258 instrument_atomic_read_write(v, sizeof(*v));
1259 return arch_atomic64_dec_if_positive(v);
1260 }
1261
1262 static __always_inline long
atomic_long_read(const atomic_long_t * v)1263 atomic_long_read(const atomic_long_t *v)
1264 {
1265 instrument_atomic_read(v, sizeof(*v));
1266 return arch_atomic_long_read(v);
1267 }
1268
1269 static __always_inline long
atomic_long_read_acquire(const atomic_long_t * v)1270 atomic_long_read_acquire(const atomic_long_t *v)
1271 {
1272 instrument_atomic_read(v, sizeof(*v));
1273 return arch_atomic_long_read_acquire(v);
1274 }
1275
1276 static __always_inline void
atomic_long_set(atomic_long_t * v,long i)1277 atomic_long_set(atomic_long_t *v, long i)
1278 {
1279 instrument_atomic_write(v, sizeof(*v));
1280 arch_atomic_long_set(v, i);
1281 }
1282
1283 static __always_inline void
atomic_long_set_release(atomic_long_t * v,long i)1284 atomic_long_set_release(atomic_long_t *v, long i)
1285 {
1286 kcsan_release();
1287 instrument_atomic_write(v, sizeof(*v));
1288 arch_atomic_long_set_release(v, i);
1289 }
1290
1291 static __always_inline void
atomic_long_add(long i,atomic_long_t * v)1292 atomic_long_add(long i, atomic_long_t *v)
1293 {
1294 instrument_atomic_read_write(v, sizeof(*v));
1295 arch_atomic_long_add(i, v);
1296 }
1297
1298 static __always_inline long
atomic_long_add_return(long i,atomic_long_t * v)1299 atomic_long_add_return(long i, atomic_long_t *v)
1300 {
1301 kcsan_mb();
1302 instrument_atomic_read_write(v, sizeof(*v));
1303 return arch_atomic_long_add_return(i, v);
1304 }
1305
1306 static __always_inline long
atomic_long_add_return_acquire(long i,atomic_long_t * v)1307 atomic_long_add_return_acquire(long i, atomic_long_t *v)
1308 {
1309 instrument_atomic_read_write(v, sizeof(*v));
1310 return arch_atomic_long_add_return_acquire(i, v);
1311 }
1312
1313 static __always_inline long
atomic_long_add_return_release(long i,atomic_long_t * v)1314 atomic_long_add_return_release(long i, atomic_long_t *v)
1315 {
1316 kcsan_release();
1317 instrument_atomic_read_write(v, sizeof(*v));
1318 return arch_atomic_long_add_return_release(i, v);
1319 }
1320
1321 static __always_inline long
atomic_long_add_return_relaxed(long i,atomic_long_t * v)1322 atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1323 {
1324 instrument_atomic_read_write(v, sizeof(*v));
1325 return arch_atomic_long_add_return_relaxed(i, v);
1326 }
1327
1328 static __always_inline long
atomic_long_fetch_add(long i,atomic_long_t * v)1329 atomic_long_fetch_add(long i, atomic_long_t *v)
1330 {
1331 kcsan_mb();
1332 instrument_atomic_read_write(v, sizeof(*v));
1333 return arch_atomic_long_fetch_add(i, v);
1334 }
1335
1336 static __always_inline long
atomic_long_fetch_add_acquire(long i,atomic_long_t * v)1337 atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1338 {
1339 instrument_atomic_read_write(v, sizeof(*v));
1340 return arch_atomic_long_fetch_add_acquire(i, v);
1341 }
1342
1343 static __always_inline long
atomic_long_fetch_add_release(long i,atomic_long_t * v)1344 atomic_long_fetch_add_release(long i, atomic_long_t *v)
1345 {
1346 kcsan_release();
1347 instrument_atomic_read_write(v, sizeof(*v));
1348 return arch_atomic_long_fetch_add_release(i, v);
1349 }
1350
1351 static __always_inline long
atomic_long_fetch_add_relaxed(long i,atomic_long_t * v)1352 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1353 {
1354 instrument_atomic_read_write(v, sizeof(*v));
1355 return arch_atomic_long_fetch_add_relaxed(i, v);
1356 }
1357
1358 static __always_inline void
atomic_long_sub(long i,atomic_long_t * v)1359 atomic_long_sub(long i, atomic_long_t *v)
1360 {
1361 instrument_atomic_read_write(v, sizeof(*v));
1362 arch_atomic_long_sub(i, v);
1363 }
1364
1365 static __always_inline long
atomic_long_sub_return(long i,atomic_long_t * v)1366 atomic_long_sub_return(long i, atomic_long_t *v)
1367 {
1368 kcsan_mb();
1369 instrument_atomic_read_write(v, sizeof(*v));
1370 return arch_atomic_long_sub_return(i, v);
1371 }
1372
1373 static __always_inline long
atomic_long_sub_return_acquire(long i,atomic_long_t * v)1374 atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1375 {
1376 instrument_atomic_read_write(v, sizeof(*v));
1377 return arch_atomic_long_sub_return_acquire(i, v);
1378 }
1379
1380 static __always_inline long
atomic_long_sub_return_release(long i,atomic_long_t * v)1381 atomic_long_sub_return_release(long i, atomic_long_t *v)
1382 {
1383 kcsan_release();
1384 instrument_atomic_read_write(v, sizeof(*v));
1385 return arch_atomic_long_sub_return_release(i, v);
1386 }
1387
1388 static __always_inline long
atomic_long_sub_return_relaxed(long i,atomic_long_t * v)1389 atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1390 {
1391 instrument_atomic_read_write(v, sizeof(*v));
1392 return arch_atomic_long_sub_return_relaxed(i, v);
1393 }
1394
1395 static __always_inline long
atomic_long_fetch_sub(long i,atomic_long_t * v)1396 atomic_long_fetch_sub(long i, atomic_long_t *v)
1397 {
1398 kcsan_mb();
1399 instrument_atomic_read_write(v, sizeof(*v));
1400 return arch_atomic_long_fetch_sub(i, v);
1401 }
1402
1403 static __always_inline long
atomic_long_fetch_sub_acquire(long i,atomic_long_t * v)1404 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1405 {
1406 instrument_atomic_read_write(v, sizeof(*v));
1407 return arch_atomic_long_fetch_sub_acquire(i, v);
1408 }
1409
1410 static __always_inline long
atomic_long_fetch_sub_release(long i,atomic_long_t * v)1411 atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1412 {
1413 kcsan_release();
1414 instrument_atomic_read_write(v, sizeof(*v));
1415 return arch_atomic_long_fetch_sub_release(i, v);
1416 }
1417
1418 static __always_inline long
atomic_long_fetch_sub_relaxed(long i,atomic_long_t * v)1419 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1420 {
1421 instrument_atomic_read_write(v, sizeof(*v));
1422 return arch_atomic_long_fetch_sub_relaxed(i, v);
1423 }
1424
1425 static __always_inline void
atomic_long_inc(atomic_long_t * v)1426 atomic_long_inc(atomic_long_t *v)
1427 {
1428 instrument_atomic_read_write(v, sizeof(*v));
1429 arch_atomic_long_inc(v);
1430 }
1431
1432 static __always_inline long
atomic_long_inc_return(atomic_long_t * v)1433 atomic_long_inc_return(atomic_long_t *v)
1434 {
1435 kcsan_mb();
1436 instrument_atomic_read_write(v, sizeof(*v));
1437 return arch_atomic_long_inc_return(v);
1438 }
1439
1440 static __always_inline long
atomic_long_inc_return_acquire(atomic_long_t * v)1441 atomic_long_inc_return_acquire(atomic_long_t *v)
1442 {
1443 instrument_atomic_read_write(v, sizeof(*v));
1444 return arch_atomic_long_inc_return_acquire(v);
1445 }
1446
1447 static __always_inline long
atomic_long_inc_return_release(atomic_long_t * v)1448 atomic_long_inc_return_release(atomic_long_t *v)
1449 {
1450 kcsan_release();
1451 instrument_atomic_read_write(v, sizeof(*v));
1452 return arch_atomic_long_inc_return_release(v);
1453 }
1454
1455 static __always_inline long
atomic_long_inc_return_relaxed(atomic_long_t * v)1456 atomic_long_inc_return_relaxed(atomic_long_t *v)
1457 {
1458 instrument_atomic_read_write(v, sizeof(*v));
1459 return arch_atomic_long_inc_return_relaxed(v);
1460 }
1461
1462 static __always_inline long
atomic_long_fetch_inc(atomic_long_t * v)1463 atomic_long_fetch_inc(atomic_long_t *v)
1464 {
1465 kcsan_mb();
1466 instrument_atomic_read_write(v, sizeof(*v));
1467 return arch_atomic_long_fetch_inc(v);
1468 }
1469
1470 static __always_inline long
atomic_long_fetch_inc_acquire(atomic_long_t * v)1471 atomic_long_fetch_inc_acquire(atomic_long_t *v)
1472 {
1473 instrument_atomic_read_write(v, sizeof(*v));
1474 return arch_atomic_long_fetch_inc_acquire(v);
1475 }
1476
1477 static __always_inline long
atomic_long_fetch_inc_release(atomic_long_t * v)1478 atomic_long_fetch_inc_release(atomic_long_t *v)
1479 {
1480 kcsan_release();
1481 instrument_atomic_read_write(v, sizeof(*v));
1482 return arch_atomic_long_fetch_inc_release(v);
1483 }
1484
1485 static __always_inline long
atomic_long_fetch_inc_relaxed(atomic_long_t * v)1486 atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1487 {
1488 instrument_atomic_read_write(v, sizeof(*v));
1489 return arch_atomic_long_fetch_inc_relaxed(v);
1490 }
1491
1492 static __always_inline void
atomic_long_dec(atomic_long_t * v)1493 atomic_long_dec(atomic_long_t *v)
1494 {
1495 instrument_atomic_read_write(v, sizeof(*v));
1496 arch_atomic_long_dec(v);
1497 }
1498
1499 static __always_inline long
atomic_long_dec_return(atomic_long_t * v)1500 atomic_long_dec_return(atomic_long_t *v)
1501 {
1502 kcsan_mb();
1503 instrument_atomic_read_write(v, sizeof(*v));
1504 return arch_atomic_long_dec_return(v);
1505 }
1506
1507 static __always_inline long
atomic_long_dec_return_acquire(atomic_long_t * v)1508 atomic_long_dec_return_acquire(atomic_long_t *v)
1509 {
1510 instrument_atomic_read_write(v, sizeof(*v));
1511 return arch_atomic_long_dec_return_acquire(v);
1512 }
1513
1514 static __always_inline long
atomic_long_dec_return_release(atomic_long_t * v)1515 atomic_long_dec_return_release(atomic_long_t *v)
1516 {
1517 kcsan_release();
1518 instrument_atomic_read_write(v, sizeof(*v));
1519 return arch_atomic_long_dec_return_release(v);
1520 }
1521
1522 static __always_inline long
atomic_long_dec_return_relaxed(atomic_long_t * v)1523 atomic_long_dec_return_relaxed(atomic_long_t *v)
1524 {
1525 instrument_atomic_read_write(v, sizeof(*v));
1526 return arch_atomic_long_dec_return_relaxed(v);
1527 }
1528
1529 static __always_inline long
atomic_long_fetch_dec(atomic_long_t * v)1530 atomic_long_fetch_dec(atomic_long_t *v)
1531 {
1532 kcsan_mb();
1533 instrument_atomic_read_write(v, sizeof(*v));
1534 return arch_atomic_long_fetch_dec(v);
1535 }
1536
1537 static __always_inline long
atomic_long_fetch_dec_acquire(atomic_long_t * v)1538 atomic_long_fetch_dec_acquire(atomic_long_t *v)
1539 {
1540 instrument_atomic_read_write(v, sizeof(*v));
1541 return arch_atomic_long_fetch_dec_acquire(v);
1542 }
1543
1544 static __always_inline long
atomic_long_fetch_dec_release(atomic_long_t * v)1545 atomic_long_fetch_dec_release(atomic_long_t *v)
1546 {
1547 kcsan_release();
1548 instrument_atomic_read_write(v, sizeof(*v));
1549 return arch_atomic_long_fetch_dec_release(v);
1550 }
1551
1552 static __always_inline long
atomic_long_fetch_dec_relaxed(atomic_long_t * v)1553 atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1554 {
1555 instrument_atomic_read_write(v, sizeof(*v));
1556 return arch_atomic_long_fetch_dec_relaxed(v);
1557 }
1558
1559 static __always_inline void
atomic_long_and(long i,atomic_long_t * v)1560 atomic_long_and(long i, atomic_long_t *v)
1561 {
1562 instrument_atomic_read_write(v, sizeof(*v));
1563 arch_atomic_long_and(i, v);
1564 }
1565
1566 static __always_inline long
atomic_long_fetch_and(long i,atomic_long_t * v)1567 atomic_long_fetch_and(long i, atomic_long_t *v)
1568 {
1569 kcsan_mb();
1570 instrument_atomic_read_write(v, sizeof(*v));
1571 return arch_atomic_long_fetch_and(i, v);
1572 }
1573
1574 static __always_inline long
atomic_long_fetch_and_acquire(long i,atomic_long_t * v)1575 atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1576 {
1577 instrument_atomic_read_write(v, sizeof(*v));
1578 return arch_atomic_long_fetch_and_acquire(i, v);
1579 }
1580
1581 static __always_inline long
atomic_long_fetch_and_release(long i,atomic_long_t * v)1582 atomic_long_fetch_and_release(long i, atomic_long_t *v)
1583 {
1584 kcsan_release();
1585 instrument_atomic_read_write(v, sizeof(*v));
1586 return arch_atomic_long_fetch_and_release(i, v);
1587 }
1588
1589 static __always_inline long
atomic_long_fetch_and_relaxed(long i,atomic_long_t * v)1590 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1591 {
1592 instrument_atomic_read_write(v, sizeof(*v));
1593 return arch_atomic_long_fetch_and_relaxed(i, v);
1594 }
1595
1596 static __always_inline void
atomic_long_andnot(long i,atomic_long_t * v)1597 atomic_long_andnot(long i, atomic_long_t *v)
1598 {
1599 instrument_atomic_read_write(v, sizeof(*v));
1600 arch_atomic_long_andnot(i, v);
1601 }
1602
1603 static __always_inline long
atomic_long_fetch_andnot(long i,atomic_long_t * v)1604 atomic_long_fetch_andnot(long i, atomic_long_t *v)
1605 {
1606 kcsan_mb();
1607 instrument_atomic_read_write(v, sizeof(*v));
1608 return arch_atomic_long_fetch_andnot(i, v);
1609 }
1610
1611 static __always_inline long
atomic_long_fetch_andnot_acquire(long i,atomic_long_t * v)1612 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1613 {
1614 instrument_atomic_read_write(v, sizeof(*v));
1615 return arch_atomic_long_fetch_andnot_acquire(i, v);
1616 }
1617
1618 static __always_inline long
atomic_long_fetch_andnot_release(long i,atomic_long_t * v)1619 atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1620 {
1621 kcsan_release();
1622 instrument_atomic_read_write(v, sizeof(*v));
1623 return arch_atomic_long_fetch_andnot_release(i, v);
1624 }
1625
1626 static __always_inline long
atomic_long_fetch_andnot_relaxed(long i,atomic_long_t * v)1627 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1628 {
1629 instrument_atomic_read_write(v, sizeof(*v));
1630 return arch_atomic_long_fetch_andnot_relaxed(i, v);
1631 }
1632
1633 static __always_inline void
atomic_long_or(long i,atomic_long_t * v)1634 atomic_long_or(long i, atomic_long_t *v)
1635 {
1636 instrument_atomic_read_write(v, sizeof(*v));
1637 arch_atomic_long_or(i, v);
1638 }
1639
1640 static __always_inline long
atomic_long_fetch_or(long i,atomic_long_t * v)1641 atomic_long_fetch_or(long i, atomic_long_t *v)
1642 {
1643 kcsan_mb();
1644 instrument_atomic_read_write(v, sizeof(*v));
1645 return arch_atomic_long_fetch_or(i, v);
1646 }
1647
1648 static __always_inline long
atomic_long_fetch_or_acquire(long i,atomic_long_t * v)1649 atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1650 {
1651 instrument_atomic_read_write(v, sizeof(*v));
1652 return arch_atomic_long_fetch_or_acquire(i, v);
1653 }
1654
1655 static __always_inline long
atomic_long_fetch_or_release(long i,atomic_long_t * v)1656 atomic_long_fetch_or_release(long i, atomic_long_t *v)
1657 {
1658 kcsan_release();
1659 instrument_atomic_read_write(v, sizeof(*v));
1660 return arch_atomic_long_fetch_or_release(i, v);
1661 }
1662
1663 static __always_inline long
atomic_long_fetch_or_relaxed(long i,atomic_long_t * v)1664 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1665 {
1666 instrument_atomic_read_write(v, sizeof(*v));
1667 return arch_atomic_long_fetch_or_relaxed(i, v);
1668 }
1669
1670 static __always_inline void
atomic_long_xor(long i,atomic_long_t * v)1671 atomic_long_xor(long i, atomic_long_t *v)
1672 {
1673 instrument_atomic_read_write(v, sizeof(*v));
1674 arch_atomic_long_xor(i, v);
1675 }
1676
1677 static __always_inline long
atomic_long_fetch_xor(long i,atomic_long_t * v)1678 atomic_long_fetch_xor(long i, atomic_long_t *v)
1679 {
1680 kcsan_mb();
1681 instrument_atomic_read_write(v, sizeof(*v));
1682 return arch_atomic_long_fetch_xor(i, v);
1683 }
1684
1685 static __always_inline long
atomic_long_fetch_xor_acquire(long i,atomic_long_t * v)1686 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1687 {
1688 instrument_atomic_read_write(v, sizeof(*v));
1689 return arch_atomic_long_fetch_xor_acquire(i, v);
1690 }
1691
1692 static __always_inline long
atomic_long_fetch_xor_release(long i,atomic_long_t * v)1693 atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1694 {
1695 kcsan_release();
1696 instrument_atomic_read_write(v, sizeof(*v));
1697 return arch_atomic_long_fetch_xor_release(i, v);
1698 }
1699
1700 static __always_inline long
atomic_long_fetch_xor_relaxed(long i,atomic_long_t * v)1701 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1702 {
1703 instrument_atomic_read_write(v, sizeof(*v));
1704 return arch_atomic_long_fetch_xor_relaxed(i, v);
1705 }
1706
1707 static __always_inline long
atomic_long_xchg(atomic_long_t * v,long i)1708 atomic_long_xchg(atomic_long_t *v, long i)
1709 {
1710 kcsan_mb();
1711 instrument_atomic_read_write(v, sizeof(*v));
1712 return arch_atomic_long_xchg(v, i);
1713 }
1714
1715 static __always_inline long
atomic_long_xchg_acquire(atomic_long_t * v,long i)1716 atomic_long_xchg_acquire(atomic_long_t *v, long i)
1717 {
1718 instrument_atomic_read_write(v, sizeof(*v));
1719 return arch_atomic_long_xchg_acquire(v, i);
1720 }
1721
1722 static __always_inline long
atomic_long_xchg_release(atomic_long_t * v,long i)1723 atomic_long_xchg_release(atomic_long_t *v, long i)
1724 {
1725 kcsan_release();
1726 instrument_atomic_read_write(v, sizeof(*v));
1727 return arch_atomic_long_xchg_release(v, i);
1728 }
1729
1730 static __always_inline long
atomic_long_xchg_relaxed(atomic_long_t * v,long i)1731 atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1732 {
1733 instrument_atomic_read_write(v, sizeof(*v));
1734 return arch_atomic_long_xchg_relaxed(v, i);
1735 }
1736
1737 static __always_inline long
atomic_long_cmpxchg(atomic_long_t * v,long old,long new)1738 atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1739 {
1740 kcsan_mb();
1741 instrument_atomic_read_write(v, sizeof(*v));
1742 return arch_atomic_long_cmpxchg(v, old, new);
1743 }
1744
1745 static __always_inline long
atomic_long_cmpxchg_acquire(atomic_long_t * v,long old,long new)1746 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1747 {
1748 instrument_atomic_read_write(v, sizeof(*v));
1749 return arch_atomic_long_cmpxchg_acquire(v, old, new);
1750 }
1751
1752 static __always_inline long
atomic_long_cmpxchg_release(atomic_long_t * v,long old,long new)1753 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1754 {
1755 kcsan_release();
1756 instrument_atomic_read_write(v, sizeof(*v));
1757 return arch_atomic_long_cmpxchg_release(v, old, new);
1758 }
1759
1760 static __always_inline long
atomic_long_cmpxchg_relaxed(atomic_long_t * v,long old,long new)1761 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1762 {
1763 instrument_atomic_read_write(v, sizeof(*v));
1764 return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1765 }
1766
1767 static __always_inline bool
atomic_long_try_cmpxchg(atomic_long_t * v,long * old,long new)1768 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1769 {
1770 kcsan_mb();
1771 instrument_atomic_read_write(v, sizeof(*v));
1772 instrument_atomic_read_write(old, sizeof(*old));
1773 return arch_atomic_long_try_cmpxchg(v, old, new);
1774 }
1775
1776 static __always_inline bool
atomic_long_try_cmpxchg_acquire(atomic_long_t * v,long * old,long new)1777 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1778 {
1779 instrument_atomic_read_write(v, sizeof(*v));
1780 instrument_atomic_read_write(old, sizeof(*old));
1781 return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1782 }
1783
1784 static __always_inline bool
atomic_long_try_cmpxchg_release(atomic_long_t * v,long * old,long new)1785 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1786 {
1787 kcsan_release();
1788 instrument_atomic_read_write(v, sizeof(*v));
1789 instrument_atomic_read_write(old, sizeof(*old));
1790 return arch_atomic_long_try_cmpxchg_release(v, old, new);
1791 }
1792
1793 static __always_inline bool
atomic_long_try_cmpxchg_relaxed(atomic_long_t * v,long * old,long new)1794 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1795 {
1796 instrument_atomic_read_write(v, sizeof(*v));
1797 instrument_atomic_read_write(old, sizeof(*old));
1798 return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1799 }
1800
1801 static __always_inline bool
atomic_long_sub_and_test(long i,atomic_long_t * v)1802 atomic_long_sub_and_test(long i, atomic_long_t *v)
1803 {
1804 kcsan_mb();
1805 instrument_atomic_read_write(v, sizeof(*v));
1806 return arch_atomic_long_sub_and_test(i, v);
1807 }
1808
1809 static __always_inline bool
atomic_long_dec_and_test(atomic_long_t * v)1810 atomic_long_dec_and_test(atomic_long_t *v)
1811 {
1812 kcsan_mb();
1813 instrument_atomic_read_write(v, sizeof(*v));
1814 return arch_atomic_long_dec_and_test(v);
1815 }
1816
1817 static __always_inline bool
atomic_long_inc_and_test(atomic_long_t * v)1818 atomic_long_inc_and_test(atomic_long_t *v)
1819 {
1820 kcsan_mb();
1821 instrument_atomic_read_write(v, sizeof(*v));
1822 return arch_atomic_long_inc_and_test(v);
1823 }
1824
1825 static __always_inline bool
atomic_long_add_negative(long i,atomic_long_t * v)1826 atomic_long_add_negative(long i, atomic_long_t *v)
1827 {
1828 kcsan_mb();
1829 instrument_atomic_read_write(v, sizeof(*v));
1830 return arch_atomic_long_add_negative(i, v);
1831 }
1832
1833 static __always_inline long
atomic_long_fetch_add_unless(atomic_long_t * v,long a,long u)1834 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1835 {
1836 kcsan_mb();
1837 instrument_atomic_read_write(v, sizeof(*v));
1838 return arch_atomic_long_fetch_add_unless(v, a, u);
1839 }
1840
1841 static __always_inline bool
atomic_long_add_unless(atomic_long_t * v,long a,long u)1842 atomic_long_add_unless(atomic_long_t *v, long a, long u)
1843 {
1844 kcsan_mb();
1845 instrument_atomic_read_write(v, sizeof(*v));
1846 return arch_atomic_long_add_unless(v, a, u);
1847 }
1848
1849 static __always_inline bool
atomic_long_inc_not_zero(atomic_long_t * v)1850 atomic_long_inc_not_zero(atomic_long_t *v)
1851 {
1852 kcsan_mb();
1853 instrument_atomic_read_write(v, sizeof(*v));
1854 return arch_atomic_long_inc_not_zero(v);
1855 }
1856
1857 static __always_inline bool
atomic_long_inc_unless_negative(atomic_long_t * v)1858 atomic_long_inc_unless_negative(atomic_long_t *v)
1859 {
1860 kcsan_mb();
1861 instrument_atomic_read_write(v, sizeof(*v));
1862 return arch_atomic_long_inc_unless_negative(v);
1863 }
1864
1865 static __always_inline bool
atomic_long_dec_unless_positive(atomic_long_t * v)1866 atomic_long_dec_unless_positive(atomic_long_t *v)
1867 {
1868 kcsan_mb();
1869 instrument_atomic_read_write(v, sizeof(*v));
1870 return arch_atomic_long_dec_unless_positive(v);
1871 }
1872
1873 static __always_inline long
atomic_long_dec_if_positive(atomic_long_t * v)1874 atomic_long_dec_if_positive(atomic_long_t *v)
1875 {
1876 kcsan_mb();
1877 instrument_atomic_read_write(v, sizeof(*v));
1878 return arch_atomic_long_dec_if_positive(v);
1879 }
1880
1881 #define xchg(ptr, ...) \
1882 ({ \
1883 typeof(ptr) __ai_ptr = (ptr); \
1884 kcsan_mb(); \
1885 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1886 arch_xchg(__ai_ptr, __VA_ARGS__); \
1887 })
1888
1889 #define xchg_acquire(ptr, ...) \
1890 ({ \
1891 typeof(ptr) __ai_ptr = (ptr); \
1892 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1893 arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1894 })
1895
1896 #define xchg_release(ptr, ...) \
1897 ({ \
1898 typeof(ptr) __ai_ptr = (ptr); \
1899 kcsan_release(); \
1900 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1901 arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1902 })
1903
1904 #define xchg_relaxed(ptr, ...) \
1905 ({ \
1906 typeof(ptr) __ai_ptr = (ptr); \
1907 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1908 arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1909 })
1910
1911 #define cmpxchg(ptr, ...) \
1912 ({ \
1913 typeof(ptr) __ai_ptr = (ptr); \
1914 kcsan_mb(); \
1915 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1916 arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1917 })
1918
1919 #define cmpxchg_acquire(ptr, ...) \
1920 ({ \
1921 typeof(ptr) __ai_ptr = (ptr); \
1922 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1923 arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1924 })
1925
1926 #define cmpxchg_release(ptr, ...) \
1927 ({ \
1928 typeof(ptr) __ai_ptr = (ptr); \
1929 kcsan_release(); \
1930 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1931 arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1932 })
1933
1934 #define cmpxchg_relaxed(ptr, ...) \
1935 ({ \
1936 typeof(ptr) __ai_ptr = (ptr); \
1937 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1938 arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1939 })
1940
1941 #define cmpxchg64(ptr, ...) \
1942 ({ \
1943 typeof(ptr) __ai_ptr = (ptr); \
1944 kcsan_mb(); \
1945 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1946 arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1947 })
1948
1949 #define cmpxchg64_acquire(ptr, ...) \
1950 ({ \
1951 typeof(ptr) __ai_ptr = (ptr); \
1952 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1953 arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1954 })
1955
1956 #define cmpxchg64_release(ptr, ...) \
1957 ({ \
1958 typeof(ptr) __ai_ptr = (ptr); \
1959 kcsan_release(); \
1960 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1961 arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1962 })
1963
1964 #define cmpxchg64_relaxed(ptr, ...) \
1965 ({ \
1966 typeof(ptr) __ai_ptr = (ptr); \
1967 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1968 arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1969 })
1970
1971 #define try_cmpxchg(ptr, oldp, ...) \
1972 ({ \
1973 typeof(ptr) __ai_ptr = (ptr); \
1974 typeof(oldp) __ai_oldp = (oldp); \
1975 kcsan_mb(); \
1976 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1977 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1978 arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1979 })
1980
1981 #define try_cmpxchg_acquire(ptr, oldp, ...) \
1982 ({ \
1983 typeof(ptr) __ai_ptr = (ptr); \
1984 typeof(oldp) __ai_oldp = (oldp); \
1985 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1986 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1987 arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1988 })
1989
1990 #define try_cmpxchg_release(ptr, oldp, ...) \
1991 ({ \
1992 typeof(ptr) __ai_ptr = (ptr); \
1993 typeof(oldp) __ai_oldp = (oldp); \
1994 kcsan_release(); \
1995 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1996 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1997 arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1998 })
1999
2000 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
2001 ({ \
2002 typeof(ptr) __ai_ptr = (ptr); \
2003 typeof(oldp) __ai_oldp = (oldp); \
2004 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2005 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2006 arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2007 })
2008
2009 #define try_cmpxchg64(ptr, oldp, ...) \
2010 ({ \
2011 typeof(ptr) __ai_ptr = (ptr); \
2012 typeof(oldp) __ai_oldp = (oldp); \
2013 kcsan_mb(); \
2014 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2015 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2016 arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2017 })
2018
2019 #define try_cmpxchg64_acquire(ptr, oldp, ...) \
2020 ({ \
2021 typeof(ptr) __ai_ptr = (ptr); \
2022 typeof(oldp) __ai_oldp = (oldp); \
2023 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2024 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2025 arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2026 })
2027
2028 #define try_cmpxchg64_release(ptr, oldp, ...) \
2029 ({ \
2030 typeof(ptr) __ai_ptr = (ptr); \
2031 typeof(oldp) __ai_oldp = (oldp); \
2032 kcsan_release(); \
2033 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2034 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2035 arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2036 })
2037
2038 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
2039 ({ \
2040 typeof(ptr) __ai_ptr = (ptr); \
2041 typeof(oldp) __ai_oldp = (oldp); \
2042 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2044 arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2045 })
2046
2047 #define cmpxchg_local(ptr, ...) \
2048 ({ \
2049 typeof(ptr) __ai_ptr = (ptr); \
2050 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2051 arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
2052 })
2053
2054 #define cmpxchg64_local(ptr, ...) \
2055 ({ \
2056 typeof(ptr) __ai_ptr = (ptr); \
2057 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2058 arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
2059 })
2060
2061 #define sync_cmpxchg(ptr, ...) \
2062 ({ \
2063 typeof(ptr) __ai_ptr = (ptr); \
2064 kcsan_mb(); \
2065 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2066 arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
2067 })
2068
2069 #define cmpxchg_double(ptr, ...) \
2070 ({ \
2071 typeof(ptr) __ai_ptr = (ptr); \
2072 kcsan_mb(); \
2073 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2074 arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2075 })
2076
2077
2078 #define cmpxchg_double_local(ptr, ...) \
2079 ({ \
2080 typeof(ptr) __ai_ptr = (ptr); \
2081 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2082 arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2083 })
2084
2085 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2086 // 764f741eb77a7ad565dc8d99ce2837d5542e8aee
2087