1 // SPDX-License-Identifier: GPL-2.0
2 
3 // Generated by scripts/atomic/gen-atomic-instrumented.sh
4 // DO NOT MODIFY THIS FILE DIRECTLY
5 
6 /*
7  * This file provides wrappers with KASAN instrumentation for atomic operations.
8  * To use this functionality an arch's atomic.h file needs to define all
9  * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
10  * this file at the end. This file provides atomic_read() that forwards to
11  * arch_atomic_read() for actual atomic operation.
12  * Note: if an arch atomic operation is implemented by means of other atomic
13  * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
14  * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15  * double instrumentation.
16  */
17 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
18 #define _LINUX_ATOMIC_INSTRUMENTED_H
19 
20 #include <linux/build_bug.h>
21 #include <linux/compiler.h>
22 #include <linux/instrumented.h>
23 
24 static __always_inline int
atomic_read(const atomic_t * v)25 atomic_read(const atomic_t *v)
26 {
27 	instrument_atomic_read(v, sizeof(*v));
28 	return arch_atomic_read(v);
29 }
30 
31 static __always_inline int
atomic_read_acquire(const atomic_t * v)32 atomic_read_acquire(const atomic_t *v)
33 {
34 	instrument_atomic_read(v, sizeof(*v));
35 	return arch_atomic_read_acquire(v);
36 }
37 
38 static __always_inline void
atomic_set(atomic_t * v,int i)39 atomic_set(atomic_t *v, int i)
40 {
41 	instrument_atomic_write(v, sizeof(*v));
42 	arch_atomic_set(v, i);
43 }
44 
45 static __always_inline void
atomic_set_release(atomic_t * v,int i)46 atomic_set_release(atomic_t *v, int i)
47 {
48 	kcsan_release();
49 	instrument_atomic_write(v, sizeof(*v));
50 	arch_atomic_set_release(v, i);
51 }
52 
53 static __always_inline void
atomic_add(int i,atomic_t * v)54 atomic_add(int i, atomic_t *v)
55 {
56 	instrument_atomic_read_write(v, sizeof(*v));
57 	arch_atomic_add(i, v);
58 }
59 
60 static __always_inline int
atomic_add_return(int i,atomic_t * v)61 atomic_add_return(int i, atomic_t *v)
62 {
63 	kcsan_mb();
64 	instrument_atomic_read_write(v, sizeof(*v));
65 	return arch_atomic_add_return(i, v);
66 }
67 
68 static __always_inline int
atomic_add_return_acquire(int i,atomic_t * v)69 atomic_add_return_acquire(int i, atomic_t *v)
70 {
71 	instrument_atomic_read_write(v, sizeof(*v));
72 	return arch_atomic_add_return_acquire(i, v);
73 }
74 
75 static __always_inline int
atomic_add_return_release(int i,atomic_t * v)76 atomic_add_return_release(int i, atomic_t *v)
77 {
78 	kcsan_release();
79 	instrument_atomic_read_write(v, sizeof(*v));
80 	return arch_atomic_add_return_release(i, v);
81 }
82 
83 static __always_inline int
atomic_add_return_relaxed(int i,atomic_t * v)84 atomic_add_return_relaxed(int i, atomic_t *v)
85 {
86 	instrument_atomic_read_write(v, sizeof(*v));
87 	return arch_atomic_add_return_relaxed(i, v);
88 }
89 
90 static __always_inline int
atomic_fetch_add(int i,atomic_t * v)91 atomic_fetch_add(int i, atomic_t *v)
92 {
93 	kcsan_mb();
94 	instrument_atomic_read_write(v, sizeof(*v));
95 	return arch_atomic_fetch_add(i, v);
96 }
97 
98 static __always_inline int
atomic_fetch_add_acquire(int i,atomic_t * v)99 atomic_fetch_add_acquire(int i, atomic_t *v)
100 {
101 	instrument_atomic_read_write(v, sizeof(*v));
102 	return arch_atomic_fetch_add_acquire(i, v);
103 }
104 
105 static __always_inline int
atomic_fetch_add_release(int i,atomic_t * v)106 atomic_fetch_add_release(int i, atomic_t *v)
107 {
108 	kcsan_release();
109 	instrument_atomic_read_write(v, sizeof(*v));
110 	return arch_atomic_fetch_add_release(i, v);
111 }
112 
113 static __always_inline int
atomic_fetch_add_relaxed(int i,atomic_t * v)114 atomic_fetch_add_relaxed(int i, atomic_t *v)
115 {
116 	instrument_atomic_read_write(v, sizeof(*v));
117 	return arch_atomic_fetch_add_relaxed(i, v);
118 }
119 
120 static __always_inline void
atomic_sub(int i,atomic_t * v)121 atomic_sub(int i, atomic_t *v)
122 {
123 	instrument_atomic_read_write(v, sizeof(*v));
124 	arch_atomic_sub(i, v);
125 }
126 
127 static __always_inline int
atomic_sub_return(int i,atomic_t * v)128 atomic_sub_return(int i, atomic_t *v)
129 {
130 	kcsan_mb();
131 	instrument_atomic_read_write(v, sizeof(*v));
132 	return arch_atomic_sub_return(i, v);
133 }
134 
135 static __always_inline int
atomic_sub_return_acquire(int i,atomic_t * v)136 atomic_sub_return_acquire(int i, atomic_t *v)
137 {
138 	instrument_atomic_read_write(v, sizeof(*v));
139 	return arch_atomic_sub_return_acquire(i, v);
140 }
141 
142 static __always_inline int
atomic_sub_return_release(int i,atomic_t * v)143 atomic_sub_return_release(int i, atomic_t *v)
144 {
145 	kcsan_release();
146 	instrument_atomic_read_write(v, sizeof(*v));
147 	return arch_atomic_sub_return_release(i, v);
148 }
149 
150 static __always_inline int
atomic_sub_return_relaxed(int i,atomic_t * v)151 atomic_sub_return_relaxed(int i, atomic_t *v)
152 {
153 	instrument_atomic_read_write(v, sizeof(*v));
154 	return arch_atomic_sub_return_relaxed(i, v);
155 }
156 
157 static __always_inline int
atomic_fetch_sub(int i,atomic_t * v)158 atomic_fetch_sub(int i, atomic_t *v)
159 {
160 	kcsan_mb();
161 	instrument_atomic_read_write(v, sizeof(*v));
162 	return arch_atomic_fetch_sub(i, v);
163 }
164 
165 static __always_inline int
atomic_fetch_sub_acquire(int i,atomic_t * v)166 atomic_fetch_sub_acquire(int i, atomic_t *v)
167 {
168 	instrument_atomic_read_write(v, sizeof(*v));
169 	return arch_atomic_fetch_sub_acquire(i, v);
170 }
171 
172 static __always_inline int
atomic_fetch_sub_release(int i,atomic_t * v)173 atomic_fetch_sub_release(int i, atomic_t *v)
174 {
175 	kcsan_release();
176 	instrument_atomic_read_write(v, sizeof(*v));
177 	return arch_atomic_fetch_sub_release(i, v);
178 }
179 
180 static __always_inline int
atomic_fetch_sub_relaxed(int i,atomic_t * v)181 atomic_fetch_sub_relaxed(int i, atomic_t *v)
182 {
183 	instrument_atomic_read_write(v, sizeof(*v));
184 	return arch_atomic_fetch_sub_relaxed(i, v);
185 }
186 
187 static __always_inline void
atomic_inc(atomic_t * v)188 atomic_inc(atomic_t *v)
189 {
190 	instrument_atomic_read_write(v, sizeof(*v));
191 	arch_atomic_inc(v);
192 }
193 
194 static __always_inline int
atomic_inc_return(atomic_t * v)195 atomic_inc_return(atomic_t *v)
196 {
197 	kcsan_mb();
198 	instrument_atomic_read_write(v, sizeof(*v));
199 	return arch_atomic_inc_return(v);
200 }
201 
202 static __always_inline int
atomic_inc_return_acquire(atomic_t * v)203 atomic_inc_return_acquire(atomic_t *v)
204 {
205 	instrument_atomic_read_write(v, sizeof(*v));
206 	return arch_atomic_inc_return_acquire(v);
207 }
208 
209 static __always_inline int
atomic_inc_return_release(atomic_t * v)210 atomic_inc_return_release(atomic_t *v)
211 {
212 	kcsan_release();
213 	instrument_atomic_read_write(v, sizeof(*v));
214 	return arch_atomic_inc_return_release(v);
215 }
216 
217 static __always_inline int
atomic_inc_return_relaxed(atomic_t * v)218 atomic_inc_return_relaxed(atomic_t *v)
219 {
220 	instrument_atomic_read_write(v, sizeof(*v));
221 	return arch_atomic_inc_return_relaxed(v);
222 }
223 
224 static __always_inline int
atomic_fetch_inc(atomic_t * v)225 atomic_fetch_inc(atomic_t *v)
226 {
227 	kcsan_mb();
228 	instrument_atomic_read_write(v, sizeof(*v));
229 	return arch_atomic_fetch_inc(v);
230 }
231 
232 static __always_inline int
atomic_fetch_inc_acquire(atomic_t * v)233 atomic_fetch_inc_acquire(atomic_t *v)
234 {
235 	instrument_atomic_read_write(v, sizeof(*v));
236 	return arch_atomic_fetch_inc_acquire(v);
237 }
238 
239 static __always_inline int
atomic_fetch_inc_release(atomic_t * v)240 atomic_fetch_inc_release(atomic_t *v)
241 {
242 	kcsan_release();
243 	instrument_atomic_read_write(v, sizeof(*v));
244 	return arch_atomic_fetch_inc_release(v);
245 }
246 
247 static __always_inline int
atomic_fetch_inc_relaxed(atomic_t * v)248 atomic_fetch_inc_relaxed(atomic_t *v)
249 {
250 	instrument_atomic_read_write(v, sizeof(*v));
251 	return arch_atomic_fetch_inc_relaxed(v);
252 }
253 
254 static __always_inline void
atomic_dec(atomic_t * v)255 atomic_dec(atomic_t *v)
256 {
257 	instrument_atomic_read_write(v, sizeof(*v));
258 	arch_atomic_dec(v);
259 }
260 
261 static __always_inline int
atomic_dec_return(atomic_t * v)262 atomic_dec_return(atomic_t *v)
263 {
264 	kcsan_mb();
265 	instrument_atomic_read_write(v, sizeof(*v));
266 	return arch_atomic_dec_return(v);
267 }
268 
269 static __always_inline int
atomic_dec_return_acquire(atomic_t * v)270 atomic_dec_return_acquire(atomic_t *v)
271 {
272 	instrument_atomic_read_write(v, sizeof(*v));
273 	return arch_atomic_dec_return_acquire(v);
274 }
275 
276 static __always_inline int
atomic_dec_return_release(atomic_t * v)277 atomic_dec_return_release(atomic_t *v)
278 {
279 	kcsan_release();
280 	instrument_atomic_read_write(v, sizeof(*v));
281 	return arch_atomic_dec_return_release(v);
282 }
283 
284 static __always_inline int
atomic_dec_return_relaxed(atomic_t * v)285 atomic_dec_return_relaxed(atomic_t *v)
286 {
287 	instrument_atomic_read_write(v, sizeof(*v));
288 	return arch_atomic_dec_return_relaxed(v);
289 }
290 
291 static __always_inline int
atomic_fetch_dec(atomic_t * v)292 atomic_fetch_dec(atomic_t *v)
293 {
294 	kcsan_mb();
295 	instrument_atomic_read_write(v, sizeof(*v));
296 	return arch_atomic_fetch_dec(v);
297 }
298 
299 static __always_inline int
atomic_fetch_dec_acquire(atomic_t * v)300 atomic_fetch_dec_acquire(atomic_t *v)
301 {
302 	instrument_atomic_read_write(v, sizeof(*v));
303 	return arch_atomic_fetch_dec_acquire(v);
304 }
305 
306 static __always_inline int
atomic_fetch_dec_release(atomic_t * v)307 atomic_fetch_dec_release(atomic_t *v)
308 {
309 	kcsan_release();
310 	instrument_atomic_read_write(v, sizeof(*v));
311 	return arch_atomic_fetch_dec_release(v);
312 }
313 
314 static __always_inline int
atomic_fetch_dec_relaxed(atomic_t * v)315 atomic_fetch_dec_relaxed(atomic_t *v)
316 {
317 	instrument_atomic_read_write(v, sizeof(*v));
318 	return arch_atomic_fetch_dec_relaxed(v);
319 }
320 
321 static __always_inline void
atomic_and(int i,atomic_t * v)322 atomic_and(int i, atomic_t *v)
323 {
324 	instrument_atomic_read_write(v, sizeof(*v));
325 	arch_atomic_and(i, v);
326 }
327 
328 static __always_inline int
atomic_fetch_and(int i,atomic_t * v)329 atomic_fetch_and(int i, atomic_t *v)
330 {
331 	kcsan_mb();
332 	instrument_atomic_read_write(v, sizeof(*v));
333 	return arch_atomic_fetch_and(i, v);
334 }
335 
336 static __always_inline int
atomic_fetch_and_acquire(int i,atomic_t * v)337 atomic_fetch_and_acquire(int i, atomic_t *v)
338 {
339 	instrument_atomic_read_write(v, sizeof(*v));
340 	return arch_atomic_fetch_and_acquire(i, v);
341 }
342 
343 static __always_inline int
atomic_fetch_and_release(int i,atomic_t * v)344 atomic_fetch_and_release(int i, atomic_t *v)
345 {
346 	kcsan_release();
347 	instrument_atomic_read_write(v, sizeof(*v));
348 	return arch_atomic_fetch_and_release(i, v);
349 }
350 
351 static __always_inline int
atomic_fetch_and_relaxed(int i,atomic_t * v)352 atomic_fetch_and_relaxed(int i, atomic_t *v)
353 {
354 	instrument_atomic_read_write(v, sizeof(*v));
355 	return arch_atomic_fetch_and_relaxed(i, v);
356 }
357 
358 static __always_inline void
atomic_andnot(int i,atomic_t * v)359 atomic_andnot(int i, atomic_t *v)
360 {
361 	instrument_atomic_read_write(v, sizeof(*v));
362 	arch_atomic_andnot(i, v);
363 }
364 
365 static __always_inline int
atomic_fetch_andnot(int i,atomic_t * v)366 atomic_fetch_andnot(int i, atomic_t *v)
367 {
368 	kcsan_mb();
369 	instrument_atomic_read_write(v, sizeof(*v));
370 	return arch_atomic_fetch_andnot(i, v);
371 }
372 
373 static __always_inline int
atomic_fetch_andnot_acquire(int i,atomic_t * v)374 atomic_fetch_andnot_acquire(int i, atomic_t *v)
375 {
376 	instrument_atomic_read_write(v, sizeof(*v));
377 	return arch_atomic_fetch_andnot_acquire(i, v);
378 }
379 
380 static __always_inline int
atomic_fetch_andnot_release(int i,atomic_t * v)381 atomic_fetch_andnot_release(int i, atomic_t *v)
382 {
383 	kcsan_release();
384 	instrument_atomic_read_write(v, sizeof(*v));
385 	return arch_atomic_fetch_andnot_release(i, v);
386 }
387 
388 static __always_inline int
atomic_fetch_andnot_relaxed(int i,atomic_t * v)389 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
390 {
391 	instrument_atomic_read_write(v, sizeof(*v));
392 	return arch_atomic_fetch_andnot_relaxed(i, v);
393 }
394 
395 static __always_inline void
atomic_or(int i,atomic_t * v)396 atomic_or(int i, atomic_t *v)
397 {
398 	instrument_atomic_read_write(v, sizeof(*v));
399 	arch_atomic_or(i, v);
400 }
401 
402 static __always_inline int
atomic_fetch_or(int i,atomic_t * v)403 atomic_fetch_or(int i, atomic_t *v)
404 {
405 	kcsan_mb();
406 	instrument_atomic_read_write(v, sizeof(*v));
407 	return arch_atomic_fetch_or(i, v);
408 }
409 
410 static __always_inline int
atomic_fetch_or_acquire(int i,atomic_t * v)411 atomic_fetch_or_acquire(int i, atomic_t *v)
412 {
413 	instrument_atomic_read_write(v, sizeof(*v));
414 	return arch_atomic_fetch_or_acquire(i, v);
415 }
416 
417 static __always_inline int
atomic_fetch_or_release(int i,atomic_t * v)418 atomic_fetch_or_release(int i, atomic_t *v)
419 {
420 	kcsan_release();
421 	instrument_atomic_read_write(v, sizeof(*v));
422 	return arch_atomic_fetch_or_release(i, v);
423 }
424 
425 static __always_inline int
atomic_fetch_or_relaxed(int i,atomic_t * v)426 atomic_fetch_or_relaxed(int i, atomic_t *v)
427 {
428 	instrument_atomic_read_write(v, sizeof(*v));
429 	return arch_atomic_fetch_or_relaxed(i, v);
430 }
431 
432 static __always_inline void
atomic_xor(int i,atomic_t * v)433 atomic_xor(int i, atomic_t *v)
434 {
435 	instrument_atomic_read_write(v, sizeof(*v));
436 	arch_atomic_xor(i, v);
437 }
438 
439 static __always_inline int
atomic_fetch_xor(int i,atomic_t * v)440 atomic_fetch_xor(int i, atomic_t *v)
441 {
442 	kcsan_mb();
443 	instrument_atomic_read_write(v, sizeof(*v));
444 	return arch_atomic_fetch_xor(i, v);
445 }
446 
447 static __always_inline int
atomic_fetch_xor_acquire(int i,atomic_t * v)448 atomic_fetch_xor_acquire(int i, atomic_t *v)
449 {
450 	instrument_atomic_read_write(v, sizeof(*v));
451 	return arch_atomic_fetch_xor_acquire(i, v);
452 }
453 
454 static __always_inline int
atomic_fetch_xor_release(int i,atomic_t * v)455 atomic_fetch_xor_release(int i, atomic_t *v)
456 {
457 	kcsan_release();
458 	instrument_atomic_read_write(v, sizeof(*v));
459 	return arch_atomic_fetch_xor_release(i, v);
460 }
461 
462 static __always_inline int
atomic_fetch_xor_relaxed(int i,atomic_t * v)463 atomic_fetch_xor_relaxed(int i, atomic_t *v)
464 {
465 	instrument_atomic_read_write(v, sizeof(*v));
466 	return arch_atomic_fetch_xor_relaxed(i, v);
467 }
468 
469 static __always_inline int
atomic_xchg(atomic_t * v,int i)470 atomic_xchg(atomic_t *v, int i)
471 {
472 	kcsan_mb();
473 	instrument_atomic_read_write(v, sizeof(*v));
474 	return arch_atomic_xchg(v, i);
475 }
476 
477 static __always_inline int
atomic_xchg_acquire(atomic_t * v,int i)478 atomic_xchg_acquire(atomic_t *v, int i)
479 {
480 	instrument_atomic_read_write(v, sizeof(*v));
481 	return arch_atomic_xchg_acquire(v, i);
482 }
483 
484 static __always_inline int
atomic_xchg_release(atomic_t * v,int i)485 atomic_xchg_release(atomic_t *v, int i)
486 {
487 	kcsan_release();
488 	instrument_atomic_read_write(v, sizeof(*v));
489 	return arch_atomic_xchg_release(v, i);
490 }
491 
492 static __always_inline int
atomic_xchg_relaxed(atomic_t * v,int i)493 atomic_xchg_relaxed(atomic_t *v, int i)
494 {
495 	instrument_atomic_read_write(v, sizeof(*v));
496 	return arch_atomic_xchg_relaxed(v, i);
497 }
498 
499 static __always_inline int
atomic_cmpxchg(atomic_t * v,int old,int new)500 atomic_cmpxchg(atomic_t *v, int old, int new)
501 {
502 	kcsan_mb();
503 	instrument_atomic_read_write(v, sizeof(*v));
504 	return arch_atomic_cmpxchg(v, old, new);
505 }
506 
507 static __always_inline int
atomic_cmpxchg_acquire(atomic_t * v,int old,int new)508 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
509 {
510 	instrument_atomic_read_write(v, sizeof(*v));
511 	return arch_atomic_cmpxchg_acquire(v, old, new);
512 }
513 
514 static __always_inline int
atomic_cmpxchg_release(atomic_t * v,int old,int new)515 atomic_cmpxchg_release(atomic_t *v, int old, int new)
516 {
517 	kcsan_release();
518 	instrument_atomic_read_write(v, sizeof(*v));
519 	return arch_atomic_cmpxchg_release(v, old, new);
520 }
521 
522 static __always_inline int
atomic_cmpxchg_relaxed(atomic_t * v,int old,int new)523 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
524 {
525 	instrument_atomic_read_write(v, sizeof(*v));
526 	return arch_atomic_cmpxchg_relaxed(v, old, new);
527 }
528 
529 static __always_inline bool
atomic_try_cmpxchg(atomic_t * v,int * old,int new)530 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
531 {
532 	kcsan_mb();
533 	instrument_atomic_read_write(v, sizeof(*v));
534 	instrument_atomic_read_write(old, sizeof(*old));
535 	return arch_atomic_try_cmpxchg(v, old, new);
536 }
537 
538 static __always_inline bool
atomic_try_cmpxchg_acquire(atomic_t * v,int * old,int new)539 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
540 {
541 	instrument_atomic_read_write(v, sizeof(*v));
542 	instrument_atomic_read_write(old, sizeof(*old));
543 	return arch_atomic_try_cmpxchg_acquire(v, old, new);
544 }
545 
546 static __always_inline bool
atomic_try_cmpxchg_release(atomic_t * v,int * old,int new)547 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
548 {
549 	kcsan_release();
550 	instrument_atomic_read_write(v, sizeof(*v));
551 	instrument_atomic_read_write(old, sizeof(*old));
552 	return arch_atomic_try_cmpxchg_release(v, old, new);
553 }
554 
555 static __always_inline bool
atomic_try_cmpxchg_relaxed(atomic_t * v,int * old,int new)556 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
557 {
558 	instrument_atomic_read_write(v, sizeof(*v));
559 	instrument_atomic_read_write(old, sizeof(*old));
560 	return arch_atomic_try_cmpxchg_relaxed(v, old, new);
561 }
562 
563 static __always_inline bool
atomic_sub_and_test(int i,atomic_t * v)564 atomic_sub_and_test(int i, atomic_t *v)
565 {
566 	kcsan_mb();
567 	instrument_atomic_read_write(v, sizeof(*v));
568 	return arch_atomic_sub_and_test(i, v);
569 }
570 
571 static __always_inline bool
atomic_dec_and_test(atomic_t * v)572 atomic_dec_and_test(atomic_t *v)
573 {
574 	kcsan_mb();
575 	instrument_atomic_read_write(v, sizeof(*v));
576 	return arch_atomic_dec_and_test(v);
577 }
578 
579 static __always_inline bool
atomic_inc_and_test(atomic_t * v)580 atomic_inc_and_test(atomic_t *v)
581 {
582 	kcsan_mb();
583 	instrument_atomic_read_write(v, sizeof(*v));
584 	return arch_atomic_inc_and_test(v);
585 }
586 
587 static __always_inline bool
atomic_add_negative(int i,atomic_t * v)588 atomic_add_negative(int i, atomic_t *v)
589 {
590 	kcsan_mb();
591 	instrument_atomic_read_write(v, sizeof(*v));
592 	return arch_atomic_add_negative(i, v);
593 }
594 
595 static __always_inline int
atomic_fetch_add_unless(atomic_t * v,int a,int u)596 atomic_fetch_add_unless(atomic_t *v, int a, int u)
597 {
598 	kcsan_mb();
599 	instrument_atomic_read_write(v, sizeof(*v));
600 	return arch_atomic_fetch_add_unless(v, a, u);
601 }
602 
603 static __always_inline bool
atomic_add_unless(atomic_t * v,int a,int u)604 atomic_add_unless(atomic_t *v, int a, int u)
605 {
606 	kcsan_mb();
607 	instrument_atomic_read_write(v, sizeof(*v));
608 	return arch_atomic_add_unless(v, a, u);
609 }
610 
611 static __always_inline bool
atomic_inc_not_zero(atomic_t * v)612 atomic_inc_not_zero(atomic_t *v)
613 {
614 	kcsan_mb();
615 	instrument_atomic_read_write(v, sizeof(*v));
616 	return arch_atomic_inc_not_zero(v);
617 }
618 
619 static __always_inline bool
atomic_inc_unless_negative(atomic_t * v)620 atomic_inc_unless_negative(atomic_t *v)
621 {
622 	kcsan_mb();
623 	instrument_atomic_read_write(v, sizeof(*v));
624 	return arch_atomic_inc_unless_negative(v);
625 }
626 
627 static __always_inline bool
atomic_dec_unless_positive(atomic_t * v)628 atomic_dec_unless_positive(atomic_t *v)
629 {
630 	kcsan_mb();
631 	instrument_atomic_read_write(v, sizeof(*v));
632 	return arch_atomic_dec_unless_positive(v);
633 }
634 
635 static __always_inline int
atomic_dec_if_positive(atomic_t * v)636 atomic_dec_if_positive(atomic_t *v)
637 {
638 	kcsan_mb();
639 	instrument_atomic_read_write(v, sizeof(*v));
640 	return arch_atomic_dec_if_positive(v);
641 }
642 
643 static __always_inline s64
atomic64_read(const atomic64_t * v)644 atomic64_read(const atomic64_t *v)
645 {
646 	instrument_atomic_read(v, sizeof(*v));
647 	return arch_atomic64_read(v);
648 }
649 
650 static __always_inline s64
atomic64_read_acquire(const atomic64_t * v)651 atomic64_read_acquire(const atomic64_t *v)
652 {
653 	instrument_atomic_read(v, sizeof(*v));
654 	return arch_atomic64_read_acquire(v);
655 }
656 
657 static __always_inline void
atomic64_set(atomic64_t * v,s64 i)658 atomic64_set(atomic64_t *v, s64 i)
659 {
660 	instrument_atomic_write(v, sizeof(*v));
661 	arch_atomic64_set(v, i);
662 }
663 
664 static __always_inline void
atomic64_set_release(atomic64_t * v,s64 i)665 atomic64_set_release(atomic64_t *v, s64 i)
666 {
667 	kcsan_release();
668 	instrument_atomic_write(v, sizeof(*v));
669 	arch_atomic64_set_release(v, i);
670 }
671 
672 static __always_inline void
atomic64_add(s64 i,atomic64_t * v)673 atomic64_add(s64 i, atomic64_t *v)
674 {
675 	instrument_atomic_read_write(v, sizeof(*v));
676 	arch_atomic64_add(i, v);
677 }
678 
679 static __always_inline s64
atomic64_add_return(s64 i,atomic64_t * v)680 atomic64_add_return(s64 i, atomic64_t *v)
681 {
682 	kcsan_mb();
683 	instrument_atomic_read_write(v, sizeof(*v));
684 	return arch_atomic64_add_return(i, v);
685 }
686 
687 static __always_inline s64
atomic64_add_return_acquire(s64 i,atomic64_t * v)688 atomic64_add_return_acquire(s64 i, atomic64_t *v)
689 {
690 	instrument_atomic_read_write(v, sizeof(*v));
691 	return arch_atomic64_add_return_acquire(i, v);
692 }
693 
694 static __always_inline s64
atomic64_add_return_release(s64 i,atomic64_t * v)695 atomic64_add_return_release(s64 i, atomic64_t *v)
696 {
697 	kcsan_release();
698 	instrument_atomic_read_write(v, sizeof(*v));
699 	return arch_atomic64_add_return_release(i, v);
700 }
701 
702 static __always_inline s64
atomic64_add_return_relaxed(s64 i,atomic64_t * v)703 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
704 {
705 	instrument_atomic_read_write(v, sizeof(*v));
706 	return arch_atomic64_add_return_relaxed(i, v);
707 }
708 
709 static __always_inline s64
atomic64_fetch_add(s64 i,atomic64_t * v)710 atomic64_fetch_add(s64 i, atomic64_t *v)
711 {
712 	kcsan_mb();
713 	instrument_atomic_read_write(v, sizeof(*v));
714 	return arch_atomic64_fetch_add(i, v);
715 }
716 
717 static __always_inline s64
atomic64_fetch_add_acquire(s64 i,atomic64_t * v)718 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
719 {
720 	instrument_atomic_read_write(v, sizeof(*v));
721 	return arch_atomic64_fetch_add_acquire(i, v);
722 }
723 
724 static __always_inline s64
atomic64_fetch_add_release(s64 i,atomic64_t * v)725 atomic64_fetch_add_release(s64 i, atomic64_t *v)
726 {
727 	kcsan_release();
728 	instrument_atomic_read_write(v, sizeof(*v));
729 	return arch_atomic64_fetch_add_release(i, v);
730 }
731 
732 static __always_inline s64
atomic64_fetch_add_relaxed(s64 i,atomic64_t * v)733 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
734 {
735 	instrument_atomic_read_write(v, sizeof(*v));
736 	return arch_atomic64_fetch_add_relaxed(i, v);
737 }
738 
739 static __always_inline void
atomic64_sub(s64 i,atomic64_t * v)740 atomic64_sub(s64 i, atomic64_t *v)
741 {
742 	instrument_atomic_read_write(v, sizeof(*v));
743 	arch_atomic64_sub(i, v);
744 }
745 
746 static __always_inline s64
atomic64_sub_return(s64 i,atomic64_t * v)747 atomic64_sub_return(s64 i, atomic64_t *v)
748 {
749 	kcsan_mb();
750 	instrument_atomic_read_write(v, sizeof(*v));
751 	return arch_atomic64_sub_return(i, v);
752 }
753 
754 static __always_inline s64
atomic64_sub_return_acquire(s64 i,atomic64_t * v)755 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
756 {
757 	instrument_atomic_read_write(v, sizeof(*v));
758 	return arch_atomic64_sub_return_acquire(i, v);
759 }
760 
761 static __always_inline s64
atomic64_sub_return_release(s64 i,atomic64_t * v)762 atomic64_sub_return_release(s64 i, atomic64_t *v)
763 {
764 	kcsan_release();
765 	instrument_atomic_read_write(v, sizeof(*v));
766 	return arch_atomic64_sub_return_release(i, v);
767 }
768 
769 static __always_inline s64
atomic64_sub_return_relaxed(s64 i,atomic64_t * v)770 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
771 {
772 	instrument_atomic_read_write(v, sizeof(*v));
773 	return arch_atomic64_sub_return_relaxed(i, v);
774 }
775 
776 static __always_inline s64
atomic64_fetch_sub(s64 i,atomic64_t * v)777 atomic64_fetch_sub(s64 i, atomic64_t *v)
778 {
779 	kcsan_mb();
780 	instrument_atomic_read_write(v, sizeof(*v));
781 	return arch_atomic64_fetch_sub(i, v);
782 }
783 
784 static __always_inline s64
atomic64_fetch_sub_acquire(s64 i,atomic64_t * v)785 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
786 {
787 	instrument_atomic_read_write(v, sizeof(*v));
788 	return arch_atomic64_fetch_sub_acquire(i, v);
789 }
790 
791 static __always_inline s64
atomic64_fetch_sub_release(s64 i,atomic64_t * v)792 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
793 {
794 	kcsan_release();
795 	instrument_atomic_read_write(v, sizeof(*v));
796 	return arch_atomic64_fetch_sub_release(i, v);
797 }
798 
799 static __always_inline s64
atomic64_fetch_sub_relaxed(s64 i,atomic64_t * v)800 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
801 {
802 	instrument_atomic_read_write(v, sizeof(*v));
803 	return arch_atomic64_fetch_sub_relaxed(i, v);
804 }
805 
806 static __always_inline void
atomic64_inc(atomic64_t * v)807 atomic64_inc(atomic64_t *v)
808 {
809 	instrument_atomic_read_write(v, sizeof(*v));
810 	arch_atomic64_inc(v);
811 }
812 
813 static __always_inline s64
atomic64_inc_return(atomic64_t * v)814 atomic64_inc_return(atomic64_t *v)
815 {
816 	kcsan_mb();
817 	instrument_atomic_read_write(v, sizeof(*v));
818 	return arch_atomic64_inc_return(v);
819 }
820 
821 static __always_inline s64
atomic64_inc_return_acquire(atomic64_t * v)822 atomic64_inc_return_acquire(atomic64_t *v)
823 {
824 	instrument_atomic_read_write(v, sizeof(*v));
825 	return arch_atomic64_inc_return_acquire(v);
826 }
827 
828 static __always_inline s64
atomic64_inc_return_release(atomic64_t * v)829 atomic64_inc_return_release(atomic64_t *v)
830 {
831 	kcsan_release();
832 	instrument_atomic_read_write(v, sizeof(*v));
833 	return arch_atomic64_inc_return_release(v);
834 }
835 
836 static __always_inline s64
atomic64_inc_return_relaxed(atomic64_t * v)837 atomic64_inc_return_relaxed(atomic64_t *v)
838 {
839 	instrument_atomic_read_write(v, sizeof(*v));
840 	return arch_atomic64_inc_return_relaxed(v);
841 }
842 
843 static __always_inline s64
atomic64_fetch_inc(atomic64_t * v)844 atomic64_fetch_inc(atomic64_t *v)
845 {
846 	kcsan_mb();
847 	instrument_atomic_read_write(v, sizeof(*v));
848 	return arch_atomic64_fetch_inc(v);
849 }
850 
851 static __always_inline s64
atomic64_fetch_inc_acquire(atomic64_t * v)852 atomic64_fetch_inc_acquire(atomic64_t *v)
853 {
854 	instrument_atomic_read_write(v, sizeof(*v));
855 	return arch_atomic64_fetch_inc_acquire(v);
856 }
857 
858 static __always_inline s64
atomic64_fetch_inc_release(atomic64_t * v)859 atomic64_fetch_inc_release(atomic64_t *v)
860 {
861 	kcsan_release();
862 	instrument_atomic_read_write(v, sizeof(*v));
863 	return arch_atomic64_fetch_inc_release(v);
864 }
865 
866 static __always_inline s64
atomic64_fetch_inc_relaxed(atomic64_t * v)867 atomic64_fetch_inc_relaxed(atomic64_t *v)
868 {
869 	instrument_atomic_read_write(v, sizeof(*v));
870 	return arch_atomic64_fetch_inc_relaxed(v);
871 }
872 
873 static __always_inline void
atomic64_dec(atomic64_t * v)874 atomic64_dec(atomic64_t *v)
875 {
876 	instrument_atomic_read_write(v, sizeof(*v));
877 	arch_atomic64_dec(v);
878 }
879 
880 static __always_inline s64
atomic64_dec_return(atomic64_t * v)881 atomic64_dec_return(atomic64_t *v)
882 {
883 	kcsan_mb();
884 	instrument_atomic_read_write(v, sizeof(*v));
885 	return arch_atomic64_dec_return(v);
886 }
887 
888 static __always_inline s64
atomic64_dec_return_acquire(atomic64_t * v)889 atomic64_dec_return_acquire(atomic64_t *v)
890 {
891 	instrument_atomic_read_write(v, sizeof(*v));
892 	return arch_atomic64_dec_return_acquire(v);
893 }
894 
895 static __always_inline s64
atomic64_dec_return_release(atomic64_t * v)896 atomic64_dec_return_release(atomic64_t *v)
897 {
898 	kcsan_release();
899 	instrument_atomic_read_write(v, sizeof(*v));
900 	return arch_atomic64_dec_return_release(v);
901 }
902 
903 static __always_inline s64
atomic64_dec_return_relaxed(atomic64_t * v)904 atomic64_dec_return_relaxed(atomic64_t *v)
905 {
906 	instrument_atomic_read_write(v, sizeof(*v));
907 	return arch_atomic64_dec_return_relaxed(v);
908 }
909 
910 static __always_inline s64
atomic64_fetch_dec(atomic64_t * v)911 atomic64_fetch_dec(atomic64_t *v)
912 {
913 	kcsan_mb();
914 	instrument_atomic_read_write(v, sizeof(*v));
915 	return arch_atomic64_fetch_dec(v);
916 }
917 
918 static __always_inline s64
atomic64_fetch_dec_acquire(atomic64_t * v)919 atomic64_fetch_dec_acquire(atomic64_t *v)
920 {
921 	instrument_atomic_read_write(v, sizeof(*v));
922 	return arch_atomic64_fetch_dec_acquire(v);
923 }
924 
925 static __always_inline s64
atomic64_fetch_dec_release(atomic64_t * v)926 atomic64_fetch_dec_release(atomic64_t *v)
927 {
928 	kcsan_release();
929 	instrument_atomic_read_write(v, sizeof(*v));
930 	return arch_atomic64_fetch_dec_release(v);
931 }
932 
933 static __always_inline s64
atomic64_fetch_dec_relaxed(atomic64_t * v)934 atomic64_fetch_dec_relaxed(atomic64_t *v)
935 {
936 	instrument_atomic_read_write(v, sizeof(*v));
937 	return arch_atomic64_fetch_dec_relaxed(v);
938 }
939 
940 static __always_inline void
atomic64_and(s64 i,atomic64_t * v)941 atomic64_and(s64 i, atomic64_t *v)
942 {
943 	instrument_atomic_read_write(v, sizeof(*v));
944 	arch_atomic64_and(i, v);
945 }
946 
947 static __always_inline s64
atomic64_fetch_and(s64 i,atomic64_t * v)948 atomic64_fetch_and(s64 i, atomic64_t *v)
949 {
950 	kcsan_mb();
951 	instrument_atomic_read_write(v, sizeof(*v));
952 	return arch_atomic64_fetch_and(i, v);
953 }
954 
955 static __always_inline s64
atomic64_fetch_and_acquire(s64 i,atomic64_t * v)956 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
957 {
958 	instrument_atomic_read_write(v, sizeof(*v));
959 	return arch_atomic64_fetch_and_acquire(i, v);
960 }
961 
962 static __always_inline s64
atomic64_fetch_and_release(s64 i,atomic64_t * v)963 atomic64_fetch_and_release(s64 i, atomic64_t *v)
964 {
965 	kcsan_release();
966 	instrument_atomic_read_write(v, sizeof(*v));
967 	return arch_atomic64_fetch_and_release(i, v);
968 }
969 
970 static __always_inline s64
atomic64_fetch_and_relaxed(s64 i,atomic64_t * v)971 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
972 {
973 	instrument_atomic_read_write(v, sizeof(*v));
974 	return arch_atomic64_fetch_and_relaxed(i, v);
975 }
976 
977 static __always_inline void
atomic64_andnot(s64 i,atomic64_t * v)978 atomic64_andnot(s64 i, atomic64_t *v)
979 {
980 	instrument_atomic_read_write(v, sizeof(*v));
981 	arch_atomic64_andnot(i, v);
982 }
983 
984 static __always_inline s64
atomic64_fetch_andnot(s64 i,atomic64_t * v)985 atomic64_fetch_andnot(s64 i, atomic64_t *v)
986 {
987 	kcsan_mb();
988 	instrument_atomic_read_write(v, sizeof(*v));
989 	return arch_atomic64_fetch_andnot(i, v);
990 }
991 
992 static __always_inline s64
atomic64_fetch_andnot_acquire(s64 i,atomic64_t * v)993 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
994 {
995 	instrument_atomic_read_write(v, sizeof(*v));
996 	return arch_atomic64_fetch_andnot_acquire(i, v);
997 }
998 
999 static __always_inline s64
atomic64_fetch_andnot_release(s64 i,atomic64_t * v)1000 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1001 {
1002 	kcsan_release();
1003 	instrument_atomic_read_write(v, sizeof(*v));
1004 	return arch_atomic64_fetch_andnot_release(i, v);
1005 }
1006 
1007 static __always_inline s64
atomic64_fetch_andnot_relaxed(s64 i,atomic64_t * v)1008 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1009 {
1010 	instrument_atomic_read_write(v, sizeof(*v));
1011 	return arch_atomic64_fetch_andnot_relaxed(i, v);
1012 }
1013 
1014 static __always_inline void
atomic64_or(s64 i,atomic64_t * v)1015 atomic64_or(s64 i, atomic64_t *v)
1016 {
1017 	instrument_atomic_read_write(v, sizeof(*v));
1018 	arch_atomic64_or(i, v);
1019 }
1020 
1021 static __always_inline s64
atomic64_fetch_or(s64 i,atomic64_t * v)1022 atomic64_fetch_or(s64 i, atomic64_t *v)
1023 {
1024 	kcsan_mb();
1025 	instrument_atomic_read_write(v, sizeof(*v));
1026 	return arch_atomic64_fetch_or(i, v);
1027 }
1028 
1029 static __always_inline s64
atomic64_fetch_or_acquire(s64 i,atomic64_t * v)1030 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1031 {
1032 	instrument_atomic_read_write(v, sizeof(*v));
1033 	return arch_atomic64_fetch_or_acquire(i, v);
1034 }
1035 
1036 static __always_inline s64
atomic64_fetch_or_release(s64 i,atomic64_t * v)1037 atomic64_fetch_or_release(s64 i, atomic64_t *v)
1038 {
1039 	kcsan_release();
1040 	instrument_atomic_read_write(v, sizeof(*v));
1041 	return arch_atomic64_fetch_or_release(i, v);
1042 }
1043 
1044 static __always_inline s64
atomic64_fetch_or_relaxed(s64 i,atomic64_t * v)1045 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1046 {
1047 	instrument_atomic_read_write(v, sizeof(*v));
1048 	return arch_atomic64_fetch_or_relaxed(i, v);
1049 }
1050 
1051 static __always_inline void
atomic64_xor(s64 i,atomic64_t * v)1052 atomic64_xor(s64 i, atomic64_t *v)
1053 {
1054 	instrument_atomic_read_write(v, sizeof(*v));
1055 	arch_atomic64_xor(i, v);
1056 }
1057 
1058 static __always_inline s64
atomic64_fetch_xor(s64 i,atomic64_t * v)1059 atomic64_fetch_xor(s64 i, atomic64_t *v)
1060 {
1061 	kcsan_mb();
1062 	instrument_atomic_read_write(v, sizeof(*v));
1063 	return arch_atomic64_fetch_xor(i, v);
1064 }
1065 
1066 static __always_inline s64
atomic64_fetch_xor_acquire(s64 i,atomic64_t * v)1067 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1068 {
1069 	instrument_atomic_read_write(v, sizeof(*v));
1070 	return arch_atomic64_fetch_xor_acquire(i, v);
1071 }
1072 
1073 static __always_inline s64
atomic64_fetch_xor_release(s64 i,atomic64_t * v)1074 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1075 {
1076 	kcsan_release();
1077 	instrument_atomic_read_write(v, sizeof(*v));
1078 	return arch_atomic64_fetch_xor_release(i, v);
1079 }
1080 
1081 static __always_inline s64
atomic64_fetch_xor_relaxed(s64 i,atomic64_t * v)1082 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1083 {
1084 	instrument_atomic_read_write(v, sizeof(*v));
1085 	return arch_atomic64_fetch_xor_relaxed(i, v);
1086 }
1087 
1088 static __always_inline s64
atomic64_xchg(atomic64_t * v,s64 i)1089 atomic64_xchg(atomic64_t *v, s64 i)
1090 {
1091 	kcsan_mb();
1092 	instrument_atomic_read_write(v, sizeof(*v));
1093 	return arch_atomic64_xchg(v, i);
1094 }
1095 
1096 static __always_inline s64
atomic64_xchg_acquire(atomic64_t * v,s64 i)1097 atomic64_xchg_acquire(atomic64_t *v, s64 i)
1098 {
1099 	instrument_atomic_read_write(v, sizeof(*v));
1100 	return arch_atomic64_xchg_acquire(v, i);
1101 }
1102 
1103 static __always_inline s64
atomic64_xchg_release(atomic64_t * v,s64 i)1104 atomic64_xchg_release(atomic64_t *v, s64 i)
1105 {
1106 	kcsan_release();
1107 	instrument_atomic_read_write(v, sizeof(*v));
1108 	return arch_atomic64_xchg_release(v, i);
1109 }
1110 
1111 static __always_inline s64
atomic64_xchg_relaxed(atomic64_t * v,s64 i)1112 atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1113 {
1114 	instrument_atomic_read_write(v, sizeof(*v));
1115 	return arch_atomic64_xchg_relaxed(v, i);
1116 }
1117 
1118 static __always_inline s64
atomic64_cmpxchg(atomic64_t * v,s64 old,s64 new)1119 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1120 {
1121 	kcsan_mb();
1122 	instrument_atomic_read_write(v, sizeof(*v));
1123 	return arch_atomic64_cmpxchg(v, old, new);
1124 }
1125 
1126 static __always_inline s64
atomic64_cmpxchg_acquire(atomic64_t * v,s64 old,s64 new)1127 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1128 {
1129 	instrument_atomic_read_write(v, sizeof(*v));
1130 	return arch_atomic64_cmpxchg_acquire(v, old, new);
1131 }
1132 
1133 static __always_inline s64
atomic64_cmpxchg_release(atomic64_t * v,s64 old,s64 new)1134 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1135 {
1136 	kcsan_release();
1137 	instrument_atomic_read_write(v, sizeof(*v));
1138 	return arch_atomic64_cmpxchg_release(v, old, new);
1139 }
1140 
1141 static __always_inline s64
atomic64_cmpxchg_relaxed(atomic64_t * v,s64 old,s64 new)1142 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1143 {
1144 	instrument_atomic_read_write(v, sizeof(*v));
1145 	return arch_atomic64_cmpxchg_relaxed(v, old, new);
1146 }
1147 
1148 static __always_inline bool
atomic64_try_cmpxchg(atomic64_t * v,s64 * old,s64 new)1149 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1150 {
1151 	kcsan_mb();
1152 	instrument_atomic_read_write(v, sizeof(*v));
1153 	instrument_atomic_read_write(old, sizeof(*old));
1154 	return arch_atomic64_try_cmpxchg(v, old, new);
1155 }
1156 
1157 static __always_inline bool
atomic64_try_cmpxchg_acquire(atomic64_t * v,s64 * old,s64 new)1158 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1159 {
1160 	instrument_atomic_read_write(v, sizeof(*v));
1161 	instrument_atomic_read_write(old, sizeof(*old));
1162 	return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1163 }
1164 
1165 static __always_inline bool
atomic64_try_cmpxchg_release(atomic64_t * v,s64 * old,s64 new)1166 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1167 {
1168 	kcsan_release();
1169 	instrument_atomic_read_write(v, sizeof(*v));
1170 	instrument_atomic_read_write(old, sizeof(*old));
1171 	return arch_atomic64_try_cmpxchg_release(v, old, new);
1172 }
1173 
1174 static __always_inline bool
atomic64_try_cmpxchg_relaxed(atomic64_t * v,s64 * old,s64 new)1175 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1176 {
1177 	instrument_atomic_read_write(v, sizeof(*v));
1178 	instrument_atomic_read_write(old, sizeof(*old));
1179 	return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1180 }
1181 
1182 static __always_inline bool
atomic64_sub_and_test(s64 i,atomic64_t * v)1183 atomic64_sub_and_test(s64 i, atomic64_t *v)
1184 {
1185 	kcsan_mb();
1186 	instrument_atomic_read_write(v, sizeof(*v));
1187 	return arch_atomic64_sub_and_test(i, v);
1188 }
1189 
1190 static __always_inline bool
atomic64_dec_and_test(atomic64_t * v)1191 atomic64_dec_and_test(atomic64_t *v)
1192 {
1193 	kcsan_mb();
1194 	instrument_atomic_read_write(v, sizeof(*v));
1195 	return arch_atomic64_dec_and_test(v);
1196 }
1197 
1198 static __always_inline bool
atomic64_inc_and_test(atomic64_t * v)1199 atomic64_inc_and_test(atomic64_t *v)
1200 {
1201 	kcsan_mb();
1202 	instrument_atomic_read_write(v, sizeof(*v));
1203 	return arch_atomic64_inc_and_test(v);
1204 }
1205 
1206 static __always_inline bool
atomic64_add_negative(s64 i,atomic64_t * v)1207 atomic64_add_negative(s64 i, atomic64_t *v)
1208 {
1209 	kcsan_mb();
1210 	instrument_atomic_read_write(v, sizeof(*v));
1211 	return arch_atomic64_add_negative(i, v);
1212 }
1213 
1214 static __always_inline s64
atomic64_fetch_add_unless(atomic64_t * v,s64 a,s64 u)1215 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1216 {
1217 	kcsan_mb();
1218 	instrument_atomic_read_write(v, sizeof(*v));
1219 	return arch_atomic64_fetch_add_unless(v, a, u);
1220 }
1221 
1222 static __always_inline bool
atomic64_add_unless(atomic64_t * v,s64 a,s64 u)1223 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1224 {
1225 	kcsan_mb();
1226 	instrument_atomic_read_write(v, sizeof(*v));
1227 	return arch_atomic64_add_unless(v, a, u);
1228 }
1229 
1230 static __always_inline bool
atomic64_inc_not_zero(atomic64_t * v)1231 atomic64_inc_not_zero(atomic64_t *v)
1232 {
1233 	kcsan_mb();
1234 	instrument_atomic_read_write(v, sizeof(*v));
1235 	return arch_atomic64_inc_not_zero(v);
1236 }
1237 
1238 static __always_inline bool
atomic64_inc_unless_negative(atomic64_t * v)1239 atomic64_inc_unless_negative(atomic64_t *v)
1240 {
1241 	kcsan_mb();
1242 	instrument_atomic_read_write(v, sizeof(*v));
1243 	return arch_atomic64_inc_unless_negative(v);
1244 }
1245 
1246 static __always_inline bool
atomic64_dec_unless_positive(atomic64_t * v)1247 atomic64_dec_unless_positive(atomic64_t *v)
1248 {
1249 	kcsan_mb();
1250 	instrument_atomic_read_write(v, sizeof(*v));
1251 	return arch_atomic64_dec_unless_positive(v);
1252 }
1253 
1254 static __always_inline s64
atomic64_dec_if_positive(atomic64_t * v)1255 atomic64_dec_if_positive(atomic64_t *v)
1256 {
1257 	kcsan_mb();
1258 	instrument_atomic_read_write(v, sizeof(*v));
1259 	return arch_atomic64_dec_if_positive(v);
1260 }
1261 
1262 static __always_inline long
atomic_long_read(const atomic_long_t * v)1263 atomic_long_read(const atomic_long_t *v)
1264 {
1265 	instrument_atomic_read(v, sizeof(*v));
1266 	return arch_atomic_long_read(v);
1267 }
1268 
1269 static __always_inline long
atomic_long_read_acquire(const atomic_long_t * v)1270 atomic_long_read_acquire(const atomic_long_t *v)
1271 {
1272 	instrument_atomic_read(v, sizeof(*v));
1273 	return arch_atomic_long_read_acquire(v);
1274 }
1275 
1276 static __always_inline void
atomic_long_set(atomic_long_t * v,long i)1277 atomic_long_set(atomic_long_t *v, long i)
1278 {
1279 	instrument_atomic_write(v, sizeof(*v));
1280 	arch_atomic_long_set(v, i);
1281 }
1282 
1283 static __always_inline void
atomic_long_set_release(atomic_long_t * v,long i)1284 atomic_long_set_release(atomic_long_t *v, long i)
1285 {
1286 	kcsan_release();
1287 	instrument_atomic_write(v, sizeof(*v));
1288 	arch_atomic_long_set_release(v, i);
1289 }
1290 
1291 static __always_inline void
atomic_long_add(long i,atomic_long_t * v)1292 atomic_long_add(long i, atomic_long_t *v)
1293 {
1294 	instrument_atomic_read_write(v, sizeof(*v));
1295 	arch_atomic_long_add(i, v);
1296 }
1297 
1298 static __always_inline long
atomic_long_add_return(long i,atomic_long_t * v)1299 atomic_long_add_return(long i, atomic_long_t *v)
1300 {
1301 	kcsan_mb();
1302 	instrument_atomic_read_write(v, sizeof(*v));
1303 	return arch_atomic_long_add_return(i, v);
1304 }
1305 
1306 static __always_inline long
atomic_long_add_return_acquire(long i,atomic_long_t * v)1307 atomic_long_add_return_acquire(long i, atomic_long_t *v)
1308 {
1309 	instrument_atomic_read_write(v, sizeof(*v));
1310 	return arch_atomic_long_add_return_acquire(i, v);
1311 }
1312 
1313 static __always_inline long
atomic_long_add_return_release(long i,atomic_long_t * v)1314 atomic_long_add_return_release(long i, atomic_long_t *v)
1315 {
1316 	kcsan_release();
1317 	instrument_atomic_read_write(v, sizeof(*v));
1318 	return arch_atomic_long_add_return_release(i, v);
1319 }
1320 
1321 static __always_inline long
atomic_long_add_return_relaxed(long i,atomic_long_t * v)1322 atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1323 {
1324 	instrument_atomic_read_write(v, sizeof(*v));
1325 	return arch_atomic_long_add_return_relaxed(i, v);
1326 }
1327 
1328 static __always_inline long
atomic_long_fetch_add(long i,atomic_long_t * v)1329 atomic_long_fetch_add(long i, atomic_long_t *v)
1330 {
1331 	kcsan_mb();
1332 	instrument_atomic_read_write(v, sizeof(*v));
1333 	return arch_atomic_long_fetch_add(i, v);
1334 }
1335 
1336 static __always_inline long
atomic_long_fetch_add_acquire(long i,atomic_long_t * v)1337 atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1338 {
1339 	instrument_atomic_read_write(v, sizeof(*v));
1340 	return arch_atomic_long_fetch_add_acquire(i, v);
1341 }
1342 
1343 static __always_inline long
atomic_long_fetch_add_release(long i,atomic_long_t * v)1344 atomic_long_fetch_add_release(long i, atomic_long_t *v)
1345 {
1346 	kcsan_release();
1347 	instrument_atomic_read_write(v, sizeof(*v));
1348 	return arch_atomic_long_fetch_add_release(i, v);
1349 }
1350 
1351 static __always_inline long
atomic_long_fetch_add_relaxed(long i,atomic_long_t * v)1352 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1353 {
1354 	instrument_atomic_read_write(v, sizeof(*v));
1355 	return arch_atomic_long_fetch_add_relaxed(i, v);
1356 }
1357 
1358 static __always_inline void
atomic_long_sub(long i,atomic_long_t * v)1359 atomic_long_sub(long i, atomic_long_t *v)
1360 {
1361 	instrument_atomic_read_write(v, sizeof(*v));
1362 	arch_atomic_long_sub(i, v);
1363 }
1364 
1365 static __always_inline long
atomic_long_sub_return(long i,atomic_long_t * v)1366 atomic_long_sub_return(long i, atomic_long_t *v)
1367 {
1368 	kcsan_mb();
1369 	instrument_atomic_read_write(v, sizeof(*v));
1370 	return arch_atomic_long_sub_return(i, v);
1371 }
1372 
1373 static __always_inline long
atomic_long_sub_return_acquire(long i,atomic_long_t * v)1374 atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1375 {
1376 	instrument_atomic_read_write(v, sizeof(*v));
1377 	return arch_atomic_long_sub_return_acquire(i, v);
1378 }
1379 
1380 static __always_inline long
atomic_long_sub_return_release(long i,atomic_long_t * v)1381 atomic_long_sub_return_release(long i, atomic_long_t *v)
1382 {
1383 	kcsan_release();
1384 	instrument_atomic_read_write(v, sizeof(*v));
1385 	return arch_atomic_long_sub_return_release(i, v);
1386 }
1387 
1388 static __always_inline long
atomic_long_sub_return_relaxed(long i,atomic_long_t * v)1389 atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1390 {
1391 	instrument_atomic_read_write(v, sizeof(*v));
1392 	return arch_atomic_long_sub_return_relaxed(i, v);
1393 }
1394 
1395 static __always_inline long
atomic_long_fetch_sub(long i,atomic_long_t * v)1396 atomic_long_fetch_sub(long i, atomic_long_t *v)
1397 {
1398 	kcsan_mb();
1399 	instrument_atomic_read_write(v, sizeof(*v));
1400 	return arch_atomic_long_fetch_sub(i, v);
1401 }
1402 
1403 static __always_inline long
atomic_long_fetch_sub_acquire(long i,atomic_long_t * v)1404 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1405 {
1406 	instrument_atomic_read_write(v, sizeof(*v));
1407 	return arch_atomic_long_fetch_sub_acquire(i, v);
1408 }
1409 
1410 static __always_inline long
atomic_long_fetch_sub_release(long i,atomic_long_t * v)1411 atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1412 {
1413 	kcsan_release();
1414 	instrument_atomic_read_write(v, sizeof(*v));
1415 	return arch_atomic_long_fetch_sub_release(i, v);
1416 }
1417 
1418 static __always_inline long
atomic_long_fetch_sub_relaxed(long i,atomic_long_t * v)1419 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1420 {
1421 	instrument_atomic_read_write(v, sizeof(*v));
1422 	return arch_atomic_long_fetch_sub_relaxed(i, v);
1423 }
1424 
1425 static __always_inline void
atomic_long_inc(atomic_long_t * v)1426 atomic_long_inc(atomic_long_t *v)
1427 {
1428 	instrument_atomic_read_write(v, sizeof(*v));
1429 	arch_atomic_long_inc(v);
1430 }
1431 
1432 static __always_inline long
atomic_long_inc_return(atomic_long_t * v)1433 atomic_long_inc_return(atomic_long_t *v)
1434 {
1435 	kcsan_mb();
1436 	instrument_atomic_read_write(v, sizeof(*v));
1437 	return arch_atomic_long_inc_return(v);
1438 }
1439 
1440 static __always_inline long
atomic_long_inc_return_acquire(atomic_long_t * v)1441 atomic_long_inc_return_acquire(atomic_long_t *v)
1442 {
1443 	instrument_atomic_read_write(v, sizeof(*v));
1444 	return arch_atomic_long_inc_return_acquire(v);
1445 }
1446 
1447 static __always_inline long
atomic_long_inc_return_release(atomic_long_t * v)1448 atomic_long_inc_return_release(atomic_long_t *v)
1449 {
1450 	kcsan_release();
1451 	instrument_atomic_read_write(v, sizeof(*v));
1452 	return arch_atomic_long_inc_return_release(v);
1453 }
1454 
1455 static __always_inline long
atomic_long_inc_return_relaxed(atomic_long_t * v)1456 atomic_long_inc_return_relaxed(atomic_long_t *v)
1457 {
1458 	instrument_atomic_read_write(v, sizeof(*v));
1459 	return arch_atomic_long_inc_return_relaxed(v);
1460 }
1461 
1462 static __always_inline long
atomic_long_fetch_inc(atomic_long_t * v)1463 atomic_long_fetch_inc(atomic_long_t *v)
1464 {
1465 	kcsan_mb();
1466 	instrument_atomic_read_write(v, sizeof(*v));
1467 	return arch_atomic_long_fetch_inc(v);
1468 }
1469 
1470 static __always_inline long
atomic_long_fetch_inc_acquire(atomic_long_t * v)1471 atomic_long_fetch_inc_acquire(atomic_long_t *v)
1472 {
1473 	instrument_atomic_read_write(v, sizeof(*v));
1474 	return arch_atomic_long_fetch_inc_acquire(v);
1475 }
1476 
1477 static __always_inline long
atomic_long_fetch_inc_release(atomic_long_t * v)1478 atomic_long_fetch_inc_release(atomic_long_t *v)
1479 {
1480 	kcsan_release();
1481 	instrument_atomic_read_write(v, sizeof(*v));
1482 	return arch_atomic_long_fetch_inc_release(v);
1483 }
1484 
1485 static __always_inline long
atomic_long_fetch_inc_relaxed(atomic_long_t * v)1486 atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1487 {
1488 	instrument_atomic_read_write(v, sizeof(*v));
1489 	return arch_atomic_long_fetch_inc_relaxed(v);
1490 }
1491 
1492 static __always_inline void
atomic_long_dec(atomic_long_t * v)1493 atomic_long_dec(atomic_long_t *v)
1494 {
1495 	instrument_atomic_read_write(v, sizeof(*v));
1496 	arch_atomic_long_dec(v);
1497 }
1498 
1499 static __always_inline long
atomic_long_dec_return(atomic_long_t * v)1500 atomic_long_dec_return(atomic_long_t *v)
1501 {
1502 	kcsan_mb();
1503 	instrument_atomic_read_write(v, sizeof(*v));
1504 	return arch_atomic_long_dec_return(v);
1505 }
1506 
1507 static __always_inline long
atomic_long_dec_return_acquire(atomic_long_t * v)1508 atomic_long_dec_return_acquire(atomic_long_t *v)
1509 {
1510 	instrument_atomic_read_write(v, sizeof(*v));
1511 	return arch_atomic_long_dec_return_acquire(v);
1512 }
1513 
1514 static __always_inline long
atomic_long_dec_return_release(atomic_long_t * v)1515 atomic_long_dec_return_release(atomic_long_t *v)
1516 {
1517 	kcsan_release();
1518 	instrument_atomic_read_write(v, sizeof(*v));
1519 	return arch_atomic_long_dec_return_release(v);
1520 }
1521 
1522 static __always_inline long
atomic_long_dec_return_relaxed(atomic_long_t * v)1523 atomic_long_dec_return_relaxed(atomic_long_t *v)
1524 {
1525 	instrument_atomic_read_write(v, sizeof(*v));
1526 	return arch_atomic_long_dec_return_relaxed(v);
1527 }
1528 
1529 static __always_inline long
atomic_long_fetch_dec(atomic_long_t * v)1530 atomic_long_fetch_dec(atomic_long_t *v)
1531 {
1532 	kcsan_mb();
1533 	instrument_atomic_read_write(v, sizeof(*v));
1534 	return arch_atomic_long_fetch_dec(v);
1535 }
1536 
1537 static __always_inline long
atomic_long_fetch_dec_acquire(atomic_long_t * v)1538 atomic_long_fetch_dec_acquire(atomic_long_t *v)
1539 {
1540 	instrument_atomic_read_write(v, sizeof(*v));
1541 	return arch_atomic_long_fetch_dec_acquire(v);
1542 }
1543 
1544 static __always_inline long
atomic_long_fetch_dec_release(atomic_long_t * v)1545 atomic_long_fetch_dec_release(atomic_long_t *v)
1546 {
1547 	kcsan_release();
1548 	instrument_atomic_read_write(v, sizeof(*v));
1549 	return arch_atomic_long_fetch_dec_release(v);
1550 }
1551 
1552 static __always_inline long
atomic_long_fetch_dec_relaxed(atomic_long_t * v)1553 atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1554 {
1555 	instrument_atomic_read_write(v, sizeof(*v));
1556 	return arch_atomic_long_fetch_dec_relaxed(v);
1557 }
1558 
1559 static __always_inline void
atomic_long_and(long i,atomic_long_t * v)1560 atomic_long_and(long i, atomic_long_t *v)
1561 {
1562 	instrument_atomic_read_write(v, sizeof(*v));
1563 	arch_atomic_long_and(i, v);
1564 }
1565 
1566 static __always_inline long
atomic_long_fetch_and(long i,atomic_long_t * v)1567 atomic_long_fetch_and(long i, atomic_long_t *v)
1568 {
1569 	kcsan_mb();
1570 	instrument_atomic_read_write(v, sizeof(*v));
1571 	return arch_atomic_long_fetch_and(i, v);
1572 }
1573 
1574 static __always_inline long
atomic_long_fetch_and_acquire(long i,atomic_long_t * v)1575 atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1576 {
1577 	instrument_atomic_read_write(v, sizeof(*v));
1578 	return arch_atomic_long_fetch_and_acquire(i, v);
1579 }
1580 
1581 static __always_inline long
atomic_long_fetch_and_release(long i,atomic_long_t * v)1582 atomic_long_fetch_and_release(long i, atomic_long_t *v)
1583 {
1584 	kcsan_release();
1585 	instrument_atomic_read_write(v, sizeof(*v));
1586 	return arch_atomic_long_fetch_and_release(i, v);
1587 }
1588 
1589 static __always_inline long
atomic_long_fetch_and_relaxed(long i,atomic_long_t * v)1590 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1591 {
1592 	instrument_atomic_read_write(v, sizeof(*v));
1593 	return arch_atomic_long_fetch_and_relaxed(i, v);
1594 }
1595 
1596 static __always_inline void
atomic_long_andnot(long i,atomic_long_t * v)1597 atomic_long_andnot(long i, atomic_long_t *v)
1598 {
1599 	instrument_atomic_read_write(v, sizeof(*v));
1600 	arch_atomic_long_andnot(i, v);
1601 }
1602 
1603 static __always_inline long
atomic_long_fetch_andnot(long i,atomic_long_t * v)1604 atomic_long_fetch_andnot(long i, atomic_long_t *v)
1605 {
1606 	kcsan_mb();
1607 	instrument_atomic_read_write(v, sizeof(*v));
1608 	return arch_atomic_long_fetch_andnot(i, v);
1609 }
1610 
1611 static __always_inline long
atomic_long_fetch_andnot_acquire(long i,atomic_long_t * v)1612 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1613 {
1614 	instrument_atomic_read_write(v, sizeof(*v));
1615 	return arch_atomic_long_fetch_andnot_acquire(i, v);
1616 }
1617 
1618 static __always_inline long
atomic_long_fetch_andnot_release(long i,atomic_long_t * v)1619 atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1620 {
1621 	kcsan_release();
1622 	instrument_atomic_read_write(v, sizeof(*v));
1623 	return arch_atomic_long_fetch_andnot_release(i, v);
1624 }
1625 
1626 static __always_inline long
atomic_long_fetch_andnot_relaxed(long i,atomic_long_t * v)1627 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1628 {
1629 	instrument_atomic_read_write(v, sizeof(*v));
1630 	return arch_atomic_long_fetch_andnot_relaxed(i, v);
1631 }
1632 
1633 static __always_inline void
atomic_long_or(long i,atomic_long_t * v)1634 atomic_long_or(long i, atomic_long_t *v)
1635 {
1636 	instrument_atomic_read_write(v, sizeof(*v));
1637 	arch_atomic_long_or(i, v);
1638 }
1639 
1640 static __always_inline long
atomic_long_fetch_or(long i,atomic_long_t * v)1641 atomic_long_fetch_or(long i, atomic_long_t *v)
1642 {
1643 	kcsan_mb();
1644 	instrument_atomic_read_write(v, sizeof(*v));
1645 	return arch_atomic_long_fetch_or(i, v);
1646 }
1647 
1648 static __always_inline long
atomic_long_fetch_or_acquire(long i,atomic_long_t * v)1649 atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1650 {
1651 	instrument_atomic_read_write(v, sizeof(*v));
1652 	return arch_atomic_long_fetch_or_acquire(i, v);
1653 }
1654 
1655 static __always_inline long
atomic_long_fetch_or_release(long i,atomic_long_t * v)1656 atomic_long_fetch_or_release(long i, atomic_long_t *v)
1657 {
1658 	kcsan_release();
1659 	instrument_atomic_read_write(v, sizeof(*v));
1660 	return arch_atomic_long_fetch_or_release(i, v);
1661 }
1662 
1663 static __always_inline long
atomic_long_fetch_or_relaxed(long i,atomic_long_t * v)1664 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1665 {
1666 	instrument_atomic_read_write(v, sizeof(*v));
1667 	return arch_atomic_long_fetch_or_relaxed(i, v);
1668 }
1669 
1670 static __always_inline void
atomic_long_xor(long i,atomic_long_t * v)1671 atomic_long_xor(long i, atomic_long_t *v)
1672 {
1673 	instrument_atomic_read_write(v, sizeof(*v));
1674 	arch_atomic_long_xor(i, v);
1675 }
1676 
1677 static __always_inline long
atomic_long_fetch_xor(long i,atomic_long_t * v)1678 atomic_long_fetch_xor(long i, atomic_long_t *v)
1679 {
1680 	kcsan_mb();
1681 	instrument_atomic_read_write(v, sizeof(*v));
1682 	return arch_atomic_long_fetch_xor(i, v);
1683 }
1684 
1685 static __always_inline long
atomic_long_fetch_xor_acquire(long i,atomic_long_t * v)1686 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1687 {
1688 	instrument_atomic_read_write(v, sizeof(*v));
1689 	return arch_atomic_long_fetch_xor_acquire(i, v);
1690 }
1691 
1692 static __always_inline long
atomic_long_fetch_xor_release(long i,atomic_long_t * v)1693 atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1694 {
1695 	kcsan_release();
1696 	instrument_atomic_read_write(v, sizeof(*v));
1697 	return arch_atomic_long_fetch_xor_release(i, v);
1698 }
1699 
1700 static __always_inline long
atomic_long_fetch_xor_relaxed(long i,atomic_long_t * v)1701 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1702 {
1703 	instrument_atomic_read_write(v, sizeof(*v));
1704 	return arch_atomic_long_fetch_xor_relaxed(i, v);
1705 }
1706 
1707 static __always_inline long
atomic_long_xchg(atomic_long_t * v,long i)1708 atomic_long_xchg(atomic_long_t *v, long i)
1709 {
1710 	kcsan_mb();
1711 	instrument_atomic_read_write(v, sizeof(*v));
1712 	return arch_atomic_long_xchg(v, i);
1713 }
1714 
1715 static __always_inline long
atomic_long_xchg_acquire(atomic_long_t * v,long i)1716 atomic_long_xchg_acquire(atomic_long_t *v, long i)
1717 {
1718 	instrument_atomic_read_write(v, sizeof(*v));
1719 	return arch_atomic_long_xchg_acquire(v, i);
1720 }
1721 
1722 static __always_inline long
atomic_long_xchg_release(atomic_long_t * v,long i)1723 atomic_long_xchg_release(atomic_long_t *v, long i)
1724 {
1725 	kcsan_release();
1726 	instrument_atomic_read_write(v, sizeof(*v));
1727 	return arch_atomic_long_xchg_release(v, i);
1728 }
1729 
1730 static __always_inline long
atomic_long_xchg_relaxed(atomic_long_t * v,long i)1731 atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1732 {
1733 	instrument_atomic_read_write(v, sizeof(*v));
1734 	return arch_atomic_long_xchg_relaxed(v, i);
1735 }
1736 
1737 static __always_inline long
atomic_long_cmpxchg(atomic_long_t * v,long old,long new)1738 atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1739 {
1740 	kcsan_mb();
1741 	instrument_atomic_read_write(v, sizeof(*v));
1742 	return arch_atomic_long_cmpxchg(v, old, new);
1743 }
1744 
1745 static __always_inline long
atomic_long_cmpxchg_acquire(atomic_long_t * v,long old,long new)1746 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1747 {
1748 	instrument_atomic_read_write(v, sizeof(*v));
1749 	return arch_atomic_long_cmpxchg_acquire(v, old, new);
1750 }
1751 
1752 static __always_inline long
atomic_long_cmpxchg_release(atomic_long_t * v,long old,long new)1753 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1754 {
1755 	kcsan_release();
1756 	instrument_atomic_read_write(v, sizeof(*v));
1757 	return arch_atomic_long_cmpxchg_release(v, old, new);
1758 }
1759 
1760 static __always_inline long
atomic_long_cmpxchg_relaxed(atomic_long_t * v,long old,long new)1761 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1762 {
1763 	instrument_atomic_read_write(v, sizeof(*v));
1764 	return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1765 }
1766 
1767 static __always_inline bool
atomic_long_try_cmpxchg(atomic_long_t * v,long * old,long new)1768 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1769 {
1770 	kcsan_mb();
1771 	instrument_atomic_read_write(v, sizeof(*v));
1772 	instrument_atomic_read_write(old, sizeof(*old));
1773 	return arch_atomic_long_try_cmpxchg(v, old, new);
1774 }
1775 
1776 static __always_inline bool
atomic_long_try_cmpxchg_acquire(atomic_long_t * v,long * old,long new)1777 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1778 {
1779 	instrument_atomic_read_write(v, sizeof(*v));
1780 	instrument_atomic_read_write(old, sizeof(*old));
1781 	return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1782 }
1783 
1784 static __always_inline bool
atomic_long_try_cmpxchg_release(atomic_long_t * v,long * old,long new)1785 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1786 {
1787 	kcsan_release();
1788 	instrument_atomic_read_write(v, sizeof(*v));
1789 	instrument_atomic_read_write(old, sizeof(*old));
1790 	return arch_atomic_long_try_cmpxchg_release(v, old, new);
1791 }
1792 
1793 static __always_inline bool
atomic_long_try_cmpxchg_relaxed(atomic_long_t * v,long * old,long new)1794 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1795 {
1796 	instrument_atomic_read_write(v, sizeof(*v));
1797 	instrument_atomic_read_write(old, sizeof(*old));
1798 	return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1799 }
1800 
1801 static __always_inline bool
atomic_long_sub_and_test(long i,atomic_long_t * v)1802 atomic_long_sub_and_test(long i, atomic_long_t *v)
1803 {
1804 	kcsan_mb();
1805 	instrument_atomic_read_write(v, sizeof(*v));
1806 	return arch_atomic_long_sub_and_test(i, v);
1807 }
1808 
1809 static __always_inline bool
atomic_long_dec_and_test(atomic_long_t * v)1810 atomic_long_dec_and_test(atomic_long_t *v)
1811 {
1812 	kcsan_mb();
1813 	instrument_atomic_read_write(v, sizeof(*v));
1814 	return arch_atomic_long_dec_and_test(v);
1815 }
1816 
1817 static __always_inline bool
atomic_long_inc_and_test(atomic_long_t * v)1818 atomic_long_inc_and_test(atomic_long_t *v)
1819 {
1820 	kcsan_mb();
1821 	instrument_atomic_read_write(v, sizeof(*v));
1822 	return arch_atomic_long_inc_and_test(v);
1823 }
1824 
1825 static __always_inline bool
atomic_long_add_negative(long i,atomic_long_t * v)1826 atomic_long_add_negative(long i, atomic_long_t *v)
1827 {
1828 	kcsan_mb();
1829 	instrument_atomic_read_write(v, sizeof(*v));
1830 	return arch_atomic_long_add_negative(i, v);
1831 }
1832 
1833 static __always_inline long
atomic_long_fetch_add_unless(atomic_long_t * v,long a,long u)1834 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1835 {
1836 	kcsan_mb();
1837 	instrument_atomic_read_write(v, sizeof(*v));
1838 	return arch_atomic_long_fetch_add_unless(v, a, u);
1839 }
1840 
1841 static __always_inline bool
atomic_long_add_unless(atomic_long_t * v,long a,long u)1842 atomic_long_add_unless(atomic_long_t *v, long a, long u)
1843 {
1844 	kcsan_mb();
1845 	instrument_atomic_read_write(v, sizeof(*v));
1846 	return arch_atomic_long_add_unless(v, a, u);
1847 }
1848 
1849 static __always_inline bool
atomic_long_inc_not_zero(atomic_long_t * v)1850 atomic_long_inc_not_zero(atomic_long_t *v)
1851 {
1852 	kcsan_mb();
1853 	instrument_atomic_read_write(v, sizeof(*v));
1854 	return arch_atomic_long_inc_not_zero(v);
1855 }
1856 
1857 static __always_inline bool
atomic_long_inc_unless_negative(atomic_long_t * v)1858 atomic_long_inc_unless_negative(atomic_long_t *v)
1859 {
1860 	kcsan_mb();
1861 	instrument_atomic_read_write(v, sizeof(*v));
1862 	return arch_atomic_long_inc_unless_negative(v);
1863 }
1864 
1865 static __always_inline bool
atomic_long_dec_unless_positive(atomic_long_t * v)1866 atomic_long_dec_unless_positive(atomic_long_t *v)
1867 {
1868 	kcsan_mb();
1869 	instrument_atomic_read_write(v, sizeof(*v));
1870 	return arch_atomic_long_dec_unless_positive(v);
1871 }
1872 
1873 static __always_inline long
atomic_long_dec_if_positive(atomic_long_t * v)1874 atomic_long_dec_if_positive(atomic_long_t *v)
1875 {
1876 	kcsan_mb();
1877 	instrument_atomic_read_write(v, sizeof(*v));
1878 	return arch_atomic_long_dec_if_positive(v);
1879 }
1880 
1881 #define xchg(ptr, ...) \
1882 ({ \
1883 	typeof(ptr) __ai_ptr = (ptr); \
1884 	kcsan_mb(); \
1885 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1886 	arch_xchg(__ai_ptr, __VA_ARGS__); \
1887 })
1888 
1889 #define xchg_acquire(ptr, ...) \
1890 ({ \
1891 	typeof(ptr) __ai_ptr = (ptr); \
1892 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1893 	arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1894 })
1895 
1896 #define xchg_release(ptr, ...) \
1897 ({ \
1898 	typeof(ptr) __ai_ptr = (ptr); \
1899 	kcsan_release(); \
1900 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1901 	arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1902 })
1903 
1904 #define xchg_relaxed(ptr, ...) \
1905 ({ \
1906 	typeof(ptr) __ai_ptr = (ptr); \
1907 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1908 	arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1909 })
1910 
1911 #define cmpxchg(ptr, ...) \
1912 ({ \
1913 	typeof(ptr) __ai_ptr = (ptr); \
1914 	kcsan_mb(); \
1915 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1916 	arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1917 })
1918 
1919 #define cmpxchg_acquire(ptr, ...) \
1920 ({ \
1921 	typeof(ptr) __ai_ptr = (ptr); \
1922 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1923 	arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1924 })
1925 
1926 #define cmpxchg_release(ptr, ...) \
1927 ({ \
1928 	typeof(ptr) __ai_ptr = (ptr); \
1929 	kcsan_release(); \
1930 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1931 	arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1932 })
1933 
1934 #define cmpxchg_relaxed(ptr, ...) \
1935 ({ \
1936 	typeof(ptr) __ai_ptr = (ptr); \
1937 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1938 	arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1939 })
1940 
1941 #define cmpxchg64(ptr, ...) \
1942 ({ \
1943 	typeof(ptr) __ai_ptr = (ptr); \
1944 	kcsan_mb(); \
1945 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1946 	arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1947 })
1948 
1949 #define cmpxchg64_acquire(ptr, ...) \
1950 ({ \
1951 	typeof(ptr) __ai_ptr = (ptr); \
1952 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1953 	arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1954 })
1955 
1956 #define cmpxchg64_release(ptr, ...) \
1957 ({ \
1958 	typeof(ptr) __ai_ptr = (ptr); \
1959 	kcsan_release(); \
1960 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1961 	arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1962 })
1963 
1964 #define cmpxchg64_relaxed(ptr, ...) \
1965 ({ \
1966 	typeof(ptr) __ai_ptr = (ptr); \
1967 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1968 	arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1969 })
1970 
1971 #define try_cmpxchg(ptr, oldp, ...) \
1972 ({ \
1973 	typeof(ptr) __ai_ptr = (ptr); \
1974 	typeof(oldp) __ai_oldp = (oldp); \
1975 	kcsan_mb(); \
1976 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1977 	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1978 	arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1979 })
1980 
1981 #define try_cmpxchg_acquire(ptr, oldp, ...) \
1982 ({ \
1983 	typeof(ptr) __ai_ptr = (ptr); \
1984 	typeof(oldp) __ai_oldp = (oldp); \
1985 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1986 	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1987 	arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1988 })
1989 
1990 #define try_cmpxchg_release(ptr, oldp, ...) \
1991 ({ \
1992 	typeof(ptr) __ai_ptr = (ptr); \
1993 	typeof(oldp) __ai_oldp = (oldp); \
1994 	kcsan_release(); \
1995 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1996 	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1997 	arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1998 })
1999 
2000 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
2001 ({ \
2002 	typeof(ptr) __ai_ptr = (ptr); \
2003 	typeof(oldp) __ai_oldp = (oldp); \
2004 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2005 	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2006 	arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2007 })
2008 
2009 #define try_cmpxchg64(ptr, oldp, ...) \
2010 ({ \
2011 	typeof(ptr) __ai_ptr = (ptr); \
2012 	typeof(oldp) __ai_oldp = (oldp); \
2013 	kcsan_mb(); \
2014 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2015 	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2016 	arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2017 })
2018 
2019 #define try_cmpxchg64_acquire(ptr, oldp, ...) \
2020 ({ \
2021 	typeof(ptr) __ai_ptr = (ptr); \
2022 	typeof(oldp) __ai_oldp = (oldp); \
2023 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2024 	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2025 	arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2026 })
2027 
2028 #define try_cmpxchg64_release(ptr, oldp, ...) \
2029 ({ \
2030 	typeof(ptr) __ai_ptr = (ptr); \
2031 	typeof(oldp) __ai_oldp = (oldp); \
2032 	kcsan_release(); \
2033 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2034 	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2035 	arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2036 })
2037 
2038 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
2039 ({ \
2040 	typeof(ptr) __ai_ptr = (ptr); \
2041 	typeof(oldp) __ai_oldp = (oldp); \
2042 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043 	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2044 	arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2045 })
2046 
2047 #define cmpxchg_local(ptr, ...) \
2048 ({ \
2049 	typeof(ptr) __ai_ptr = (ptr); \
2050 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2051 	arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
2052 })
2053 
2054 #define cmpxchg64_local(ptr, ...) \
2055 ({ \
2056 	typeof(ptr) __ai_ptr = (ptr); \
2057 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2058 	arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
2059 })
2060 
2061 #define sync_cmpxchg(ptr, ...) \
2062 ({ \
2063 	typeof(ptr) __ai_ptr = (ptr); \
2064 	kcsan_mb(); \
2065 	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2066 	arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
2067 })
2068 
2069 #define cmpxchg_double(ptr, ...) \
2070 ({ \
2071 	typeof(ptr) __ai_ptr = (ptr); \
2072 	kcsan_mb(); \
2073 	instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2074 	arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2075 })
2076 
2077 
2078 #define cmpxchg_double_local(ptr, ...) \
2079 ({ \
2080 	typeof(ptr) __ai_ptr = (ptr); \
2081 	instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2082 	arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2083 })
2084 
2085 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2086 // 764f741eb77a7ad565dc8d99ce2837d5542e8aee
2087