1 /* Copyright (C) 2003-2022 Free Software Foundation, Inc. 2 This file is part of the GNU C Library. 3 4 The GNU C Library is free software; you can redistribute it and/or 5 modify it under the terms of the GNU Lesser General Public 6 License as published by the Free Software Foundation; either 7 version 2.1 of the License, or (at your option) any later version. 8 9 The GNU C Library is distributed in the hope that it will be useful, 10 but WITHOUT ANY WARRANTY; without even the implied warranty of 11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 Lesser General Public License for more details. 13 14 You should have received a copy of the GNU Lesser General Public 15 License along with the GNU C Library; if not, see 16 <https://www.gnu.org/licenses/>. */ 17 18 /* Activate all C11 atomic builtins. 19 20 Note: 21 E.g. in nptl/pthread_key_delete.c if compiled with GCCs 6 and before, 22 an extra stack-frame is generated and the old value is stored on stack 23 before cs instruction but it never loads this value from stack. 24 An unreleased GCC 7 omit those stack operations. 25 26 E.g. in nptl/pthread_once.c the condition code of cs instruction is 27 evaluated by a sequence of ipm, sra, compare and jump instructions instead 28 of one conditional jump instruction. This also occurs with an unreleased 29 GCC 7. 30 31 The atomic_fetch_abc_def C11 builtins are now using load-and-abc instructions 32 on z196 zarch and higher cpus instead of a loop with compare-and-swap 33 instruction. */ 34 #define USE_ATOMIC_COMPILER_BUILTINS 1 35 36 #ifdef __s390x__ 37 # define __HAVE_64B_ATOMICS 1 38 #else 39 # define __HAVE_64B_ATOMICS 0 40 #endif 41 42 #define ATOMIC_EXCHANGE_USES_CAS 1 43 44 /* Implement some of the non-C11 atomic macros from include/atomic.h 45 with help of the C11 atomic builtins. The other non-C11 atomic macros 46 are using the macros defined here. */ 47 48 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL. 49 Return the old *MEM value. */ 50 #define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \ 51 ({ __atomic_check_size((mem)); \ 52 typeof ((__typeof (*(mem))) *(mem)) __atg1_oldval = (oldval); \ 53 __atomic_compare_exchange_n (mem, (void *) &__atg1_oldval, \ 54 newval, 1, __ATOMIC_ACQUIRE, \ 55 __ATOMIC_RELAXED); \ 56 __atg1_oldval; }) 57 #define atomic_compare_and_exchange_val_rel(mem, newval, oldval) \ 58 ({ __atomic_check_size((mem)); \ 59 typeof ((__typeof (*(mem))) *(mem)) __atg1_2_oldval = (oldval); \ 60 __atomic_compare_exchange_n (mem, (void *) &__atg1_2_oldval, \ 61 newval, 1, __ATOMIC_RELEASE, \ 62 __ATOMIC_RELAXED); \ 63 __atg1_2_oldval; }) 64 65 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL. 66 Return zero if *MEM was changed or non-zero if no exchange happened. */ 67 #define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ 68 ({ __atomic_check_size((mem)); \ 69 typeof ((__typeof (*(mem))) *(mem)) __atg2_oldval = (oldval); \ 70 !__atomic_compare_exchange_n (mem, (void *) &__atg2_oldval, newval, \ 71 1, __ATOMIC_ACQUIRE, \ 72 __ATOMIC_RELAXED); }) 73 #define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ 74 atomic_compare_and_exchange_bool_acq (mem, newval, oldval) 75 76 /* Store NEWVALUE in *MEM and return the old value. */ 77 #define atomic_exchange_acq(mem, newvalue) \ 78 ({ __atomic_check_size((mem)); \ 79 __atomic_exchange_n (mem, newvalue, __ATOMIC_ACQUIRE); }) 80 #define atomic_exchange_rel(mem, newvalue) \ 81 ({ __atomic_check_size((mem)); \ 82 __atomic_exchange_n (mem, newvalue, __ATOMIC_RELEASE); }) 83 84 /* Add VALUE to *MEM and return the old value of *MEM. */ 85 /* The gcc builtin uses load-and-add instruction on z196 zarch and higher cpus 86 instead of a loop with compare-and-swap instruction. */ 87 # define atomic_exchange_and_add_acq(mem, operand) \ 88 ({ __atomic_check_size((mem)); \ 89 __atomic_fetch_add ((mem), (operand), __ATOMIC_ACQUIRE); }) 90 # define atomic_exchange_and_add_rel(mem, operand) \ 91 ({ __atomic_check_size((mem)); \ 92 __atomic_fetch_add ((mem), (operand), __ATOMIC_RELEASE); }) 93 #define catomic_exchange_and_add(mem, value) \ 94 atomic_exchange_and_add (mem, value) 95 96 /* Atomically *mem |= mask and return the old value of *mem. */ 97 /* The gcc builtin uses load-and-or instruction on z196 zarch and higher cpus 98 instead of a loop with compare-and-swap instruction. */ 99 #define atomic_or_val(mem, operand) \ 100 ({ __atomic_check_size((mem)); \ 101 __atomic_fetch_or ((mem), (operand), __ATOMIC_ACQUIRE); }) 102 /* Atomically *mem |= mask. */ 103 #define atomic_or(mem, mask) \ 104 do { \ 105 atomic_or_val (mem, mask); \ 106 } while (0) 107 #define catomic_or(mem, mask) \ 108 atomic_or (mem, mask) 109 110 /* Atomically *mem |= 1 << bit and return true if the bit was set in old value 111 of *mem. */ 112 /* The load-and-or instruction is used on z196 zarch and higher cpus 113 instead of a loop with compare-and-swap instruction. */ 114 #define atomic_bit_test_set(mem, bit) \ 115 ({ __typeof (*(mem)) __atg14_old; \ 116 __typeof (mem) __atg14_memp = (mem); \ 117 __typeof (*(mem)) __atg14_mask = ((__typeof (*(mem))) 1 << (bit)); \ 118 __atg14_old = atomic_or_val (__atg14_memp, __atg14_mask); \ 119 __atg14_old & __atg14_mask; }) 120 121 /* Atomically *mem &= mask and return the old value of *mem. */ 122 /* The gcc builtin uses load-and-and instruction on z196 zarch and higher cpus 123 instead of a loop with compare-and-swap instruction. */ 124 #define atomic_and_val(mem, operand) \ 125 ({ __atomic_check_size((mem)); \ 126 __atomic_fetch_and ((mem), (operand), __ATOMIC_ACQUIRE); }) 127 /* Atomically *mem &= mask. */ 128 #define atomic_and(mem, mask) \ 129 do { \ 130 atomic_and_val (mem, mask); \ 131 } while (0) 132 #define catomic_and(mem, mask) \ 133 atomic_and(mem, mask) 134