1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_CMPXCHG_32_H
3 #define _ASM_X86_CMPXCHG_32_H
4
5 /*
6 * Note: if you use set64_bit(), __cmpxchg64(), or their variants,
7 * you need to test for the feature in boot_cpu_data.
8 */
9
10 /*
11 * CMPXCHG8B only writes to the target if we had the previous
12 * value in registers, otherwise it acts as a read and gives us the
13 * "new previous" value. That is why there is a loop. Preloading
14 * EDX:EAX is a performance optimization: in the common case it means
15 * we need only one locked operation.
16 *
17 * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
18 * least an FPU save and/or %cr0.ts manipulation.
19 *
20 * cmpxchg8b must be used with the lock prefix here to allow the
21 * instruction to be executed atomically. We need to have the reader
22 * side to see the coherent 64bit value.
23 */
set_64bit(volatile u64 * ptr,u64 value)24 static inline void set_64bit(volatile u64 *ptr, u64 value)
25 {
26 u32 low = value;
27 u32 high = value >> 32;
28 u64 prev = *ptr;
29
30 asm volatile("\n1:\t"
31 LOCK_PREFIX "cmpxchg8b %0\n\t"
32 "jnz 1b"
33 : "=m" (*ptr), "+A" (prev)
34 : "b" (low), "c" (high)
35 : "memory");
36 }
37
38 #ifdef CONFIG_X86_CMPXCHG64
39 #define arch_cmpxchg64(ptr, o, n) \
40 ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
41 (unsigned long long)(n)))
42 #define arch_cmpxchg64_local(ptr, o, n) \
43 ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
44 (unsigned long long)(n)))
45 #define arch_try_cmpxchg64(ptr, po, n) \
46 __try_cmpxchg64((ptr), (unsigned long long *)(po), \
47 (unsigned long long)(n))
48 #endif
49
__cmpxchg64(volatile u64 * ptr,u64 old,u64 new)50 static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
51 {
52 u64 prev;
53 asm volatile(LOCK_PREFIX "cmpxchg8b %1"
54 : "=A" (prev),
55 "+m" (*ptr)
56 : "b" ((u32)new),
57 "c" ((u32)(new >> 32)),
58 "0" (old)
59 : "memory");
60 return prev;
61 }
62
__cmpxchg64_local(volatile u64 * ptr,u64 old,u64 new)63 static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
64 {
65 u64 prev;
66 asm volatile("cmpxchg8b %1"
67 : "=A" (prev),
68 "+m" (*ptr)
69 : "b" ((u32)new),
70 "c" ((u32)(new >> 32)),
71 "0" (old)
72 : "memory");
73 return prev;
74 }
75
__try_cmpxchg64(volatile u64 * ptr,u64 * pold,u64 new)76 static inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *pold, u64 new)
77 {
78 bool success;
79 u64 old = *pold;
80 asm volatile(LOCK_PREFIX "cmpxchg8b %[ptr]"
81 CC_SET(z)
82 : CC_OUT(z) (success),
83 [ptr] "+m" (*ptr),
84 "+A" (old)
85 : "b" ((u32)new),
86 "c" ((u32)(new >> 32))
87 : "memory");
88
89 if (unlikely(!success))
90 *pold = old;
91 return success;
92 }
93
94 #ifndef CONFIG_X86_CMPXCHG64
95 /*
96 * Building a kernel capable running on 80386 and 80486. It may be necessary
97 * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
98 */
99
100 #define arch_cmpxchg64(ptr, o, n) \
101 ({ \
102 __typeof__(*(ptr)) __ret; \
103 __typeof__(*(ptr)) __old = (o); \
104 __typeof__(*(ptr)) __new = (n); \
105 alternative_io(LOCK_PREFIX_HERE \
106 "call cmpxchg8b_emu", \
107 "lock; cmpxchg8b (%%esi)" , \
108 X86_FEATURE_CX8, \
109 "=A" (__ret), \
110 "S" ((ptr)), "0" (__old), \
111 "b" ((unsigned int)__new), \
112 "c" ((unsigned int)(__new>>32)) \
113 : "memory"); \
114 __ret; })
115
116
117 #define arch_cmpxchg64_local(ptr, o, n) \
118 ({ \
119 __typeof__(*(ptr)) __ret; \
120 __typeof__(*(ptr)) __old = (o); \
121 __typeof__(*(ptr)) __new = (n); \
122 alternative_io("call cmpxchg8b_emu", \
123 "cmpxchg8b (%%esi)" , \
124 X86_FEATURE_CX8, \
125 "=A" (__ret), \
126 "S" ((ptr)), "0" (__old), \
127 "b" ((unsigned int)__new), \
128 "c" ((unsigned int)(__new>>32)) \
129 : "memory"); \
130 __ret; })
131
132 #endif
133
134 #define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX8)
135
136 #endif /* _ASM_X86_CMPXCHG_32_H */
137