1 #ifndef __ALPHA_SYSTEM_H
2 #error Do not include xchg.h directly!
3 #else
4 /*
5 * xchg/xchg_local and cmpxchg/cmpxchg_local share the same code
6 * except that local version do not have the expensive memory barrier.
7 * So this file is included twice from asm/system.h.
8 */
9
10 /*
11 * Atomic exchange.
12 * Since it can be used to implement critical sections
13 * it must clobber "memory" (also for interrupts in UP).
14 */
15
16 static inline unsigned long
____xchg(_u8,volatile char * m,unsigned long val)17 ____xchg(_u8, volatile char *m, unsigned long val)
18 {
19 unsigned long ret, tmp, addr64;
20
21 __asm__ __volatile__(
22 " andnot %4,7,%3\n"
23 " insbl %1,%4,%1\n"
24 "1: ldq_l %2,0(%3)\n"
25 " extbl %2,%4,%0\n"
26 " mskbl %2,%4,%2\n"
27 " or %1,%2,%2\n"
28 " stq_c %2,0(%3)\n"
29 " beq %2,2f\n"
30 __ASM__MB
31 ".subsection 2\n"
32 "2: br 1b\n"
33 ".previous"
34 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
35 : "r" ((long)m), "1" (val) : "memory");
36
37 return ret;
38 }
39
40 static inline unsigned long
____xchg(_u16,volatile short * m,unsigned long val)41 ____xchg(_u16, volatile short *m, unsigned long val)
42 {
43 unsigned long ret, tmp, addr64;
44
45 __asm__ __volatile__(
46 " andnot %4,7,%3\n"
47 " inswl %1,%4,%1\n"
48 "1: ldq_l %2,0(%3)\n"
49 " extwl %2,%4,%0\n"
50 " mskwl %2,%4,%2\n"
51 " or %1,%2,%2\n"
52 " stq_c %2,0(%3)\n"
53 " beq %2,2f\n"
54 __ASM__MB
55 ".subsection 2\n"
56 "2: br 1b\n"
57 ".previous"
58 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
59 : "r" ((long)m), "1" (val) : "memory");
60
61 return ret;
62 }
63
64 static inline unsigned long
____xchg(_u32,volatile int * m,unsigned long val)65 ____xchg(_u32, volatile int *m, unsigned long val)
66 {
67 unsigned long dummy;
68
69 __asm__ __volatile__(
70 "1: ldl_l %0,%4\n"
71 " bis $31,%3,%1\n"
72 " stl_c %1,%2\n"
73 " beq %1,2f\n"
74 __ASM__MB
75 ".subsection 2\n"
76 "2: br 1b\n"
77 ".previous"
78 : "=&r" (val), "=&r" (dummy), "=m" (*m)
79 : "rI" (val), "m" (*m) : "memory");
80
81 return val;
82 }
83
84 static inline unsigned long
____xchg(_u64,volatile long * m,unsigned long val)85 ____xchg(_u64, volatile long *m, unsigned long val)
86 {
87 unsigned long dummy;
88
89 __asm__ __volatile__(
90 "1: ldq_l %0,%4\n"
91 " bis $31,%3,%1\n"
92 " stq_c %1,%2\n"
93 " beq %1,2f\n"
94 __ASM__MB
95 ".subsection 2\n"
96 "2: br 1b\n"
97 ".previous"
98 : "=&r" (val), "=&r" (dummy), "=m" (*m)
99 : "rI" (val), "m" (*m) : "memory");
100
101 return val;
102 }
103
104 /* This function doesn't exist, so you'll get a linker error
105 if something tries to do an invalid xchg(). */
106 extern void __xchg_called_with_bad_pointer(void);
107
108 static __always_inline unsigned long
109 ____xchg(, volatile void *ptr, unsigned long x, int size)
110 {
111 switch (size) {
112 case 1:
113 return ____xchg(_u8, ptr, x);
114 case 2:
115 return ____xchg(_u16, ptr, x);
116 case 4:
117 return ____xchg(_u32, ptr, x);
118 case 8:
119 return ____xchg(_u64, ptr, x);
120 }
121 __xchg_called_with_bad_pointer();
122 return x;
123 }
124
125 /*
126 * Atomic compare and exchange. Compare OLD with MEM, if identical,
127 * store NEW in MEM. Return the initial value in MEM. Success is
128 * indicated by comparing RETURN with OLD.
129 *
130 * The memory barrier should be placed in SMP only when we actually
131 * make the change. If we don't change anything (so if the returned
132 * prev is equal to old) then we aren't acquiring anything new and
133 * we don't need any memory barrier as far I can tell.
134 */
135
136 static inline unsigned long
____cmpxchg(_u8,volatile char * m,unsigned char old,unsigned char new)137 ____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new)
138 {
139 unsigned long prev, tmp, cmp, addr64;
140
141 __asm__ __volatile__(
142 " andnot %5,7,%4\n"
143 " insbl %1,%5,%1\n"
144 "1: ldq_l %2,0(%4)\n"
145 " extbl %2,%5,%0\n"
146 " cmpeq %0,%6,%3\n"
147 " beq %3,2f\n"
148 " mskbl %2,%5,%2\n"
149 " or %1,%2,%2\n"
150 " stq_c %2,0(%4)\n"
151 " beq %2,3f\n"
152 __ASM__MB
153 "2:\n"
154 ".subsection 2\n"
155 "3: br 1b\n"
156 ".previous"
157 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
158 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
159
160 return prev;
161 }
162
163 static inline unsigned long
____cmpxchg(_u16,volatile short * m,unsigned short old,unsigned short new)164 ____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new)
165 {
166 unsigned long prev, tmp, cmp, addr64;
167
168 __asm__ __volatile__(
169 " andnot %5,7,%4\n"
170 " inswl %1,%5,%1\n"
171 "1: ldq_l %2,0(%4)\n"
172 " extwl %2,%5,%0\n"
173 " cmpeq %0,%6,%3\n"
174 " beq %3,2f\n"
175 " mskwl %2,%5,%2\n"
176 " or %1,%2,%2\n"
177 " stq_c %2,0(%4)\n"
178 " beq %2,3f\n"
179 __ASM__MB
180 "2:\n"
181 ".subsection 2\n"
182 "3: br 1b\n"
183 ".previous"
184 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
185 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
186
187 return prev;
188 }
189
190 static inline unsigned long
____cmpxchg(_u32,volatile int * m,int old,int new)191 ____cmpxchg(_u32, volatile int *m, int old, int new)
192 {
193 unsigned long prev, cmp;
194
195 __asm__ __volatile__(
196 "1: ldl_l %0,%5\n"
197 " cmpeq %0,%3,%1\n"
198 " beq %1,2f\n"
199 " mov %4,%1\n"
200 " stl_c %1,%2\n"
201 " beq %1,3f\n"
202 __ASM__MB
203 "2:\n"
204 ".subsection 2\n"
205 "3: br 1b\n"
206 ".previous"
207 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
208 : "r"((long) old), "r"(new), "m"(*m) : "memory");
209
210 return prev;
211 }
212
213 static inline unsigned long
____cmpxchg(_u64,volatile long * m,unsigned long old,unsigned long new)214 ____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new)
215 {
216 unsigned long prev, cmp;
217
218 __asm__ __volatile__(
219 "1: ldq_l %0,%5\n"
220 " cmpeq %0,%3,%1\n"
221 " beq %1,2f\n"
222 " mov %4,%1\n"
223 " stq_c %1,%2\n"
224 " beq %1,3f\n"
225 __ASM__MB
226 "2:\n"
227 ".subsection 2\n"
228 "3: br 1b\n"
229 ".previous"
230 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
231 : "r"((long) old), "r"(new), "m"(*m) : "memory");
232
233 return prev;
234 }
235
236 /* This function doesn't exist, so you'll get a linker error
237 if something tries to do an invalid cmpxchg(). */
238 extern void __cmpxchg_called_with_bad_pointer(void);
239
240 static __always_inline unsigned long
241 ____cmpxchg(, volatile void *ptr, unsigned long old, unsigned long new,
242 int size)
243 {
244 switch (size) {
245 case 1:
246 return ____cmpxchg(_u8, ptr, old, new);
247 case 2:
248 return ____cmpxchg(_u16, ptr, old, new);
249 case 4:
250 return ____cmpxchg(_u32, ptr, old, new);
251 case 8:
252 return ____cmpxchg(_u64, ptr, old, new);
253 }
254 __cmpxchg_called_with_bad_pointer();
255 return old;
256 }
257
258 #endif
259