1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2
3 /*
4 * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
5 * access-register mode nor the linkage stack this instruction will always
6 * cause a special-operation exception (the trap-enabled bit in the DUCT
7 * is and will stay 0). The instruction pattern is
8 * b2 ff 0f ff trap4 4095(%r0)
9 */
10 #define RSEQ_SIG 0xB2FF0FFF
11
12 #define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory")
13 #define rseq_smp_rmb() rseq_smp_mb()
14 #define rseq_smp_wmb() rseq_smp_mb()
15
16 #define rseq_smp_load_acquire(p) \
17 __extension__ ({ \
18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
19 rseq_barrier(); \
20 ____p1; \
21 })
22
23 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
24
25 #define rseq_smp_store_release(p, v) \
26 do { \
27 rseq_barrier(); \
28 RSEQ_WRITE_ONCE(*p, v); \
29 } while (0)
30
31 #ifdef RSEQ_SKIP_FASTPATH
32 #include "rseq-skip.h"
33 #else /* !RSEQ_SKIP_FASTPATH */
34
35 #ifdef __s390x__
36
37 #define LONG_L "lg"
38 #define LONG_S "stg"
39 #define LONG_LT_R "ltgr"
40 #define LONG_CMP "cg"
41 #define LONG_CMP_R "cgr"
42 #define LONG_ADDI "aghi"
43 #define LONG_ADD_R "agr"
44
45 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
46 start_ip, post_commit_offset, abort_ip) \
47 ".pushsection __rseq_cs, \"aw\"\n\t" \
48 ".balign 32\n\t" \
49 __rseq_str(label) ":\n\t" \
50 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
51 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
52 ".popsection\n\t" \
53 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
54 ".quad " __rseq_str(label) "b\n\t" \
55 ".popsection\n\t"
56
57 /*
58 * Exit points of a rseq critical section consist of all instructions outside
59 * of the critical section where a critical section can either branch to or
60 * reach through the normal course of its execution. The abort IP and the
61 * post-commit IP are already part of the __rseq_cs section and should not be
62 * explicitly defined as additional exit points. Knowing all exit points is
63 * useful to assist debuggers stepping over the critical section.
64 */
65 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
66 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
67 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
68 ".popsection\n\t"
69
70 #elif __s390__
71
72 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
73 start_ip, post_commit_offset, abort_ip) \
74 ".pushsection __rseq_cs, \"aw\"\n\t" \
75 ".balign 32\n\t" \
76 __rseq_str(label) ":\n\t" \
77 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
78 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
79 ".popsection\n\t" \
80 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
81 ".long 0x0, " __rseq_str(label) "b\n\t" \
82 ".popsection\n\t"
83
84 /*
85 * Exit points of a rseq critical section consist of all instructions outside
86 * of the critical section where a critical section can either branch to or
87 * reach through the normal course of its execution. The abort IP and the
88 * post-commit IP are already part of the __rseq_cs section and should not be
89 * explicitly defined as additional exit points. Knowing all exit points is
90 * useful to assist debuggers stepping over the critical section.
91 */
92 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
93 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
94 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
95 ".popsection\n\t"
96
97 #define LONG_L "l"
98 #define LONG_S "st"
99 #define LONG_LT_R "ltr"
100 #define LONG_CMP "c"
101 #define LONG_CMP_R "cr"
102 #define LONG_ADDI "ahi"
103 #define LONG_ADD_R "ar"
104
105 #endif
106
107 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
108 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
109 (post_commit_ip - start_ip), abort_ip)
110
111 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
112 RSEQ_INJECT_ASM(1) \
113 "larl %%r0, " __rseq_str(cs_label) "\n\t" \
114 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \
115 __rseq_str(label) ":\n\t"
116
117 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
118 RSEQ_INJECT_ASM(2) \
119 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
120 "jnz " __rseq_str(label) "\n\t"
121
122 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
123 ".pushsection __rseq_failure, \"ax\"\n\t" \
124 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
125 __rseq_str(label) ":\n\t" \
126 teardown \
127 "jg %l[" __rseq_str(abort_label) "]\n\t" \
128 ".popsection\n\t"
129
130 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
131 ".pushsection __rseq_failure, \"ax\"\n\t" \
132 __rseq_str(label) ":\n\t" \
133 teardown \
134 "jg %l[" __rseq_str(cmpfail_label) "]\n\t" \
135 ".popsection\n\t"
136
137 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)138 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
139 {
140 RSEQ_INJECT_C(9)
141
142 __asm__ __volatile__ goto (
143 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
144 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
145 #ifdef RSEQ_COMPARE_TWICE
146 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
147 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
148 #endif
149 /* Start rseq by storing table entry pointer into rseq_cs. */
150 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
151 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
152 RSEQ_INJECT_ASM(3)
153 LONG_CMP " %[expect], %[v]\n\t"
154 "jnz %l[cmpfail]\n\t"
155 RSEQ_INJECT_ASM(4)
156 #ifdef RSEQ_COMPARE_TWICE
157 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
158 LONG_CMP " %[expect], %[v]\n\t"
159 "jnz %l[error2]\n\t"
160 #endif
161 /* final store */
162 LONG_S " %[newv], %[v]\n\t"
163 "2:\n\t"
164 RSEQ_INJECT_ASM(5)
165 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
166 : /* gcc asm goto does not allow outputs */
167 : [cpu_id] "r" (cpu),
168 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
169 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
170 [v] "m" (*v),
171 [expect] "r" (expect),
172 [newv] "r" (newv)
173 RSEQ_INJECT_INPUT
174 : "memory", "cc", "r0"
175 RSEQ_INJECT_CLOBBER
176 : abort, cmpfail
177 #ifdef RSEQ_COMPARE_TWICE
178 , error1, error2
179 #endif
180 );
181 rseq_after_asm_goto();
182 return 0;
183 abort:
184 rseq_after_asm_goto();
185 RSEQ_INJECT_FAILED
186 return -1;
187 cmpfail:
188 rseq_after_asm_goto();
189 return 1;
190 #ifdef RSEQ_COMPARE_TWICE
191 error1:
192 rseq_after_asm_goto();
193 rseq_bug("cpu_id comparison failed");
194 error2:
195 rseq_after_asm_goto();
196 rseq_bug("expected value comparison failed");
197 #endif
198 }
199
200 /*
201 * Compare @v against @expectnot. When it does _not_ match, load @v
202 * into @load, and store the content of *@v + voffp into @v.
203 */
204 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,long voffp,intptr_t * load,int cpu)205 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
206 long voffp, intptr_t *load, int cpu)
207 {
208 RSEQ_INJECT_C(9)
209
210 __asm__ __volatile__ goto (
211 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
212 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
213 #ifdef RSEQ_COMPARE_TWICE
214 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
216 #endif
217 /* Start rseq by storing table entry pointer into rseq_cs. */
218 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
219 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
220 RSEQ_INJECT_ASM(3)
221 LONG_L " %%r1, %[v]\n\t"
222 LONG_CMP_R " %%r1, %[expectnot]\n\t"
223 "je %l[cmpfail]\n\t"
224 RSEQ_INJECT_ASM(4)
225 #ifdef RSEQ_COMPARE_TWICE
226 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
227 LONG_L " %%r1, %[v]\n\t"
228 LONG_CMP_R " %%r1, %[expectnot]\n\t"
229 "je %l[error2]\n\t"
230 #endif
231 LONG_S " %%r1, %[load]\n\t"
232 LONG_ADD_R " %%r1, %[voffp]\n\t"
233 LONG_L " %%r1, 0(%%r1)\n\t"
234 /* final store */
235 LONG_S " %%r1, %[v]\n\t"
236 "2:\n\t"
237 RSEQ_INJECT_ASM(5)
238 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
239 : /* gcc asm goto does not allow outputs */
240 : [cpu_id] "r" (cpu),
241 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
242 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
243 /* final store input */
244 [v] "m" (*v),
245 [expectnot] "r" (expectnot),
246 [voffp] "r" (voffp),
247 [load] "m" (*load)
248 RSEQ_INJECT_INPUT
249 : "memory", "cc", "r0", "r1"
250 RSEQ_INJECT_CLOBBER
251 : abort, cmpfail
252 #ifdef RSEQ_COMPARE_TWICE
253 , error1, error2
254 #endif
255 );
256 rseq_after_asm_goto();
257 return 0;
258 abort:
259 rseq_after_asm_goto();
260 RSEQ_INJECT_FAILED
261 return -1;
262 cmpfail:
263 rseq_after_asm_goto();
264 return 1;
265 #ifdef RSEQ_COMPARE_TWICE
266 error1:
267 rseq_after_asm_goto();
268 rseq_bug("cpu_id comparison failed");
269 error2:
270 rseq_after_asm_goto();
271 rseq_bug("expected value comparison failed");
272 #endif
273 }
274
275 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)276 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
277 {
278 RSEQ_INJECT_C(9)
279
280 __asm__ __volatile__ goto (
281 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
282 #ifdef RSEQ_COMPARE_TWICE
283 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
284 #endif
285 /* Start rseq by storing table entry pointer into rseq_cs. */
286 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
287 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
288 RSEQ_INJECT_ASM(3)
289 #ifdef RSEQ_COMPARE_TWICE
290 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
291 #endif
292 LONG_L " %%r0, %[v]\n\t"
293 LONG_ADD_R " %%r0, %[count]\n\t"
294 /* final store */
295 LONG_S " %%r0, %[v]\n\t"
296 "2:\n\t"
297 RSEQ_INJECT_ASM(4)
298 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
299 : /* gcc asm goto does not allow outputs */
300 : [cpu_id] "r" (cpu),
301 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
302 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
303 /* final store input */
304 [v] "m" (*v),
305 [count] "r" (count)
306 RSEQ_INJECT_INPUT
307 : "memory", "cc", "r0"
308 RSEQ_INJECT_CLOBBER
309 : abort
310 #ifdef RSEQ_COMPARE_TWICE
311 , error1
312 #endif
313 );
314 rseq_after_asm_goto();
315 return 0;
316 abort:
317 rseq_after_asm_goto();
318 RSEQ_INJECT_FAILED
319 return -1;
320 #ifdef RSEQ_COMPARE_TWICE
321 error1:
322 rseq_after_asm_goto();
323 rseq_bug("cpu_id comparison failed");
324 #endif
325 }
326
327 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)328 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
329 intptr_t *v2, intptr_t newv2,
330 intptr_t newv, int cpu)
331 {
332 RSEQ_INJECT_C(9)
333
334 __asm__ __volatile__ goto (
335 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
336 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
337 #ifdef RSEQ_COMPARE_TWICE
338 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
339 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
340 #endif
341 /* Start rseq by storing table entry pointer into rseq_cs. */
342 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
343 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
344 RSEQ_INJECT_ASM(3)
345 LONG_CMP " %[expect], %[v]\n\t"
346 "jnz %l[cmpfail]\n\t"
347 RSEQ_INJECT_ASM(4)
348 #ifdef RSEQ_COMPARE_TWICE
349 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
350 LONG_CMP " %[expect], %[v]\n\t"
351 "jnz %l[error2]\n\t"
352 #endif
353 /* try store */
354 LONG_S " %[newv2], %[v2]\n\t"
355 RSEQ_INJECT_ASM(5)
356 /* final store */
357 LONG_S " %[newv], %[v]\n\t"
358 "2:\n\t"
359 RSEQ_INJECT_ASM(6)
360 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
361 : /* gcc asm goto does not allow outputs */
362 : [cpu_id] "r" (cpu),
363 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
364 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
365 /* try store input */
366 [v2] "m" (*v2),
367 [newv2] "r" (newv2),
368 /* final store input */
369 [v] "m" (*v),
370 [expect] "r" (expect),
371 [newv] "r" (newv)
372 RSEQ_INJECT_INPUT
373 : "memory", "cc", "r0"
374 RSEQ_INJECT_CLOBBER
375 : abort, cmpfail
376 #ifdef RSEQ_COMPARE_TWICE
377 , error1, error2
378 #endif
379 );
380 rseq_after_asm_goto();
381 return 0;
382 abort:
383 rseq_after_asm_goto();
384 RSEQ_INJECT_FAILED
385 return -1;
386 cmpfail:
387 rseq_after_asm_goto();
388 return 1;
389 #ifdef RSEQ_COMPARE_TWICE
390 error1:
391 rseq_after_asm_goto();
392 rseq_bug("cpu_id comparison failed");
393 error2:
394 rseq_after_asm_goto();
395 rseq_bug("expected value comparison failed");
396 #endif
397 }
398
399 /* s390 is TSO. */
400 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)401 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
402 intptr_t *v2, intptr_t newv2,
403 intptr_t newv, int cpu)
404 {
405 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
406 }
407
408 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)409 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
410 intptr_t *v2, intptr_t expect2,
411 intptr_t newv, int cpu)
412 {
413 RSEQ_INJECT_C(9)
414
415 __asm__ __volatile__ goto (
416 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
417 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
418 #ifdef RSEQ_COMPARE_TWICE
419 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
420 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
421 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
422 #endif
423 /* Start rseq by storing table entry pointer into rseq_cs. */
424 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
425 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
426 RSEQ_INJECT_ASM(3)
427 LONG_CMP " %[expect], %[v]\n\t"
428 "jnz %l[cmpfail]\n\t"
429 RSEQ_INJECT_ASM(4)
430 LONG_CMP " %[expect2], %[v2]\n\t"
431 "jnz %l[cmpfail]\n\t"
432 RSEQ_INJECT_ASM(5)
433 #ifdef RSEQ_COMPARE_TWICE
434 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
435 LONG_CMP " %[expect], %[v]\n\t"
436 "jnz %l[error2]\n\t"
437 LONG_CMP " %[expect2], %[v2]\n\t"
438 "jnz %l[error3]\n\t"
439 #endif
440 /* final store */
441 LONG_S " %[newv], %[v]\n\t"
442 "2:\n\t"
443 RSEQ_INJECT_ASM(6)
444 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
445 : /* gcc asm goto does not allow outputs */
446 : [cpu_id] "r" (cpu),
447 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
448 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
449 /* cmp2 input */
450 [v2] "m" (*v2),
451 [expect2] "r" (expect2),
452 /* final store input */
453 [v] "m" (*v),
454 [expect] "r" (expect),
455 [newv] "r" (newv)
456 RSEQ_INJECT_INPUT
457 : "memory", "cc", "r0"
458 RSEQ_INJECT_CLOBBER
459 : abort, cmpfail
460 #ifdef RSEQ_COMPARE_TWICE
461 , error1, error2, error3
462 #endif
463 );
464 rseq_after_asm_goto();
465 return 0;
466 abort:
467 rseq_after_asm_goto();
468 RSEQ_INJECT_FAILED
469 return -1;
470 cmpfail:
471 rseq_after_asm_goto();
472 return 1;
473 #ifdef RSEQ_COMPARE_TWICE
474 error1:
475 rseq_after_asm_goto();
476 rseq_bug("cpu_id comparison failed");
477 error2:
478 rseq_after_asm_goto();
479 rseq_bug("1st expected value comparison failed");
480 error3:
481 rseq_after_asm_goto();
482 rseq_bug("2nd expected value comparison failed");
483 #endif
484 }
485
486 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)487 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
488 void *dst, void *src, size_t len,
489 intptr_t newv, int cpu)
490 {
491 uint64_t rseq_scratch[3];
492
493 RSEQ_INJECT_C(9)
494
495 __asm__ __volatile__ goto (
496 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
497 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
498 #ifdef RSEQ_COMPARE_TWICE
499 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
500 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
501 #endif
502 LONG_S " %[src], %[rseq_scratch0]\n\t"
503 LONG_S " %[dst], %[rseq_scratch1]\n\t"
504 LONG_S " %[len], %[rseq_scratch2]\n\t"
505 /* Start rseq by storing table entry pointer into rseq_cs. */
506 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
507 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
508 RSEQ_INJECT_ASM(3)
509 LONG_CMP " %[expect], %[v]\n\t"
510 "jnz 5f\n\t"
511 RSEQ_INJECT_ASM(4)
512 #ifdef RSEQ_COMPARE_TWICE
513 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
514 LONG_CMP " %[expect], %[v]\n\t"
515 "jnz 7f\n\t"
516 #endif
517 /* try memcpy */
518 LONG_LT_R " %[len], %[len]\n\t"
519 "jz 333f\n\t"
520 "222:\n\t"
521 "ic %%r0,0(%[src])\n\t"
522 "stc %%r0,0(%[dst])\n\t"
523 LONG_ADDI " %[src], 1\n\t"
524 LONG_ADDI " %[dst], 1\n\t"
525 LONG_ADDI " %[len], -1\n\t"
526 "jnz 222b\n\t"
527 "333:\n\t"
528 RSEQ_INJECT_ASM(5)
529 /* final store */
530 LONG_S " %[newv], %[v]\n\t"
531 "2:\n\t"
532 RSEQ_INJECT_ASM(6)
533 /* teardown */
534 LONG_L " %[len], %[rseq_scratch2]\n\t"
535 LONG_L " %[dst], %[rseq_scratch1]\n\t"
536 LONG_L " %[src], %[rseq_scratch0]\n\t"
537 RSEQ_ASM_DEFINE_ABORT(4,
538 LONG_L " %[len], %[rseq_scratch2]\n\t"
539 LONG_L " %[dst], %[rseq_scratch1]\n\t"
540 LONG_L " %[src], %[rseq_scratch0]\n\t",
541 abort)
542 RSEQ_ASM_DEFINE_CMPFAIL(5,
543 LONG_L " %[len], %[rseq_scratch2]\n\t"
544 LONG_L " %[dst], %[rseq_scratch1]\n\t"
545 LONG_L " %[src], %[rseq_scratch0]\n\t",
546 cmpfail)
547 #ifdef RSEQ_COMPARE_TWICE
548 RSEQ_ASM_DEFINE_CMPFAIL(6,
549 LONG_L " %[len], %[rseq_scratch2]\n\t"
550 LONG_L " %[dst], %[rseq_scratch1]\n\t"
551 LONG_L " %[src], %[rseq_scratch0]\n\t",
552 error1)
553 RSEQ_ASM_DEFINE_CMPFAIL(7,
554 LONG_L " %[len], %[rseq_scratch2]\n\t"
555 LONG_L " %[dst], %[rseq_scratch1]\n\t"
556 LONG_L " %[src], %[rseq_scratch0]\n\t",
557 error2)
558 #endif
559 : /* gcc asm goto does not allow outputs */
560 : [cpu_id] "r" (cpu),
561 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
562 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
563 /* final store input */
564 [v] "m" (*v),
565 [expect] "r" (expect),
566 [newv] "r" (newv),
567 /* try memcpy input */
568 [dst] "r" (dst),
569 [src] "r" (src),
570 [len] "r" (len),
571 [rseq_scratch0] "m" (rseq_scratch[0]),
572 [rseq_scratch1] "m" (rseq_scratch[1]),
573 [rseq_scratch2] "m" (rseq_scratch[2])
574 RSEQ_INJECT_INPUT
575 : "memory", "cc", "r0"
576 RSEQ_INJECT_CLOBBER
577 : abort, cmpfail
578 #ifdef RSEQ_COMPARE_TWICE
579 , error1, error2
580 #endif
581 );
582 rseq_after_asm_goto();
583 return 0;
584 abort:
585 rseq_after_asm_goto();
586 RSEQ_INJECT_FAILED
587 return -1;
588 cmpfail:
589 rseq_after_asm_goto();
590 return 1;
591 #ifdef RSEQ_COMPARE_TWICE
592 error1:
593 rseq_after_asm_goto();
594 rseq_bug("cpu_id comparison failed");
595 error2:
596 rseq_after_asm_goto();
597 rseq_bug("expected value comparison failed");
598 #endif
599 }
600
601 /* s390 is TSO. */
602 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)603 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
604 void *dst, void *src, size_t len,
605 intptr_t newv, int cpu)
606 {
607 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
608 newv, cpu);
609 }
610 #endif /* !RSEQ_SKIP_FASTPATH */
611