1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * rseq-x86.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8 #include <stdint.h>
9
10 /*
11 * RSEQ_SIG is used with the following reserved undefined instructions, which
12 * trap in user-space:
13 *
14 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
15 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
16 */
17 #define RSEQ_SIG 0x53053053
18
19 /*
20 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
21 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
22 * address through a "r" input operand.
23 */
24
25 /* Offset of cpu_id and rseq_cs fields in struct rseq. */
26 #define RSEQ_CPU_ID_OFFSET 4
27 #define RSEQ_CS_OFFSET 8
28
29 #ifdef __x86_64__
30
31 #define RSEQ_ASM_TP_SEGMENT %%fs
32
33 #define rseq_smp_mb() \
34 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
35 #define rseq_smp_rmb() rseq_barrier()
36 #define rseq_smp_wmb() rseq_barrier()
37
38 #define rseq_smp_load_acquire(p) \
39 __extension__ ({ \
40 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
41 rseq_barrier(); \
42 ____p1; \
43 })
44
45 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
46
47 #define rseq_smp_store_release(p, v) \
48 do { \
49 rseq_barrier(); \
50 RSEQ_WRITE_ONCE(*p, v); \
51 } while (0)
52
53 #ifdef RSEQ_SKIP_FASTPATH
54 #include "rseq-skip.h"
55 #else /* !RSEQ_SKIP_FASTPATH */
56
57 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
58 start_ip, post_commit_offset, abort_ip) \
59 ".pushsection __rseq_cs, \"aw\"\n\t" \
60 ".balign 32\n\t" \
61 __rseq_str(label) ":\n\t" \
62 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
63 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
64 ".popsection\n\t" \
65 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
66 ".quad " __rseq_str(label) "b\n\t" \
67 ".popsection\n\t"
68
69
70 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
71 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
72 (post_commit_ip - start_ip), abort_ip)
73
74 /*
75 * Exit points of a rseq critical section consist of all instructions outside
76 * of the critical section where a critical section can either branch to or
77 * reach through the normal course of its execution. The abort IP and the
78 * post-commit IP are already part of the __rseq_cs section and should not be
79 * explicitly defined as additional exit points. Knowing all exit points is
80 * useful to assist debuggers stepping over the critical section.
81 */
82 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
83 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
84 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
85 ".popsection\n\t"
86
87 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
88 RSEQ_INJECT_ASM(1) \
89 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
90 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
91 __rseq_str(label) ":\n\t"
92
93 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
94 RSEQ_INJECT_ASM(2) \
95 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
96 "jnz " __rseq_str(label) "\n\t"
97
98 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
99 ".pushsection __rseq_failure, \"ax\"\n\t" \
100 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
101 ".byte 0x0f, 0xb9, 0x3d\n\t" \
102 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
103 __rseq_str(label) ":\n\t" \
104 teardown \
105 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
106 ".popsection\n\t"
107
108 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
109 ".pushsection __rseq_failure, \"ax\"\n\t" \
110 __rseq_str(label) ":\n\t" \
111 teardown \
112 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
113 ".popsection\n\t"
114
115 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)116 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
117 {
118 RSEQ_INJECT_C(9)
119
120 __asm__ __volatile__ goto (
121 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
122 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
123 #ifdef RSEQ_COMPARE_TWICE
124 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
125 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
126 #endif
127 /* Start rseq by storing table entry pointer into rseq_cs. */
128 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
129 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
130 RSEQ_INJECT_ASM(3)
131 "cmpq %[v], %[expect]\n\t"
132 "jnz %l[cmpfail]\n\t"
133 RSEQ_INJECT_ASM(4)
134 #ifdef RSEQ_COMPARE_TWICE
135 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
136 "cmpq %[v], %[expect]\n\t"
137 "jnz %l[error2]\n\t"
138 #endif
139 /* final store */
140 "movq %[newv], %[v]\n\t"
141 "2:\n\t"
142 RSEQ_INJECT_ASM(5)
143 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
144 : /* gcc asm goto does not allow outputs */
145 : [cpu_id] "r" (cpu),
146 [rseq_offset] "r" (rseq_offset),
147 [v] "m" (*v),
148 [expect] "r" (expect),
149 [newv] "r" (newv)
150 : "memory", "cc", "rax"
151 RSEQ_INJECT_CLOBBER
152 : abort, cmpfail
153 #ifdef RSEQ_COMPARE_TWICE
154 , error1, error2
155 #endif
156 );
157 rseq_after_asm_goto();
158 return 0;
159 abort:
160 rseq_after_asm_goto();
161 RSEQ_INJECT_FAILED
162 return -1;
163 cmpfail:
164 rseq_after_asm_goto();
165 return 1;
166 #ifdef RSEQ_COMPARE_TWICE
167 error1:
168 rseq_after_asm_goto();
169 rseq_bug("cpu_id comparison failed");
170 error2:
171 rseq_after_asm_goto();
172 rseq_bug("expected value comparison failed");
173 #endif
174 }
175
176 /*
177 * Compare @v against @expectnot. When it does _not_ match, load @v
178 * into @load, and store the content of *@v + voffp into @v.
179 */
180 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,long voffp,intptr_t * load,int cpu)181 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
182 long voffp, intptr_t *load, int cpu)
183 {
184 RSEQ_INJECT_C(9)
185
186 __asm__ __volatile__ goto (
187 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
188 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
189 #ifdef RSEQ_COMPARE_TWICE
190 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
191 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
192 #endif
193 /* Start rseq by storing table entry pointer into rseq_cs. */
194 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
195 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
196 RSEQ_INJECT_ASM(3)
197 "movq %[v], %%rbx\n\t"
198 "cmpq %%rbx, %[expectnot]\n\t"
199 "je %l[cmpfail]\n\t"
200 RSEQ_INJECT_ASM(4)
201 #ifdef RSEQ_COMPARE_TWICE
202 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
203 "movq %[v], %%rbx\n\t"
204 "cmpq %%rbx, %[expectnot]\n\t"
205 "je %l[error2]\n\t"
206 #endif
207 "movq %%rbx, %[load]\n\t"
208 "addq %[voffp], %%rbx\n\t"
209 "movq (%%rbx), %%rbx\n\t"
210 /* final store */
211 "movq %%rbx, %[v]\n\t"
212 "2:\n\t"
213 RSEQ_INJECT_ASM(5)
214 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
215 : /* gcc asm goto does not allow outputs */
216 : [cpu_id] "r" (cpu),
217 [rseq_offset] "r" (rseq_offset),
218 /* final store input */
219 [v] "m" (*v),
220 [expectnot] "r" (expectnot),
221 [voffp] "er" (voffp),
222 [load] "m" (*load)
223 : "memory", "cc", "rax", "rbx"
224 RSEQ_INJECT_CLOBBER
225 : abort, cmpfail
226 #ifdef RSEQ_COMPARE_TWICE
227 , error1, error2
228 #endif
229 );
230 rseq_after_asm_goto();
231 return 0;
232 abort:
233 rseq_after_asm_goto();
234 RSEQ_INJECT_FAILED
235 return -1;
236 cmpfail:
237 rseq_after_asm_goto();
238 return 1;
239 #ifdef RSEQ_COMPARE_TWICE
240 error1:
241 rseq_after_asm_goto();
242 rseq_bug("cpu_id comparison failed");
243 error2:
244 rseq_after_asm_goto();
245 rseq_bug("expected value comparison failed");
246 #endif
247 }
248
249 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)250 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
251 {
252 RSEQ_INJECT_C(9)
253
254 __asm__ __volatile__ goto (
255 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
256 #ifdef RSEQ_COMPARE_TWICE
257 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
258 #endif
259 /* Start rseq by storing table entry pointer into rseq_cs. */
260 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
261 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
262 RSEQ_INJECT_ASM(3)
263 #ifdef RSEQ_COMPARE_TWICE
264 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
265 #endif
266 /* final store */
267 "addq %[count], %[v]\n\t"
268 "2:\n\t"
269 RSEQ_INJECT_ASM(4)
270 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
271 : /* gcc asm goto does not allow outputs */
272 : [cpu_id] "r" (cpu),
273 [rseq_offset] "r" (rseq_offset),
274 /* final store input */
275 [v] "m" (*v),
276 [count] "er" (count)
277 : "memory", "cc", "rax"
278 RSEQ_INJECT_CLOBBER
279 : abort
280 #ifdef RSEQ_COMPARE_TWICE
281 , error1
282 #endif
283 );
284 rseq_after_asm_goto();
285 return 0;
286 abort:
287 rseq_after_asm_goto();
288 RSEQ_INJECT_FAILED
289 return -1;
290 #ifdef RSEQ_COMPARE_TWICE
291 error1:
292 rseq_after_asm_goto();
293 rseq_bug("cpu_id comparison failed");
294 #endif
295 }
296
297 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
298
299 /*
300 * pval = *(ptr+off)
301 * *pval += inc;
302 */
303 static inline __attribute__((always_inline))
rseq_offset_deref_addv(intptr_t * ptr,long off,intptr_t inc,int cpu)304 int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu)
305 {
306 RSEQ_INJECT_C(9)
307
308 __asm__ __volatile__ goto (
309 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
310 #ifdef RSEQ_COMPARE_TWICE
311 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
312 #endif
313 /* Start rseq by storing table entry pointer into rseq_cs. */
314 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
315 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
316 RSEQ_INJECT_ASM(3)
317 #ifdef RSEQ_COMPARE_TWICE
318 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
319 #endif
320 /* get p+v */
321 "movq %[ptr], %%rbx\n\t"
322 "addq %[off], %%rbx\n\t"
323 /* get pv */
324 "movq (%%rbx), %%rcx\n\t"
325 /* *pv += inc */
326 "addq %[inc], (%%rcx)\n\t"
327 "2:\n\t"
328 RSEQ_INJECT_ASM(4)
329 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
330 : /* gcc asm goto does not allow outputs */
331 : [cpu_id] "r" (cpu),
332 [rseq_offset] "r" (rseq_offset),
333 /* final store input */
334 [ptr] "m" (*ptr),
335 [off] "er" (off),
336 [inc] "er" (inc)
337 : "memory", "cc", "rax", "rbx", "rcx"
338 RSEQ_INJECT_CLOBBER
339 : abort
340 #ifdef RSEQ_COMPARE_TWICE
341 , error1
342 #endif
343 );
344 return 0;
345 abort:
346 RSEQ_INJECT_FAILED
347 return -1;
348 #ifdef RSEQ_COMPARE_TWICE
349 error1:
350 rseq_bug("cpu_id comparison failed");
351 #endif
352 }
353
354 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)355 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
356 intptr_t *v2, intptr_t newv2,
357 intptr_t newv, int cpu)
358 {
359 RSEQ_INJECT_C(9)
360
361 __asm__ __volatile__ goto (
362 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
363 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
364 #ifdef RSEQ_COMPARE_TWICE
365 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
366 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
367 #endif
368 /* Start rseq by storing table entry pointer into rseq_cs. */
369 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
370 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
371 RSEQ_INJECT_ASM(3)
372 "cmpq %[v], %[expect]\n\t"
373 "jnz %l[cmpfail]\n\t"
374 RSEQ_INJECT_ASM(4)
375 #ifdef RSEQ_COMPARE_TWICE
376 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
377 "cmpq %[v], %[expect]\n\t"
378 "jnz %l[error2]\n\t"
379 #endif
380 /* try store */
381 "movq %[newv2], %[v2]\n\t"
382 RSEQ_INJECT_ASM(5)
383 /* final store */
384 "movq %[newv], %[v]\n\t"
385 "2:\n\t"
386 RSEQ_INJECT_ASM(6)
387 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
388 : /* gcc asm goto does not allow outputs */
389 : [cpu_id] "r" (cpu),
390 [rseq_offset] "r" (rseq_offset),
391 /* try store input */
392 [v2] "m" (*v2),
393 [newv2] "r" (newv2),
394 /* final store input */
395 [v] "m" (*v),
396 [expect] "r" (expect),
397 [newv] "r" (newv)
398 : "memory", "cc", "rax"
399 RSEQ_INJECT_CLOBBER
400 : abort, cmpfail
401 #ifdef RSEQ_COMPARE_TWICE
402 , error1, error2
403 #endif
404 );
405 rseq_after_asm_goto();
406 return 0;
407 abort:
408 rseq_after_asm_goto();
409 RSEQ_INJECT_FAILED
410 return -1;
411 cmpfail:
412 rseq_after_asm_goto();
413 return 1;
414 #ifdef RSEQ_COMPARE_TWICE
415 error1:
416 rseq_after_asm_goto();
417 rseq_bug("cpu_id comparison failed");
418 error2:
419 rseq_after_asm_goto();
420 rseq_bug("expected value comparison failed");
421 #endif
422 }
423
424 /* x86-64 is TSO. */
425 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)426 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
427 intptr_t *v2, intptr_t newv2,
428 intptr_t newv, int cpu)
429 {
430 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
431 }
432
433 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)434 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
435 intptr_t *v2, intptr_t expect2,
436 intptr_t newv, int cpu)
437 {
438 RSEQ_INJECT_C(9)
439
440 __asm__ __volatile__ goto (
441 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
442 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
443 #ifdef RSEQ_COMPARE_TWICE
444 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
445 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
446 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
447 #endif
448 /* Start rseq by storing table entry pointer into rseq_cs. */
449 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
450 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
451 RSEQ_INJECT_ASM(3)
452 "cmpq %[v], %[expect]\n\t"
453 "jnz %l[cmpfail]\n\t"
454 RSEQ_INJECT_ASM(4)
455 "cmpq %[v2], %[expect2]\n\t"
456 "jnz %l[cmpfail]\n\t"
457 RSEQ_INJECT_ASM(5)
458 #ifdef RSEQ_COMPARE_TWICE
459 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
460 "cmpq %[v], %[expect]\n\t"
461 "jnz %l[error2]\n\t"
462 "cmpq %[v2], %[expect2]\n\t"
463 "jnz %l[error3]\n\t"
464 #endif
465 /* final store */
466 "movq %[newv], %[v]\n\t"
467 "2:\n\t"
468 RSEQ_INJECT_ASM(6)
469 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
470 : /* gcc asm goto does not allow outputs */
471 : [cpu_id] "r" (cpu),
472 [rseq_offset] "r" (rseq_offset),
473 /* cmp2 input */
474 [v2] "m" (*v2),
475 [expect2] "r" (expect2),
476 /* final store input */
477 [v] "m" (*v),
478 [expect] "r" (expect),
479 [newv] "r" (newv)
480 : "memory", "cc", "rax"
481 RSEQ_INJECT_CLOBBER
482 : abort, cmpfail
483 #ifdef RSEQ_COMPARE_TWICE
484 , error1, error2, error3
485 #endif
486 );
487 rseq_after_asm_goto();
488 return 0;
489 abort:
490 rseq_after_asm_goto();
491 RSEQ_INJECT_FAILED
492 return -1;
493 cmpfail:
494 rseq_after_asm_goto();
495 return 1;
496 #ifdef RSEQ_COMPARE_TWICE
497 error1:
498 rseq_after_asm_goto();
499 rseq_bug("cpu_id comparison failed");
500 error2:
501 rseq_after_asm_goto();
502 rseq_bug("1st expected value comparison failed");
503 error3:
504 rseq_after_asm_goto();
505 rseq_bug("2nd expected value comparison failed");
506 #endif
507 }
508
509 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)510 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
511 void *dst, void *src, size_t len,
512 intptr_t newv, int cpu)
513 {
514 uint64_t rseq_scratch[3];
515
516 RSEQ_INJECT_C(9)
517
518 __asm__ __volatile__ goto (
519 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
520 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
521 #ifdef RSEQ_COMPARE_TWICE
522 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
523 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
524 #endif
525 "movq %[src], %[rseq_scratch0]\n\t"
526 "movq %[dst], %[rseq_scratch1]\n\t"
527 "movq %[len], %[rseq_scratch2]\n\t"
528 /* Start rseq by storing table entry pointer into rseq_cs. */
529 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
530 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
531 RSEQ_INJECT_ASM(3)
532 "cmpq %[v], %[expect]\n\t"
533 "jnz 5f\n\t"
534 RSEQ_INJECT_ASM(4)
535 #ifdef RSEQ_COMPARE_TWICE
536 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
537 "cmpq %[v], %[expect]\n\t"
538 "jnz 7f\n\t"
539 #endif
540 /* try memcpy */
541 "test %[len], %[len]\n\t" \
542 "jz 333f\n\t" \
543 "222:\n\t" \
544 "movb (%[src]), %%al\n\t" \
545 "movb %%al, (%[dst])\n\t" \
546 "inc %[src]\n\t" \
547 "inc %[dst]\n\t" \
548 "dec %[len]\n\t" \
549 "jnz 222b\n\t" \
550 "333:\n\t" \
551 RSEQ_INJECT_ASM(5)
552 /* final store */
553 "movq %[newv], %[v]\n\t"
554 "2:\n\t"
555 RSEQ_INJECT_ASM(6)
556 /* teardown */
557 "movq %[rseq_scratch2], %[len]\n\t"
558 "movq %[rseq_scratch1], %[dst]\n\t"
559 "movq %[rseq_scratch0], %[src]\n\t"
560 RSEQ_ASM_DEFINE_ABORT(4,
561 "movq %[rseq_scratch2], %[len]\n\t"
562 "movq %[rseq_scratch1], %[dst]\n\t"
563 "movq %[rseq_scratch0], %[src]\n\t",
564 abort)
565 RSEQ_ASM_DEFINE_CMPFAIL(5,
566 "movq %[rseq_scratch2], %[len]\n\t"
567 "movq %[rseq_scratch1], %[dst]\n\t"
568 "movq %[rseq_scratch0], %[src]\n\t",
569 cmpfail)
570 #ifdef RSEQ_COMPARE_TWICE
571 RSEQ_ASM_DEFINE_CMPFAIL(6,
572 "movq %[rseq_scratch2], %[len]\n\t"
573 "movq %[rseq_scratch1], %[dst]\n\t"
574 "movq %[rseq_scratch0], %[src]\n\t",
575 error1)
576 RSEQ_ASM_DEFINE_CMPFAIL(7,
577 "movq %[rseq_scratch2], %[len]\n\t"
578 "movq %[rseq_scratch1], %[dst]\n\t"
579 "movq %[rseq_scratch0], %[src]\n\t",
580 error2)
581 #endif
582 : /* gcc asm goto does not allow outputs */
583 : [cpu_id] "r" (cpu),
584 [rseq_offset] "r" (rseq_offset),
585 /* final store input */
586 [v] "m" (*v),
587 [expect] "r" (expect),
588 [newv] "r" (newv),
589 /* try memcpy input */
590 [dst] "r" (dst),
591 [src] "r" (src),
592 [len] "r" (len),
593 [rseq_scratch0] "m" (rseq_scratch[0]),
594 [rseq_scratch1] "m" (rseq_scratch[1]),
595 [rseq_scratch2] "m" (rseq_scratch[2])
596 : "memory", "cc", "rax"
597 RSEQ_INJECT_CLOBBER
598 : abort, cmpfail
599 #ifdef RSEQ_COMPARE_TWICE
600 , error1, error2
601 #endif
602 );
603 rseq_after_asm_goto();
604 return 0;
605 abort:
606 rseq_after_asm_goto();
607 RSEQ_INJECT_FAILED
608 return -1;
609 cmpfail:
610 rseq_after_asm_goto();
611 return 1;
612 #ifdef RSEQ_COMPARE_TWICE
613 error1:
614 rseq_after_asm_goto();
615 rseq_bug("cpu_id comparison failed");
616 error2:
617 rseq_after_asm_goto();
618 rseq_bug("expected value comparison failed");
619 #endif
620 }
621
622 /* x86-64 is TSO. */
623 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)624 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
625 void *dst, void *src, size_t len,
626 intptr_t newv, int cpu)
627 {
628 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
629 newv, cpu);
630 }
631
632 #endif /* !RSEQ_SKIP_FASTPATH */
633
634 #elif defined(__i386__)
635
636 #define RSEQ_ASM_TP_SEGMENT %%gs
637
638 #define rseq_smp_mb() \
639 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
640 #define rseq_smp_rmb() \
641 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
642 #define rseq_smp_wmb() \
643 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
644
645 #define rseq_smp_load_acquire(p) \
646 __extension__ ({ \
647 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
648 rseq_smp_mb(); \
649 ____p1; \
650 })
651
652 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
653
654 #define rseq_smp_store_release(p, v) \
655 do { \
656 rseq_smp_mb(); \
657 RSEQ_WRITE_ONCE(*p, v); \
658 } while (0)
659
660 #ifdef RSEQ_SKIP_FASTPATH
661 #include "rseq-skip.h"
662 #else /* !RSEQ_SKIP_FASTPATH */
663
664 /*
665 * Use eax as scratch register and take memory operands as input to
666 * lessen register pressure. Especially needed when compiling in O0.
667 */
668 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
669 start_ip, post_commit_offset, abort_ip) \
670 ".pushsection __rseq_cs, \"aw\"\n\t" \
671 ".balign 32\n\t" \
672 __rseq_str(label) ":\n\t" \
673 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
674 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
675 ".popsection\n\t" \
676 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
677 ".long " __rseq_str(label) "b, 0x0\n\t" \
678 ".popsection\n\t"
679
680 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
681 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
682 (post_commit_ip - start_ip), abort_ip)
683
684 /*
685 * Exit points of a rseq critical section consist of all instructions outside
686 * of the critical section where a critical section can either branch to or
687 * reach through the normal course of its execution. The abort IP and the
688 * post-commit IP are already part of the __rseq_cs section and should not be
689 * explicitly defined as additional exit points. Knowing all exit points is
690 * useful to assist debuggers stepping over the critical section.
691 */
692 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
693 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
694 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
695 ".popsection\n\t"
696
697 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
698 RSEQ_INJECT_ASM(1) \
699 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
700 __rseq_str(label) ":\n\t"
701
702 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
703 RSEQ_INJECT_ASM(2) \
704 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
705 "jnz " __rseq_str(label) "\n\t"
706
707 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
708 ".pushsection __rseq_failure, \"ax\"\n\t" \
709 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
710 ".byte 0x0f, 0xb9, 0x3d\n\t" \
711 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
712 __rseq_str(label) ":\n\t" \
713 teardown \
714 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
715 ".popsection\n\t"
716
717 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
718 ".pushsection __rseq_failure, \"ax\"\n\t" \
719 __rseq_str(label) ":\n\t" \
720 teardown \
721 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
722 ".popsection\n\t"
723
724 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)725 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
726 {
727 RSEQ_INJECT_C(9)
728
729 __asm__ __volatile__ goto (
730 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
731 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
732 #ifdef RSEQ_COMPARE_TWICE
733 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
734 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
735 #endif
736 /* Start rseq by storing table entry pointer into rseq_cs. */
737 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
738 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
739 RSEQ_INJECT_ASM(3)
740 "cmpl %[v], %[expect]\n\t"
741 "jnz %l[cmpfail]\n\t"
742 RSEQ_INJECT_ASM(4)
743 #ifdef RSEQ_COMPARE_TWICE
744 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
745 "cmpl %[v], %[expect]\n\t"
746 "jnz %l[error2]\n\t"
747 #endif
748 /* final store */
749 "movl %[newv], %[v]\n\t"
750 "2:\n\t"
751 RSEQ_INJECT_ASM(5)
752 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
753 : /* gcc asm goto does not allow outputs */
754 : [cpu_id] "r" (cpu),
755 [rseq_offset] "r" (rseq_offset),
756 [v] "m" (*v),
757 [expect] "r" (expect),
758 [newv] "r" (newv)
759 : "memory", "cc", "eax"
760 RSEQ_INJECT_CLOBBER
761 : abort, cmpfail
762 #ifdef RSEQ_COMPARE_TWICE
763 , error1, error2
764 #endif
765 );
766 rseq_after_asm_goto();
767 return 0;
768 abort:
769 rseq_after_asm_goto();
770 RSEQ_INJECT_FAILED
771 return -1;
772 cmpfail:
773 rseq_after_asm_goto();
774 return 1;
775 #ifdef RSEQ_COMPARE_TWICE
776 error1:
777 rseq_after_asm_goto();
778 rseq_bug("cpu_id comparison failed");
779 error2:
780 rseq_after_asm_goto();
781 rseq_bug("expected value comparison failed");
782 #endif
783 }
784
785 /*
786 * Compare @v against @expectnot. When it does _not_ match, load @v
787 * into @load, and store the content of *@v + voffp into @v.
788 */
789 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,long voffp,intptr_t * load,int cpu)790 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
791 long voffp, intptr_t *load, int cpu)
792 {
793 RSEQ_INJECT_C(9)
794
795 __asm__ __volatile__ goto (
796 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
797 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
798 #ifdef RSEQ_COMPARE_TWICE
799 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
800 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
801 #endif
802 /* Start rseq by storing table entry pointer into rseq_cs. */
803 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
804 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
805 RSEQ_INJECT_ASM(3)
806 "movl %[v], %%ebx\n\t"
807 "cmpl %%ebx, %[expectnot]\n\t"
808 "je %l[cmpfail]\n\t"
809 RSEQ_INJECT_ASM(4)
810 #ifdef RSEQ_COMPARE_TWICE
811 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
812 "movl %[v], %%ebx\n\t"
813 "cmpl %%ebx, %[expectnot]\n\t"
814 "je %l[error2]\n\t"
815 #endif
816 "movl %%ebx, %[load]\n\t"
817 "addl %[voffp], %%ebx\n\t"
818 "movl (%%ebx), %%ebx\n\t"
819 /* final store */
820 "movl %%ebx, %[v]\n\t"
821 "2:\n\t"
822 RSEQ_INJECT_ASM(5)
823 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
824 : /* gcc asm goto does not allow outputs */
825 : [cpu_id] "r" (cpu),
826 [rseq_offset] "r" (rseq_offset),
827 /* final store input */
828 [v] "m" (*v),
829 [expectnot] "r" (expectnot),
830 [voffp] "ir" (voffp),
831 [load] "m" (*load)
832 : "memory", "cc", "eax", "ebx"
833 RSEQ_INJECT_CLOBBER
834 : abort, cmpfail
835 #ifdef RSEQ_COMPARE_TWICE
836 , error1, error2
837 #endif
838 );
839 rseq_after_asm_goto();
840 return 0;
841 abort:
842 rseq_after_asm_goto();
843 RSEQ_INJECT_FAILED
844 return -1;
845 cmpfail:
846 rseq_after_asm_goto();
847 return 1;
848 #ifdef RSEQ_COMPARE_TWICE
849 error1:
850 rseq_after_asm_goto();
851 rseq_bug("cpu_id comparison failed");
852 error2:
853 rseq_after_asm_goto();
854 rseq_bug("expected value comparison failed");
855 #endif
856 }
857
858 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)859 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
860 {
861 RSEQ_INJECT_C(9)
862
863 __asm__ __volatile__ goto (
864 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
865 #ifdef RSEQ_COMPARE_TWICE
866 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
867 #endif
868 /* Start rseq by storing table entry pointer into rseq_cs. */
869 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
870 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
871 RSEQ_INJECT_ASM(3)
872 #ifdef RSEQ_COMPARE_TWICE
873 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
874 #endif
875 /* final store */
876 "addl %[count], %[v]\n\t"
877 "2:\n\t"
878 RSEQ_INJECT_ASM(4)
879 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
880 : /* gcc asm goto does not allow outputs */
881 : [cpu_id] "r" (cpu),
882 [rseq_offset] "r" (rseq_offset),
883 /* final store input */
884 [v] "m" (*v),
885 [count] "ir" (count)
886 : "memory", "cc", "eax"
887 RSEQ_INJECT_CLOBBER
888 : abort
889 #ifdef RSEQ_COMPARE_TWICE
890 , error1
891 #endif
892 );
893 rseq_after_asm_goto();
894 return 0;
895 abort:
896 rseq_after_asm_goto();
897 RSEQ_INJECT_FAILED
898 return -1;
899 #ifdef RSEQ_COMPARE_TWICE
900 error1:
901 rseq_after_asm_goto();
902 rseq_bug("cpu_id comparison failed");
903 #endif
904 }
905
906 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)907 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
908 intptr_t *v2, intptr_t newv2,
909 intptr_t newv, int cpu)
910 {
911 RSEQ_INJECT_C(9)
912
913 __asm__ __volatile__ goto (
914 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
915 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
916 #ifdef RSEQ_COMPARE_TWICE
917 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
918 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
919 #endif
920 /* Start rseq by storing table entry pointer into rseq_cs. */
921 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
922 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
923 RSEQ_INJECT_ASM(3)
924 "cmpl %[v], %[expect]\n\t"
925 "jnz %l[cmpfail]\n\t"
926 RSEQ_INJECT_ASM(4)
927 #ifdef RSEQ_COMPARE_TWICE
928 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
929 "cmpl %[v], %[expect]\n\t"
930 "jnz %l[error2]\n\t"
931 #endif
932 /* try store */
933 "movl %[newv2], %%eax\n\t"
934 "movl %%eax, %[v2]\n\t"
935 RSEQ_INJECT_ASM(5)
936 /* final store */
937 "movl %[newv], %[v]\n\t"
938 "2:\n\t"
939 RSEQ_INJECT_ASM(6)
940 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
941 : /* gcc asm goto does not allow outputs */
942 : [cpu_id] "r" (cpu),
943 [rseq_offset] "r" (rseq_offset),
944 /* try store input */
945 [v2] "m" (*v2),
946 [newv2] "m" (newv2),
947 /* final store input */
948 [v] "m" (*v),
949 [expect] "r" (expect),
950 [newv] "r" (newv)
951 : "memory", "cc", "eax"
952 RSEQ_INJECT_CLOBBER
953 : abort, cmpfail
954 #ifdef RSEQ_COMPARE_TWICE
955 , error1, error2
956 #endif
957 );
958 rseq_after_asm_goto();
959 return 0;
960 abort:
961 rseq_after_asm_goto();
962 RSEQ_INJECT_FAILED
963 return -1;
964 cmpfail:
965 rseq_after_asm_goto();
966 return 1;
967 #ifdef RSEQ_COMPARE_TWICE
968 error1:
969 rseq_after_asm_goto();
970 rseq_bug("cpu_id comparison failed");
971 error2:
972 rseq_after_asm_goto();
973 rseq_bug("expected value comparison failed");
974 #endif
975 }
976
977 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)978 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
979 intptr_t *v2, intptr_t newv2,
980 intptr_t newv, int cpu)
981 {
982 RSEQ_INJECT_C(9)
983
984 __asm__ __volatile__ goto (
985 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
986 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
987 #ifdef RSEQ_COMPARE_TWICE
988 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
989 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
990 #endif
991 /* Start rseq by storing table entry pointer into rseq_cs. */
992 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
993 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
994 RSEQ_INJECT_ASM(3)
995 "movl %[expect], %%eax\n\t"
996 "cmpl %[v], %%eax\n\t"
997 "jnz %l[cmpfail]\n\t"
998 RSEQ_INJECT_ASM(4)
999 #ifdef RSEQ_COMPARE_TWICE
1000 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
1001 "movl %[expect], %%eax\n\t"
1002 "cmpl %[v], %%eax\n\t"
1003 "jnz %l[error2]\n\t"
1004 #endif
1005 /* try store */
1006 "movl %[newv2], %[v2]\n\t"
1007 RSEQ_INJECT_ASM(5)
1008 "lock; addl $0,-128(%%esp)\n\t"
1009 /* final store */
1010 "movl %[newv], %[v]\n\t"
1011 "2:\n\t"
1012 RSEQ_INJECT_ASM(6)
1013 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1014 : /* gcc asm goto does not allow outputs */
1015 : [cpu_id] "r" (cpu),
1016 [rseq_offset] "r" (rseq_offset),
1017 /* try store input */
1018 [v2] "m" (*v2),
1019 [newv2] "r" (newv2),
1020 /* final store input */
1021 [v] "m" (*v),
1022 [expect] "m" (expect),
1023 [newv] "r" (newv)
1024 : "memory", "cc", "eax"
1025 RSEQ_INJECT_CLOBBER
1026 : abort, cmpfail
1027 #ifdef RSEQ_COMPARE_TWICE
1028 , error1, error2
1029 #endif
1030 );
1031 rseq_after_asm_goto();
1032 return 0;
1033 abort:
1034 rseq_after_asm_goto();
1035 RSEQ_INJECT_FAILED
1036 return -1;
1037 cmpfail:
1038 rseq_after_asm_goto();
1039 return 1;
1040 #ifdef RSEQ_COMPARE_TWICE
1041 error1:
1042 rseq_after_asm_goto();
1043 rseq_bug("cpu_id comparison failed");
1044 error2:
1045 rseq_after_asm_goto();
1046 rseq_bug("expected value comparison failed");
1047 #endif
1048
1049 }
1050
1051 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)1052 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
1053 intptr_t *v2, intptr_t expect2,
1054 intptr_t newv, int cpu)
1055 {
1056 RSEQ_INJECT_C(9)
1057
1058 __asm__ __volatile__ goto (
1059 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1060 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1061 #ifdef RSEQ_COMPARE_TWICE
1062 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1063 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1064 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
1065 #endif
1066 /* Start rseq by storing table entry pointer into rseq_cs. */
1067 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1068 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1069 RSEQ_INJECT_ASM(3)
1070 "cmpl %[v], %[expect]\n\t"
1071 "jnz %l[cmpfail]\n\t"
1072 RSEQ_INJECT_ASM(4)
1073 "cmpl %[expect2], %[v2]\n\t"
1074 "jnz %l[cmpfail]\n\t"
1075 RSEQ_INJECT_ASM(5)
1076 #ifdef RSEQ_COMPARE_TWICE
1077 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
1078 "cmpl %[v], %[expect]\n\t"
1079 "jnz %l[error2]\n\t"
1080 "cmpl %[expect2], %[v2]\n\t"
1081 "jnz %l[error3]\n\t"
1082 #endif
1083 "movl %[newv], %%eax\n\t"
1084 /* final store */
1085 "movl %%eax, %[v]\n\t"
1086 "2:\n\t"
1087 RSEQ_INJECT_ASM(6)
1088 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1089 : /* gcc asm goto does not allow outputs */
1090 : [cpu_id] "r" (cpu),
1091 [rseq_offset] "r" (rseq_offset),
1092 /* cmp2 input */
1093 [v2] "m" (*v2),
1094 [expect2] "r" (expect2),
1095 /* final store input */
1096 [v] "m" (*v),
1097 [expect] "r" (expect),
1098 [newv] "m" (newv)
1099 : "memory", "cc", "eax"
1100 RSEQ_INJECT_CLOBBER
1101 : abort, cmpfail
1102 #ifdef RSEQ_COMPARE_TWICE
1103 , error1, error2, error3
1104 #endif
1105 );
1106 rseq_after_asm_goto();
1107 return 0;
1108 abort:
1109 rseq_after_asm_goto();
1110 RSEQ_INJECT_FAILED
1111 return -1;
1112 cmpfail:
1113 rseq_after_asm_goto();
1114 return 1;
1115 #ifdef RSEQ_COMPARE_TWICE
1116 error1:
1117 rseq_after_asm_goto();
1118 rseq_bug("cpu_id comparison failed");
1119 error2:
1120 rseq_after_asm_goto();
1121 rseq_bug("1st expected value comparison failed");
1122 error3:
1123 rseq_after_asm_goto();
1124 rseq_bug("2nd expected value comparison failed");
1125 #endif
1126 }
1127
1128 /* TODO: implement a faster memcpy. */
1129 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)1130 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1131 void *dst, void *src, size_t len,
1132 intptr_t newv, int cpu)
1133 {
1134 uint32_t rseq_scratch[3];
1135
1136 RSEQ_INJECT_C(9)
1137
1138 __asm__ __volatile__ goto (
1139 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1140 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1141 #ifdef RSEQ_COMPARE_TWICE
1142 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1143 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1144 #endif
1145 "movl %[src], %[rseq_scratch0]\n\t"
1146 "movl %[dst], %[rseq_scratch1]\n\t"
1147 "movl %[len], %[rseq_scratch2]\n\t"
1148 /* Start rseq by storing table entry pointer into rseq_cs. */
1149 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1150 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1151 RSEQ_INJECT_ASM(3)
1152 "movl %[expect], %%eax\n\t"
1153 "cmpl %%eax, %[v]\n\t"
1154 "jnz 5f\n\t"
1155 RSEQ_INJECT_ASM(4)
1156 #ifdef RSEQ_COMPARE_TWICE
1157 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
1158 "movl %[expect], %%eax\n\t"
1159 "cmpl %%eax, %[v]\n\t"
1160 "jnz 7f\n\t"
1161 #endif
1162 /* try memcpy */
1163 "test %[len], %[len]\n\t" \
1164 "jz 333f\n\t" \
1165 "222:\n\t" \
1166 "movb (%[src]), %%al\n\t" \
1167 "movb %%al, (%[dst])\n\t" \
1168 "inc %[src]\n\t" \
1169 "inc %[dst]\n\t" \
1170 "dec %[len]\n\t" \
1171 "jnz 222b\n\t" \
1172 "333:\n\t" \
1173 RSEQ_INJECT_ASM(5)
1174 "movl %[newv], %%eax\n\t"
1175 /* final store */
1176 "movl %%eax, %[v]\n\t"
1177 "2:\n\t"
1178 RSEQ_INJECT_ASM(6)
1179 /* teardown */
1180 "movl %[rseq_scratch2], %[len]\n\t"
1181 "movl %[rseq_scratch1], %[dst]\n\t"
1182 "movl %[rseq_scratch0], %[src]\n\t"
1183 RSEQ_ASM_DEFINE_ABORT(4,
1184 "movl %[rseq_scratch2], %[len]\n\t"
1185 "movl %[rseq_scratch1], %[dst]\n\t"
1186 "movl %[rseq_scratch0], %[src]\n\t",
1187 abort)
1188 RSEQ_ASM_DEFINE_CMPFAIL(5,
1189 "movl %[rseq_scratch2], %[len]\n\t"
1190 "movl %[rseq_scratch1], %[dst]\n\t"
1191 "movl %[rseq_scratch0], %[src]\n\t",
1192 cmpfail)
1193 #ifdef RSEQ_COMPARE_TWICE
1194 RSEQ_ASM_DEFINE_CMPFAIL(6,
1195 "movl %[rseq_scratch2], %[len]\n\t"
1196 "movl %[rseq_scratch1], %[dst]\n\t"
1197 "movl %[rseq_scratch0], %[src]\n\t",
1198 error1)
1199 RSEQ_ASM_DEFINE_CMPFAIL(7,
1200 "movl %[rseq_scratch2], %[len]\n\t"
1201 "movl %[rseq_scratch1], %[dst]\n\t"
1202 "movl %[rseq_scratch0], %[src]\n\t",
1203 error2)
1204 #endif
1205 : /* gcc asm goto does not allow outputs */
1206 : [cpu_id] "r" (cpu),
1207 [rseq_offset] "r" (rseq_offset),
1208 /* final store input */
1209 [v] "m" (*v),
1210 [expect] "m" (expect),
1211 [newv] "m" (newv),
1212 /* try memcpy input */
1213 [dst] "r" (dst),
1214 [src] "r" (src),
1215 [len] "r" (len),
1216 [rseq_scratch0] "m" (rseq_scratch[0]),
1217 [rseq_scratch1] "m" (rseq_scratch[1]),
1218 [rseq_scratch2] "m" (rseq_scratch[2])
1219 : "memory", "cc", "eax"
1220 RSEQ_INJECT_CLOBBER
1221 : abort, cmpfail
1222 #ifdef RSEQ_COMPARE_TWICE
1223 , error1, error2
1224 #endif
1225 );
1226 rseq_after_asm_goto();
1227 return 0;
1228 abort:
1229 rseq_after_asm_goto();
1230 RSEQ_INJECT_FAILED
1231 return -1;
1232 cmpfail:
1233 rseq_after_asm_goto();
1234 return 1;
1235 #ifdef RSEQ_COMPARE_TWICE
1236 error1:
1237 rseq_after_asm_goto();
1238 rseq_bug("cpu_id comparison failed");
1239 error2:
1240 rseq_after_asm_goto();
1241 rseq_bug("expected value comparison failed");
1242 #endif
1243 }
1244
1245 /* TODO: implement a faster memcpy. */
1246 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)1247 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1248 void *dst, void *src, size_t len,
1249 intptr_t newv, int cpu)
1250 {
1251 uint32_t rseq_scratch[3];
1252
1253 RSEQ_INJECT_C(9)
1254
1255 __asm__ __volatile__ goto (
1256 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1257 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1258 #ifdef RSEQ_COMPARE_TWICE
1259 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1260 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1261 #endif
1262 "movl %[src], %[rseq_scratch0]\n\t"
1263 "movl %[dst], %[rseq_scratch1]\n\t"
1264 "movl %[len], %[rseq_scratch2]\n\t"
1265 /* Start rseq by storing table entry pointer into rseq_cs. */
1266 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1267 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1268 RSEQ_INJECT_ASM(3)
1269 "movl %[expect], %%eax\n\t"
1270 "cmpl %%eax, %[v]\n\t"
1271 "jnz 5f\n\t"
1272 RSEQ_INJECT_ASM(4)
1273 #ifdef RSEQ_COMPARE_TWICE
1274 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
1275 "movl %[expect], %%eax\n\t"
1276 "cmpl %%eax, %[v]\n\t"
1277 "jnz 7f\n\t"
1278 #endif
1279 /* try memcpy */
1280 "test %[len], %[len]\n\t" \
1281 "jz 333f\n\t" \
1282 "222:\n\t" \
1283 "movb (%[src]), %%al\n\t" \
1284 "movb %%al, (%[dst])\n\t" \
1285 "inc %[src]\n\t" \
1286 "inc %[dst]\n\t" \
1287 "dec %[len]\n\t" \
1288 "jnz 222b\n\t" \
1289 "333:\n\t" \
1290 RSEQ_INJECT_ASM(5)
1291 "lock; addl $0,-128(%%esp)\n\t"
1292 "movl %[newv], %%eax\n\t"
1293 /* final store */
1294 "movl %%eax, %[v]\n\t"
1295 "2:\n\t"
1296 RSEQ_INJECT_ASM(6)
1297 /* teardown */
1298 "movl %[rseq_scratch2], %[len]\n\t"
1299 "movl %[rseq_scratch1], %[dst]\n\t"
1300 "movl %[rseq_scratch0], %[src]\n\t"
1301 RSEQ_ASM_DEFINE_ABORT(4,
1302 "movl %[rseq_scratch2], %[len]\n\t"
1303 "movl %[rseq_scratch1], %[dst]\n\t"
1304 "movl %[rseq_scratch0], %[src]\n\t",
1305 abort)
1306 RSEQ_ASM_DEFINE_CMPFAIL(5,
1307 "movl %[rseq_scratch2], %[len]\n\t"
1308 "movl %[rseq_scratch1], %[dst]\n\t"
1309 "movl %[rseq_scratch0], %[src]\n\t",
1310 cmpfail)
1311 #ifdef RSEQ_COMPARE_TWICE
1312 RSEQ_ASM_DEFINE_CMPFAIL(6,
1313 "movl %[rseq_scratch2], %[len]\n\t"
1314 "movl %[rseq_scratch1], %[dst]\n\t"
1315 "movl %[rseq_scratch0], %[src]\n\t",
1316 error1)
1317 RSEQ_ASM_DEFINE_CMPFAIL(7,
1318 "movl %[rseq_scratch2], %[len]\n\t"
1319 "movl %[rseq_scratch1], %[dst]\n\t"
1320 "movl %[rseq_scratch0], %[src]\n\t",
1321 error2)
1322 #endif
1323 : /* gcc asm goto does not allow outputs */
1324 : [cpu_id] "r" (cpu),
1325 [rseq_offset] "r" (rseq_offset),
1326 /* final store input */
1327 [v] "m" (*v),
1328 [expect] "m" (expect),
1329 [newv] "m" (newv),
1330 /* try memcpy input */
1331 [dst] "r" (dst),
1332 [src] "r" (src),
1333 [len] "r" (len),
1334 [rseq_scratch0] "m" (rseq_scratch[0]),
1335 [rseq_scratch1] "m" (rseq_scratch[1]),
1336 [rseq_scratch2] "m" (rseq_scratch[2])
1337 : "memory", "cc", "eax"
1338 RSEQ_INJECT_CLOBBER
1339 : abort, cmpfail
1340 #ifdef RSEQ_COMPARE_TWICE
1341 , error1, error2
1342 #endif
1343 );
1344 rseq_after_asm_goto();
1345 return 0;
1346 abort:
1347 rseq_after_asm_goto();
1348 RSEQ_INJECT_FAILED
1349 return -1;
1350 cmpfail:
1351 rseq_after_asm_goto();
1352 return 1;
1353 #ifdef RSEQ_COMPARE_TWICE
1354 error1:
1355 rseq_after_asm_goto();
1356 rseq_bug("cpu_id comparison failed");
1357 error2:
1358 rseq_after_asm_goto();
1359 rseq_bug("expected value comparison failed");
1360 #endif
1361 }
1362
1363 #endif /* !RSEQ_SKIP_FASTPATH */
1364
1365 #endif
1366