1 // SPDX-License-Identifier: LGPL-2.1
2 /*
3 * rseq.c
4 *
5 * Copyright (C) 2016 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; only
10 * version 2.1 of the License.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 */
17
18 #define _GNU_SOURCE
19 #include <errno.h>
20 #include <sched.h>
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <string.h>
24 #include <unistd.h>
25 #include <syscall.h>
26 #include <assert.h>
27 #include <signal.h>
28 #include <limits.h>
29 #include <dlfcn.h>
30 #include <stddef.h>
31 #include <sys/auxv.h>
32 #include <linux/auxvec.h>
33
34 #include <linux/compiler.h>
35
36 #include "../kselftest.h"
37 #include "rseq.h"
38
39 /*
40 * Define weak versions to play nice with binaries that are statically linked
41 * against a libc that doesn't support registering its own rseq.
42 */
43 __weak ptrdiff_t __rseq_offset;
44 __weak unsigned int __rseq_size;
45 __weak unsigned int __rseq_flags;
46
47 static const ptrdiff_t *libc_rseq_offset_p = &__rseq_offset;
48 static const unsigned int *libc_rseq_size_p = &__rseq_size;
49 static const unsigned int *libc_rseq_flags_p = &__rseq_flags;
50
51 /* Offset from the thread pointer to the rseq area. */
52 ptrdiff_t rseq_offset;
53
54 /*
55 * Size of the registered rseq area. 0 if the registration was
56 * unsuccessful.
57 */
58 unsigned int rseq_size = -1U;
59
60 /* Flags used during rseq registration. */
61 unsigned int rseq_flags;
62
63 /*
64 * rseq feature size supported by the kernel. 0 if the registration was
65 * unsuccessful.
66 */
67 unsigned int rseq_feature_size = -1U;
68
69 static int rseq_ownership;
70 static int rseq_reg_success; /* At least one rseq registration has succeded. */
71
72 /* Allocate a large area for the TLS. */
73 #define RSEQ_THREAD_AREA_ALLOC_SIZE 1024
74
75 /* Original struct rseq feature size is 20 bytes. */
76 #define ORIG_RSEQ_FEATURE_SIZE 20
77
78 /* Original struct rseq allocation size is 32 bytes. */
79 #define ORIG_RSEQ_ALLOC_SIZE 32
80
81 static
82 __thread struct rseq_abi __rseq_abi __attribute__((tls_model("initial-exec"), aligned(RSEQ_THREAD_AREA_ALLOC_SIZE))) = {
83 .cpu_id = RSEQ_ABI_CPU_ID_UNINITIALIZED,
84 };
85
sys_rseq(struct rseq_abi * rseq_abi,uint32_t rseq_len,int flags,uint32_t sig)86 static int sys_rseq(struct rseq_abi *rseq_abi, uint32_t rseq_len,
87 int flags, uint32_t sig)
88 {
89 return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig);
90 }
91
sys_getcpu(unsigned * cpu,unsigned * node)92 static int sys_getcpu(unsigned *cpu, unsigned *node)
93 {
94 return syscall(__NR_getcpu, cpu, node, NULL);
95 }
96
rseq_available(void)97 int rseq_available(void)
98 {
99 int rc;
100
101 rc = sys_rseq(NULL, 0, 0, 0);
102 if (rc != -1)
103 abort();
104 switch (errno) {
105 case ENOSYS:
106 return 0;
107 case EINVAL:
108 return 1;
109 default:
110 abort();
111 }
112 }
113
rseq_register_current_thread(void)114 int rseq_register_current_thread(void)
115 {
116 int rc;
117
118 if (!rseq_ownership) {
119 /* Treat libc's ownership as a successful registration. */
120 return 0;
121 }
122 rc = sys_rseq(&__rseq_abi, rseq_size, 0, RSEQ_SIG);
123 if (rc) {
124 if (RSEQ_READ_ONCE(rseq_reg_success)) {
125 /* Incoherent success/failure within process. */
126 abort();
127 }
128 return -1;
129 }
130 assert(rseq_current_cpu_raw() >= 0);
131 RSEQ_WRITE_ONCE(rseq_reg_success, 1);
132 return 0;
133 }
134
rseq_unregister_current_thread(void)135 int rseq_unregister_current_thread(void)
136 {
137 int rc;
138
139 if (!rseq_ownership) {
140 /* Treat libc's ownership as a successful unregistration. */
141 return 0;
142 }
143 rc = sys_rseq(&__rseq_abi, rseq_size, RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG);
144 if (rc)
145 return -1;
146 return 0;
147 }
148
149 static
get_rseq_feature_size(void)150 unsigned int get_rseq_feature_size(void)
151 {
152 unsigned long auxv_rseq_feature_size, auxv_rseq_align;
153
154 auxv_rseq_align = getauxval(AT_RSEQ_ALIGN);
155 assert(!auxv_rseq_align || auxv_rseq_align <= RSEQ_THREAD_AREA_ALLOC_SIZE);
156
157 auxv_rseq_feature_size = getauxval(AT_RSEQ_FEATURE_SIZE);
158 assert(!auxv_rseq_feature_size || auxv_rseq_feature_size <= RSEQ_THREAD_AREA_ALLOC_SIZE);
159 if (auxv_rseq_feature_size)
160 return auxv_rseq_feature_size;
161 else
162 return ORIG_RSEQ_FEATURE_SIZE;
163 }
164
165 static __attribute__((constructor))
rseq_init(void)166 void rseq_init(void)
167 {
168 /*
169 * If the libc's registered rseq size isn't already valid, it may be
170 * because the binary is dynamically linked and not necessarily due to
171 * libc not having registered a restartable sequence. Try to find the
172 * symbols if that's the case.
173 */
174 if (!*libc_rseq_size_p) {
175 libc_rseq_offset_p = dlsym(RTLD_NEXT, "__rseq_offset");
176 libc_rseq_size_p = dlsym(RTLD_NEXT, "__rseq_size");
177 libc_rseq_flags_p = dlsym(RTLD_NEXT, "__rseq_flags");
178 }
179 if (libc_rseq_size_p && libc_rseq_offset_p && libc_rseq_flags_p &&
180 *libc_rseq_size_p != 0) {
181 /* rseq registration owned by glibc */
182 rseq_offset = *libc_rseq_offset_p;
183 rseq_size = *libc_rseq_size_p;
184 rseq_flags = *libc_rseq_flags_p;
185 rseq_feature_size = get_rseq_feature_size();
186 if (rseq_feature_size > rseq_size)
187 rseq_feature_size = rseq_size;
188 return;
189 }
190 rseq_ownership = 1;
191 if (!rseq_available()) {
192 rseq_size = 0;
193 rseq_feature_size = 0;
194 return;
195 }
196 rseq_offset = (void *)&__rseq_abi - rseq_thread_pointer();
197 rseq_flags = 0;
198 rseq_feature_size = get_rseq_feature_size();
199 if (rseq_feature_size == ORIG_RSEQ_FEATURE_SIZE)
200 rseq_size = ORIG_RSEQ_ALLOC_SIZE;
201 else
202 rseq_size = RSEQ_THREAD_AREA_ALLOC_SIZE;
203 }
204
205 static __attribute__((destructor))
rseq_exit(void)206 void rseq_exit(void)
207 {
208 if (!rseq_ownership)
209 return;
210 rseq_offset = 0;
211 rseq_size = -1U;
212 rseq_feature_size = -1U;
213 rseq_ownership = 0;
214 }
215
rseq_fallback_current_cpu(void)216 int32_t rseq_fallback_current_cpu(void)
217 {
218 int32_t cpu;
219
220 cpu = sched_getcpu();
221 if (cpu < 0) {
222 perror("sched_getcpu()");
223 abort();
224 }
225 return cpu;
226 }
227
rseq_fallback_current_node(void)228 int32_t rseq_fallback_current_node(void)
229 {
230 uint32_t cpu_id, node_id;
231 int ret;
232
233 ret = sys_getcpu(&cpu_id, &node_id);
234 if (ret) {
235 perror("sys_getcpu()");
236 return ret;
237 }
238 return (int32_t) node_id;
239 }
240