1 // SPDX-License-Identifier: GPL-2.0
2 
3 #define pr_fmt(fmt) "rethook: " fmt
4 
5 #include <linux/bug.h>
6 #include <linux/kallsyms.h>
7 #include <linux/kprobes.h>
8 #include <linux/preempt.h>
9 #include <linux/rethook.h>
10 #include <linux/slab.h>
11 #include <linux/sort.h>
12 
13 /* Return hook list (shadow stack by list) */
14 
15 /*
16  * This function is called from delayed_put_task_struct() when a task is
17  * dead and cleaned up to recycle any kretprobe instances associated with
18  * this task. These left over instances represent probed functions that
19  * have been called but will never return.
20  */
rethook_flush_task(struct task_struct * tk)21 void rethook_flush_task(struct task_struct *tk)
22 {
23 	struct rethook_node *rhn;
24 	struct llist_node *node;
25 
26 	node = __llist_del_all(&tk->rethooks);
27 	while (node) {
28 		rhn = container_of(node, struct rethook_node, llist);
29 		node = node->next;
30 		preempt_disable();
31 		rethook_recycle(rhn);
32 		preempt_enable();
33 	}
34 }
35 
rethook_free_rcu(struct rcu_head * head)36 static void rethook_free_rcu(struct rcu_head *head)
37 {
38 	struct rethook *rh = container_of(head, struct rethook, rcu);
39 	struct rethook_node *rhn;
40 	struct freelist_node *node;
41 	int count = 1;
42 
43 	node = rh->pool.head;
44 	while (node) {
45 		rhn = container_of(node, struct rethook_node, freelist);
46 		node = node->next;
47 		kfree(rhn);
48 		count++;
49 	}
50 
51 	/* The rh->ref is the number of pooled node + 1 */
52 	if (refcount_sub_and_test(count, &rh->ref))
53 		kfree(rh);
54 }
55 
56 /**
57  * rethook_stop() - Stop using a rethook.
58  * @rh: the struct rethook to stop.
59  *
60  * Stop using a rethook to prepare for freeing it. If you want to wait for
61  * all running rethook handler before calling rethook_free(), you need to
62  * call this first and wait RCU, and call rethook_free().
63  */
rethook_stop(struct rethook * rh)64 void rethook_stop(struct rethook *rh)
65 {
66 	rcu_assign_pointer(rh->handler, NULL);
67 }
68 
69 /**
70  * rethook_free() - Free struct rethook.
71  * @rh: the struct rethook to be freed.
72  *
73  * Free the rethook. Before calling this function, user must ensure the
74  * @rh::data is cleaned if needed (or, the handler can access it after
75  * calling this function.) This function will set the @rh to be freed
76  * after all rethook_node are freed (not soon). And the caller must
77  * not touch @rh after calling this.
78  */
rethook_free(struct rethook * rh)79 void rethook_free(struct rethook *rh)
80 {
81 	rethook_stop(rh);
82 
83 	call_rcu(&rh->rcu, rethook_free_rcu);
84 }
85 
rethook_get_handler(struct rethook * rh)86 static inline rethook_handler_t rethook_get_handler(struct rethook *rh)
87 {
88 	return (rethook_handler_t)rcu_dereference_check(rh->handler,
89 							rcu_read_lock_any_held());
90 }
91 
92 /**
93  * rethook_alloc() - Allocate struct rethook.
94  * @data: a data to pass the @handler when hooking the return.
95  * @handler: the return hook callback function.
96  *
97  * Allocate and initialize a new rethook with @data and @handler.
98  * Return NULL if memory allocation fails or @handler is NULL.
99  * Note that @handler == NULL means this rethook is going to be freed.
100  */
rethook_alloc(void * data,rethook_handler_t handler)101 struct rethook *rethook_alloc(void *data, rethook_handler_t handler)
102 {
103 	struct rethook *rh = kzalloc(sizeof(struct rethook), GFP_KERNEL);
104 
105 	if (!rh || !handler) {
106 		kfree(rh);
107 		return NULL;
108 	}
109 
110 	rh->data = data;
111 	rcu_assign_pointer(rh->handler, handler);
112 	rh->pool.head = NULL;
113 	refcount_set(&rh->ref, 1);
114 
115 	return rh;
116 }
117 
118 /**
119  * rethook_add_node() - Add a new node to the rethook.
120  * @rh: the struct rethook.
121  * @node: the struct rethook_node to be added.
122  *
123  * Add @node to @rh. User must allocate @node (as a part of user's
124  * data structure.) The @node fields are initialized in this function.
125  */
rethook_add_node(struct rethook * rh,struct rethook_node * node)126 void rethook_add_node(struct rethook *rh, struct rethook_node *node)
127 {
128 	node->rethook = rh;
129 	freelist_add(&node->freelist, &rh->pool);
130 	refcount_inc(&rh->ref);
131 }
132 
free_rethook_node_rcu(struct rcu_head * head)133 static void free_rethook_node_rcu(struct rcu_head *head)
134 {
135 	struct rethook_node *node = container_of(head, struct rethook_node, rcu);
136 
137 	if (refcount_dec_and_test(&node->rethook->ref))
138 		kfree(node->rethook);
139 	kfree(node);
140 }
141 
142 /**
143  * rethook_recycle() - return the node to rethook.
144  * @node: The struct rethook_node to be returned.
145  *
146  * Return back the @node to @node::rethook. If the @node::rethook is already
147  * marked as freed, this will free the @node.
148  */
rethook_recycle(struct rethook_node * node)149 void rethook_recycle(struct rethook_node *node)
150 {
151 	rethook_handler_t handler;
152 
153 	handler = rethook_get_handler(node->rethook);
154 	if (likely(handler))
155 		freelist_add(&node->freelist, &node->rethook->pool);
156 	else
157 		call_rcu(&node->rcu, free_rethook_node_rcu);
158 }
159 NOKPROBE_SYMBOL(rethook_recycle);
160 
161 /**
162  * rethook_try_get() - get an unused rethook node.
163  * @rh: The struct rethook which pools the nodes.
164  *
165  * Get an unused rethook node from @rh. If the node pool is empty, this
166  * will return NULL. Caller must disable preemption.
167  */
rethook_try_get(struct rethook * rh)168 struct rethook_node *rethook_try_get(struct rethook *rh)
169 {
170 	rethook_handler_t handler = rethook_get_handler(rh);
171 	struct freelist_node *fn;
172 
173 	/* Check whether @rh is going to be freed. */
174 	if (unlikely(!handler))
175 		return NULL;
176 
177 	/*
178 	 * This expects the caller will set up a rethook on a function entry.
179 	 * When the function returns, the rethook will eventually be reclaimed
180 	 * or released in the rethook_recycle() with call_rcu().
181 	 * This means the caller must be run in the RCU-availabe context.
182 	 */
183 	if (unlikely(!rcu_is_watching()))
184 		return NULL;
185 
186 	fn = freelist_try_get(&rh->pool);
187 	if (!fn)
188 		return NULL;
189 
190 	return container_of(fn, struct rethook_node, freelist);
191 }
192 NOKPROBE_SYMBOL(rethook_try_get);
193 
194 /**
195  * rethook_hook() - Hook the current function return.
196  * @node: The struct rethook node to hook the function return.
197  * @regs: The struct pt_regs for the function entry.
198  * @mcount: True if this is called from mcount(ftrace) context.
199  *
200  * Hook the current running function return. This must be called when the
201  * function entry (or at least @regs must be the registers of the function
202  * entry.) @mcount is used for identifying the context. If this is called
203  * from ftrace (mcount) callback, @mcount must be set true. If this is called
204  * from the real function entry (e.g. kprobes) @mcount must be set false.
205  * This is because the way to hook the function return depends on the context.
206  */
rethook_hook(struct rethook_node * node,struct pt_regs * regs,bool mcount)207 void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount)
208 {
209 	arch_rethook_prepare(node, regs, mcount);
210 	__llist_add(&node->llist, &current->rethooks);
211 }
212 NOKPROBE_SYMBOL(rethook_hook);
213 
214 /* This assumes the 'tsk' is the current task or is not running. */
__rethook_find_ret_addr(struct task_struct * tsk,struct llist_node ** cur)215 static unsigned long __rethook_find_ret_addr(struct task_struct *tsk,
216 					     struct llist_node **cur)
217 {
218 	struct rethook_node *rh = NULL;
219 	struct llist_node *node = *cur;
220 
221 	if (!node)
222 		node = tsk->rethooks.first;
223 	else
224 		node = node->next;
225 
226 	while (node) {
227 		rh = container_of(node, struct rethook_node, llist);
228 		if (rh->ret_addr != (unsigned long)arch_rethook_trampoline) {
229 			*cur = node;
230 			return rh->ret_addr;
231 		}
232 		node = node->next;
233 	}
234 	return 0;
235 }
236 NOKPROBE_SYMBOL(__rethook_find_ret_addr);
237 
238 /**
239  * rethook_find_ret_addr -- Find correct return address modified by rethook
240  * @tsk: Target task
241  * @frame: A frame pointer
242  * @cur: a storage of the loop cursor llist_node pointer for next call
243  *
244  * Find the correct return address modified by a rethook on @tsk in unsigned
245  * long type.
246  * The @tsk must be 'current' or a task which is not running. @frame is a hint
247  * to get the currect return address - which is compared with the
248  * rethook::frame field. The @cur is a loop cursor for searching the
249  * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
250  * first call, but '@cur' itself must NOT NULL.
251  *
252  * Returns found address value or zero if not found.
253  */
rethook_find_ret_addr(struct task_struct * tsk,unsigned long frame,struct llist_node ** cur)254 unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame,
255 				    struct llist_node **cur)
256 {
257 	struct rethook_node *rhn = NULL;
258 	unsigned long ret;
259 
260 	if (WARN_ON_ONCE(!cur))
261 		return 0;
262 
263 	if (WARN_ON_ONCE(tsk != current && task_is_running(tsk)))
264 		return 0;
265 
266 	do {
267 		ret = __rethook_find_ret_addr(tsk, cur);
268 		if (!ret)
269 			break;
270 		rhn = container_of(*cur, struct rethook_node, llist);
271 	} while (rhn->frame != frame);
272 
273 	return ret;
274 }
275 NOKPROBE_SYMBOL(rethook_find_ret_addr);
276 
arch_rethook_fixup_return(struct pt_regs * regs,unsigned long correct_ret_addr)277 void __weak arch_rethook_fixup_return(struct pt_regs *regs,
278 				      unsigned long correct_ret_addr)
279 {
280 	/*
281 	 * Do nothing by default. If the architecture which uses a
282 	 * frame pointer to record real return address on the stack,
283 	 * it should fill this function to fixup the return address
284 	 * so that stacktrace works from the rethook handler.
285 	 */
286 }
287 
288 /* This function will be called from each arch-defined trampoline. */
rethook_trampoline_handler(struct pt_regs * regs,unsigned long frame)289 unsigned long rethook_trampoline_handler(struct pt_regs *regs,
290 					 unsigned long frame)
291 {
292 	struct llist_node *first, *node = NULL;
293 	unsigned long correct_ret_addr;
294 	rethook_handler_t handler;
295 	struct rethook_node *rhn;
296 
297 	correct_ret_addr = __rethook_find_ret_addr(current, &node);
298 	if (!correct_ret_addr) {
299 		pr_err("rethook: Return address not found! Maybe there is a bug in the kernel\n");
300 		BUG_ON(1);
301 	}
302 
303 	instruction_pointer_set(regs, correct_ret_addr);
304 
305 	/*
306 	 * These loops must be protected from rethook_free_rcu() because those
307 	 * are accessing 'rhn->rethook'.
308 	 */
309 	preempt_disable_notrace();
310 
311 	/*
312 	 * Run the handler on the shadow stack. Do not unlink the list here because
313 	 * stackdump inside the handlers needs to decode it.
314 	 */
315 	first = current->rethooks.first;
316 	while (first) {
317 		rhn = container_of(first, struct rethook_node, llist);
318 		if (WARN_ON_ONCE(rhn->frame != frame))
319 			break;
320 		handler = rethook_get_handler(rhn->rethook);
321 		if (handler)
322 			handler(rhn, rhn->rethook->data,
323 				correct_ret_addr, regs);
324 
325 		if (first == node)
326 			break;
327 		first = first->next;
328 	}
329 
330 	/* Fixup registers for returning to correct address. */
331 	arch_rethook_fixup_return(regs, correct_ret_addr);
332 
333 	/* Unlink used shadow stack */
334 	first = current->rethooks.first;
335 	current->rethooks.first = node->next;
336 	node->next = NULL;
337 
338 	while (first) {
339 		rhn = container_of(first, struct rethook_node, llist);
340 		first = first->next;
341 		rethook_recycle(rhn);
342 	}
343 	preempt_enable_notrace();
344 
345 	return correct_ret_addr;
346 }
347 NOKPROBE_SYMBOL(rethook_trampoline_handler);
348