1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2008-2014 Mathieu Desnoyers
4  */
5 #include <linux/module.h>
6 #include <linux/mutex.h>
7 #include <linux/types.h>
8 #include <linux/jhash.h>
9 #include <linux/list.h>
10 #include <linux/rcupdate.h>
11 #include <linux/tracepoint.h>
12 #include <linux/err.h>
13 #include <linux/slab.h>
14 #include <linux/sched/signal.h>
15 #include <linux/sched/task.h>
16 #include <linux/static_key.h>
17 
18 enum tp_func_state {
19 	TP_FUNC_0,
20 	TP_FUNC_1,
21 	TP_FUNC_2,
22 	TP_FUNC_N,
23 };
24 
25 extern tracepoint_ptr_t __start___tracepoints_ptrs[];
26 extern tracepoint_ptr_t __stop___tracepoints_ptrs[];
27 
28 DEFINE_SRCU(tracepoint_srcu);
29 EXPORT_SYMBOL_GPL(tracepoint_srcu);
30 
31 enum tp_transition_sync {
32 	TP_TRANSITION_SYNC_1_0_1,
33 	TP_TRANSITION_SYNC_N_2_1,
34 
35 	_NR_TP_TRANSITION_SYNC,
36 };
37 
38 struct tp_transition_snapshot {
39 	unsigned long rcu;
40 	unsigned long srcu;
41 	bool ongoing;
42 };
43 
44 /* Protected by tracepoints_mutex */
45 static struct tp_transition_snapshot tp_transition_snapshot[_NR_TP_TRANSITION_SYNC];
46 
tp_rcu_get_state(enum tp_transition_sync sync)47 static void tp_rcu_get_state(enum tp_transition_sync sync)
48 {
49 	struct tp_transition_snapshot *snapshot = &tp_transition_snapshot[sync];
50 
51 	/* Keep the latest get_state snapshot. */
52 	snapshot->rcu = get_state_synchronize_rcu();
53 	snapshot->srcu = start_poll_synchronize_srcu(&tracepoint_srcu);
54 	snapshot->ongoing = true;
55 }
56 
tp_rcu_cond_sync(enum tp_transition_sync sync)57 static void tp_rcu_cond_sync(enum tp_transition_sync sync)
58 {
59 	struct tp_transition_snapshot *snapshot = &tp_transition_snapshot[sync];
60 
61 	if (!snapshot->ongoing)
62 		return;
63 	cond_synchronize_rcu(snapshot->rcu);
64 	if (!poll_state_synchronize_srcu(&tracepoint_srcu, snapshot->srcu))
65 		synchronize_srcu(&tracepoint_srcu);
66 	snapshot->ongoing = false;
67 }
68 
69 /* Set to 1 to enable tracepoint debug output */
70 static const int tracepoint_debug;
71 
72 #ifdef CONFIG_MODULES
73 /*
74  * Tracepoint module list mutex protects the local module list.
75  */
76 static DEFINE_MUTEX(tracepoint_module_list_mutex);
77 
78 /* Local list of struct tp_module */
79 static LIST_HEAD(tracepoint_module_list);
80 #endif /* CONFIG_MODULES */
81 
82 /*
83  * tracepoints_mutex protects the builtin and module tracepoints.
84  * tracepoints_mutex nests inside tracepoint_module_list_mutex.
85  */
86 static DEFINE_MUTEX(tracepoints_mutex);
87 
88 static struct rcu_head *early_probes;
89 static bool ok_to_free_tracepoints;
90 
91 /*
92  * Note about RCU :
93  * It is used to delay the free of multiple probes array until a quiescent
94  * state is reached.
95  */
96 struct tp_probes {
97 	struct rcu_head rcu;
98 	struct tracepoint_func probes[];
99 };
100 
101 /* Called in removal of a func but failed to allocate a new tp_funcs */
tp_stub_func(void)102 static void tp_stub_func(void)
103 {
104 	return;
105 }
106 
allocate_probes(int count)107 static inline void *allocate_probes(int count)
108 {
109 	struct tp_probes *p  = kmalloc(struct_size(p, probes, count),
110 				       GFP_KERNEL);
111 	return p == NULL ? NULL : p->probes;
112 }
113 
srcu_free_old_probes(struct rcu_head * head)114 static void srcu_free_old_probes(struct rcu_head *head)
115 {
116 	kfree(container_of(head, struct tp_probes, rcu));
117 }
118 
rcu_free_old_probes(struct rcu_head * head)119 static void rcu_free_old_probes(struct rcu_head *head)
120 {
121 	call_srcu(&tracepoint_srcu, head, srcu_free_old_probes);
122 }
123 
release_early_probes(void)124 static __init int release_early_probes(void)
125 {
126 	struct rcu_head *tmp;
127 
128 	ok_to_free_tracepoints = true;
129 
130 	while (early_probes) {
131 		tmp = early_probes;
132 		early_probes = tmp->next;
133 		call_rcu(tmp, rcu_free_old_probes);
134 	}
135 
136 	return 0;
137 }
138 
139 /* SRCU is initialized at core_initcall */
140 postcore_initcall(release_early_probes);
141 
release_probes(struct tracepoint_func * old)142 static inline void release_probes(struct tracepoint_func *old)
143 {
144 	if (old) {
145 		struct tp_probes *tp_probes = container_of(old,
146 			struct tp_probes, probes[0]);
147 
148 		/*
149 		 * We can't free probes if SRCU is not initialized yet.
150 		 * Postpone the freeing till after SRCU is initialized.
151 		 */
152 		if (unlikely(!ok_to_free_tracepoints)) {
153 			tp_probes->rcu.next = early_probes;
154 			early_probes = &tp_probes->rcu;
155 			return;
156 		}
157 
158 		/*
159 		 * Tracepoint probes are protected by both sched RCU and SRCU,
160 		 * by calling the SRCU callback in the sched RCU callback we
161 		 * cover both cases. So let us chain the SRCU and sched RCU
162 		 * callbacks to wait for both grace periods.
163 		 */
164 		call_rcu(&tp_probes->rcu, rcu_free_old_probes);
165 	}
166 }
167 
debug_print_probes(struct tracepoint_func * funcs)168 static void debug_print_probes(struct tracepoint_func *funcs)
169 {
170 	int i;
171 
172 	if (!tracepoint_debug || !funcs)
173 		return;
174 
175 	for (i = 0; funcs[i].func; i++)
176 		printk(KERN_DEBUG "Probe %d : %p\n", i, funcs[i].func);
177 }
178 
179 static struct tracepoint_func *
func_add(struct tracepoint_func ** funcs,struct tracepoint_func * tp_func,int prio)180 func_add(struct tracepoint_func **funcs, struct tracepoint_func *tp_func,
181 	 int prio)
182 {
183 	struct tracepoint_func *old, *new;
184 	int iter_probes;	/* Iterate over old probe array. */
185 	int nr_probes = 0;	/* Counter for probes */
186 	int pos = -1;		/* Insertion position into new array */
187 
188 	if (WARN_ON(!tp_func->func))
189 		return ERR_PTR(-EINVAL);
190 
191 	debug_print_probes(*funcs);
192 	old = *funcs;
193 	if (old) {
194 		/* (N -> N+1), (N != 0, 1) probes */
195 		for (iter_probes = 0; old[iter_probes].func; iter_probes++) {
196 			if (old[iter_probes].func == tp_stub_func)
197 				continue;	/* Skip stub functions. */
198 			if (old[iter_probes].func == tp_func->func &&
199 			    old[iter_probes].data == tp_func->data)
200 				return ERR_PTR(-EEXIST);
201 			nr_probes++;
202 		}
203 	}
204 	/* + 2 : one for new probe, one for NULL func */
205 	new = allocate_probes(nr_probes + 2);
206 	if (new == NULL)
207 		return ERR_PTR(-ENOMEM);
208 	if (old) {
209 		nr_probes = 0;
210 		for (iter_probes = 0; old[iter_probes].func; iter_probes++) {
211 			if (old[iter_probes].func == tp_stub_func)
212 				continue;
213 			/* Insert before probes of lower priority */
214 			if (pos < 0 && old[iter_probes].prio < prio)
215 				pos = nr_probes++;
216 			new[nr_probes++] = old[iter_probes];
217 		}
218 		if (pos < 0)
219 			pos = nr_probes++;
220 		/* nr_probes now points to the end of the new array */
221 	} else {
222 		pos = 0;
223 		nr_probes = 1; /* must point at end of array */
224 	}
225 	new[pos] = *tp_func;
226 	new[nr_probes].func = NULL;
227 	*funcs = new;
228 	debug_print_probes(*funcs);
229 	return old;
230 }
231 
func_remove(struct tracepoint_func ** funcs,struct tracepoint_func * tp_func)232 static void *func_remove(struct tracepoint_func **funcs,
233 		struct tracepoint_func *tp_func)
234 {
235 	int nr_probes = 0, nr_del = 0, i;
236 	struct tracepoint_func *old, *new;
237 
238 	old = *funcs;
239 
240 	if (!old)
241 		return ERR_PTR(-ENOENT);
242 
243 	debug_print_probes(*funcs);
244 	/* (N -> M), (N > 1, M >= 0) probes */
245 	if (tp_func->func) {
246 		for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
247 			if ((old[nr_probes].func == tp_func->func &&
248 			     old[nr_probes].data == tp_func->data) ||
249 			    old[nr_probes].func == tp_stub_func)
250 				nr_del++;
251 		}
252 	}
253 
254 	/*
255 	 * If probe is NULL, then nr_probes = nr_del = 0, and then the
256 	 * entire entry will be removed.
257 	 */
258 	if (nr_probes - nr_del == 0) {
259 		/* N -> 0, (N > 1) */
260 		*funcs = NULL;
261 		debug_print_probes(*funcs);
262 		return old;
263 	} else {
264 		int j = 0;
265 		/* N -> M, (N > 1, M > 0) */
266 		/* + 1 for NULL */
267 		new = allocate_probes(nr_probes - nr_del + 1);
268 		if (new) {
269 			for (i = 0; old[i].func; i++) {
270 				if ((old[i].func != tp_func->func ||
271 				     old[i].data != tp_func->data) &&
272 				    old[i].func != tp_stub_func)
273 					new[j++] = old[i];
274 			}
275 			new[nr_probes - nr_del].func = NULL;
276 			*funcs = new;
277 		} else {
278 			/*
279 			 * Failed to allocate, replace the old function
280 			 * with calls to tp_stub_func.
281 			 */
282 			for (i = 0; old[i].func; i++) {
283 				if (old[i].func == tp_func->func &&
284 				    old[i].data == tp_func->data)
285 					WRITE_ONCE(old[i].func, tp_stub_func);
286 			}
287 			*funcs = old;
288 		}
289 	}
290 	debug_print_probes(*funcs);
291 	return old;
292 }
293 
294 /*
295  * Count the number of functions (enum tp_func_state) in a tp_funcs array.
296  */
nr_func_state(const struct tracepoint_func * tp_funcs)297 static enum tp_func_state nr_func_state(const struct tracepoint_func *tp_funcs)
298 {
299 	if (!tp_funcs)
300 		return TP_FUNC_0;
301 	if (!tp_funcs[1].func)
302 		return TP_FUNC_1;
303 	if (!tp_funcs[2].func)
304 		return TP_FUNC_2;
305 	return TP_FUNC_N;	/* 3 or more */
306 }
307 
tracepoint_update_call(struct tracepoint * tp,struct tracepoint_func * tp_funcs)308 static void tracepoint_update_call(struct tracepoint *tp, struct tracepoint_func *tp_funcs)
309 {
310 	void *func = tp->iterator;
311 
312 	/* Synthetic events do not have static call sites */
313 	if (!tp->static_call_key)
314 		return;
315 	if (nr_func_state(tp_funcs) == TP_FUNC_1)
316 		func = tp_funcs[0].func;
317 	__static_call_update(tp->static_call_key, tp->static_call_tramp, func);
318 }
319 
320 /*
321  * Add the probe function to a tracepoint.
322  */
tracepoint_add_func(struct tracepoint * tp,struct tracepoint_func * func,int prio,bool warn)323 static int tracepoint_add_func(struct tracepoint *tp,
324 			       struct tracepoint_func *func, int prio,
325 			       bool warn)
326 {
327 	struct tracepoint_func *old, *tp_funcs;
328 	int ret;
329 
330 	if (tp->regfunc && !static_key_enabled(&tp->key)) {
331 		ret = tp->regfunc();
332 		if (ret < 0)
333 			return ret;
334 	}
335 
336 	tp_funcs = rcu_dereference_protected(tp->funcs,
337 			lockdep_is_held(&tracepoints_mutex));
338 	old = func_add(&tp_funcs, func, prio);
339 	if (IS_ERR(old)) {
340 		WARN_ON_ONCE(warn && PTR_ERR(old) != -ENOMEM);
341 		return PTR_ERR(old);
342 	}
343 
344 	/*
345 	 * rcu_assign_pointer has as smp_store_release() which makes sure
346 	 * that the new probe callbacks array is consistent before setting
347 	 * a pointer to it.  This array is referenced by __DO_TRACE from
348 	 * include/linux/tracepoint.h using rcu_dereference_sched().
349 	 */
350 	switch (nr_func_state(tp_funcs)) {
351 	case TP_FUNC_1:		/* 0->1 */
352 		/*
353 		 * Make sure new static func never uses old data after a
354 		 * 1->0->1 transition sequence.
355 		 */
356 		tp_rcu_cond_sync(TP_TRANSITION_SYNC_1_0_1);
357 		/* Set static call to first function */
358 		tracepoint_update_call(tp, tp_funcs);
359 		/* Both iterator and static call handle NULL tp->funcs */
360 		rcu_assign_pointer(tp->funcs, tp_funcs);
361 		static_key_enable(&tp->key);
362 		break;
363 	case TP_FUNC_2:		/* 1->2 */
364 		/* Set iterator static call */
365 		tracepoint_update_call(tp, tp_funcs);
366 		/*
367 		 * Iterator callback installed before updating tp->funcs.
368 		 * Requires ordering between RCU assign/dereference and
369 		 * static call update/call.
370 		 */
371 		fallthrough;
372 	case TP_FUNC_N:		/* N->N+1 (N>1) */
373 		rcu_assign_pointer(tp->funcs, tp_funcs);
374 		/*
375 		 * Make sure static func never uses incorrect data after a
376 		 * N->...->2->1 (N>1) transition sequence.
377 		 */
378 		if (tp_funcs[0].data != old[0].data)
379 			tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
380 		break;
381 	default:
382 		WARN_ON_ONCE(1);
383 		break;
384 	}
385 
386 	release_probes(old);
387 	return 0;
388 }
389 
390 /*
391  * Remove a probe function from a tracepoint.
392  * Note: only waiting an RCU period after setting elem->call to the empty
393  * function insures that the original callback is not used anymore. This insured
394  * by preempt_disable around the call site.
395  */
tracepoint_remove_func(struct tracepoint * tp,struct tracepoint_func * func)396 static int tracepoint_remove_func(struct tracepoint *tp,
397 		struct tracepoint_func *func)
398 {
399 	struct tracepoint_func *old, *tp_funcs;
400 
401 	tp_funcs = rcu_dereference_protected(tp->funcs,
402 			lockdep_is_held(&tracepoints_mutex));
403 	old = func_remove(&tp_funcs, func);
404 	if (WARN_ON_ONCE(IS_ERR(old)))
405 		return PTR_ERR(old);
406 
407 	if (tp_funcs == old)
408 		/* Failed allocating new tp_funcs, replaced func with stub */
409 		return 0;
410 
411 	switch (nr_func_state(tp_funcs)) {
412 	case TP_FUNC_0:		/* 1->0 */
413 		/* Removed last function */
414 		if (tp->unregfunc && static_key_enabled(&tp->key))
415 			tp->unregfunc();
416 
417 		static_key_disable(&tp->key);
418 		/* Set iterator static call */
419 		tracepoint_update_call(tp, tp_funcs);
420 		/* Both iterator and static call handle NULL tp->funcs */
421 		rcu_assign_pointer(tp->funcs, NULL);
422 		/*
423 		 * Make sure new static func never uses old data after a
424 		 * 1->0->1 transition sequence.
425 		 */
426 		tp_rcu_get_state(TP_TRANSITION_SYNC_1_0_1);
427 		break;
428 	case TP_FUNC_1:		/* 2->1 */
429 		rcu_assign_pointer(tp->funcs, tp_funcs);
430 		/*
431 		 * Make sure static func never uses incorrect data after a
432 		 * N->...->2->1 (N>2) transition sequence. If the first
433 		 * element's data has changed, then force the synchronization
434 		 * to prevent current readers that have loaded the old data
435 		 * from calling the new function.
436 		 */
437 		if (tp_funcs[0].data != old[0].data)
438 			tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
439 		tp_rcu_cond_sync(TP_TRANSITION_SYNC_N_2_1);
440 		/* Set static call to first function */
441 		tracepoint_update_call(tp, tp_funcs);
442 		break;
443 	case TP_FUNC_2:		/* N->N-1 (N>2) */
444 		fallthrough;
445 	case TP_FUNC_N:
446 		rcu_assign_pointer(tp->funcs, tp_funcs);
447 		/*
448 		 * Make sure static func never uses incorrect data after a
449 		 * N->...->2->1 (N>2) transition sequence.
450 		 */
451 		if (tp_funcs[0].data != old[0].data)
452 			tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
453 		break;
454 	default:
455 		WARN_ON_ONCE(1);
456 		break;
457 	}
458 	release_probes(old);
459 	return 0;
460 }
461 
462 /**
463  * tracepoint_probe_register_prio_may_exist -  Connect a probe to a tracepoint with priority
464  * @tp: tracepoint
465  * @probe: probe handler
466  * @data: tracepoint data
467  * @prio: priority of this function over other registered functions
468  *
469  * Same as tracepoint_probe_register_prio() except that it will not warn
470  * if the tracepoint is already registered.
471  */
tracepoint_probe_register_prio_may_exist(struct tracepoint * tp,void * probe,void * data,int prio)472 int tracepoint_probe_register_prio_may_exist(struct tracepoint *tp, void *probe,
473 					     void *data, int prio)
474 {
475 	struct tracepoint_func tp_func;
476 	int ret;
477 
478 	mutex_lock(&tracepoints_mutex);
479 	tp_func.func = probe;
480 	tp_func.data = data;
481 	tp_func.prio = prio;
482 	ret = tracepoint_add_func(tp, &tp_func, prio, false);
483 	mutex_unlock(&tracepoints_mutex);
484 	return ret;
485 }
486 EXPORT_SYMBOL_GPL(tracepoint_probe_register_prio_may_exist);
487 
488 /**
489  * tracepoint_probe_register_prio -  Connect a probe to a tracepoint with priority
490  * @tp: tracepoint
491  * @probe: probe handler
492  * @data: tracepoint data
493  * @prio: priority of this function over other registered functions
494  *
495  * Returns 0 if ok, error value on error.
496  * Note: if @tp is within a module, the caller is responsible for
497  * unregistering the probe before the module is gone. This can be
498  * performed either with a tracepoint module going notifier, or from
499  * within module exit functions.
500  */
tracepoint_probe_register_prio(struct tracepoint * tp,void * probe,void * data,int prio)501 int tracepoint_probe_register_prio(struct tracepoint *tp, void *probe,
502 				   void *data, int prio)
503 {
504 	struct tracepoint_func tp_func;
505 	int ret;
506 
507 	mutex_lock(&tracepoints_mutex);
508 	tp_func.func = probe;
509 	tp_func.data = data;
510 	tp_func.prio = prio;
511 	ret = tracepoint_add_func(tp, &tp_func, prio, true);
512 	mutex_unlock(&tracepoints_mutex);
513 	return ret;
514 }
515 EXPORT_SYMBOL_GPL(tracepoint_probe_register_prio);
516 
517 /**
518  * tracepoint_probe_register -  Connect a probe to a tracepoint
519  * @tp: tracepoint
520  * @probe: probe handler
521  * @data: tracepoint data
522  *
523  * Returns 0 if ok, error value on error.
524  * Note: if @tp is within a module, the caller is responsible for
525  * unregistering the probe before the module is gone. This can be
526  * performed either with a tracepoint module going notifier, or from
527  * within module exit functions.
528  */
tracepoint_probe_register(struct tracepoint * tp,void * probe,void * data)529 int tracepoint_probe_register(struct tracepoint *tp, void *probe, void *data)
530 {
531 	return tracepoint_probe_register_prio(tp, probe, data, TRACEPOINT_DEFAULT_PRIO);
532 }
533 EXPORT_SYMBOL_GPL(tracepoint_probe_register);
534 
535 /**
536  * tracepoint_probe_unregister -  Disconnect a probe from a tracepoint
537  * @tp: tracepoint
538  * @probe: probe function pointer
539  * @data: tracepoint data
540  *
541  * Returns 0 if ok, error value on error.
542  */
tracepoint_probe_unregister(struct tracepoint * tp,void * probe,void * data)543 int tracepoint_probe_unregister(struct tracepoint *tp, void *probe, void *data)
544 {
545 	struct tracepoint_func tp_func;
546 	int ret;
547 
548 	mutex_lock(&tracepoints_mutex);
549 	tp_func.func = probe;
550 	tp_func.data = data;
551 	ret = tracepoint_remove_func(tp, &tp_func);
552 	mutex_unlock(&tracepoints_mutex);
553 	return ret;
554 }
555 EXPORT_SYMBOL_GPL(tracepoint_probe_unregister);
556 
for_each_tracepoint_range(tracepoint_ptr_t * begin,tracepoint_ptr_t * end,void (* fct)(struct tracepoint * tp,void * priv),void * priv)557 static void for_each_tracepoint_range(
558 		tracepoint_ptr_t *begin, tracepoint_ptr_t *end,
559 		void (*fct)(struct tracepoint *tp, void *priv),
560 		void *priv)
561 {
562 	tracepoint_ptr_t *iter;
563 
564 	if (!begin)
565 		return;
566 	for (iter = begin; iter < end; iter++)
567 		fct(tracepoint_ptr_deref(iter), priv);
568 }
569 
570 #ifdef CONFIG_MODULES
trace_module_has_bad_taint(struct module * mod)571 bool trace_module_has_bad_taint(struct module *mod)
572 {
573 	return mod->taints & ~((1 << TAINT_OOT_MODULE) | (1 << TAINT_CRAP) |
574 				(1 << TAINT_UNSIGNED_MODULE) | (1 << TAINT_TEST) |
575 				(1 << TAINT_LIVEPATCH));
576 }
577 
578 static BLOCKING_NOTIFIER_HEAD(tracepoint_notify_list);
579 
580 /**
581  * register_tracepoint_module_notifier - register tracepoint coming/going notifier
582  * @nb: notifier block
583  *
584  * Notifiers registered with this function are called on module
585  * coming/going with the tracepoint_module_list_mutex held.
586  * The notifier block callback should expect a "struct tp_module" data
587  * pointer.
588  */
register_tracepoint_module_notifier(struct notifier_block * nb)589 int register_tracepoint_module_notifier(struct notifier_block *nb)
590 {
591 	struct tp_module *tp_mod;
592 	int ret;
593 
594 	mutex_lock(&tracepoint_module_list_mutex);
595 	ret = blocking_notifier_chain_register(&tracepoint_notify_list, nb);
596 	if (ret)
597 		goto end;
598 	list_for_each_entry(tp_mod, &tracepoint_module_list, list)
599 		(void) nb->notifier_call(nb, MODULE_STATE_COMING, tp_mod);
600 end:
601 	mutex_unlock(&tracepoint_module_list_mutex);
602 	return ret;
603 }
604 EXPORT_SYMBOL_GPL(register_tracepoint_module_notifier);
605 
606 /**
607  * unregister_tracepoint_module_notifier - unregister tracepoint coming/going notifier
608  * @nb: notifier block
609  *
610  * The notifier block callback should expect a "struct tp_module" data
611  * pointer.
612  */
unregister_tracepoint_module_notifier(struct notifier_block * nb)613 int unregister_tracepoint_module_notifier(struct notifier_block *nb)
614 {
615 	struct tp_module *tp_mod;
616 	int ret;
617 
618 	mutex_lock(&tracepoint_module_list_mutex);
619 	ret = blocking_notifier_chain_unregister(&tracepoint_notify_list, nb);
620 	if (ret)
621 		goto end;
622 	list_for_each_entry(tp_mod, &tracepoint_module_list, list)
623 		(void) nb->notifier_call(nb, MODULE_STATE_GOING, tp_mod);
624 end:
625 	mutex_unlock(&tracepoint_module_list_mutex);
626 	return ret;
627 
628 }
629 EXPORT_SYMBOL_GPL(unregister_tracepoint_module_notifier);
630 
631 /*
632  * Ensure the tracer unregistered the module's probes before the module
633  * teardown is performed. Prevents leaks of probe and data pointers.
634  */
tp_module_going_check_quiescent(struct tracepoint * tp,void * priv)635 static void tp_module_going_check_quiescent(struct tracepoint *tp, void *priv)
636 {
637 	WARN_ON_ONCE(tp->funcs);
638 }
639 
tracepoint_module_coming(struct module * mod)640 static int tracepoint_module_coming(struct module *mod)
641 {
642 	struct tp_module *tp_mod;
643 
644 	if (!mod->num_tracepoints)
645 		return 0;
646 
647 	/*
648 	 * We skip modules that taint the kernel, especially those with different
649 	 * module headers (for forced load), to make sure we don't cause a crash.
650 	 * Staging, out-of-tree, unsigned GPL, and test modules are fine.
651 	 */
652 	if (trace_module_has_bad_taint(mod))
653 		return 0;
654 
655 	tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL);
656 	if (!tp_mod)
657 		return -ENOMEM;
658 	tp_mod->mod = mod;
659 
660 	mutex_lock(&tracepoint_module_list_mutex);
661 	list_add_tail(&tp_mod->list, &tracepoint_module_list);
662 	blocking_notifier_call_chain(&tracepoint_notify_list,
663 			MODULE_STATE_COMING, tp_mod);
664 	mutex_unlock(&tracepoint_module_list_mutex);
665 	return 0;
666 }
667 
tracepoint_module_going(struct module * mod)668 static void tracepoint_module_going(struct module *mod)
669 {
670 	struct tp_module *tp_mod;
671 
672 	if (!mod->num_tracepoints)
673 		return;
674 
675 	mutex_lock(&tracepoint_module_list_mutex);
676 	list_for_each_entry(tp_mod, &tracepoint_module_list, list) {
677 		if (tp_mod->mod == mod) {
678 			blocking_notifier_call_chain(&tracepoint_notify_list,
679 					MODULE_STATE_GOING, tp_mod);
680 			list_del(&tp_mod->list);
681 			kfree(tp_mod);
682 			/*
683 			 * Called the going notifier before checking for
684 			 * quiescence.
685 			 */
686 			for_each_tracepoint_range(mod->tracepoints_ptrs,
687 				mod->tracepoints_ptrs + mod->num_tracepoints,
688 				tp_module_going_check_quiescent, NULL);
689 			break;
690 		}
691 	}
692 	/*
693 	 * In the case of modules that were tainted at "coming", we'll simply
694 	 * walk through the list without finding it. We cannot use the "tainted"
695 	 * flag on "going", in case a module taints the kernel only after being
696 	 * loaded.
697 	 */
698 	mutex_unlock(&tracepoint_module_list_mutex);
699 }
700 
tracepoint_module_notify(struct notifier_block * self,unsigned long val,void * data)701 static int tracepoint_module_notify(struct notifier_block *self,
702 		unsigned long val, void *data)
703 {
704 	struct module *mod = data;
705 	int ret = 0;
706 
707 	switch (val) {
708 	case MODULE_STATE_COMING:
709 		ret = tracepoint_module_coming(mod);
710 		break;
711 	case MODULE_STATE_LIVE:
712 		break;
713 	case MODULE_STATE_GOING:
714 		tracepoint_module_going(mod);
715 		break;
716 	case MODULE_STATE_UNFORMED:
717 		break;
718 	}
719 	return notifier_from_errno(ret);
720 }
721 
722 static struct notifier_block tracepoint_module_nb = {
723 	.notifier_call = tracepoint_module_notify,
724 	.priority = 0,
725 };
726 
init_tracepoints(void)727 static __init int init_tracepoints(void)
728 {
729 	int ret;
730 
731 	ret = register_module_notifier(&tracepoint_module_nb);
732 	if (ret)
733 		pr_warn("Failed to register tracepoint module enter notifier\n");
734 
735 	return ret;
736 }
737 __initcall(init_tracepoints);
738 #endif /* CONFIG_MODULES */
739 
740 /**
741  * for_each_kernel_tracepoint - iteration on all kernel tracepoints
742  * @fct: callback
743  * @priv: private data
744  */
for_each_kernel_tracepoint(void (* fct)(struct tracepoint * tp,void * priv),void * priv)745 void for_each_kernel_tracepoint(void (*fct)(struct tracepoint *tp, void *priv),
746 		void *priv)
747 {
748 	for_each_tracepoint_range(__start___tracepoints_ptrs,
749 		__stop___tracepoints_ptrs, fct, priv);
750 }
751 EXPORT_SYMBOL_GPL(for_each_kernel_tracepoint);
752 
753 #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
754 
755 /* NB: reg/unreg are called while guarded with the tracepoints_mutex */
756 static int sys_tracepoint_refcount;
757 
syscall_regfunc(void)758 int syscall_regfunc(void)
759 {
760 	struct task_struct *p, *t;
761 
762 	if (!sys_tracepoint_refcount) {
763 		read_lock(&tasklist_lock);
764 		for_each_process_thread(p, t) {
765 			set_task_syscall_work(t, SYSCALL_TRACEPOINT);
766 		}
767 		read_unlock(&tasklist_lock);
768 	}
769 	sys_tracepoint_refcount++;
770 
771 	return 0;
772 }
773 
syscall_unregfunc(void)774 void syscall_unregfunc(void)
775 {
776 	struct task_struct *p, *t;
777 
778 	sys_tracepoint_refcount--;
779 	if (!sys_tracepoint_refcount) {
780 		read_lock(&tasklist_lock);
781 		for_each_process_thread(p, t) {
782 			clear_task_syscall_work(t, SYSCALL_TRACEPOINT);
783 		}
784 		read_unlock(&tasklist_lock);
785 	}
786 }
787 #endif
788