1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Infrastructure for profiling code inserted by 'gcc -pg'.
4  *
5  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
7  *
8  * Originally ported from the -rt patch by:
9  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
10  *
11  * Based on code in the latency_tracer, that is:
12  *
13  *  Copyright (C) 2004-2006 Ingo Molnar
14  *  Copyright (C) 2004 Nadia Yvette Chambers
15  */
16 
17 #include <linux/stop_machine.h>
18 #include <linux/clocksource.h>
19 #include <linux/sched/task.h>
20 #include <linux/kallsyms.h>
21 #include <linux/security.h>
22 #include <linux/seq_file.h>
23 #include <linux/tracefs.h>
24 #include <linux/hardirq.h>
25 #include <linux/kthread.h>
26 #include <linux/uaccess.h>
27 #include <linux/bsearch.h>
28 #include <linux/module.h>
29 #include <linux/ftrace.h>
30 #include <linux/sysctl.h>
31 #include <linux/slab.h>
32 #include <linux/ctype.h>
33 #include <linux/sort.h>
34 #include <linux/list.h>
35 #include <linux/hash.h>
36 #include <linux/rcupdate.h>
37 #include <linux/kprobes.h>
38 
39 #include <trace/events/sched.h>
40 
41 #include <asm/sections.h>
42 #include <asm/setup.h>
43 
44 #include "ftrace_internal.h"
45 #include "trace_output.h"
46 #include "trace_stat.h"
47 
48 /* Flags that do not get reset */
49 #define FTRACE_NOCLEAR_FLAGS	(FTRACE_FL_DISABLED | FTRACE_FL_TOUCHED | \
50 				 FTRACE_FL_MODIFIED)
51 
52 #define FTRACE_INVALID_FUNCTION		"__ftrace_invalid_address__"
53 
54 #define FTRACE_WARN_ON(cond)			\
55 	({					\
56 		int ___r = cond;		\
57 		if (WARN_ON(___r))		\
58 			ftrace_kill();		\
59 		___r;				\
60 	})
61 
62 #define FTRACE_WARN_ON_ONCE(cond)		\
63 	({					\
64 		int ___r = cond;		\
65 		if (WARN_ON_ONCE(___r))		\
66 			ftrace_kill();		\
67 		___r;				\
68 	})
69 
70 /* hash bits for specific function selection */
71 #define FTRACE_HASH_DEFAULT_BITS 10
72 #define FTRACE_HASH_MAX_BITS 12
73 
74 #ifdef CONFIG_DYNAMIC_FTRACE
75 #define INIT_OPS_HASH(opsname)	\
76 	.func_hash		= &opsname.local_hash,			\
77 	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
78 #else
79 #define INIT_OPS_HASH(opsname)
80 #endif
81 
82 enum {
83 	FTRACE_MODIFY_ENABLE_FL		= (1 << 0),
84 	FTRACE_MODIFY_MAY_SLEEP_FL	= (1 << 1),
85 };
86 
87 struct ftrace_ops ftrace_list_end __read_mostly = {
88 	.func		= ftrace_stub,
89 	.flags		= FTRACE_OPS_FL_STUB,
90 	INIT_OPS_HASH(ftrace_list_end)
91 };
92 
93 /* ftrace_enabled is a method to turn ftrace on or off */
94 int ftrace_enabled __read_mostly;
95 static int __maybe_unused last_ftrace_enabled;
96 
97 /* Current function tracing op */
98 struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
99 /* What to set function_trace_op to */
100 static struct ftrace_ops *set_function_trace_op;
101 
ftrace_pids_enabled(struct ftrace_ops * ops)102 static bool ftrace_pids_enabled(struct ftrace_ops *ops)
103 {
104 	struct trace_array *tr;
105 
106 	if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
107 		return false;
108 
109 	tr = ops->private;
110 
111 	return tr->function_pids != NULL || tr->function_no_pids != NULL;
112 }
113 
114 static void ftrace_update_trampoline(struct ftrace_ops *ops);
115 
116 /*
117  * ftrace_disabled is set when an anomaly is discovered.
118  * ftrace_disabled is much stronger than ftrace_enabled.
119  */
120 static int ftrace_disabled __read_mostly;
121 
122 DEFINE_MUTEX(ftrace_lock);
123 
124 struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end;
125 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
126 struct ftrace_ops global_ops;
127 
128 /* Defined by vmlinux.lds.h see the comment above arch_ftrace_ops_list_func for details */
129 void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
130 			  struct ftrace_ops *op, struct ftrace_regs *fregs);
131 
132 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
133 /*
134  * Stub used to invoke the list ops without requiring a separate trampoline.
135  */
136 const struct ftrace_ops ftrace_list_ops = {
137 	.func	= ftrace_ops_list_func,
138 	.flags	= FTRACE_OPS_FL_STUB,
139 };
140 
ftrace_ops_nop_func(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * op,struct ftrace_regs * fregs)141 static void ftrace_ops_nop_func(unsigned long ip, unsigned long parent_ip,
142 				struct ftrace_ops *op,
143 				struct ftrace_regs *fregs)
144 {
145 	/* do nothing */
146 }
147 
148 /*
149  * Stub used when a call site is disabled. May be called transiently by threads
150  * which have made it into ftrace_caller but haven't yet recovered the ops at
151  * the point the call site is disabled.
152  */
153 const struct ftrace_ops ftrace_nop_ops = {
154 	.func	= ftrace_ops_nop_func,
155 	.flags  = FTRACE_OPS_FL_STUB,
156 };
157 #endif
158 
ftrace_ops_init(struct ftrace_ops * ops)159 static inline void ftrace_ops_init(struct ftrace_ops *ops)
160 {
161 #ifdef CONFIG_DYNAMIC_FTRACE
162 	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
163 		mutex_init(&ops->local_hash.regex_lock);
164 		ops->func_hash = &ops->local_hash;
165 		ops->flags |= FTRACE_OPS_FL_INITIALIZED;
166 	}
167 #endif
168 }
169 
ftrace_pid_func(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * op,struct ftrace_regs * fregs)170 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
171 			    struct ftrace_ops *op, struct ftrace_regs *fregs)
172 {
173 	struct trace_array *tr = op->private;
174 	int pid;
175 
176 	if (tr) {
177 		pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid);
178 		if (pid == FTRACE_PID_IGNORE)
179 			return;
180 		if (pid != FTRACE_PID_TRACE &&
181 		    pid != current->pid)
182 			return;
183 	}
184 
185 	op->saved_func(ip, parent_ip, op, fregs);
186 }
187 
ftrace_sync_ipi(void * data)188 static void ftrace_sync_ipi(void *data)
189 {
190 	/* Probably not needed, but do it anyway */
191 	smp_rmb();
192 }
193 
ftrace_ops_get_list_func(struct ftrace_ops * ops)194 static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
195 {
196 	/*
197 	 * If this is a dynamic or RCU ops, or we force list func,
198 	 * then it needs to call the list anyway.
199 	 */
200 	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) ||
201 	    FTRACE_FORCE_LIST_FUNC)
202 		return ftrace_ops_list_func;
203 
204 	return ftrace_ops_get_func(ops);
205 }
206 
update_ftrace_function(void)207 static void update_ftrace_function(void)
208 {
209 	ftrace_func_t func;
210 
211 	/*
212 	 * Prepare the ftrace_ops that the arch callback will use.
213 	 * If there's only one ftrace_ops registered, the ftrace_ops_list
214 	 * will point to the ops we want.
215 	 */
216 	set_function_trace_op = rcu_dereference_protected(ftrace_ops_list,
217 						lockdep_is_held(&ftrace_lock));
218 
219 	/* If there's no ftrace_ops registered, just call the stub function */
220 	if (set_function_trace_op == &ftrace_list_end) {
221 		func = ftrace_stub;
222 
223 	/*
224 	 * If we are at the end of the list and this ops is
225 	 * recursion safe and not dynamic and the arch supports passing ops,
226 	 * then have the mcount trampoline call the function directly.
227 	 */
228 	} else if (rcu_dereference_protected(ftrace_ops_list->next,
229 			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
230 		func = ftrace_ops_get_list_func(ftrace_ops_list);
231 
232 	} else {
233 		/* Just use the default ftrace_ops */
234 		set_function_trace_op = &ftrace_list_end;
235 		func = ftrace_ops_list_func;
236 	}
237 
238 	update_function_graph_func();
239 
240 	/* If there's no change, then do nothing more here */
241 	if (ftrace_trace_function == func)
242 		return;
243 
244 	/*
245 	 * If we are using the list function, it doesn't care
246 	 * about the function_trace_ops.
247 	 */
248 	if (func == ftrace_ops_list_func) {
249 		ftrace_trace_function = func;
250 		/*
251 		 * Don't even bother setting function_trace_ops,
252 		 * it would be racy to do so anyway.
253 		 */
254 		return;
255 	}
256 
257 #ifndef CONFIG_DYNAMIC_FTRACE
258 	/*
259 	 * For static tracing, we need to be a bit more careful.
260 	 * The function change takes affect immediately. Thus,
261 	 * we need to coordinate the setting of the function_trace_ops
262 	 * with the setting of the ftrace_trace_function.
263 	 *
264 	 * Set the function to the list ops, which will call the
265 	 * function we want, albeit indirectly, but it handles the
266 	 * ftrace_ops and doesn't depend on function_trace_op.
267 	 */
268 	ftrace_trace_function = ftrace_ops_list_func;
269 	/*
270 	 * Make sure all CPUs see this. Yes this is slow, but static
271 	 * tracing is slow and nasty to have enabled.
272 	 */
273 	synchronize_rcu_tasks_rude();
274 	/* Now all cpus are using the list ops. */
275 	function_trace_op = set_function_trace_op;
276 	/* Make sure the function_trace_op is visible on all CPUs */
277 	smp_wmb();
278 	/* Nasty way to force a rmb on all cpus */
279 	smp_call_function(ftrace_sync_ipi, NULL, 1);
280 	/* OK, we are all set to update the ftrace_trace_function now! */
281 #endif /* !CONFIG_DYNAMIC_FTRACE */
282 
283 	ftrace_trace_function = func;
284 }
285 
add_ftrace_ops(struct ftrace_ops __rcu ** list,struct ftrace_ops * ops)286 static void add_ftrace_ops(struct ftrace_ops __rcu **list,
287 			   struct ftrace_ops *ops)
288 {
289 	rcu_assign_pointer(ops->next, *list);
290 
291 	/*
292 	 * We are entering ops into the list but another
293 	 * CPU might be walking that list. We need to make sure
294 	 * the ops->next pointer is valid before another CPU sees
295 	 * the ops pointer included into the list.
296 	 */
297 	rcu_assign_pointer(*list, ops);
298 }
299 
remove_ftrace_ops(struct ftrace_ops __rcu ** list,struct ftrace_ops * ops)300 static int remove_ftrace_ops(struct ftrace_ops __rcu **list,
301 			     struct ftrace_ops *ops)
302 {
303 	struct ftrace_ops **p;
304 
305 	/*
306 	 * If we are removing the last function, then simply point
307 	 * to the ftrace_stub.
308 	 */
309 	if (rcu_dereference_protected(*list,
310 			lockdep_is_held(&ftrace_lock)) == ops &&
311 	    rcu_dereference_protected(ops->next,
312 			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
313 		*list = &ftrace_list_end;
314 		return 0;
315 	}
316 
317 	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
318 		if (*p == ops)
319 			break;
320 
321 	if (*p != ops)
322 		return -1;
323 
324 	*p = (*p)->next;
325 	return 0;
326 }
327 
328 static void ftrace_update_trampoline(struct ftrace_ops *ops);
329 
__register_ftrace_function(struct ftrace_ops * ops)330 int __register_ftrace_function(struct ftrace_ops *ops)
331 {
332 	if (ops->flags & FTRACE_OPS_FL_DELETED)
333 		return -EINVAL;
334 
335 	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
336 		return -EBUSY;
337 
338 #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
339 	/*
340 	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
341 	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
342 	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
343 	 */
344 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
345 	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
346 		return -EINVAL;
347 
348 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
349 		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
350 #endif
351 	if (!ftrace_enabled && (ops->flags & FTRACE_OPS_FL_PERMANENT))
352 		return -EBUSY;
353 
354 	if (!is_kernel_core_data((unsigned long)ops))
355 		ops->flags |= FTRACE_OPS_FL_DYNAMIC;
356 
357 	add_ftrace_ops(&ftrace_ops_list, ops);
358 
359 	/* Always save the function, and reset at unregistering */
360 	ops->saved_func = ops->func;
361 
362 	if (ftrace_pids_enabled(ops))
363 		ops->func = ftrace_pid_func;
364 
365 	ftrace_update_trampoline(ops);
366 
367 	if (ftrace_enabled)
368 		update_ftrace_function();
369 
370 	return 0;
371 }
372 
__unregister_ftrace_function(struct ftrace_ops * ops)373 int __unregister_ftrace_function(struct ftrace_ops *ops)
374 {
375 	int ret;
376 
377 	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
378 		return -EBUSY;
379 
380 	ret = remove_ftrace_ops(&ftrace_ops_list, ops);
381 
382 	if (ret < 0)
383 		return ret;
384 
385 	if (ftrace_enabled)
386 		update_ftrace_function();
387 
388 	ops->func = ops->saved_func;
389 
390 	return 0;
391 }
392 
ftrace_update_pid_func(void)393 static void ftrace_update_pid_func(void)
394 {
395 	struct ftrace_ops *op;
396 
397 	/* Only do something if we are tracing something */
398 	if (ftrace_trace_function == ftrace_stub)
399 		return;
400 
401 	do_for_each_ftrace_op(op, ftrace_ops_list) {
402 		if (op->flags & FTRACE_OPS_FL_PID) {
403 			op->func = ftrace_pids_enabled(op) ?
404 				ftrace_pid_func : op->saved_func;
405 			ftrace_update_trampoline(op);
406 		}
407 	} while_for_each_ftrace_op(op);
408 
409 	update_ftrace_function();
410 }
411 
412 #ifdef CONFIG_FUNCTION_PROFILER
413 struct ftrace_profile {
414 	struct hlist_node		node;
415 	unsigned long			ip;
416 	unsigned long			counter;
417 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
418 	unsigned long long		time;
419 	unsigned long long		time_squared;
420 #endif
421 };
422 
423 struct ftrace_profile_page {
424 	struct ftrace_profile_page	*next;
425 	unsigned long			index;
426 	struct ftrace_profile		records[];
427 };
428 
429 struct ftrace_profile_stat {
430 	atomic_t			disabled;
431 	struct hlist_head		*hash;
432 	struct ftrace_profile_page	*pages;
433 	struct ftrace_profile_page	*start;
434 	struct tracer_stat		stat;
435 };
436 
437 #define PROFILE_RECORDS_SIZE						\
438 	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
439 
440 #define PROFILES_PER_PAGE					\
441 	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
442 
443 static int ftrace_profile_enabled __read_mostly;
444 
445 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
446 static DEFINE_MUTEX(ftrace_profile_lock);
447 
448 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
449 
450 #define FTRACE_PROFILE_HASH_BITS 10
451 #define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
452 
453 static void *
function_stat_next(void * v,int idx)454 function_stat_next(void *v, int idx)
455 {
456 	struct ftrace_profile *rec = v;
457 	struct ftrace_profile_page *pg;
458 
459 	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
460 
461  again:
462 	if (idx != 0)
463 		rec++;
464 
465 	if ((void *)rec >= (void *)&pg->records[pg->index]) {
466 		pg = pg->next;
467 		if (!pg)
468 			return NULL;
469 		rec = &pg->records[0];
470 		if (!rec->counter)
471 			goto again;
472 	}
473 
474 	return rec;
475 }
476 
function_stat_start(struct tracer_stat * trace)477 static void *function_stat_start(struct tracer_stat *trace)
478 {
479 	struct ftrace_profile_stat *stat =
480 		container_of(trace, struct ftrace_profile_stat, stat);
481 
482 	if (!stat || !stat->start)
483 		return NULL;
484 
485 	return function_stat_next(&stat->start->records[0], 0);
486 }
487 
488 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
489 /* function graph compares on total time */
function_stat_cmp(const void * p1,const void * p2)490 static int function_stat_cmp(const void *p1, const void *p2)
491 {
492 	const struct ftrace_profile *a = p1;
493 	const struct ftrace_profile *b = p2;
494 
495 	if (a->time < b->time)
496 		return -1;
497 	if (a->time > b->time)
498 		return 1;
499 	else
500 		return 0;
501 }
502 #else
503 /* not function graph compares against hits */
function_stat_cmp(const void * p1,const void * p2)504 static int function_stat_cmp(const void *p1, const void *p2)
505 {
506 	const struct ftrace_profile *a = p1;
507 	const struct ftrace_profile *b = p2;
508 
509 	if (a->counter < b->counter)
510 		return -1;
511 	if (a->counter > b->counter)
512 		return 1;
513 	else
514 		return 0;
515 }
516 #endif
517 
function_stat_headers(struct seq_file * m)518 static int function_stat_headers(struct seq_file *m)
519 {
520 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
521 	seq_puts(m, "  Function                               "
522 		 "Hit    Time            Avg             s^2\n"
523 		    "  --------                               "
524 		 "---    ----            ---             ---\n");
525 #else
526 	seq_puts(m, "  Function                               Hit\n"
527 		    "  --------                               ---\n");
528 #endif
529 	return 0;
530 }
531 
function_stat_show(struct seq_file * m,void * v)532 static int function_stat_show(struct seq_file *m, void *v)
533 {
534 	struct ftrace_profile *rec = v;
535 	char str[KSYM_SYMBOL_LEN];
536 	int ret = 0;
537 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
538 	static struct trace_seq s;
539 	unsigned long long avg;
540 	unsigned long long stddev;
541 #endif
542 	mutex_lock(&ftrace_profile_lock);
543 
544 	/* we raced with function_profile_reset() */
545 	if (unlikely(rec->counter == 0)) {
546 		ret = -EBUSY;
547 		goto out;
548 	}
549 
550 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
551 	avg = div64_ul(rec->time, rec->counter);
552 	if (tracing_thresh && (avg < tracing_thresh))
553 		goto out;
554 #endif
555 
556 	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
557 	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
558 
559 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
560 	seq_puts(m, "    ");
561 
562 	/* Sample standard deviation (s^2) */
563 	if (rec->counter <= 1)
564 		stddev = 0;
565 	else {
566 		/*
567 		 * Apply Welford's method:
568 		 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
569 		 */
570 		stddev = rec->counter * rec->time_squared -
571 			 rec->time * rec->time;
572 
573 		/*
574 		 * Divide only 1000 for ns^2 -> us^2 conversion.
575 		 * trace_print_graph_duration will divide 1000 again.
576 		 */
577 		stddev = div64_ul(stddev,
578 				  rec->counter * (rec->counter - 1) * 1000);
579 	}
580 
581 	trace_seq_init(&s);
582 	trace_print_graph_duration(rec->time, &s);
583 	trace_seq_puts(&s, "    ");
584 	trace_print_graph_duration(avg, &s);
585 	trace_seq_puts(&s, "    ");
586 	trace_print_graph_duration(stddev, &s);
587 	trace_print_seq(m, &s);
588 #endif
589 	seq_putc(m, '\n');
590 out:
591 	mutex_unlock(&ftrace_profile_lock);
592 
593 	return ret;
594 }
595 
ftrace_profile_reset(struct ftrace_profile_stat * stat)596 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
597 {
598 	struct ftrace_profile_page *pg;
599 
600 	pg = stat->pages = stat->start;
601 
602 	while (pg) {
603 		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
604 		pg->index = 0;
605 		pg = pg->next;
606 	}
607 
608 	memset(stat->hash, 0,
609 	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
610 }
611 
ftrace_profile_pages_init(struct ftrace_profile_stat * stat)612 static int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
613 {
614 	struct ftrace_profile_page *pg;
615 	int functions;
616 	int pages;
617 	int i;
618 
619 	/* If we already allocated, do nothing */
620 	if (stat->pages)
621 		return 0;
622 
623 	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
624 	if (!stat->pages)
625 		return -ENOMEM;
626 
627 #ifdef CONFIG_DYNAMIC_FTRACE
628 	functions = ftrace_update_tot_cnt;
629 #else
630 	/*
631 	 * We do not know the number of functions that exist because
632 	 * dynamic tracing is what counts them. With past experience
633 	 * we have around 20K functions. That should be more than enough.
634 	 * It is highly unlikely we will execute every function in
635 	 * the kernel.
636 	 */
637 	functions = 20000;
638 #endif
639 
640 	pg = stat->start = stat->pages;
641 
642 	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
643 
644 	for (i = 1; i < pages; i++) {
645 		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
646 		if (!pg->next)
647 			goto out_free;
648 		pg = pg->next;
649 	}
650 
651 	return 0;
652 
653  out_free:
654 	pg = stat->start;
655 	while (pg) {
656 		unsigned long tmp = (unsigned long)pg;
657 
658 		pg = pg->next;
659 		free_page(tmp);
660 	}
661 
662 	stat->pages = NULL;
663 	stat->start = NULL;
664 
665 	return -ENOMEM;
666 }
667 
ftrace_profile_init_cpu(int cpu)668 static int ftrace_profile_init_cpu(int cpu)
669 {
670 	struct ftrace_profile_stat *stat;
671 	int size;
672 
673 	stat = &per_cpu(ftrace_profile_stats, cpu);
674 
675 	if (stat->hash) {
676 		/* If the profile is already created, simply reset it */
677 		ftrace_profile_reset(stat);
678 		return 0;
679 	}
680 
681 	/*
682 	 * We are profiling all functions, but usually only a few thousand
683 	 * functions are hit. We'll make a hash of 1024 items.
684 	 */
685 	size = FTRACE_PROFILE_HASH_SIZE;
686 
687 	stat->hash = kcalloc(size, sizeof(struct hlist_head), GFP_KERNEL);
688 
689 	if (!stat->hash)
690 		return -ENOMEM;
691 
692 	/* Preallocate the function profiling pages */
693 	if (ftrace_profile_pages_init(stat) < 0) {
694 		kfree(stat->hash);
695 		stat->hash = NULL;
696 		return -ENOMEM;
697 	}
698 
699 	return 0;
700 }
701 
ftrace_profile_init(void)702 static int ftrace_profile_init(void)
703 {
704 	int cpu;
705 	int ret = 0;
706 
707 	for_each_possible_cpu(cpu) {
708 		ret = ftrace_profile_init_cpu(cpu);
709 		if (ret)
710 			break;
711 	}
712 
713 	return ret;
714 }
715 
716 /* interrupts must be disabled */
717 static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat * stat,unsigned long ip)718 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
719 {
720 	struct ftrace_profile *rec;
721 	struct hlist_head *hhd;
722 	unsigned long key;
723 
724 	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
725 	hhd = &stat->hash[key];
726 
727 	if (hlist_empty(hhd))
728 		return NULL;
729 
730 	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
731 		if (rec->ip == ip)
732 			return rec;
733 	}
734 
735 	return NULL;
736 }
737 
ftrace_add_profile(struct ftrace_profile_stat * stat,struct ftrace_profile * rec)738 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
739 			       struct ftrace_profile *rec)
740 {
741 	unsigned long key;
742 
743 	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
744 	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
745 }
746 
747 /*
748  * The memory is already allocated, this simply finds a new record to use.
749  */
750 static struct ftrace_profile *
ftrace_profile_alloc(struct ftrace_profile_stat * stat,unsigned long ip)751 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
752 {
753 	struct ftrace_profile *rec = NULL;
754 
755 	/* prevent recursion (from NMIs) */
756 	if (atomic_inc_return(&stat->disabled) != 1)
757 		goto out;
758 
759 	/*
760 	 * Try to find the function again since an NMI
761 	 * could have added it
762 	 */
763 	rec = ftrace_find_profiled_func(stat, ip);
764 	if (rec)
765 		goto out;
766 
767 	if (stat->pages->index == PROFILES_PER_PAGE) {
768 		if (!stat->pages->next)
769 			goto out;
770 		stat->pages = stat->pages->next;
771 	}
772 
773 	rec = &stat->pages->records[stat->pages->index++];
774 	rec->ip = ip;
775 	ftrace_add_profile(stat, rec);
776 
777  out:
778 	atomic_dec(&stat->disabled);
779 
780 	return rec;
781 }
782 
783 static void
function_profile_call(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * ops,struct ftrace_regs * fregs)784 function_profile_call(unsigned long ip, unsigned long parent_ip,
785 		      struct ftrace_ops *ops, struct ftrace_regs *fregs)
786 {
787 	struct ftrace_profile_stat *stat;
788 	struct ftrace_profile *rec;
789 	unsigned long flags;
790 
791 	if (!ftrace_profile_enabled)
792 		return;
793 
794 	local_irq_save(flags);
795 
796 	stat = this_cpu_ptr(&ftrace_profile_stats);
797 	if (!stat->hash || !ftrace_profile_enabled)
798 		goto out;
799 
800 	rec = ftrace_find_profiled_func(stat, ip);
801 	if (!rec) {
802 		rec = ftrace_profile_alloc(stat, ip);
803 		if (!rec)
804 			goto out;
805 	}
806 
807 	rec->counter++;
808  out:
809 	local_irq_restore(flags);
810 }
811 
812 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
813 static bool fgraph_graph_time = true;
814 
ftrace_graph_graph_time_control(bool enable)815 void ftrace_graph_graph_time_control(bool enable)
816 {
817 	fgraph_graph_time = enable;
818 }
819 
profile_graph_entry(struct ftrace_graph_ent * trace)820 static int profile_graph_entry(struct ftrace_graph_ent *trace)
821 {
822 	struct ftrace_ret_stack *ret_stack;
823 
824 	function_profile_call(trace->func, 0, NULL, NULL);
825 
826 	/* If function graph is shutting down, ret_stack can be NULL */
827 	if (!current->ret_stack)
828 		return 0;
829 
830 	ret_stack = ftrace_graph_get_ret_stack(current, 0);
831 	if (ret_stack)
832 		ret_stack->subtime = 0;
833 
834 	return 1;
835 }
836 
profile_graph_return(struct ftrace_graph_ret * trace)837 static void profile_graph_return(struct ftrace_graph_ret *trace)
838 {
839 	struct ftrace_ret_stack *ret_stack;
840 	struct ftrace_profile_stat *stat;
841 	unsigned long long calltime;
842 	struct ftrace_profile *rec;
843 	unsigned long flags;
844 
845 	local_irq_save(flags);
846 	stat = this_cpu_ptr(&ftrace_profile_stats);
847 	if (!stat->hash || !ftrace_profile_enabled)
848 		goto out;
849 
850 	/* If the calltime was zero'd ignore it */
851 	if (!trace->calltime)
852 		goto out;
853 
854 	calltime = trace->rettime - trace->calltime;
855 
856 	if (!fgraph_graph_time) {
857 
858 		/* Append this call time to the parent time to subtract */
859 		ret_stack = ftrace_graph_get_ret_stack(current, 1);
860 		if (ret_stack)
861 			ret_stack->subtime += calltime;
862 
863 		ret_stack = ftrace_graph_get_ret_stack(current, 0);
864 		if (ret_stack && ret_stack->subtime < calltime)
865 			calltime -= ret_stack->subtime;
866 		else
867 			calltime = 0;
868 	}
869 
870 	rec = ftrace_find_profiled_func(stat, trace->func);
871 	if (rec) {
872 		rec->time += calltime;
873 		rec->time_squared += calltime * calltime;
874 	}
875 
876  out:
877 	local_irq_restore(flags);
878 }
879 
880 static struct fgraph_ops fprofiler_ops = {
881 	.entryfunc = &profile_graph_entry,
882 	.retfunc = &profile_graph_return,
883 };
884 
register_ftrace_profiler(void)885 static int register_ftrace_profiler(void)
886 {
887 	return register_ftrace_graph(&fprofiler_ops);
888 }
889 
unregister_ftrace_profiler(void)890 static void unregister_ftrace_profiler(void)
891 {
892 	unregister_ftrace_graph(&fprofiler_ops);
893 }
894 #else
895 static struct ftrace_ops ftrace_profile_ops __read_mostly = {
896 	.func		= function_profile_call,
897 	.flags		= FTRACE_OPS_FL_INITIALIZED,
898 	INIT_OPS_HASH(ftrace_profile_ops)
899 };
900 
register_ftrace_profiler(void)901 static int register_ftrace_profiler(void)
902 {
903 	return register_ftrace_function(&ftrace_profile_ops);
904 }
905 
unregister_ftrace_profiler(void)906 static void unregister_ftrace_profiler(void)
907 {
908 	unregister_ftrace_function(&ftrace_profile_ops);
909 }
910 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
911 
912 static ssize_t
ftrace_profile_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)913 ftrace_profile_write(struct file *filp, const char __user *ubuf,
914 		     size_t cnt, loff_t *ppos)
915 {
916 	unsigned long val;
917 	int ret;
918 
919 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
920 	if (ret)
921 		return ret;
922 
923 	val = !!val;
924 
925 	mutex_lock(&ftrace_profile_lock);
926 	if (ftrace_profile_enabled ^ val) {
927 		if (val) {
928 			ret = ftrace_profile_init();
929 			if (ret < 0) {
930 				cnt = ret;
931 				goto out;
932 			}
933 
934 			ret = register_ftrace_profiler();
935 			if (ret < 0) {
936 				cnt = ret;
937 				goto out;
938 			}
939 			ftrace_profile_enabled = 1;
940 		} else {
941 			ftrace_profile_enabled = 0;
942 			/*
943 			 * unregister_ftrace_profiler calls stop_machine
944 			 * so this acts like an synchronize_rcu.
945 			 */
946 			unregister_ftrace_profiler();
947 		}
948 	}
949  out:
950 	mutex_unlock(&ftrace_profile_lock);
951 
952 	*ppos += cnt;
953 
954 	return cnt;
955 }
956 
957 static ssize_t
ftrace_profile_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)958 ftrace_profile_read(struct file *filp, char __user *ubuf,
959 		     size_t cnt, loff_t *ppos)
960 {
961 	char buf[64];		/* big enough to hold a number */
962 	int r;
963 
964 	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
965 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
966 }
967 
968 static const struct file_operations ftrace_profile_fops = {
969 	.open		= tracing_open_generic,
970 	.read		= ftrace_profile_read,
971 	.write		= ftrace_profile_write,
972 	.llseek		= default_llseek,
973 };
974 
975 /* used to initialize the real stat files */
976 static struct tracer_stat function_stats __initdata = {
977 	.name		= "functions",
978 	.stat_start	= function_stat_start,
979 	.stat_next	= function_stat_next,
980 	.stat_cmp	= function_stat_cmp,
981 	.stat_headers	= function_stat_headers,
982 	.stat_show	= function_stat_show
983 };
984 
ftrace_profile_tracefs(struct dentry * d_tracer)985 static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
986 {
987 	struct ftrace_profile_stat *stat;
988 	char *name;
989 	int ret;
990 	int cpu;
991 
992 	for_each_possible_cpu(cpu) {
993 		stat = &per_cpu(ftrace_profile_stats, cpu);
994 
995 		name = kasprintf(GFP_KERNEL, "function%d", cpu);
996 		if (!name) {
997 			/*
998 			 * The files created are permanent, if something happens
999 			 * we still do not free memory.
1000 			 */
1001 			WARN(1,
1002 			     "Could not allocate stat file for cpu %d\n",
1003 			     cpu);
1004 			return;
1005 		}
1006 		stat->stat = function_stats;
1007 		stat->stat.name = name;
1008 		ret = register_stat_tracer(&stat->stat);
1009 		if (ret) {
1010 			WARN(1,
1011 			     "Could not register function stat for cpu %d\n",
1012 			     cpu);
1013 			kfree(name);
1014 			return;
1015 		}
1016 	}
1017 
1018 	trace_create_file("function_profile_enabled",
1019 			  TRACE_MODE_WRITE, d_tracer, NULL,
1020 			  &ftrace_profile_fops);
1021 }
1022 
1023 #else /* CONFIG_FUNCTION_PROFILER */
ftrace_profile_tracefs(struct dentry * d_tracer)1024 static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
1025 {
1026 }
1027 #endif /* CONFIG_FUNCTION_PROFILER */
1028 
1029 #ifdef CONFIG_DYNAMIC_FTRACE
1030 
1031 static struct ftrace_ops *removed_ops;
1032 
1033 /*
1034  * Set when doing a global update, like enabling all recs or disabling them.
1035  * It is not set when just updating a single ftrace_ops.
1036  */
1037 static bool update_all_ops;
1038 
1039 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
1040 # error Dynamic ftrace depends on MCOUNT_RECORD
1041 #endif
1042 
1043 struct ftrace_func_probe {
1044 	struct ftrace_probe_ops	*probe_ops;
1045 	struct ftrace_ops	ops;
1046 	struct trace_array	*tr;
1047 	struct list_head	list;
1048 	void			*data;
1049 	int			ref;
1050 };
1051 
1052 /*
1053  * We make these constant because no one should touch them,
1054  * but they are used as the default "empty hash", to avoid allocating
1055  * it all the time. These are in a read only section such that if
1056  * anyone does try to modify it, it will cause an exception.
1057  */
1058 static const struct hlist_head empty_buckets[1];
1059 static const struct ftrace_hash empty_hash = {
1060 	.buckets = (struct hlist_head *)empty_buckets,
1061 };
1062 #define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1063 
1064 struct ftrace_ops global_ops = {
1065 	.func				= ftrace_stub,
1066 	.local_hash.notrace_hash	= EMPTY_HASH,
1067 	.local_hash.filter_hash		= EMPTY_HASH,
1068 	INIT_OPS_HASH(global_ops)
1069 	.flags				= FTRACE_OPS_FL_INITIALIZED |
1070 					  FTRACE_OPS_FL_PID,
1071 };
1072 
1073 /*
1074  * Used by the stack unwinder to know about dynamic ftrace trampolines.
1075  */
ftrace_ops_trampoline(unsigned long addr)1076 struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr)
1077 {
1078 	struct ftrace_ops *op = NULL;
1079 
1080 	/*
1081 	 * Some of the ops may be dynamically allocated,
1082 	 * they are freed after a synchronize_rcu().
1083 	 */
1084 	preempt_disable_notrace();
1085 
1086 	do_for_each_ftrace_op(op, ftrace_ops_list) {
1087 		/*
1088 		 * This is to check for dynamically allocated trampolines.
1089 		 * Trampolines that are in kernel text will have
1090 		 * core_kernel_text() return true.
1091 		 */
1092 		if (op->trampoline && op->trampoline_size)
1093 			if (addr >= op->trampoline &&
1094 			    addr < op->trampoline + op->trampoline_size) {
1095 				preempt_enable_notrace();
1096 				return op;
1097 			}
1098 	} while_for_each_ftrace_op(op);
1099 	preempt_enable_notrace();
1100 
1101 	return NULL;
1102 }
1103 
1104 /*
1105  * This is used by __kernel_text_address() to return true if the
1106  * address is on a dynamically allocated trampoline that would
1107  * not return true for either core_kernel_text() or
1108  * is_module_text_address().
1109  */
is_ftrace_trampoline(unsigned long addr)1110 bool is_ftrace_trampoline(unsigned long addr)
1111 {
1112 	return ftrace_ops_trampoline(addr) != NULL;
1113 }
1114 
1115 struct ftrace_page {
1116 	struct ftrace_page	*next;
1117 	struct dyn_ftrace	*records;
1118 	int			index;
1119 	int			order;
1120 };
1121 
1122 #define ENTRY_SIZE sizeof(struct dyn_ftrace)
1123 #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1124 
1125 static struct ftrace_page	*ftrace_pages_start;
1126 static struct ftrace_page	*ftrace_pages;
1127 
1128 static __always_inline unsigned long
ftrace_hash_key(struct ftrace_hash * hash,unsigned long ip)1129 ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip)
1130 {
1131 	if (hash->size_bits > 0)
1132 		return hash_long(ip, hash->size_bits);
1133 
1134 	return 0;
1135 }
1136 
1137 /* Only use this function if ftrace_hash_empty() has already been tested */
1138 static __always_inline struct ftrace_func_entry *
__ftrace_lookup_ip(struct ftrace_hash * hash,unsigned long ip)1139 __ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1140 {
1141 	unsigned long key;
1142 	struct ftrace_func_entry *entry;
1143 	struct hlist_head *hhd;
1144 
1145 	key = ftrace_hash_key(hash, ip);
1146 	hhd = &hash->buckets[key];
1147 
1148 	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1149 		if (entry->ip == ip)
1150 			return entry;
1151 	}
1152 	return NULL;
1153 }
1154 
1155 /**
1156  * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
1157  * @hash: The hash to look at
1158  * @ip: The instruction pointer to test
1159  *
1160  * Search a given @hash to see if a given instruction pointer (@ip)
1161  * exists in it.
1162  *
1163  * Returns the entry that holds the @ip if found. NULL otherwise.
1164  */
1165 struct ftrace_func_entry *
ftrace_lookup_ip(struct ftrace_hash * hash,unsigned long ip)1166 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1167 {
1168 	if (ftrace_hash_empty(hash))
1169 		return NULL;
1170 
1171 	return __ftrace_lookup_ip(hash, ip);
1172 }
1173 
__add_hash_entry(struct ftrace_hash * hash,struct ftrace_func_entry * entry)1174 static void __add_hash_entry(struct ftrace_hash *hash,
1175 			     struct ftrace_func_entry *entry)
1176 {
1177 	struct hlist_head *hhd;
1178 	unsigned long key;
1179 
1180 	key = ftrace_hash_key(hash, entry->ip);
1181 	hhd = &hash->buckets[key];
1182 	hlist_add_head(&entry->hlist, hhd);
1183 	hash->count++;
1184 }
1185 
1186 static struct ftrace_func_entry *
add_hash_entry(struct ftrace_hash * hash,unsigned long ip)1187 add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1188 {
1189 	struct ftrace_func_entry *entry;
1190 
1191 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1192 	if (!entry)
1193 		return NULL;
1194 
1195 	entry->ip = ip;
1196 	__add_hash_entry(hash, entry);
1197 
1198 	return entry;
1199 }
1200 
1201 static void
free_hash_entry(struct ftrace_hash * hash,struct ftrace_func_entry * entry)1202 free_hash_entry(struct ftrace_hash *hash,
1203 		  struct ftrace_func_entry *entry)
1204 {
1205 	hlist_del(&entry->hlist);
1206 	kfree(entry);
1207 	hash->count--;
1208 }
1209 
1210 static void
remove_hash_entry(struct ftrace_hash * hash,struct ftrace_func_entry * entry)1211 remove_hash_entry(struct ftrace_hash *hash,
1212 		  struct ftrace_func_entry *entry)
1213 {
1214 	hlist_del_rcu(&entry->hlist);
1215 	hash->count--;
1216 }
1217 
ftrace_hash_clear(struct ftrace_hash * hash)1218 static void ftrace_hash_clear(struct ftrace_hash *hash)
1219 {
1220 	struct hlist_head *hhd;
1221 	struct hlist_node *tn;
1222 	struct ftrace_func_entry *entry;
1223 	int size = 1 << hash->size_bits;
1224 	int i;
1225 
1226 	if (!hash->count)
1227 		return;
1228 
1229 	for (i = 0; i < size; i++) {
1230 		hhd = &hash->buckets[i];
1231 		hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1232 			free_hash_entry(hash, entry);
1233 	}
1234 	FTRACE_WARN_ON(hash->count);
1235 }
1236 
free_ftrace_mod(struct ftrace_mod_load * ftrace_mod)1237 static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod)
1238 {
1239 	list_del(&ftrace_mod->list);
1240 	kfree(ftrace_mod->module);
1241 	kfree(ftrace_mod->func);
1242 	kfree(ftrace_mod);
1243 }
1244 
clear_ftrace_mod_list(struct list_head * head)1245 static void clear_ftrace_mod_list(struct list_head *head)
1246 {
1247 	struct ftrace_mod_load *p, *n;
1248 
1249 	/* stack tracer isn't supported yet */
1250 	if (!head)
1251 		return;
1252 
1253 	mutex_lock(&ftrace_lock);
1254 	list_for_each_entry_safe(p, n, head, list)
1255 		free_ftrace_mod(p);
1256 	mutex_unlock(&ftrace_lock);
1257 }
1258 
free_ftrace_hash(struct ftrace_hash * hash)1259 static void free_ftrace_hash(struct ftrace_hash *hash)
1260 {
1261 	if (!hash || hash == EMPTY_HASH)
1262 		return;
1263 	ftrace_hash_clear(hash);
1264 	kfree(hash->buckets);
1265 	kfree(hash);
1266 }
1267 
__free_ftrace_hash_rcu(struct rcu_head * rcu)1268 static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1269 {
1270 	struct ftrace_hash *hash;
1271 
1272 	hash = container_of(rcu, struct ftrace_hash, rcu);
1273 	free_ftrace_hash(hash);
1274 }
1275 
free_ftrace_hash_rcu(struct ftrace_hash * hash)1276 static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1277 {
1278 	if (!hash || hash == EMPTY_HASH)
1279 		return;
1280 	call_rcu(&hash->rcu, __free_ftrace_hash_rcu);
1281 }
1282 
1283 /**
1284  * ftrace_free_filter - remove all filters for an ftrace_ops
1285  * @ops - the ops to remove the filters from
1286  */
ftrace_free_filter(struct ftrace_ops * ops)1287 void ftrace_free_filter(struct ftrace_ops *ops)
1288 {
1289 	ftrace_ops_init(ops);
1290 	free_ftrace_hash(ops->func_hash->filter_hash);
1291 	free_ftrace_hash(ops->func_hash->notrace_hash);
1292 }
1293 EXPORT_SYMBOL_GPL(ftrace_free_filter);
1294 
alloc_ftrace_hash(int size_bits)1295 static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1296 {
1297 	struct ftrace_hash *hash;
1298 	int size;
1299 
1300 	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1301 	if (!hash)
1302 		return NULL;
1303 
1304 	size = 1 << size_bits;
1305 	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1306 
1307 	if (!hash->buckets) {
1308 		kfree(hash);
1309 		return NULL;
1310 	}
1311 
1312 	hash->size_bits = size_bits;
1313 
1314 	return hash;
1315 }
1316 
1317 
ftrace_add_mod(struct trace_array * tr,const char * func,const char * module,int enable)1318 static int ftrace_add_mod(struct trace_array *tr,
1319 			  const char *func, const char *module,
1320 			  int enable)
1321 {
1322 	struct ftrace_mod_load *ftrace_mod;
1323 	struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace;
1324 
1325 	ftrace_mod = kzalloc(sizeof(*ftrace_mod), GFP_KERNEL);
1326 	if (!ftrace_mod)
1327 		return -ENOMEM;
1328 
1329 	INIT_LIST_HEAD(&ftrace_mod->list);
1330 	ftrace_mod->func = kstrdup(func, GFP_KERNEL);
1331 	ftrace_mod->module = kstrdup(module, GFP_KERNEL);
1332 	ftrace_mod->enable = enable;
1333 
1334 	if (!ftrace_mod->func || !ftrace_mod->module)
1335 		goto out_free;
1336 
1337 	list_add(&ftrace_mod->list, mod_head);
1338 
1339 	return 0;
1340 
1341  out_free:
1342 	free_ftrace_mod(ftrace_mod);
1343 
1344 	return -ENOMEM;
1345 }
1346 
1347 static struct ftrace_hash *
alloc_and_copy_ftrace_hash(int size_bits,struct ftrace_hash * hash)1348 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1349 {
1350 	struct ftrace_func_entry *entry;
1351 	struct ftrace_hash *new_hash;
1352 	int size;
1353 	int i;
1354 
1355 	new_hash = alloc_ftrace_hash(size_bits);
1356 	if (!new_hash)
1357 		return NULL;
1358 
1359 	if (hash)
1360 		new_hash->flags = hash->flags;
1361 
1362 	/* Empty hash? */
1363 	if (ftrace_hash_empty(hash))
1364 		return new_hash;
1365 
1366 	size = 1 << hash->size_bits;
1367 	for (i = 0; i < size; i++) {
1368 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1369 			if (add_hash_entry(new_hash, entry->ip) == NULL)
1370 				goto free_hash;
1371 		}
1372 	}
1373 
1374 	FTRACE_WARN_ON(new_hash->count != hash->count);
1375 
1376 	return new_hash;
1377 
1378  free_hash:
1379 	free_ftrace_hash(new_hash);
1380 	return NULL;
1381 }
1382 
1383 static void
1384 ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
1385 static void
1386 ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
1387 
1388 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
1389 				       struct ftrace_hash *new_hash);
1390 
dup_hash(struct ftrace_hash * src,int size)1391 static struct ftrace_hash *dup_hash(struct ftrace_hash *src, int size)
1392 {
1393 	struct ftrace_func_entry *entry;
1394 	struct ftrace_hash *new_hash;
1395 	struct hlist_head *hhd;
1396 	struct hlist_node *tn;
1397 	int bits = 0;
1398 	int i;
1399 
1400 	/*
1401 	 * Use around half the size (max bit of it), but
1402 	 * a minimum of 2 is fine (as size of 0 or 1 both give 1 for bits).
1403 	 */
1404 	bits = fls(size / 2);
1405 
1406 	/* Don't allocate too much */
1407 	if (bits > FTRACE_HASH_MAX_BITS)
1408 		bits = FTRACE_HASH_MAX_BITS;
1409 
1410 	new_hash = alloc_ftrace_hash(bits);
1411 	if (!new_hash)
1412 		return NULL;
1413 
1414 	new_hash->flags = src->flags;
1415 
1416 	size = 1 << src->size_bits;
1417 	for (i = 0; i < size; i++) {
1418 		hhd = &src->buckets[i];
1419 		hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1420 			remove_hash_entry(src, entry);
1421 			__add_hash_entry(new_hash, entry);
1422 		}
1423 	}
1424 	return new_hash;
1425 }
1426 
1427 static struct ftrace_hash *
__ftrace_hash_move(struct ftrace_hash * src)1428 __ftrace_hash_move(struct ftrace_hash *src)
1429 {
1430 	int size = src->count;
1431 
1432 	/*
1433 	 * If the new source is empty, just return the empty_hash.
1434 	 */
1435 	if (ftrace_hash_empty(src))
1436 		return EMPTY_HASH;
1437 
1438 	return dup_hash(src, size);
1439 }
1440 
1441 static int
ftrace_hash_move(struct ftrace_ops * ops,int enable,struct ftrace_hash ** dst,struct ftrace_hash * src)1442 ftrace_hash_move(struct ftrace_ops *ops, int enable,
1443 		 struct ftrace_hash **dst, struct ftrace_hash *src)
1444 {
1445 	struct ftrace_hash *new_hash;
1446 	int ret;
1447 
1448 	/* Reject setting notrace hash on IPMODIFY ftrace_ops */
1449 	if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
1450 		return -EINVAL;
1451 
1452 	new_hash = __ftrace_hash_move(src);
1453 	if (!new_hash)
1454 		return -ENOMEM;
1455 
1456 	/* Make sure this can be applied if it is IPMODIFY ftrace_ops */
1457 	if (enable) {
1458 		/* IPMODIFY should be updated only when filter_hash updating */
1459 		ret = ftrace_hash_ipmodify_update(ops, new_hash);
1460 		if (ret < 0) {
1461 			free_ftrace_hash(new_hash);
1462 			return ret;
1463 		}
1464 	}
1465 
1466 	/*
1467 	 * Remove the current set, update the hash and add
1468 	 * them back.
1469 	 */
1470 	ftrace_hash_rec_disable_modify(ops, enable);
1471 
1472 	rcu_assign_pointer(*dst, new_hash);
1473 
1474 	ftrace_hash_rec_enable_modify(ops, enable);
1475 
1476 	return 0;
1477 }
1478 
hash_contains_ip(unsigned long ip,struct ftrace_ops_hash * hash)1479 static bool hash_contains_ip(unsigned long ip,
1480 			     struct ftrace_ops_hash *hash)
1481 {
1482 	/*
1483 	 * The function record is a match if it exists in the filter
1484 	 * hash and not in the notrace hash. Note, an empty hash is
1485 	 * considered a match for the filter hash, but an empty
1486 	 * notrace hash is considered not in the notrace hash.
1487 	 */
1488 	return (ftrace_hash_empty(hash->filter_hash) ||
1489 		__ftrace_lookup_ip(hash->filter_hash, ip)) &&
1490 		(ftrace_hash_empty(hash->notrace_hash) ||
1491 		 !__ftrace_lookup_ip(hash->notrace_hash, ip));
1492 }
1493 
1494 /*
1495  * Test the hashes for this ops to see if we want to call
1496  * the ops->func or not.
1497  *
1498  * It's a match if the ip is in the ops->filter_hash or
1499  * the filter_hash does not exist or is empty,
1500  *  AND
1501  * the ip is not in the ops->notrace_hash.
1502  *
1503  * This needs to be called with preemption disabled as
1504  * the hashes are freed with call_rcu().
1505  */
1506 int
ftrace_ops_test(struct ftrace_ops * ops,unsigned long ip,void * regs)1507 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1508 {
1509 	struct ftrace_ops_hash hash;
1510 	int ret;
1511 
1512 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
1513 	/*
1514 	 * There's a small race when adding ops that the ftrace handler
1515 	 * that wants regs, may be called without them. We can not
1516 	 * allow that handler to be called if regs is NULL.
1517 	 */
1518 	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
1519 		return 0;
1520 #endif
1521 
1522 	rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash);
1523 	rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash);
1524 
1525 	if (hash_contains_ip(ip, &hash))
1526 		ret = 1;
1527 	else
1528 		ret = 0;
1529 
1530 	return ret;
1531 }
1532 
1533 /*
1534  * This is a double for. Do not use 'break' to break out of the loop,
1535  * you must use a goto.
1536  */
1537 #define do_for_each_ftrace_rec(pg, rec)					\
1538 	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
1539 		int _____i;						\
1540 		for (_____i = 0; _____i < pg->index; _____i++) {	\
1541 			rec = &pg->records[_____i];
1542 
1543 #define while_for_each_ftrace_rec()		\
1544 		}				\
1545 	}
1546 
1547 
ftrace_cmp_recs(const void * a,const void * b)1548 static int ftrace_cmp_recs(const void *a, const void *b)
1549 {
1550 	const struct dyn_ftrace *key = a;
1551 	const struct dyn_ftrace *rec = b;
1552 
1553 	if (key->flags < rec->ip)
1554 		return -1;
1555 	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
1556 		return 1;
1557 	return 0;
1558 }
1559 
lookup_rec(unsigned long start,unsigned long end)1560 static struct dyn_ftrace *lookup_rec(unsigned long start, unsigned long end)
1561 {
1562 	struct ftrace_page *pg;
1563 	struct dyn_ftrace *rec = NULL;
1564 	struct dyn_ftrace key;
1565 
1566 	key.ip = start;
1567 	key.flags = end;	/* overload flags, as it is unsigned long */
1568 
1569 	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1570 		if (pg->index == 0 ||
1571 		    end < pg->records[0].ip ||
1572 		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1573 			continue;
1574 		rec = bsearch(&key, pg->records, pg->index,
1575 			      sizeof(struct dyn_ftrace),
1576 			      ftrace_cmp_recs);
1577 		if (rec)
1578 			break;
1579 	}
1580 	return rec;
1581 }
1582 
1583 /**
1584  * ftrace_location_range - return the first address of a traced location
1585  *	if it touches the given ip range
1586  * @start: start of range to search.
1587  * @end: end of range to search (inclusive). @end points to the last byte
1588  *	to check.
1589  *
1590  * Returns rec->ip if the related ftrace location is a least partly within
1591  * the given address range. That is, the first address of the instruction
1592  * that is either a NOP or call to the function tracer. It checks the ftrace
1593  * internal tables to determine if the address belongs or not.
1594  */
ftrace_location_range(unsigned long start,unsigned long end)1595 unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1596 {
1597 	struct dyn_ftrace *rec;
1598 
1599 	rec = lookup_rec(start, end);
1600 	if (rec)
1601 		return rec->ip;
1602 
1603 	return 0;
1604 }
1605 
1606 /**
1607  * ftrace_location - return the ftrace location
1608  * @ip: the instruction pointer to check
1609  *
1610  * If @ip matches the ftrace location, return @ip.
1611  * If @ip matches sym+0, return sym's ftrace location.
1612  * Otherwise, return 0.
1613  */
ftrace_location(unsigned long ip)1614 unsigned long ftrace_location(unsigned long ip)
1615 {
1616 	struct dyn_ftrace *rec;
1617 	unsigned long offset;
1618 	unsigned long size;
1619 
1620 	rec = lookup_rec(ip, ip);
1621 	if (!rec) {
1622 		if (!kallsyms_lookup_size_offset(ip, &size, &offset))
1623 			goto out;
1624 
1625 		/* map sym+0 to __fentry__ */
1626 		if (!offset)
1627 			rec = lookup_rec(ip, ip + size - 1);
1628 	}
1629 
1630 	if (rec)
1631 		return rec->ip;
1632 
1633 out:
1634 	return 0;
1635 }
1636 
1637 /**
1638  * ftrace_text_reserved - return true if range contains an ftrace location
1639  * @start: start of range to search
1640  * @end: end of range to search (inclusive). @end points to the last byte to check.
1641  *
1642  * Returns 1 if @start and @end contains a ftrace location.
1643  * That is, the instruction that is either a NOP or call to
1644  * the function tracer. It checks the ftrace internal tables to
1645  * determine if the address belongs or not.
1646  */
ftrace_text_reserved(const void * start,const void * end)1647 int ftrace_text_reserved(const void *start, const void *end)
1648 {
1649 	unsigned long ret;
1650 
1651 	ret = ftrace_location_range((unsigned long)start,
1652 				    (unsigned long)end);
1653 
1654 	return (int)!!ret;
1655 }
1656 
1657 /* Test if ops registered to this rec needs regs */
test_rec_ops_needs_regs(struct dyn_ftrace * rec)1658 static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
1659 {
1660 	struct ftrace_ops *ops;
1661 	bool keep_regs = false;
1662 
1663 	for (ops = ftrace_ops_list;
1664 	     ops != &ftrace_list_end; ops = ops->next) {
1665 		/* pass rec in as regs to have non-NULL val */
1666 		if (ftrace_ops_test(ops, rec->ip, rec)) {
1667 			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1668 				keep_regs = true;
1669 				break;
1670 			}
1671 		}
1672 	}
1673 
1674 	return  keep_regs;
1675 }
1676 
1677 static struct ftrace_ops *
1678 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1679 static struct ftrace_ops *
1680 ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude);
1681 static struct ftrace_ops *
1682 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
1683 
skip_record(struct dyn_ftrace * rec)1684 static bool skip_record(struct dyn_ftrace *rec)
1685 {
1686 	/*
1687 	 * At boot up, weak functions are set to disable. Function tracing
1688 	 * can be enabled before they are, and they still need to be disabled now.
1689 	 * If the record is disabled, still continue if it is marked as already
1690 	 * enabled (this is needed to keep the accounting working).
1691 	 */
1692 	return rec->flags & FTRACE_FL_DISABLED &&
1693 		!(rec->flags & FTRACE_FL_ENABLED);
1694 }
1695 
__ftrace_hash_rec_update(struct ftrace_ops * ops,int filter_hash,bool inc)1696 static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
1697 				     int filter_hash,
1698 				     bool inc)
1699 {
1700 	struct ftrace_hash *hash;
1701 	struct ftrace_hash *other_hash;
1702 	struct ftrace_page *pg;
1703 	struct dyn_ftrace *rec;
1704 	bool update = false;
1705 	int count = 0;
1706 	int all = false;
1707 
1708 	/* Only update if the ops has been registered */
1709 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1710 		return false;
1711 
1712 	/*
1713 	 * In the filter_hash case:
1714 	 *   If the count is zero, we update all records.
1715 	 *   Otherwise we just update the items in the hash.
1716 	 *
1717 	 * In the notrace_hash case:
1718 	 *   We enable the update in the hash.
1719 	 *   As disabling notrace means enabling the tracing,
1720 	 *   and enabling notrace means disabling, the inc variable
1721 	 *   gets inversed.
1722 	 */
1723 	if (filter_hash) {
1724 		hash = ops->func_hash->filter_hash;
1725 		other_hash = ops->func_hash->notrace_hash;
1726 		if (ftrace_hash_empty(hash))
1727 			all = true;
1728 	} else {
1729 		inc = !inc;
1730 		hash = ops->func_hash->notrace_hash;
1731 		other_hash = ops->func_hash->filter_hash;
1732 		/*
1733 		 * If the notrace hash has no items,
1734 		 * then there's nothing to do.
1735 		 */
1736 		if (ftrace_hash_empty(hash))
1737 			return false;
1738 	}
1739 
1740 	do_for_each_ftrace_rec(pg, rec) {
1741 		int in_other_hash = 0;
1742 		int in_hash = 0;
1743 		int match = 0;
1744 
1745 		if (skip_record(rec))
1746 			continue;
1747 
1748 		if (all) {
1749 			/*
1750 			 * Only the filter_hash affects all records.
1751 			 * Update if the record is not in the notrace hash.
1752 			 */
1753 			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1754 				match = 1;
1755 		} else {
1756 			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1757 			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1758 
1759 			/*
1760 			 * If filter_hash is set, we want to match all functions
1761 			 * that are in the hash but not in the other hash.
1762 			 *
1763 			 * If filter_hash is not set, then we are decrementing.
1764 			 * That means we match anything that is in the hash
1765 			 * and also in the other_hash. That is, we need to turn
1766 			 * off functions in the other hash because they are disabled
1767 			 * by this hash.
1768 			 */
1769 			if (filter_hash && in_hash && !in_other_hash)
1770 				match = 1;
1771 			else if (!filter_hash && in_hash &&
1772 				 (in_other_hash || ftrace_hash_empty(other_hash)))
1773 				match = 1;
1774 		}
1775 		if (!match)
1776 			continue;
1777 
1778 		if (inc) {
1779 			rec->flags++;
1780 			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
1781 				return false;
1782 
1783 			if (ops->flags & FTRACE_OPS_FL_DIRECT)
1784 				rec->flags |= FTRACE_FL_DIRECT;
1785 
1786 			/*
1787 			 * If there's only a single callback registered to a
1788 			 * function, and the ops has a trampoline registered
1789 			 * for it, then we can call it directly.
1790 			 */
1791 			if (ftrace_rec_count(rec) == 1 && ops->trampoline)
1792 				rec->flags |= FTRACE_FL_TRAMP;
1793 			else
1794 				/*
1795 				 * If we are adding another function callback
1796 				 * to this function, and the previous had a
1797 				 * custom trampoline in use, then we need to go
1798 				 * back to the default trampoline.
1799 				 */
1800 				rec->flags &= ~FTRACE_FL_TRAMP;
1801 
1802 			/*
1803 			 * If any ops wants regs saved for this function
1804 			 * then all ops will get saved regs.
1805 			 */
1806 			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
1807 				rec->flags |= FTRACE_FL_REGS;
1808 		} else {
1809 			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
1810 				return false;
1811 			rec->flags--;
1812 
1813 			/*
1814 			 * Only the internal direct_ops should have the
1815 			 * DIRECT flag set. Thus, if it is removing a
1816 			 * function, then that function should no longer
1817 			 * be direct.
1818 			 */
1819 			if (ops->flags & FTRACE_OPS_FL_DIRECT)
1820 				rec->flags &= ~FTRACE_FL_DIRECT;
1821 
1822 			/*
1823 			 * If the rec had REGS enabled and the ops that is
1824 			 * being removed had REGS set, then see if there is
1825 			 * still any ops for this record that wants regs.
1826 			 * If not, we can stop recording them.
1827 			 */
1828 			if (ftrace_rec_count(rec) > 0 &&
1829 			    rec->flags & FTRACE_FL_REGS &&
1830 			    ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1831 				if (!test_rec_ops_needs_regs(rec))
1832 					rec->flags &= ~FTRACE_FL_REGS;
1833 			}
1834 
1835 			/*
1836 			 * The TRAMP needs to be set only if rec count
1837 			 * is decremented to one, and the ops that is
1838 			 * left has a trampoline. As TRAMP can only be
1839 			 * enabled if there is only a single ops attached
1840 			 * to it.
1841 			 */
1842 			if (ftrace_rec_count(rec) == 1 &&
1843 			    ftrace_find_tramp_ops_any_other(rec, ops))
1844 				rec->flags |= FTRACE_FL_TRAMP;
1845 			else
1846 				rec->flags &= ~FTRACE_FL_TRAMP;
1847 
1848 			/*
1849 			 * flags will be cleared in ftrace_check_record()
1850 			 * if rec count is zero.
1851 			 */
1852 		}
1853 
1854 		/*
1855 		 * If the rec has a single associated ops, and ops->func can be
1856 		 * called directly, allow the call site to call via the ops.
1857 		 */
1858 		if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS) &&
1859 		    ftrace_rec_count(rec) == 1 &&
1860 		    ftrace_ops_get_func(ops) == ops->func)
1861 			rec->flags |= FTRACE_FL_CALL_OPS;
1862 		else
1863 			rec->flags &= ~FTRACE_FL_CALL_OPS;
1864 
1865 		count++;
1866 
1867 		/* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
1868 		update |= ftrace_test_record(rec, true) != FTRACE_UPDATE_IGNORE;
1869 
1870 		/* Shortcut, if we handled all records, we are done. */
1871 		if (!all && count == hash->count)
1872 			return update;
1873 	} while_for_each_ftrace_rec();
1874 
1875 	return update;
1876 }
1877 
ftrace_hash_rec_disable(struct ftrace_ops * ops,int filter_hash)1878 static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
1879 				    int filter_hash)
1880 {
1881 	return __ftrace_hash_rec_update(ops, filter_hash, 0);
1882 }
1883 
ftrace_hash_rec_enable(struct ftrace_ops * ops,int filter_hash)1884 static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
1885 				   int filter_hash)
1886 {
1887 	return __ftrace_hash_rec_update(ops, filter_hash, 1);
1888 }
1889 
ftrace_hash_rec_update_modify(struct ftrace_ops * ops,int filter_hash,int inc)1890 static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
1891 					  int filter_hash, int inc)
1892 {
1893 	struct ftrace_ops *op;
1894 
1895 	__ftrace_hash_rec_update(ops, filter_hash, inc);
1896 
1897 	if (ops->func_hash != &global_ops.local_hash)
1898 		return;
1899 
1900 	/*
1901 	 * If the ops shares the global_ops hash, then we need to update
1902 	 * all ops that are enabled and use this hash.
1903 	 */
1904 	do_for_each_ftrace_op(op, ftrace_ops_list) {
1905 		/* Already done */
1906 		if (op == ops)
1907 			continue;
1908 		if (op->func_hash == &global_ops.local_hash)
1909 			__ftrace_hash_rec_update(op, filter_hash, inc);
1910 	} while_for_each_ftrace_op(op);
1911 }
1912 
ftrace_hash_rec_disable_modify(struct ftrace_ops * ops,int filter_hash)1913 static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
1914 					   int filter_hash)
1915 {
1916 	ftrace_hash_rec_update_modify(ops, filter_hash, 0);
1917 }
1918 
ftrace_hash_rec_enable_modify(struct ftrace_ops * ops,int filter_hash)1919 static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
1920 					  int filter_hash)
1921 {
1922 	ftrace_hash_rec_update_modify(ops, filter_hash, 1);
1923 }
1924 
1925 /*
1926  * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
1927  * or no-needed to update, -EBUSY if it detects a conflict of the flag
1928  * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
1929  * Note that old_hash and new_hash has below meanings
1930  *  - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
1931  *  - If the hash is EMPTY_HASH, it hits nothing
1932  *  - Anything else hits the recs which match the hash entries.
1933  *
1934  * DIRECT ops does not have IPMODIFY flag, but we still need to check it
1935  * against functions with FTRACE_FL_IPMODIFY. If there is any overlap, call
1936  * ops_func(SHARE_IPMODIFY_SELF) to make sure current ops can share with
1937  * IPMODIFY. If ops_func(SHARE_IPMODIFY_SELF) returns non-zero, propagate
1938  * the return value to the caller and eventually to the owner of the DIRECT
1939  * ops.
1940  */
__ftrace_hash_update_ipmodify(struct ftrace_ops * ops,struct ftrace_hash * old_hash,struct ftrace_hash * new_hash)1941 static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
1942 					 struct ftrace_hash *old_hash,
1943 					 struct ftrace_hash *new_hash)
1944 {
1945 	struct ftrace_page *pg;
1946 	struct dyn_ftrace *rec, *end = NULL;
1947 	int in_old, in_new;
1948 	bool is_ipmodify, is_direct;
1949 
1950 	/* Only update if the ops has been registered */
1951 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1952 		return 0;
1953 
1954 	is_ipmodify = ops->flags & FTRACE_OPS_FL_IPMODIFY;
1955 	is_direct = ops->flags & FTRACE_OPS_FL_DIRECT;
1956 
1957 	/* neither IPMODIFY nor DIRECT, skip */
1958 	if (!is_ipmodify && !is_direct)
1959 		return 0;
1960 
1961 	if (WARN_ON_ONCE(is_ipmodify && is_direct))
1962 		return 0;
1963 
1964 	/*
1965 	 * Since the IPMODIFY and DIRECT are very address sensitive
1966 	 * actions, we do not allow ftrace_ops to set all functions to new
1967 	 * hash.
1968 	 */
1969 	if (!new_hash || !old_hash)
1970 		return -EINVAL;
1971 
1972 	/* Update rec->flags */
1973 	do_for_each_ftrace_rec(pg, rec) {
1974 
1975 		if (rec->flags & FTRACE_FL_DISABLED)
1976 			continue;
1977 
1978 		/* We need to update only differences of filter_hash */
1979 		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
1980 		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
1981 		if (in_old == in_new)
1982 			continue;
1983 
1984 		if (in_new) {
1985 			if (rec->flags & FTRACE_FL_IPMODIFY) {
1986 				int ret;
1987 
1988 				/* Cannot have two ipmodify on same rec */
1989 				if (is_ipmodify)
1990 					goto rollback;
1991 
1992 				FTRACE_WARN_ON(rec->flags & FTRACE_FL_DIRECT);
1993 
1994 				/*
1995 				 * Another ops with IPMODIFY is already
1996 				 * attached. We are now attaching a direct
1997 				 * ops. Run SHARE_IPMODIFY_SELF, to check
1998 				 * whether sharing is supported.
1999 				 */
2000 				if (!ops->ops_func)
2001 					return -EBUSY;
2002 				ret = ops->ops_func(ops, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF);
2003 				if (ret)
2004 					return ret;
2005 			} else if (is_ipmodify) {
2006 				rec->flags |= FTRACE_FL_IPMODIFY;
2007 			}
2008 		} else if (is_ipmodify) {
2009 			rec->flags &= ~FTRACE_FL_IPMODIFY;
2010 		}
2011 	} while_for_each_ftrace_rec();
2012 
2013 	return 0;
2014 
2015 rollback:
2016 	end = rec;
2017 
2018 	/* Roll back what we did above */
2019 	do_for_each_ftrace_rec(pg, rec) {
2020 
2021 		if (rec->flags & FTRACE_FL_DISABLED)
2022 			continue;
2023 
2024 		if (rec == end)
2025 			goto err_out;
2026 
2027 		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
2028 		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
2029 		if (in_old == in_new)
2030 			continue;
2031 
2032 		if (in_new)
2033 			rec->flags &= ~FTRACE_FL_IPMODIFY;
2034 		else
2035 			rec->flags |= FTRACE_FL_IPMODIFY;
2036 	} while_for_each_ftrace_rec();
2037 
2038 err_out:
2039 	return -EBUSY;
2040 }
2041 
ftrace_hash_ipmodify_enable(struct ftrace_ops * ops)2042 static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
2043 {
2044 	struct ftrace_hash *hash = ops->func_hash->filter_hash;
2045 
2046 	if (ftrace_hash_empty(hash))
2047 		hash = NULL;
2048 
2049 	return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
2050 }
2051 
2052 /* Disabling always succeeds */
ftrace_hash_ipmodify_disable(struct ftrace_ops * ops)2053 static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
2054 {
2055 	struct ftrace_hash *hash = ops->func_hash->filter_hash;
2056 
2057 	if (ftrace_hash_empty(hash))
2058 		hash = NULL;
2059 
2060 	__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
2061 }
2062 
ftrace_hash_ipmodify_update(struct ftrace_ops * ops,struct ftrace_hash * new_hash)2063 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
2064 				       struct ftrace_hash *new_hash)
2065 {
2066 	struct ftrace_hash *old_hash = ops->func_hash->filter_hash;
2067 
2068 	if (ftrace_hash_empty(old_hash))
2069 		old_hash = NULL;
2070 
2071 	if (ftrace_hash_empty(new_hash))
2072 		new_hash = NULL;
2073 
2074 	return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
2075 }
2076 
print_ip_ins(const char * fmt,const unsigned char * p)2077 static void print_ip_ins(const char *fmt, const unsigned char *p)
2078 {
2079 	char ins[MCOUNT_INSN_SIZE];
2080 
2081 	if (copy_from_kernel_nofault(ins, p, MCOUNT_INSN_SIZE)) {
2082 		printk(KERN_CONT "%s[FAULT] %px\n", fmt, p);
2083 		return;
2084 	}
2085 
2086 	printk(KERN_CONT "%s", fmt);
2087 	pr_cont("%*phC", MCOUNT_INSN_SIZE, ins);
2088 }
2089 
2090 enum ftrace_bug_type ftrace_bug_type;
2091 const void *ftrace_expected;
2092 
print_bug_type(void)2093 static void print_bug_type(void)
2094 {
2095 	switch (ftrace_bug_type) {
2096 	case FTRACE_BUG_UNKNOWN:
2097 		break;
2098 	case FTRACE_BUG_INIT:
2099 		pr_info("Initializing ftrace call sites\n");
2100 		break;
2101 	case FTRACE_BUG_NOP:
2102 		pr_info("Setting ftrace call site to NOP\n");
2103 		break;
2104 	case FTRACE_BUG_CALL:
2105 		pr_info("Setting ftrace call site to call ftrace function\n");
2106 		break;
2107 	case FTRACE_BUG_UPDATE:
2108 		pr_info("Updating ftrace call site to call a different ftrace function\n");
2109 		break;
2110 	}
2111 }
2112 
2113 /**
2114  * ftrace_bug - report and shutdown function tracer
2115  * @failed: The failed type (EFAULT, EINVAL, EPERM)
2116  * @rec: The record that failed
2117  *
2118  * The arch code that enables or disables the function tracing
2119  * can call ftrace_bug() when it has detected a problem in
2120  * modifying the code. @failed should be one of either:
2121  * EFAULT - if the problem happens on reading the @ip address
2122  * EINVAL - if what is read at @ip is not what was expected
2123  * EPERM - if the problem happens on writing to the @ip address
2124  */
ftrace_bug(int failed,struct dyn_ftrace * rec)2125 void ftrace_bug(int failed, struct dyn_ftrace *rec)
2126 {
2127 	unsigned long ip = rec ? rec->ip : 0;
2128 
2129 	pr_info("------------[ ftrace bug ]------------\n");
2130 
2131 	switch (failed) {
2132 	case -EFAULT:
2133 		pr_info("ftrace faulted on modifying ");
2134 		print_ip_sym(KERN_INFO, ip);
2135 		break;
2136 	case -EINVAL:
2137 		pr_info("ftrace failed to modify ");
2138 		print_ip_sym(KERN_INFO, ip);
2139 		print_ip_ins(" actual:   ", (unsigned char *)ip);
2140 		pr_cont("\n");
2141 		if (ftrace_expected) {
2142 			print_ip_ins(" expected: ", ftrace_expected);
2143 			pr_cont("\n");
2144 		}
2145 		break;
2146 	case -EPERM:
2147 		pr_info("ftrace faulted on writing ");
2148 		print_ip_sym(KERN_INFO, ip);
2149 		break;
2150 	default:
2151 		pr_info("ftrace faulted on unknown error ");
2152 		print_ip_sym(KERN_INFO, ip);
2153 	}
2154 	print_bug_type();
2155 	if (rec) {
2156 		struct ftrace_ops *ops = NULL;
2157 
2158 		pr_info("ftrace record flags: %lx\n", rec->flags);
2159 		pr_cont(" (%ld)%s%s", ftrace_rec_count(rec),
2160 			rec->flags & FTRACE_FL_REGS ? " R" : "  ",
2161 			rec->flags & FTRACE_FL_CALL_OPS ? " O" : "  ");
2162 		if (rec->flags & FTRACE_FL_TRAMP_EN) {
2163 			ops = ftrace_find_tramp_ops_any(rec);
2164 			if (ops) {
2165 				do {
2166 					pr_cont("\ttramp: %pS (%pS)",
2167 						(void *)ops->trampoline,
2168 						(void *)ops->func);
2169 					ops = ftrace_find_tramp_ops_next(rec, ops);
2170 				} while (ops);
2171 			} else
2172 				pr_cont("\ttramp: ERROR!");
2173 
2174 		}
2175 		ip = ftrace_get_addr_curr(rec);
2176 		pr_cont("\n expected tramp: %lx\n", ip);
2177 	}
2178 
2179 	FTRACE_WARN_ON_ONCE(1);
2180 }
2181 
ftrace_check_record(struct dyn_ftrace * rec,bool enable,bool update)2182 static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update)
2183 {
2184 	unsigned long flag = 0UL;
2185 
2186 	ftrace_bug_type = FTRACE_BUG_UNKNOWN;
2187 
2188 	if (skip_record(rec))
2189 		return FTRACE_UPDATE_IGNORE;
2190 
2191 	/*
2192 	 * If we are updating calls:
2193 	 *
2194 	 *   If the record has a ref count, then we need to enable it
2195 	 *   because someone is using it.
2196 	 *
2197 	 *   Otherwise we make sure its disabled.
2198 	 *
2199 	 * If we are disabling calls, then disable all records that
2200 	 * are enabled.
2201 	 */
2202 	if (enable && ftrace_rec_count(rec))
2203 		flag = FTRACE_FL_ENABLED;
2204 
2205 	/*
2206 	 * If enabling and the REGS flag does not match the REGS_EN, or
2207 	 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
2208 	 * this record. Set flags to fail the compare against ENABLED.
2209 	 * Same for direct calls.
2210 	 */
2211 	if (flag) {
2212 		if (!(rec->flags & FTRACE_FL_REGS) !=
2213 		    !(rec->flags & FTRACE_FL_REGS_EN))
2214 			flag |= FTRACE_FL_REGS;
2215 
2216 		if (!(rec->flags & FTRACE_FL_TRAMP) !=
2217 		    !(rec->flags & FTRACE_FL_TRAMP_EN))
2218 			flag |= FTRACE_FL_TRAMP;
2219 
2220 		/*
2221 		 * Direct calls are special, as count matters.
2222 		 * We must test the record for direct, if the
2223 		 * DIRECT and DIRECT_EN do not match, but only
2224 		 * if the count is 1. That's because, if the
2225 		 * count is something other than one, we do not
2226 		 * want the direct enabled (it will be done via the
2227 		 * direct helper). But if DIRECT_EN is set, and
2228 		 * the count is not one, we need to clear it.
2229 		 *
2230 		 */
2231 		if (ftrace_rec_count(rec) == 1) {
2232 			if (!(rec->flags & FTRACE_FL_DIRECT) !=
2233 			    !(rec->flags & FTRACE_FL_DIRECT_EN))
2234 				flag |= FTRACE_FL_DIRECT;
2235 		} else if (rec->flags & FTRACE_FL_DIRECT_EN) {
2236 			flag |= FTRACE_FL_DIRECT;
2237 		}
2238 
2239 		/*
2240 		 * Ops calls are special, as count matters.
2241 		 * As with direct calls, they must only be enabled when count
2242 		 * is one, otherwise they'll be handled via the list ops.
2243 		 */
2244 		if (ftrace_rec_count(rec) == 1) {
2245 			if (!(rec->flags & FTRACE_FL_CALL_OPS) !=
2246 			    !(rec->flags & FTRACE_FL_CALL_OPS_EN))
2247 				flag |= FTRACE_FL_CALL_OPS;
2248 		} else if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
2249 			flag |= FTRACE_FL_CALL_OPS;
2250 		}
2251 	}
2252 
2253 	/* If the state of this record hasn't changed, then do nothing */
2254 	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
2255 		return FTRACE_UPDATE_IGNORE;
2256 
2257 	if (flag) {
2258 		/* Save off if rec is being enabled (for return value) */
2259 		flag ^= rec->flags & FTRACE_FL_ENABLED;
2260 
2261 		if (update) {
2262 			rec->flags |= FTRACE_FL_ENABLED | FTRACE_FL_TOUCHED;
2263 			if (flag & FTRACE_FL_REGS) {
2264 				if (rec->flags & FTRACE_FL_REGS)
2265 					rec->flags |= FTRACE_FL_REGS_EN;
2266 				else
2267 					rec->flags &= ~FTRACE_FL_REGS_EN;
2268 			}
2269 			if (flag & FTRACE_FL_TRAMP) {
2270 				if (rec->flags & FTRACE_FL_TRAMP)
2271 					rec->flags |= FTRACE_FL_TRAMP_EN;
2272 				else
2273 					rec->flags &= ~FTRACE_FL_TRAMP_EN;
2274 			}
2275 
2276 			/* Keep track of anything that modifies the function */
2277 			if (rec->flags & (FTRACE_FL_DIRECT | FTRACE_FL_IPMODIFY))
2278 				rec->flags |= FTRACE_FL_MODIFIED;
2279 
2280 			if (flag & FTRACE_FL_DIRECT) {
2281 				/*
2282 				 * If there's only one user (direct_ops helper)
2283 				 * then we can call the direct function
2284 				 * directly (no ftrace trampoline).
2285 				 */
2286 				if (ftrace_rec_count(rec) == 1) {
2287 					if (rec->flags & FTRACE_FL_DIRECT)
2288 						rec->flags |= FTRACE_FL_DIRECT_EN;
2289 					else
2290 						rec->flags &= ~FTRACE_FL_DIRECT_EN;
2291 				} else {
2292 					/*
2293 					 * Can only call directly if there's
2294 					 * only one callback to the function.
2295 					 */
2296 					rec->flags &= ~FTRACE_FL_DIRECT_EN;
2297 				}
2298 			}
2299 
2300 			if (flag & FTRACE_FL_CALL_OPS) {
2301 				if (ftrace_rec_count(rec) == 1) {
2302 					if (rec->flags & FTRACE_FL_CALL_OPS)
2303 						rec->flags |= FTRACE_FL_CALL_OPS_EN;
2304 					else
2305 						rec->flags &= ~FTRACE_FL_CALL_OPS_EN;
2306 				} else {
2307 					/*
2308 					 * Can only call directly if there's
2309 					 * only one set of associated ops.
2310 					 */
2311 					rec->flags &= ~FTRACE_FL_CALL_OPS_EN;
2312 				}
2313 			}
2314 		}
2315 
2316 		/*
2317 		 * If this record is being updated from a nop, then
2318 		 *   return UPDATE_MAKE_CALL.
2319 		 * Otherwise,
2320 		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
2321 		 *   from the save regs, to a non-save regs function or
2322 		 *   vice versa, or from a trampoline call.
2323 		 */
2324 		if (flag & FTRACE_FL_ENABLED) {
2325 			ftrace_bug_type = FTRACE_BUG_CALL;
2326 			return FTRACE_UPDATE_MAKE_CALL;
2327 		}
2328 
2329 		ftrace_bug_type = FTRACE_BUG_UPDATE;
2330 		return FTRACE_UPDATE_MODIFY_CALL;
2331 	}
2332 
2333 	if (update) {
2334 		/* If there's no more users, clear all flags */
2335 		if (!ftrace_rec_count(rec))
2336 			rec->flags &= FTRACE_NOCLEAR_FLAGS;
2337 		else
2338 			/*
2339 			 * Just disable the record, but keep the ops TRAMP
2340 			 * and REGS states. The _EN flags must be disabled though.
2341 			 */
2342 			rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
2343 					FTRACE_FL_REGS_EN | FTRACE_FL_DIRECT_EN |
2344 					FTRACE_FL_CALL_OPS_EN);
2345 	}
2346 
2347 	ftrace_bug_type = FTRACE_BUG_NOP;
2348 	return FTRACE_UPDATE_MAKE_NOP;
2349 }
2350 
2351 /**
2352  * ftrace_update_record - set a record that now is tracing or not
2353  * @rec: the record to update
2354  * @enable: set to true if the record is tracing, false to force disable
2355  *
2356  * The records that represent all functions that can be traced need
2357  * to be updated when tracing has been enabled.
2358  */
ftrace_update_record(struct dyn_ftrace * rec,bool enable)2359 int ftrace_update_record(struct dyn_ftrace *rec, bool enable)
2360 {
2361 	return ftrace_check_record(rec, enable, true);
2362 }
2363 
2364 /**
2365  * ftrace_test_record - check if the record has been enabled or not
2366  * @rec: the record to test
2367  * @enable: set to true to check if enabled, false if it is disabled
2368  *
2369  * The arch code may need to test if a record is already set to
2370  * tracing to determine how to modify the function code that it
2371  * represents.
2372  */
ftrace_test_record(struct dyn_ftrace * rec,bool enable)2373 int ftrace_test_record(struct dyn_ftrace *rec, bool enable)
2374 {
2375 	return ftrace_check_record(rec, enable, false);
2376 }
2377 
2378 static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace * rec)2379 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
2380 {
2381 	struct ftrace_ops *op;
2382 	unsigned long ip = rec->ip;
2383 
2384 	do_for_each_ftrace_op(op, ftrace_ops_list) {
2385 
2386 		if (!op->trampoline)
2387 			continue;
2388 
2389 		if (hash_contains_ip(ip, op->func_hash))
2390 			return op;
2391 	} while_for_each_ftrace_op(op);
2392 
2393 	return NULL;
2394 }
2395 
2396 static struct ftrace_ops *
ftrace_find_tramp_ops_any_other(struct dyn_ftrace * rec,struct ftrace_ops * op_exclude)2397 ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude)
2398 {
2399 	struct ftrace_ops *op;
2400 	unsigned long ip = rec->ip;
2401 
2402 	do_for_each_ftrace_op(op, ftrace_ops_list) {
2403 
2404 		if (op == op_exclude || !op->trampoline)
2405 			continue;
2406 
2407 		if (hash_contains_ip(ip, op->func_hash))
2408 			return op;
2409 	} while_for_each_ftrace_op(op);
2410 
2411 	return NULL;
2412 }
2413 
2414 static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace * rec,struct ftrace_ops * op)2415 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
2416 			   struct ftrace_ops *op)
2417 {
2418 	unsigned long ip = rec->ip;
2419 
2420 	while_for_each_ftrace_op(op) {
2421 
2422 		if (!op->trampoline)
2423 			continue;
2424 
2425 		if (hash_contains_ip(ip, op->func_hash))
2426 			return op;
2427 	}
2428 
2429 	return NULL;
2430 }
2431 
2432 static struct ftrace_ops *
ftrace_find_tramp_ops_curr(struct dyn_ftrace * rec)2433 ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
2434 {
2435 	struct ftrace_ops *op;
2436 	unsigned long ip = rec->ip;
2437 
2438 	/*
2439 	 * Need to check removed ops first.
2440 	 * If they are being removed, and this rec has a tramp,
2441 	 * and this rec is in the ops list, then it would be the
2442 	 * one with the tramp.
2443 	 */
2444 	if (removed_ops) {
2445 		if (hash_contains_ip(ip, &removed_ops->old_hash))
2446 			return removed_ops;
2447 	}
2448 
2449 	/*
2450 	 * Need to find the current trampoline for a rec.
2451 	 * Now, a trampoline is only attached to a rec if there
2452 	 * was a single 'ops' attached to it. But this can be called
2453 	 * when we are adding another op to the rec or removing the
2454 	 * current one. Thus, if the op is being added, we can
2455 	 * ignore it because it hasn't attached itself to the rec
2456 	 * yet.
2457 	 *
2458 	 * If an ops is being modified (hooking to different functions)
2459 	 * then we don't care about the new functions that are being
2460 	 * added, just the old ones (that are probably being removed).
2461 	 *
2462 	 * If we are adding an ops to a function that already is using
2463 	 * a trampoline, it needs to be removed (trampolines are only
2464 	 * for single ops connected), then an ops that is not being
2465 	 * modified also needs to be checked.
2466 	 */
2467 	do_for_each_ftrace_op(op, ftrace_ops_list) {
2468 
2469 		if (!op->trampoline)
2470 			continue;
2471 
2472 		/*
2473 		 * If the ops is being added, it hasn't gotten to
2474 		 * the point to be removed from this tree yet.
2475 		 */
2476 		if (op->flags & FTRACE_OPS_FL_ADDING)
2477 			continue;
2478 
2479 
2480 		/*
2481 		 * If the ops is being modified and is in the old
2482 		 * hash, then it is probably being removed from this
2483 		 * function.
2484 		 */
2485 		if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
2486 		    hash_contains_ip(ip, &op->old_hash))
2487 			return op;
2488 		/*
2489 		 * If the ops is not being added or modified, and it's
2490 		 * in its normal filter hash, then this must be the one
2491 		 * we want!
2492 		 */
2493 		if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
2494 		    hash_contains_ip(ip, op->func_hash))
2495 			return op;
2496 
2497 	} while_for_each_ftrace_op(op);
2498 
2499 	return NULL;
2500 }
2501 
2502 static struct ftrace_ops *
ftrace_find_tramp_ops_new(struct dyn_ftrace * rec)2503 ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
2504 {
2505 	struct ftrace_ops *op;
2506 	unsigned long ip = rec->ip;
2507 
2508 	do_for_each_ftrace_op(op, ftrace_ops_list) {
2509 		/* pass rec in as regs to have non-NULL val */
2510 		if (hash_contains_ip(ip, op->func_hash))
2511 			return op;
2512 	} while_for_each_ftrace_op(op);
2513 
2514 	return NULL;
2515 }
2516 
2517 struct ftrace_ops *
ftrace_find_unique_ops(struct dyn_ftrace * rec)2518 ftrace_find_unique_ops(struct dyn_ftrace *rec)
2519 {
2520 	struct ftrace_ops *op, *found = NULL;
2521 	unsigned long ip = rec->ip;
2522 
2523 	do_for_each_ftrace_op(op, ftrace_ops_list) {
2524 
2525 		if (hash_contains_ip(ip, op->func_hash)) {
2526 			if (found)
2527 				return NULL;
2528 			found = op;
2529 		}
2530 
2531 	} while_for_each_ftrace_op(op);
2532 
2533 	return found;
2534 }
2535 
2536 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
2537 /* Protected by rcu_tasks for reading, and direct_mutex for writing */
2538 static struct ftrace_hash __rcu *direct_functions = EMPTY_HASH;
2539 static DEFINE_MUTEX(direct_mutex);
2540 int ftrace_direct_func_count;
2541 
2542 /*
2543  * Search the direct_functions hash to see if the given instruction pointer
2544  * has a direct caller attached to it.
2545  */
ftrace_find_rec_direct(unsigned long ip)2546 unsigned long ftrace_find_rec_direct(unsigned long ip)
2547 {
2548 	struct ftrace_func_entry *entry;
2549 
2550 	entry = __ftrace_lookup_ip(direct_functions, ip);
2551 	if (!entry)
2552 		return 0;
2553 
2554 	return entry->direct;
2555 }
2556 
call_direct_funcs(unsigned long ip,unsigned long pip,struct ftrace_ops * ops,struct ftrace_regs * fregs)2557 static void call_direct_funcs(unsigned long ip, unsigned long pip,
2558 			      struct ftrace_ops *ops, struct ftrace_regs *fregs)
2559 {
2560 	unsigned long addr = READ_ONCE(ops->direct_call);
2561 
2562 	if (!addr)
2563 		return;
2564 
2565 	arch_ftrace_set_direct_caller(fregs, addr);
2566 }
2567 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
2568 
2569 /**
2570  * ftrace_get_addr_new - Get the call address to set to
2571  * @rec:  The ftrace record descriptor
2572  *
2573  * If the record has the FTRACE_FL_REGS set, that means that it
2574  * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
2575  * is not set, then it wants to convert to the normal callback.
2576  *
2577  * Returns the address of the trampoline to set to
2578  */
ftrace_get_addr_new(struct dyn_ftrace * rec)2579 unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
2580 {
2581 	struct ftrace_ops *ops;
2582 	unsigned long addr;
2583 
2584 	if ((rec->flags & FTRACE_FL_DIRECT) &&
2585 	    (ftrace_rec_count(rec) == 1)) {
2586 		addr = ftrace_find_rec_direct(rec->ip);
2587 		if (addr)
2588 			return addr;
2589 		WARN_ON_ONCE(1);
2590 	}
2591 
2592 	/* Trampolines take precedence over regs */
2593 	if (rec->flags & FTRACE_FL_TRAMP) {
2594 		ops = ftrace_find_tramp_ops_new(rec);
2595 		if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
2596 			pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
2597 				(void *)rec->ip, (void *)rec->ip, rec->flags);
2598 			/* Ftrace is shutting down, return anything */
2599 			return (unsigned long)FTRACE_ADDR;
2600 		}
2601 		return ops->trampoline;
2602 	}
2603 
2604 	if (rec->flags & FTRACE_FL_REGS)
2605 		return (unsigned long)FTRACE_REGS_ADDR;
2606 	else
2607 		return (unsigned long)FTRACE_ADDR;
2608 }
2609 
2610 /**
2611  * ftrace_get_addr_curr - Get the call address that is already there
2612  * @rec:  The ftrace record descriptor
2613  *
2614  * The FTRACE_FL_REGS_EN is set when the record already points to
2615  * a function that saves all the regs. Basically the '_EN' version
2616  * represents the current state of the function.
2617  *
2618  * Returns the address of the trampoline that is currently being called
2619  */
ftrace_get_addr_curr(struct dyn_ftrace * rec)2620 unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
2621 {
2622 	struct ftrace_ops *ops;
2623 	unsigned long addr;
2624 
2625 	/* Direct calls take precedence over trampolines */
2626 	if (rec->flags & FTRACE_FL_DIRECT_EN) {
2627 		addr = ftrace_find_rec_direct(rec->ip);
2628 		if (addr)
2629 			return addr;
2630 		WARN_ON_ONCE(1);
2631 	}
2632 
2633 	/* Trampolines take precedence over regs */
2634 	if (rec->flags & FTRACE_FL_TRAMP_EN) {
2635 		ops = ftrace_find_tramp_ops_curr(rec);
2636 		if (FTRACE_WARN_ON(!ops)) {
2637 			pr_warn("Bad trampoline accounting at: %p (%pS)\n",
2638 				(void *)rec->ip, (void *)rec->ip);
2639 			/* Ftrace is shutting down, return anything */
2640 			return (unsigned long)FTRACE_ADDR;
2641 		}
2642 		return ops->trampoline;
2643 	}
2644 
2645 	if (rec->flags & FTRACE_FL_REGS_EN)
2646 		return (unsigned long)FTRACE_REGS_ADDR;
2647 	else
2648 		return (unsigned long)FTRACE_ADDR;
2649 }
2650 
2651 static int
__ftrace_replace_code(struct dyn_ftrace * rec,bool enable)2652 __ftrace_replace_code(struct dyn_ftrace *rec, bool enable)
2653 {
2654 	unsigned long ftrace_old_addr;
2655 	unsigned long ftrace_addr;
2656 	int ret;
2657 
2658 	ftrace_addr = ftrace_get_addr_new(rec);
2659 
2660 	/* This needs to be done before we call ftrace_update_record */
2661 	ftrace_old_addr = ftrace_get_addr_curr(rec);
2662 
2663 	ret = ftrace_update_record(rec, enable);
2664 
2665 	ftrace_bug_type = FTRACE_BUG_UNKNOWN;
2666 
2667 	switch (ret) {
2668 	case FTRACE_UPDATE_IGNORE:
2669 		return 0;
2670 
2671 	case FTRACE_UPDATE_MAKE_CALL:
2672 		ftrace_bug_type = FTRACE_BUG_CALL;
2673 		return ftrace_make_call(rec, ftrace_addr);
2674 
2675 	case FTRACE_UPDATE_MAKE_NOP:
2676 		ftrace_bug_type = FTRACE_BUG_NOP;
2677 		return ftrace_make_nop(NULL, rec, ftrace_old_addr);
2678 
2679 	case FTRACE_UPDATE_MODIFY_CALL:
2680 		ftrace_bug_type = FTRACE_BUG_UPDATE;
2681 		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
2682 	}
2683 
2684 	return -1; /* unknown ftrace bug */
2685 }
2686 
ftrace_replace_code(int mod_flags)2687 void __weak ftrace_replace_code(int mod_flags)
2688 {
2689 	struct dyn_ftrace *rec;
2690 	struct ftrace_page *pg;
2691 	bool enable = mod_flags & FTRACE_MODIFY_ENABLE_FL;
2692 	int schedulable = mod_flags & FTRACE_MODIFY_MAY_SLEEP_FL;
2693 	int failed;
2694 
2695 	if (unlikely(ftrace_disabled))
2696 		return;
2697 
2698 	do_for_each_ftrace_rec(pg, rec) {
2699 
2700 		if (skip_record(rec))
2701 			continue;
2702 
2703 		failed = __ftrace_replace_code(rec, enable);
2704 		if (failed) {
2705 			ftrace_bug(failed, rec);
2706 			/* Stop processing */
2707 			return;
2708 		}
2709 		if (schedulable)
2710 			cond_resched();
2711 	} while_for_each_ftrace_rec();
2712 }
2713 
2714 struct ftrace_rec_iter {
2715 	struct ftrace_page	*pg;
2716 	int			index;
2717 };
2718 
2719 /**
2720  * ftrace_rec_iter_start - start up iterating over traced functions
2721  *
2722  * Returns an iterator handle that is used to iterate over all
2723  * the records that represent address locations where functions
2724  * are traced.
2725  *
2726  * May return NULL if no records are available.
2727  */
ftrace_rec_iter_start(void)2728 struct ftrace_rec_iter *ftrace_rec_iter_start(void)
2729 {
2730 	/*
2731 	 * We only use a single iterator.
2732 	 * Protected by the ftrace_lock mutex.
2733 	 */
2734 	static struct ftrace_rec_iter ftrace_rec_iter;
2735 	struct ftrace_rec_iter *iter = &ftrace_rec_iter;
2736 
2737 	iter->pg = ftrace_pages_start;
2738 	iter->index = 0;
2739 
2740 	/* Could have empty pages */
2741 	while (iter->pg && !iter->pg->index)
2742 		iter->pg = iter->pg->next;
2743 
2744 	if (!iter->pg)
2745 		return NULL;
2746 
2747 	return iter;
2748 }
2749 
2750 /**
2751  * ftrace_rec_iter_next - get the next record to process.
2752  * @iter: The handle to the iterator.
2753  *
2754  * Returns the next iterator after the given iterator @iter.
2755  */
ftrace_rec_iter_next(struct ftrace_rec_iter * iter)2756 struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
2757 {
2758 	iter->index++;
2759 
2760 	if (iter->index >= iter->pg->index) {
2761 		iter->pg = iter->pg->next;
2762 		iter->index = 0;
2763 
2764 		/* Could have empty pages */
2765 		while (iter->pg && !iter->pg->index)
2766 			iter->pg = iter->pg->next;
2767 	}
2768 
2769 	if (!iter->pg)
2770 		return NULL;
2771 
2772 	return iter;
2773 }
2774 
2775 /**
2776  * ftrace_rec_iter_record - get the record at the iterator location
2777  * @iter: The current iterator location
2778  *
2779  * Returns the record that the current @iter is at.
2780  */
ftrace_rec_iter_record(struct ftrace_rec_iter * iter)2781 struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
2782 {
2783 	return &iter->pg->records[iter->index];
2784 }
2785 
2786 static int
ftrace_nop_initialize(struct module * mod,struct dyn_ftrace * rec)2787 ftrace_nop_initialize(struct module *mod, struct dyn_ftrace *rec)
2788 {
2789 	int ret;
2790 
2791 	if (unlikely(ftrace_disabled))
2792 		return 0;
2793 
2794 	ret = ftrace_init_nop(mod, rec);
2795 	if (ret) {
2796 		ftrace_bug_type = FTRACE_BUG_INIT;
2797 		ftrace_bug(ret, rec);
2798 		return 0;
2799 	}
2800 	return 1;
2801 }
2802 
2803 /*
2804  * archs can override this function if they must do something
2805  * before the modifying code is performed.
2806  */
ftrace_arch_code_modify_prepare(void)2807 void __weak ftrace_arch_code_modify_prepare(void)
2808 {
2809 }
2810 
2811 /*
2812  * archs can override this function if they must do something
2813  * after the modifying code is performed.
2814  */
ftrace_arch_code_modify_post_process(void)2815 void __weak ftrace_arch_code_modify_post_process(void)
2816 {
2817 }
2818 
update_ftrace_func(ftrace_func_t func)2819 static int update_ftrace_func(ftrace_func_t func)
2820 {
2821 	static ftrace_func_t save_func;
2822 
2823 	/* Avoid updating if it hasn't changed */
2824 	if (func == save_func)
2825 		return 0;
2826 
2827 	save_func = func;
2828 
2829 	return ftrace_update_ftrace_func(func);
2830 }
2831 
ftrace_modify_all_code(int command)2832 void ftrace_modify_all_code(int command)
2833 {
2834 	int update = command & FTRACE_UPDATE_TRACE_FUNC;
2835 	int mod_flags = 0;
2836 	int err = 0;
2837 
2838 	if (command & FTRACE_MAY_SLEEP)
2839 		mod_flags = FTRACE_MODIFY_MAY_SLEEP_FL;
2840 
2841 	/*
2842 	 * If the ftrace_caller calls a ftrace_ops func directly,
2843 	 * we need to make sure that it only traces functions it
2844 	 * expects to trace. When doing the switch of functions,
2845 	 * we need to update to the ftrace_ops_list_func first
2846 	 * before the transition between old and new calls are set,
2847 	 * as the ftrace_ops_list_func will check the ops hashes
2848 	 * to make sure the ops are having the right functions
2849 	 * traced.
2850 	 */
2851 	if (update) {
2852 		err = update_ftrace_func(ftrace_ops_list_func);
2853 		if (FTRACE_WARN_ON(err))
2854 			return;
2855 	}
2856 
2857 	if (command & FTRACE_UPDATE_CALLS)
2858 		ftrace_replace_code(mod_flags | FTRACE_MODIFY_ENABLE_FL);
2859 	else if (command & FTRACE_DISABLE_CALLS)
2860 		ftrace_replace_code(mod_flags);
2861 
2862 	if (update && ftrace_trace_function != ftrace_ops_list_func) {
2863 		function_trace_op = set_function_trace_op;
2864 		smp_wmb();
2865 		/* If irqs are disabled, we are in stop machine */
2866 		if (!irqs_disabled())
2867 			smp_call_function(ftrace_sync_ipi, NULL, 1);
2868 		err = update_ftrace_func(ftrace_trace_function);
2869 		if (FTRACE_WARN_ON(err))
2870 			return;
2871 	}
2872 
2873 	if (command & FTRACE_START_FUNC_RET)
2874 		err = ftrace_enable_ftrace_graph_caller();
2875 	else if (command & FTRACE_STOP_FUNC_RET)
2876 		err = ftrace_disable_ftrace_graph_caller();
2877 	FTRACE_WARN_ON(err);
2878 }
2879 
__ftrace_modify_code(void * data)2880 static int __ftrace_modify_code(void *data)
2881 {
2882 	int *command = data;
2883 
2884 	ftrace_modify_all_code(*command);
2885 
2886 	return 0;
2887 }
2888 
2889 /**
2890  * ftrace_run_stop_machine - go back to the stop machine method
2891  * @command: The command to tell ftrace what to do
2892  *
2893  * If an arch needs to fall back to the stop machine method, the
2894  * it can call this function.
2895  */
ftrace_run_stop_machine(int command)2896 void ftrace_run_stop_machine(int command)
2897 {
2898 	stop_machine(__ftrace_modify_code, &command, NULL);
2899 }
2900 
2901 /**
2902  * arch_ftrace_update_code - modify the code to trace or not trace
2903  * @command: The command that needs to be done
2904  *
2905  * Archs can override this function if it does not need to
2906  * run stop_machine() to modify code.
2907  */
arch_ftrace_update_code(int command)2908 void __weak arch_ftrace_update_code(int command)
2909 {
2910 	ftrace_run_stop_machine(command);
2911 }
2912 
ftrace_run_update_code(int command)2913 static void ftrace_run_update_code(int command)
2914 {
2915 	ftrace_arch_code_modify_prepare();
2916 
2917 	/*
2918 	 * By default we use stop_machine() to modify the code.
2919 	 * But archs can do what ever they want as long as it
2920 	 * is safe. The stop_machine() is the safest, but also
2921 	 * produces the most overhead.
2922 	 */
2923 	arch_ftrace_update_code(command);
2924 
2925 	ftrace_arch_code_modify_post_process();
2926 }
2927 
ftrace_run_modify_code(struct ftrace_ops * ops,int command,struct ftrace_ops_hash * old_hash)2928 static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
2929 				   struct ftrace_ops_hash *old_hash)
2930 {
2931 	ops->flags |= FTRACE_OPS_FL_MODIFYING;
2932 	ops->old_hash.filter_hash = old_hash->filter_hash;
2933 	ops->old_hash.notrace_hash = old_hash->notrace_hash;
2934 	ftrace_run_update_code(command);
2935 	ops->old_hash.filter_hash = NULL;
2936 	ops->old_hash.notrace_hash = NULL;
2937 	ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
2938 }
2939 
2940 static ftrace_func_t saved_ftrace_func;
2941 static int ftrace_start_up;
2942 
arch_ftrace_trampoline_free(struct ftrace_ops * ops)2943 void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
2944 {
2945 }
2946 
2947 /* List of trace_ops that have allocated trampolines */
2948 static LIST_HEAD(ftrace_ops_trampoline_list);
2949 
ftrace_add_trampoline_to_kallsyms(struct ftrace_ops * ops)2950 static void ftrace_add_trampoline_to_kallsyms(struct ftrace_ops *ops)
2951 {
2952 	lockdep_assert_held(&ftrace_lock);
2953 	list_add_rcu(&ops->list, &ftrace_ops_trampoline_list);
2954 }
2955 
ftrace_remove_trampoline_from_kallsyms(struct ftrace_ops * ops)2956 static void ftrace_remove_trampoline_from_kallsyms(struct ftrace_ops *ops)
2957 {
2958 	lockdep_assert_held(&ftrace_lock);
2959 	list_del_rcu(&ops->list);
2960 	synchronize_rcu();
2961 }
2962 
2963 /*
2964  * "__builtin__ftrace" is used as a module name in /proc/kallsyms for symbols
2965  * for pages allocated for ftrace purposes, even though "__builtin__ftrace" is
2966  * not a module.
2967  */
2968 #define FTRACE_TRAMPOLINE_MOD "__builtin__ftrace"
2969 #define FTRACE_TRAMPOLINE_SYM "ftrace_trampoline"
2970 
ftrace_trampoline_free(struct ftrace_ops * ops)2971 static void ftrace_trampoline_free(struct ftrace_ops *ops)
2972 {
2973 	if (ops && (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP) &&
2974 	    ops->trampoline) {
2975 		/*
2976 		 * Record the text poke event before the ksymbol unregister
2977 		 * event.
2978 		 */
2979 		perf_event_text_poke((void *)ops->trampoline,
2980 				     (void *)ops->trampoline,
2981 				     ops->trampoline_size, NULL, 0);
2982 		perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
2983 				   ops->trampoline, ops->trampoline_size,
2984 				   true, FTRACE_TRAMPOLINE_SYM);
2985 		/* Remove from kallsyms after the perf events */
2986 		ftrace_remove_trampoline_from_kallsyms(ops);
2987 	}
2988 
2989 	arch_ftrace_trampoline_free(ops);
2990 }
2991 
ftrace_startup_enable(int command)2992 static void ftrace_startup_enable(int command)
2993 {
2994 	if (saved_ftrace_func != ftrace_trace_function) {
2995 		saved_ftrace_func = ftrace_trace_function;
2996 		command |= FTRACE_UPDATE_TRACE_FUNC;
2997 	}
2998 
2999 	if (!command || !ftrace_enabled)
3000 		return;
3001 
3002 	ftrace_run_update_code(command);
3003 }
3004 
ftrace_startup_all(int command)3005 static void ftrace_startup_all(int command)
3006 {
3007 	update_all_ops = true;
3008 	ftrace_startup_enable(command);
3009 	update_all_ops = false;
3010 }
3011 
ftrace_startup(struct ftrace_ops * ops,int command)3012 int ftrace_startup(struct ftrace_ops *ops, int command)
3013 {
3014 	int ret;
3015 
3016 	if (unlikely(ftrace_disabled))
3017 		return -ENODEV;
3018 
3019 	ret = __register_ftrace_function(ops);
3020 	if (ret)
3021 		return ret;
3022 
3023 	ftrace_start_up++;
3024 
3025 	/*
3026 	 * Note that ftrace probes uses this to start up
3027 	 * and modify functions it will probe. But we still
3028 	 * set the ADDING flag for modification, as probes
3029 	 * do not have trampolines. If they add them in the
3030 	 * future, then the probes will need to distinguish
3031 	 * between adding and updating probes.
3032 	 */
3033 	ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
3034 
3035 	ret = ftrace_hash_ipmodify_enable(ops);
3036 	if (ret < 0) {
3037 		/* Rollback registration process */
3038 		__unregister_ftrace_function(ops);
3039 		ftrace_start_up--;
3040 		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
3041 		if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
3042 			ftrace_trampoline_free(ops);
3043 		return ret;
3044 	}
3045 
3046 	if (ftrace_hash_rec_enable(ops, 1))
3047 		command |= FTRACE_UPDATE_CALLS;
3048 
3049 	ftrace_startup_enable(command);
3050 
3051 	/*
3052 	 * If ftrace is in an undefined state, we just remove ops from list
3053 	 * to prevent the NULL pointer, instead of totally rolling it back and
3054 	 * free trampoline, because those actions could cause further damage.
3055 	 */
3056 	if (unlikely(ftrace_disabled)) {
3057 		__unregister_ftrace_function(ops);
3058 		return -ENODEV;
3059 	}
3060 
3061 	ops->flags &= ~FTRACE_OPS_FL_ADDING;
3062 
3063 	return 0;
3064 }
3065 
ftrace_shutdown(struct ftrace_ops * ops,int command)3066 int ftrace_shutdown(struct ftrace_ops *ops, int command)
3067 {
3068 	int ret;
3069 
3070 	if (unlikely(ftrace_disabled))
3071 		return -ENODEV;
3072 
3073 	ret = __unregister_ftrace_function(ops);
3074 	if (ret)
3075 		return ret;
3076 
3077 	ftrace_start_up--;
3078 	/*
3079 	 * Just warn in case of unbalance, no need to kill ftrace, it's not
3080 	 * critical but the ftrace_call callers may be never nopped again after
3081 	 * further ftrace uses.
3082 	 */
3083 	WARN_ON_ONCE(ftrace_start_up < 0);
3084 
3085 	/* Disabling ipmodify never fails */
3086 	ftrace_hash_ipmodify_disable(ops);
3087 
3088 	if (ftrace_hash_rec_disable(ops, 1))
3089 		command |= FTRACE_UPDATE_CALLS;
3090 
3091 	ops->flags &= ~FTRACE_OPS_FL_ENABLED;
3092 
3093 	if (saved_ftrace_func != ftrace_trace_function) {
3094 		saved_ftrace_func = ftrace_trace_function;
3095 		command |= FTRACE_UPDATE_TRACE_FUNC;
3096 	}
3097 
3098 	if (!command || !ftrace_enabled)
3099 		goto out;
3100 
3101 	/*
3102 	 * If the ops uses a trampoline, then it needs to be
3103 	 * tested first on update.
3104 	 */
3105 	ops->flags |= FTRACE_OPS_FL_REMOVING;
3106 	removed_ops = ops;
3107 
3108 	/* The trampoline logic checks the old hashes */
3109 	ops->old_hash.filter_hash = ops->func_hash->filter_hash;
3110 	ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;
3111 
3112 	ftrace_run_update_code(command);
3113 
3114 	/*
3115 	 * If there's no more ops registered with ftrace, run a
3116 	 * sanity check to make sure all rec flags are cleared.
3117 	 */
3118 	if (rcu_dereference_protected(ftrace_ops_list,
3119 			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
3120 		struct ftrace_page *pg;
3121 		struct dyn_ftrace *rec;
3122 
3123 		do_for_each_ftrace_rec(pg, rec) {
3124 			if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_NOCLEAR_FLAGS))
3125 				pr_warn("  %pS flags:%lx\n",
3126 					(void *)rec->ip, rec->flags);
3127 		} while_for_each_ftrace_rec();
3128 	}
3129 
3130 	ops->old_hash.filter_hash = NULL;
3131 	ops->old_hash.notrace_hash = NULL;
3132 
3133 	removed_ops = NULL;
3134 	ops->flags &= ~FTRACE_OPS_FL_REMOVING;
3135 
3136 out:
3137 	/*
3138 	 * Dynamic ops may be freed, we must make sure that all
3139 	 * callers are done before leaving this function.
3140 	 */
3141 	if (ops->flags & FTRACE_OPS_FL_DYNAMIC) {
3142 		/*
3143 		 * We need to do a hard force of sched synchronization.
3144 		 * This is because we use preempt_disable() to do RCU, but
3145 		 * the function tracers can be called where RCU is not watching
3146 		 * (like before user_exit()). We can not rely on the RCU
3147 		 * infrastructure to do the synchronization, thus we must do it
3148 		 * ourselves.
3149 		 */
3150 		synchronize_rcu_tasks_rude();
3151 
3152 		/*
3153 		 * When the kernel is preemptive, tasks can be preempted
3154 		 * while on a ftrace trampoline. Just scheduling a task on
3155 		 * a CPU is not good enough to flush them. Calling
3156 		 * synchronize_rcu_tasks() will wait for those tasks to
3157 		 * execute and either schedule voluntarily or enter user space.
3158 		 */
3159 		if (IS_ENABLED(CONFIG_PREEMPTION))
3160 			synchronize_rcu_tasks();
3161 
3162 		ftrace_trampoline_free(ops);
3163 	}
3164 
3165 	return 0;
3166 }
3167 
3168 static u64		ftrace_update_time;
3169 unsigned long		ftrace_update_tot_cnt;
3170 unsigned long		ftrace_number_of_pages;
3171 unsigned long		ftrace_number_of_groups;
3172 
ops_traces_mod(struct ftrace_ops * ops)3173 static inline int ops_traces_mod(struct ftrace_ops *ops)
3174 {
3175 	/*
3176 	 * Filter_hash being empty will default to trace module.
3177 	 * But notrace hash requires a test of individual module functions.
3178 	 */
3179 	return ftrace_hash_empty(ops->func_hash->filter_hash) &&
3180 		ftrace_hash_empty(ops->func_hash->notrace_hash);
3181 }
3182 
ftrace_update_code(struct module * mod,struct ftrace_page * new_pgs)3183 static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
3184 {
3185 	bool init_nop = ftrace_need_init_nop();
3186 	struct ftrace_page *pg;
3187 	struct dyn_ftrace *p;
3188 	u64 start, stop;
3189 	unsigned long update_cnt = 0;
3190 	unsigned long rec_flags = 0;
3191 	int i;
3192 
3193 	start = ftrace_now(raw_smp_processor_id());
3194 
3195 	/*
3196 	 * When a module is loaded, this function is called to convert
3197 	 * the calls to mcount in its text to nops, and also to create
3198 	 * an entry in the ftrace data. Now, if ftrace is activated
3199 	 * after this call, but before the module sets its text to
3200 	 * read-only, the modification of enabling ftrace can fail if
3201 	 * the read-only is done while ftrace is converting the calls.
3202 	 * To prevent this, the module's records are set as disabled
3203 	 * and will be enabled after the call to set the module's text
3204 	 * to read-only.
3205 	 */
3206 	if (mod)
3207 		rec_flags |= FTRACE_FL_DISABLED;
3208 
3209 	for (pg = new_pgs; pg; pg = pg->next) {
3210 
3211 		for (i = 0; i < pg->index; i++) {
3212 
3213 			/* If something went wrong, bail without enabling anything */
3214 			if (unlikely(ftrace_disabled))
3215 				return -1;
3216 
3217 			p = &pg->records[i];
3218 			p->flags = rec_flags;
3219 
3220 			/*
3221 			 * Do the initial record conversion from mcount jump
3222 			 * to the NOP instructions.
3223 			 */
3224 			if (init_nop && !ftrace_nop_initialize(mod, p))
3225 				break;
3226 
3227 			update_cnt++;
3228 		}
3229 	}
3230 
3231 	stop = ftrace_now(raw_smp_processor_id());
3232 	ftrace_update_time = stop - start;
3233 	ftrace_update_tot_cnt += update_cnt;
3234 
3235 	return 0;
3236 }
3237 
ftrace_allocate_records(struct ftrace_page * pg,int count)3238 static int ftrace_allocate_records(struct ftrace_page *pg, int count)
3239 {
3240 	int order;
3241 	int pages;
3242 	int cnt;
3243 
3244 	if (WARN_ON(!count))
3245 		return -EINVAL;
3246 
3247 	/* We want to fill as much as possible, with no empty pages */
3248 	pages = DIV_ROUND_UP(count, ENTRIES_PER_PAGE);
3249 	order = fls(pages) - 1;
3250 
3251  again:
3252 	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
3253 
3254 	if (!pg->records) {
3255 		/* if we can't allocate this size, try something smaller */
3256 		if (!order)
3257 			return -ENOMEM;
3258 		order--;
3259 		goto again;
3260 	}
3261 
3262 	ftrace_number_of_pages += 1 << order;
3263 	ftrace_number_of_groups++;
3264 
3265 	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
3266 	pg->order = order;
3267 
3268 	if (cnt > count)
3269 		cnt = count;
3270 
3271 	return cnt;
3272 }
3273 
ftrace_free_pages(struct ftrace_page * pages)3274 static void ftrace_free_pages(struct ftrace_page *pages)
3275 {
3276 	struct ftrace_page *pg = pages;
3277 
3278 	while (pg) {
3279 		if (pg->records) {
3280 			free_pages((unsigned long)pg->records, pg->order);
3281 			ftrace_number_of_pages -= 1 << pg->order;
3282 		}
3283 		pages = pg->next;
3284 		kfree(pg);
3285 		pg = pages;
3286 		ftrace_number_of_groups--;
3287 	}
3288 }
3289 
3290 static struct ftrace_page *
ftrace_allocate_pages(unsigned long num_to_init)3291 ftrace_allocate_pages(unsigned long num_to_init)
3292 {
3293 	struct ftrace_page *start_pg;
3294 	struct ftrace_page *pg;
3295 	int cnt;
3296 
3297 	if (!num_to_init)
3298 		return NULL;
3299 
3300 	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
3301 	if (!pg)
3302 		return NULL;
3303 
3304 	/*
3305 	 * Try to allocate as much as possible in one continues
3306 	 * location that fills in all of the space. We want to
3307 	 * waste as little space as possible.
3308 	 */
3309 	for (;;) {
3310 		cnt = ftrace_allocate_records(pg, num_to_init);
3311 		if (cnt < 0)
3312 			goto free_pages;
3313 
3314 		num_to_init -= cnt;
3315 		if (!num_to_init)
3316 			break;
3317 
3318 		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
3319 		if (!pg->next)
3320 			goto free_pages;
3321 
3322 		pg = pg->next;
3323 	}
3324 
3325 	return start_pg;
3326 
3327  free_pages:
3328 	ftrace_free_pages(start_pg);
3329 	pr_info("ftrace: FAILED to allocate memory for functions\n");
3330 	return NULL;
3331 }
3332 
3333 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
3334 
3335 struct ftrace_iterator {
3336 	loff_t				pos;
3337 	loff_t				func_pos;
3338 	loff_t				mod_pos;
3339 	struct ftrace_page		*pg;
3340 	struct dyn_ftrace		*func;
3341 	struct ftrace_func_probe	*probe;
3342 	struct ftrace_func_entry	*probe_entry;
3343 	struct trace_parser		parser;
3344 	struct ftrace_hash		*hash;
3345 	struct ftrace_ops		*ops;
3346 	struct trace_array		*tr;
3347 	struct list_head		*mod_list;
3348 	int				pidx;
3349 	int				idx;
3350 	unsigned			flags;
3351 };
3352 
3353 static void *
t_probe_next(struct seq_file * m,loff_t * pos)3354 t_probe_next(struct seq_file *m, loff_t *pos)
3355 {
3356 	struct ftrace_iterator *iter = m->private;
3357 	struct trace_array *tr = iter->ops->private;
3358 	struct list_head *func_probes;
3359 	struct ftrace_hash *hash;
3360 	struct list_head *next;
3361 	struct hlist_node *hnd = NULL;
3362 	struct hlist_head *hhd;
3363 	int size;
3364 
3365 	(*pos)++;
3366 	iter->pos = *pos;
3367 
3368 	if (!tr)
3369 		return NULL;
3370 
3371 	func_probes = &tr->func_probes;
3372 	if (list_empty(func_probes))
3373 		return NULL;
3374 
3375 	if (!iter->probe) {
3376 		next = func_probes->next;
3377 		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3378 	}
3379 
3380 	if (iter->probe_entry)
3381 		hnd = &iter->probe_entry->hlist;
3382 
3383 	hash = iter->probe->ops.func_hash->filter_hash;
3384 
3385 	/*
3386 	 * A probe being registered may temporarily have an empty hash
3387 	 * and it's at the end of the func_probes list.
3388 	 */
3389 	if (!hash || hash == EMPTY_HASH)
3390 		return NULL;
3391 
3392 	size = 1 << hash->size_bits;
3393 
3394  retry:
3395 	if (iter->pidx >= size) {
3396 		if (iter->probe->list.next == func_probes)
3397 			return NULL;
3398 		next = iter->probe->list.next;
3399 		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3400 		hash = iter->probe->ops.func_hash->filter_hash;
3401 		size = 1 << hash->size_bits;
3402 		iter->pidx = 0;
3403 	}
3404 
3405 	hhd = &hash->buckets[iter->pidx];
3406 
3407 	if (hlist_empty(hhd)) {
3408 		iter->pidx++;
3409 		hnd = NULL;
3410 		goto retry;
3411 	}
3412 
3413 	if (!hnd)
3414 		hnd = hhd->first;
3415 	else {
3416 		hnd = hnd->next;
3417 		if (!hnd) {
3418 			iter->pidx++;
3419 			goto retry;
3420 		}
3421 	}
3422 
3423 	if (WARN_ON_ONCE(!hnd))
3424 		return NULL;
3425 
3426 	iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist);
3427 
3428 	return iter;
3429 }
3430 
t_probe_start(struct seq_file * m,loff_t * pos)3431 static void *t_probe_start(struct seq_file *m, loff_t *pos)
3432 {
3433 	struct ftrace_iterator *iter = m->private;
3434 	void *p = NULL;
3435 	loff_t l;
3436 
3437 	if (!(iter->flags & FTRACE_ITER_DO_PROBES))
3438 		return NULL;
3439 
3440 	if (iter->mod_pos > *pos)
3441 		return NULL;
3442 
3443 	iter->probe = NULL;
3444 	iter->probe_entry = NULL;
3445 	iter->pidx = 0;
3446 	for (l = 0; l <= (*pos - iter->mod_pos); ) {
3447 		p = t_probe_next(m, &l);
3448 		if (!p)
3449 			break;
3450 	}
3451 	if (!p)
3452 		return NULL;
3453 
3454 	/* Only set this if we have an item */
3455 	iter->flags |= FTRACE_ITER_PROBE;
3456 
3457 	return iter;
3458 }
3459 
3460 static int
t_probe_show(struct seq_file * m,struct ftrace_iterator * iter)3461 t_probe_show(struct seq_file *m, struct ftrace_iterator *iter)
3462 {
3463 	struct ftrace_func_entry *probe_entry;
3464 	struct ftrace_probe_ops *probe_ops;
3465 	struct ftrace_func_probe *probe;
3466 
3467 	probe = iter->probe;
3468 	probe_entry = iter->probe_entry;
3469 
3470 	if (WARN_ON_ONCE(!probe || !probe_entry))
3471 		return -EIO;
3472 
3473 	probe_ops = probe->probe_ops;
3474 
3475 	if (probe_ops->print)
3476 		return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data);
3477 
3478 	seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip,
3479 		   (void *)probe_ops->func);
3480 
3481 	return 0;
3482 }
3483 
3484 static void *
t_mod_next(struct seq_file * m,loff_t * pos)3485 t_mod_next(struct seq_file *m, loff_t *pos)
3486 {
3487 	struct ftrace_iterator *iter = m->private;
3488 	struct trace_array *tr = iter->tr;
3489 
3490 	(*pos)++;
3491 	iter->pos = *pos;
3492 
3493 	iter->mod_list = iter->mod_list->next;
3494 
3495 	if (iter->mod_list == &tr->mod_trace ||
3496 	    iter->mod_list == &tr->mod_notrace) {
3497 		iter->flags &= ~FTRACE_ITER_MOD;
3498 		return NULL;
3499 	}
3500 
3501 	iter->mod_pos = *pos;
3502 
3503 	return iter;
3504 }
3505 
t_mod_start(struct seq_file * m,loff_t * pos)3506 static void *t_mod_start(struct seq_file *m, loff_t *pos)
3507 {
3508 	struct ftrace_iterator *iter = m->private;
3509 	void *p = NULL;
3510 	loff_t l;
3511 
3512 	if (iter->func_pos > *pos)
3513 		return NULL;
3514 
3515 	iter->mod_pos = iter->func_pos;
3516 
3517 	/* probes are only available if tr is set */
3518 	if (!iter->tr)
3519 		return NULL;
3520 
3521 	for (l = 0; l <= (*pos - iter->func_pos); ) {
3522 		p = t_mod_next(m, &l);
3523 		if (!p)
3524 			break;
3525 	}
3526 	if (!p) {
3527 		iter->flags &= ~FTRACE_ITER_MOD;
3528 		return t_probe_start(m, pos);
3529 	}
3530 
3531 	/* Only set this if we have an item */
3532 	iter->flags |= FTRACE_ITER_MOD;
3533 
3534 	return iter;
3535 }
3536 
3537 static int
t_mod_show(struct seq_file * m,struct ftrace_iterator * iter)3538 t_mod_show(struct seq_file *m, struct ftrace_iterator *iter)
3539 {
3540 	struct ftrace_mod_load *ftrace_mod;
3541 	struct trace_array *tr = iter->tr;
3542 
3543 	if (WARN_ON_ONCE(!iter->mod_list) ||
3544 			 iter->mod_list == &tr->mod_trace ||
3545 			 iter->mod_list == &tr->mod_notrace)
3546 		return -EIO;
3547 
3548 	ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list);
3549 
3550 	if (ftrace_mod->func)
3551 		seq_printf(m, "%s", ftrace_mod->func);
3552 	else
3553 		seq_putc(m, '*');
3554 
3555 	seq_printf(m, ":mod:%s\n", ftrace_mod->module);
3556 
3557 	return 0;
3558 }
3559 
3560 static void *
t_func_next(struct seq_file * m,loff_t * pos)3561 t_func_next(struct seq_file *m, loff_t *pos)
3562 {
3563 	struct ftrace_iterator *iter = m->private;
3564 	struct dyn_ftrace *rec = NULL;
3565 
3566 	(*pos)++;
3567 
3568  retry:
3569 	if (iter->idx >= iter->pg->index) {
3570 		if (iter->pg->next) {
3571 			iter->pg = iter->pg->next;
3572 			iter->idx = 0;
3573 			goto retry;
3574 		}
3575 	} else {
3576 		rec = &iter->pg->records[iter->idx++];
3577 		if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
3578 		     !ftrace_lookup_ip(iter->hash, rec->ip)) ||
3579 
3580 		    ((iter->flags & FTRACE_ITER_ENABLED) &&
3581 		     !(rec->flags & FTRACE_FL_ENABLED)) ||
3582 
3583 		    ((iter->flags & FTRACE_ITER_TOUCHED) &&
3584 		     !(rec->flags & FTRACE_FL_TOUCHED))) {
3585 
3586 			rec = NULL;
3587 			goto retry;
3588 		}
3589 	}
3590 
3591 	if (!rec)
3592 		return NULL;
3593 
3594 	iter->pos = iter->func_pos = *pos;
3595 	iter->func = rec;
3596 
3597 	return iter;
3598 }
3599 
3600 static void *
t_next(struct seq_file * m,void * v,loff_t * pos)3601 t_next(struct seq_file *m, void *v, loff_t *pos)
3602 {
3603 	struct ftrace_iterator *iter = m->private;
3604 	loff_t l = *pos; /* t_probe_start() must use original pos */
3605 	void *ret;
3606 
3607 	if (unlikely(ftrace_disabled))
3608 		return NULL;
3609 
3610 	if (iter->flags & FTRACE_ITER_PROBE)
3611 		return t_probe_next(m, pos);
3612 
3613 	if (iter->flags & FTRACE_ITER_MOD)
3614 		return t_mod_next(m, pos);
3615 
3616 	if (iter->flags & FTRACE_ITER_PRINTALL) {
3617 		/* next must increment pos, and t_probe_start does not */
3618 		(*pos)++;
3619 		return t_mod_start(m, &l);
3620 	}
3621 
3622 	ret = t_func_next(m, pos);
3623 
3624 	if (!ret)
3625 		return t_mod_start(m, &l);
3626 
3627 	return ret;
3628 }
3629 
reset_iter_read(struct ftrace_iterator * iter)3630 static void reset_iter_read(struct ftrace_iterator *iter)
3631 {
3632 	iter->pos = 0;
3633 	iter->func_pos = 0;
3634 	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD);
3635 }
3636 
t_start(struct seq_file * m,loff_t * pos)3637 static void *t_start(struct seq_file *m, loff_t *pos)
3638 {
3639 	struct ftrace_iterator *iter = m->private;
3640 	void *p = NULL;
3641 	loff_t l;
3642 
3643 	mutex_lock(&ftrace_lock);
3644 
3645 	if (unlikely(ftrace_disabled))
3646 		return NULL;
3647 
3648 	/*
3649 	 * If an lseek was done, then reset and start from beginning.
3650 	 */
3651 	if (*pos < iter->pos)
3652 		reset_iter_read(iter);
3653 
3654 	/*
3655 	 * For set_ftrace_filter reading, if we have the filter
3656 	 * off, we can short cut and just print out that all
3657 	 * functions are enabled.
3658 	 */
3659 	if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
3660 	    ftrace_hash_empty(iter->hash)) {
3661 		iter->func_pos = 1; /* Account for the message */
3662 		if (*pos > 0)
3663 			return t_mod_start(m, pos);
3664 		iter->flags |= FTRACE_ITER_PRINTALL;
3665 		/* reset in case of seek/pread */
3666 		iter->flags &= ~FTRACE_ITER_PROBE;
3667 		return iter;
3668 	}
3669 
3670 	if (iter->flags & FTRACE_ITER_MOD)
3671 		return t_mod_start(m, pos);
3672 
3673 	/*
3674 	 * Unfortunately, we need to restart at ftrace_pages_start
3675 	 * every time we let go of the ftrace_mutex. This is because
3676 	 * those pointers can change without the lock.
3677 	 */
3678 	iter->pg = ftrace_pages_start;
3679 	iter->idx = 0;
3680 	for (l = 0; l <= *pos; ) {
3681 		p = t_func_next(m, &l);
3682 		if (!p)
3683 			break;
3684 	}
3685 
3686 	if (!p)
3687 		return t_mod_start(m, pos);
3688 
3689 	return iter;
3690 }
3691 
t_stop(struct seq_file * m,void * p)3692 static void t_stop(struct seq_file *m, void *p)
3693 {
3694 	mutex_unlock(&ftrace_lock);
3695 }
3696 
3697 void * __weak
arch_ftrace_trampoline_func(struct ftrace_ops * ops,struct dyn_ftrace * rec)3698 arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
3699 {
3700 	return NULL;
3701 }
3702 
add_trampoline_func(struct seq_file * m,struct ftrace_ops * ops,struct dyn_ftrace * rec)3703 static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
3704 				struct dyn_ftrace *rec)
3705 {
3706 	void *ptr;
3707 
3708 	ptr = arch_ftrace_trampoline_func(ops, rec);
3709 	if (ptr)
3710 		seq_printf(m, " ->%pS", ptr);
3711 }
3712 
3713 #ifdef FTRACE_MCOUNT_MAX_OFFSET
3714 /*
3715  * Weak functions can still have an mcount/fentry that is saved in
3716  * the __mcount_loc section. These can be detected by having a
3717  * symbol offset of greater than FTRACE_MCOUNT_MAX_OFFSET, as the
3718  * symbol found by kallsyms is not the function that the mcount/fentry
3719  * is part of. The offset is much greater in these cases.
3720  *
3721  * Test the record to make sure that the ip points to a valid kallsyms
3722  * and if not, mark it disabled.
3723  */
test_for_valid_rec(struct dyn_ftrace * rec)3724 static int test_for_valid_rec(struct dyn_ftrace *rec)
3725 {
3726 	char str[KSYM_SYMBOL_LEN];
3727 	unsigned long offset;
3728 	const char *ret;
3729 
3730 	ret = kallsyms_lookup(rec->ip, NULL, &offset, NULL, str);
3731 
3732 	/* Weak functions can cause invalid addresses */
3733 	if (!ret || offset > FTRACE_MCOUNT_MAX_OFFSET) {
3734 		rec->flags |= FTRACE_FL_DISABLED;
3735 		return 0;
3736 	}
3737 	return 1;
3738 }
3739 
3740 static struct workqueue_struct *ftrace_check_wq __initdata;
3741 static struct work_struct ftrace_check_work __initdata;
3742 
3743 /*
3744  * Scan all the mcount/fentry entries to make sure they are valid.
3745  */
ftrace_check_work_func(struct work_struct * work)3746 static __init void ftrace_check_work_func(struct work_struct *work)
3747 {
3748 	struct ftrace_page *pg;
3749 	struct dyn_ftrace *rec;
3750 
3751 	mutex_lock(&ftrace_lock);
3752 	do_for_each_ftrace_rec(pg, rec) {
3753 		test_for_valid_rec(rec);
3754 	} while_for_each_ftrace_rec();
3755 	mutex_unlock(&ftrace_lock);
3756 }
3757 
ftrace_check_for_weak_functions(void)3758 static int __init ftrace_check_for_weak_functions(void)
3759 {
3760 	INIT_WORK(&ftrace_check_work, ftrace_check_work_func);
3761 
3762 	ftrace_check_wq = alloc_workqueue("ftrace_check_wq", WQ_UNBOUND, 0);
3763 
3764 	queue_work(ftrace_check_wq, &ftrace_check_work);
3765 	return 0;
3766 }
3767 
ftrace_check_sync(void)3768 static int __init ftrace_check_sync(void)
3769 {
3770 	/* Make sure the ftrace_check updates are finished */
3771 	if (ftrace_check_wq)
3772 		destroy_workqueue(ftrace_check_wq);
3773 	return 0;
3774 }
3775 
3776 late_initcall_sync(ftrace_check_sync);
3777 subsys_initcall(ftrace_check_for_weak_functions);
3778 
print_rec(struct seq_file * m,unsigned long ip)3779 static int print_rec(struct seq_file *m, unsigned long ip)
3780 {
3781 	unsigned long offset;
3782 	char str[KSYM_SYMBOL_LEN];
3783 	char *modname;
3784 	const char *ret;
3785 
3786 	ret = kallsyms_lookup(ip, NULL, &offset, &modname, str);
3787 	/* Weak functions can cause invalid addresses */
3788 	if (!ret || offset > FTRACE_MCOUNT_MAX_OFFSET) {
3789 		snprintf(str, KSYM_SYMBOL_LEN, "%s_%ld",
3790 			 FTRACE_INVALID_FUNCTION, offset);
3791 		ret = NULL;
3792 	}
3793 
3794 	seq_puts(m, str);
3795 	if (modname)
3796 		seq_printf(m, " [%s]", modname);
3797 	return ret == NULL ? -1 : 0;
3798 }
3799 #else
test_for_valid_rec(struct dyn_ftrace * rec)3800 static inline int test_for_valid_rec(struct dyn_ftrace *rec)
3801 {
3802 	return 1;
3803 }
3804 
print_rec(struct seq_file * m,unsigned long ip)3805 static inline int print_rec(struct seq_file *m, unsigned long ip)
3806 {
3807 	seq_printf(m, "%ps", (void *)ip);
3808 	return 0;
3809 }
3810 #endif
3811 
t_show(struct seq_file * m,void * v)3812 static int t_show(struct seq_file *m, void *v)
3813 {
3814 	struct ftrace_iterator *iter = m->private;
3815 	struct dyn_ftrace *rec;
3816 
3817 	if (iter->flags & FTRACE_ITER_PROBE)
3818 		return t_probe_show(m, iter);
3819 
3820 	if (iter->flags & FTRACE_ITER_MOD)
3821 		return t_mod_show(m, iter);
3822 
3823 	if (iter->flags & FTRACE_ITER_PRINTALL) {
3824 		if (iter->flags & FTRACE_ITER_NOTRACE)
3825 			seq_puts(m, "#### no functions disabled ####\n");
3826 		else
3827 			seq_puts(m, "#### all functions enabled ####\n");
3828 		return 0;
3829 	}
3830 
3831 	rec = iter->func;
3832 
3833 	if (!rec)
3834 		return 0;
3835 
3836 	if (iter->flags & FTRACE_ITER_ADDRS)
3837 		seq_printf(m, "%lx ", rec->ip);
3838 
3839 	if (print_rec(m, rec->ip)) {
3840 		/* This should only happen when a rec is disabled */
3841 		WARN_ON_ONCE(!(rec->flags & FTRACE_FL_DISABLED));
3842 		seq_putc(m, '\n');
3843 		return 0;
3844 	}
3845 
3846 	if (iter->flags & (FTRACE_ITER_ENABLED | FTRACE_ITER_TOUCHED)) {
3847 		struct ftrace_ops *ops;
3848 
3849 		seq_printf(m, " (%ld)%s%s%s%s%s",
3850 			   ftrace_rec_count(rec),
3851 			   rec->flags & FTRACE_FL_REGS ? " R" : "  ",
3852 			   rec->flags & FTRACE_FL_IPMODIFY ? " I" : "  ",
3853 			   rec->flags & FTRACE_FL_DIRECT ? " D" : "  ",
3854 			   rec->flags & FTRACE_FL_CALL_OPS ? " O" : "  ",
3855 			   rec->flags & FTRACE_FL_MODIFIED ? " M " : "   ");
3856 		if (rec->flags & FTRACE_FL_TRAMP_EN) {
3857 			ops = ftrace_find_tramp_ops_any(rec);
3858 			if (ops) {
3859 				do {
3860 					seq_printf(m, "\ttramp: %pS (%pS)",
3861 						   (void *)ops->trampoline,
3862 						   (void *)ops->func);
3863 					add_trampoline_func(m, ops, rec);
3864 					ops = ftrace_find_tramp_ops_next(rec, ops);
3865 				} while (ops);
3866 			} else
3867 				seq_puts(m, "\ttramp: ERROR!");
3868 		} else {
3869 			add_trampoline_func(m, NULL, rec);
3870 		}
3871 		if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
3872 			ops = ftrace_find_unique_ops(rec);
3873 			if (ops) {
3874 				seq_printf(m, "\tops: %pS (%pS)",
3875 					   ops, ops->func);
3876 			} else {
3877 				seq_puts(m, "\tops: ERROR!");
3878 			}
3879 		}
3880 		if (rec->flags & FTRACE_FL_DIRECT) {
3881 			unsigned long direct;
3882 
3883 			direct = ftrace_find_rec_direct(rec->ip);
3884 			if (direct)
3885 				seq_printf(m, "\n\tdirect-->%pS", (void *)direct);
3886 		}
3887 	}
3888 
3889 	seq_putc(m, '\n');
3890 
3891 	return 0;
3892 }
3893 
3894 static const struct seq_operations show_ftrace_seq_ops = {
3895 	.start = t_start,
3896 	.next = t_next,
3897 	.stop = t_stop,
3898 	.show = t_show,
3899 };
3900 
3901 static int
ftrace_avail_open(struct inode * inode,struct file * file)3902 ftrace_avail_open(struct inode *inode, struct file *file)
3903 {
3904 	struct ftrace_iterator *iter;
3905 	int ret;
3906 
3907 	ret = security_locked_down(LOCKDOWN_TRACEFS);
3908 	if (ret)
3909 		return ret;
3910 
3911 	if (unlikely(ftrace_disabled))
3912 		return -ENODEV;
3913 
3914 	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3915 	if (!iter)
3916 		return -ENOMEM;
3917 
3918 	iter->pg = ftrace_pages_start;
3919 	iter->ops = &global_ops;
3920 
3921 	return 0;
3922 }
3923 
3924 static int
ftrace_enabled_open(struct inode * inode,struct file * file)3925 ftrace_enabled_open(struct inode *inode, struct file *file)
3926 {
3927 	struct ftrace_iterator *iter;
3928 
3929 	/*
3930 	 * This shows us what functions are currently being
3931 	 * traced and by what. Not sure if we want lockdown
3932 	 * to hide such critical information for an admin.
3933 	 * Although, perhaps it can show information we don't
3934 	 * want people to see, but if something is tracing
3935 	 * something, we probably want to know about it.
3936 	 */
3937 
3938 	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3939 	if (!iter)
3940 		return -ENOMEM;
3941 
3942 	iter->pg = ftrace_pages_start;
3943 	iter->flags = FTRACE_ITER_ENABLED;
3944 	iter->ops = &global_ops;
3945 
3946 	return 0;
3947 }
3948 
3949 static int
ftrace_touched_open(struct inode * inode,struct file * file)3950 ftrace_touched_open(struct inode *inode, struct file *file)
3951 {
3952 	struct ftrace_iterator *iter;
3953 
3954 	/*
3955 	 * This shows us what functions have ever been enabled
3956 	 * (traced, direct, patched, etc). Not sure if we want lockdown
3957 	 * to hide such critical information for an admin.
3958 	 * Although, perhaps it can show information we don't
3959 	 * want people to see, but if something had traced
3960 	 * something, we probably want to know about it.
3961 	 */
3962 
3963 	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3964 	if (!iter)
3965 		return -ENOMEM;
3966 
3967 	iter->pg = ftrace_pages_start;
3968 	iter->flags = FTRACE_ITER_TOUCHED;
3969 	iter->ops = &global_ops;
3970 
3971 	return 0;
3972 }
3973 
3974 static int
ftrace_avail_addrs_open(struct inode * inode,struct file * file)3975 ftrace_avail_addrs_open(struct inode *inode, struct file *file)
3976 {
3977 	struct ftrace_iterator *iter;
3978 	int ret;
3979 
3980 	ret = security_locked_down(LOCKDOWN_TRACEFS);
3981 	if (ret)
3982 		return ret;
3983 
3984 	if (unlikely(ftrace_disabled))
3985 		return -ENODEV;
3986 
3987 	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3988 	if (!iter)
3989 		return -ENOMEM;
3990 
3991 	iter->pg = ftrace_pages_start;
3992 	iter->flags = FTRACE_ITER_ADDRS;
3993 	iter->ops = &global_ops;
3994 
3995 	return 0;
3996 }
3997 
3998 /**
3999  * ftrace_regex_open - initialize function tracer filter files
4000  * @ops: The ftrace_ops that hold the hash filters
4001  * @flag: The type of filter to process
4002  * @inode: The inode, usually passed in to your open routine
4003  * @file: The file, usually passed in to your open routine
4004  *
4005  * ftrace_regex_open() initializes the filter files for the
4006  * @ops. Depending on @flag it may process the filter hash or
4007  * the notrace hash of @ops. With this called from the open
4008  * routine, you can use ftrace_filter_write() for the write
4009  * routine if @flag has FTRACE_ITER_FILTER set, or
4010  * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
4011  * tracing_lseek() should be used as the lseek routine, and
4012  * release must call ftrace_regex_release().
4013  */
4014 int
ftrace_regex_open(struct ftrace_ops * ops,int flag,struct inode * inode,struct file * file)4015 ftrace_regex_open(struct ftrace_ops *ops, int flag,
4016 		  struct inode *inode, struct file *file)
4017 {
4018 	struct ftrace_iterator *iter;
4019 	struct ftrace_hash *hash;
4020 	struct list_head *mod_head;
4021 	struct trace_array *tr = ops->private;
4022 	int ret = -ENOMEM;
4023 
4024 	ftrace_ops_init(ops);
4025 
4026 	if (unlikely(ftrace_disabled))
4027 		return -ENODEV;
4028 
4029 	if (tracing_check_open_get_tr(tr))
4030 		return -ENODEV;
4031 
4032 	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4033 	if (!iter)
4034 		goto out;
4035 
4036 	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX))
4037 		goto out;
4038 
4039 	iter->ops = ops;
4040 	iter->flags = flag;
4041 	iter->tr = tr;
4042 
4043 	mutex_lock(&ops->func_hash->regex_lock);
4044 
4045 	if (flag & FTRACE_ITER_NOTRACE) {
4046 		hash = ops->func_hash->notrace_hash;
4047 		mod_head = tr ? &tr->mod_notrace : NULL;
4048 	} else {
4049 		hash = ops->func_hash->filter_hash;
4050 		mod_head = tr ? &tr->mod_trace : NULL;
4051 	}
4052 
4053 	iter->mod_list = mod_head;
4054 
4055 	if (file->f_mode & FMODE_WRITE) {
4056 		const int size_bits = FTRACE_HASH_DEFAULT_BITS;
4057 
4058 		if (file->f_flags & O_TRUNC) {
4059 			iter->hash = alloc_ftrace_hash(size_bits);
4060 			clear_ftrace_mod_list(mod_head);
4061 	        } else {
4062 			iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);
4063 		}
4064 
4065 		if (!iter->hash) {
4066 			trace_parser_put(&iter->parser);
4067 			goto out_unlock;
4068 		}
4069 	} else
4070 		iter->hash = hash;
4071 
4072 	ret = 0;
4073 
4074 	if (file->f_mode & FMODE_READ) {
4075 		iter->pg = ftrace_pages_start;
4076 
4077 		ret = seq_open(file, &show_ftrace_seq_ops);
4078 		if (!ret) {
4079 			struct seq_file *m = file->private_data;
4080 			m->private = iter;
4081 		} else {
4082 			/* Failed */
4083 			free_ftrace_hash(iter->hash);
4084 			trace_parser_put(&iter->parser);
4085 		}
4086 	} else
4087 		file->private_data = iter;
4088 
4089  out_unlock:
4090 	mutex_unlock(&ops->func_hash->regex_lock);
4091 
4092  out:
4093 	if (ret) {
4094 		kfree(iter);
4095 		if (tr)
4096 			trace_array_put(tr);
4097 	}
4098 
4099 	return ret;
4100 }
4101 
4102 static int
ftrace_filter_open(struct inode * inode,struct file * file)4103 ftrace_filter_open(struct inode *inode, struct file *file)
4104 {
4105 	struct ftrace_ops *ops = inode->i_private;
4106 
4107 	/* Checks for tracefs lockdown */
4108 	return ftrace_regex_open(ops,
4109 			FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES,
4110 			inode, file);
4111 }
4112 
4113 static int
ftrace_notrace_open(struct inode * inode,struct file * file)4114 ftrace_notrace_open(struct inode *inode, struct file *file)
4115 {
4116 	struct ftrace_ops *ops = inode->i_private;
4117 
4118 	/* Checks for tracefs lockdown */
4119 	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
4120 				 inode, file);
4121 }
4122 
4123 /* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
4124 struct ftrace_glob {
4125 	char *search;
4126 	unsigned len;
4127 	int type;
4128 };
4129 
4130 /*
4131  * If symbols in an architecture don't correspond exactly to the user-visible
4132  * name of what they represent, it is possible to define this function to
4133  * perform the necessary adjustments.
4134 */
arch_ftrace_match_adjust(char * str,const char * search)4135 char * __weak arch_ftrace_match_adjust(char *str, const char *search)
4136 {
4137 	return str;
4138 }
4139 
ftrace_match(char * str,struct ftrace_glob * g)4140 static int ftrace_match(char *str, struct ftrace_glob *g)
4141 {
4142 	int matched = 0;
4143 	int slen;
4144 
4145 	str = arch_ftrace_match_adjust(str, g->search);
4146 
4147 	switch (g->type) {
4148 	case MATCH_FULL:
4149 		if (strcmp(str, g->search) == 0)
4150 			matched = 1;
4151 		break;
4152 	case MATCH_FRONT_ONLY:
4153 		if (strncmp(str, g->search, g->len) == 0)
4154 			matched = 1;
4155 		break;
4156 	case MATCH_MIDDLE_ONLY:
4157 		if (strstr(str, g->search))
4158 			matched = 1;
4159 		break;
4160 	case MATCH_END_ONLY:
4161 		slen = strlen(str);
4162 		if (slen >= g->len &&
4163 		    memcmp(str + slen - g->len, g->search, g->len) == 0)
4164 			matched = 1;
4165 		break;
4166 	case MATCH_GLOB:
4167 		if (glob_match(g->search, str))
4168 			matched = 1;
4169 		break;
4170 	}
4171 
4172 	return matched;
4173 }
4174 
4175 static int
enter_record(struct ftrace_hash * hash,struct dyn_ftrace * rec,int clear_filter)4176 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
4177 {
4178 	struct ftrace_func_entry *entry;
4179 	int ret = 0;
4180 
4181 	entry = ftrace_lookup_ip(hash, rec->ip);
4182 	if (clear_filter) {
4183 		/* Do nothing if it doesn't exist */
4184 		if (!entry)
4185 			return 0;
4186 
4187 		free_hash_entry(hash, entry);
4188 	} else {
4189 		/* Do nothing if it exists */
4190 		if (entry)
4191 			return 0;
4192 		if (add_hash_entry(hash, rec->ip) == NULL)
4193 			ret = -ENOMEM;
4194 	}
4195 	return ret;
4196 }
4197 
4198 static int
add_rec_by_index(struct ftrace_hash * hash,struct ftrace_glob * func_g,int clear_filter)4199 add_rec_by_index(struct ftrace_hash *hash, struct ftrace_glob *func_g,
4200 		 int clear_filter)
4201 {
4202 	long index = simple_strtoul(func_g->search, NULL, 0);
4203 	struct ftrace_page *pg;
4204 	struct dyn_ftrace *rec;
4205 
4206 	/* The index starts at 1 */
4207 	if (--index < 0)
4208 		return 0;
4209 
4210 	do_for_each_ftrace_rec(pg, rec) {
4211 		if (pg->index <= index) {
4212 			index -= pg->index;
4213 			/* this is a double loop, break goes to the next page */
4214 			break;
4215 		}
4216 		rec = &pg->records[index];
4217 		enter_record(hash, rec, clear_filter);
4218 		return 1;
4219 	} while_for_each_ftrace_rec();
4220 	return 0;
4221 }
4222 
4223 #ifdef FTRACE_MCOUNT_MAX_OFFSET
lookup_ip(unsigned long ip,char ** modname,char * str)4224 static int lookup_ip(unsigned long ip, char **modname, char *str)
4225 {
4226 	unsigned long offset;
4227 
4228 	kallsyms_lookup(ip, NULL, &offset, modname, str);
4229 	if (offset > FTRACE_MCOUNT_MAX_OFFSET)
4230 		return -1;
4231 	return 0;
4232 }
4233 #else
lookup_ip(unsigned long ip,char ** modname,char * str)4234 static int lookup_ip(unsigned long ip, char **modname, char *str)
4235 {
4236 	kallsyms_lookup(ip, NULL, NULL, modname, str);
4237 	return 0;
4238 }
4239 #endif
4240 
4241 static int
ftrace_match_record(struct dyn_ftrace * rec,struct ftrace_glob * func_g,struct ftrace_glob * mod_g,int exclude_mod)4242 ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
4243 		struct ftrace_glob *mod_g, int exclude_mod)
4244 {
4245 	char str[KSYM_SYMBOL_LEN];
4246 	char *modname;
4247 
4248 	if (lookup_ip(rec->ip, &modname, str)) {
4249 		/* This should only happen when a rec is disabled */
4250 		WARN_ON_ONCE(system_state == SYSTEM_RUNNING &&
4251 			     !(rec->flags & FTRACE_FL_DISABLED));
4252 		return 0;
4253 	}
4254 
4255 	if (mod_g) {
4256 		int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;
4257 
4258 		/* blank module name to match all modules */
4259 		if (!mod_g->len) {
4260 			/* blank module globbing: modname xor exclude_mod */
4261 			if (!exclude_mod != !modname)
4262 				goto func_match;
4263 			return 0;
4264 		}
4265 
4266 		/*
4267 		 * exclude_mod is set to trace everything but the given
4268 		 * module. If it is set and the module matches, then
4269 		 * return 0. If it is not set, and the module doesn't match
4270 		 * also return 0. Otherwise, check the function to see if
4271 		 * that matches.
4272 		 */
4273 		if (!mod_matches == !exclude_mod)
4274 			return 0;
4275 func_match:
4276 		/* blank search means to match all funcs in the mod */
4277 		if (!func_g->len)
4278 			return 1;
4279 	}
4280 
4281 	return ftrace_match(str, func_g);
4282 }
4283 
4284 static int
match_records(struct ftrace_hash * hash,char * func,int len,char * mod)4285 match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
4286 {
4287 	struct ftrace_page *pg;
4288 	struct dyn_ftrace *rec;
4289 	struct ftrace_glob func_g = { .type = MATCH_FULL };
4290 	struct ftrace_glob mod_g = { .type = MATCH_FULL };
4291 	struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
4292 	int exclude_mod = 0;
4293 	int found = 0;
4294 	int ret;
4295 	int clear_filter = 0;
4296 
4297 	if (func) {
4298 		func_g.type = filter_parse_regex(func, len, &func_g.search,
4299 						 &clear_filter);
4300 		func_g.len = strlen(func_g.search);
4301 	}
4302 
4303 	if (mod) {
4304 		mod_g.type = filter_parse_regex(mod, strlen(mod),
4305 				&mod_g.search, &exclude_mod);
4306 		mod_g.len = strlen(mod_g.search);
4307 	}
4308 
4309 	mutex_lock(&ftrace_lock);
4310 
4311 	if (unlikely(ftrace_disabled))
4312 		goto out_unlock;
4313 
4314 	if (func_g.type == MATCH_INDEX) {
4315 		found = add_rec_by_index(hash, &func_g, clear_filter);
4316 		goto out_unlock;
4317 	}
4318 
4319 	do_for_each_ftrace_rec(pg, rec) {
4320 
4321 		if (rec->flags & FTRACE_FL_DISABLED)
4322 			continue;
4323 
4324 		if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
4325 			ret = enter_record(hash, rec, clear_filter);
4326 			if (ret < 0) {
4327 				found = ret;
4328 				goto out_unlock;
4329 			}
4330 			found = 1;
4331 		}
4332 		cond_resched();
4333 	} while_for_each_ftrace_rec();
4334  out_unlock:
4335 	mutex_unlock(&ftrace_lock);
4336 
4337 	return found;
4338 }
4339 
4340 static int
ftrace_match_records(struct ftrace_hash * hash,char * buff,int len)4341 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
4342 {
4343 	return match_records(hash, buff, len, NULL);
4344 }
4345 
ftrace_ops_update_code(struct ftrace_ops * ops,struct ftrace_ops_hash * old_hash)4346 static void ftrace_ops_update_code(struct ftrace_ops *ops,
4347 				   struct ftrace_ops_hash *old_hash)
4348 {
4349 	struct ftrace_ops *op;
4350 
4351 	if (!ftrace_enabled)
4352 		return;
4353 
4354 	if (ops->flags & FTRACE_OPS_FL_ENABLED) {
4355 		ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
4356 		return;
4357 	}
4358 
4359 	/*
4360 	 * If this is the shared global_ops filter, then we need to
4361 	 * check if there is another ops that shares it, is enabled.
4362 	 * If so, we still need to run the modify code.
4363 	 */
4364 	if (ops->func_hash != &global_ops.local_hash)
4365 		return;
4366 
4367 	do_for_each_ftrace_op(op, ftrace_ops_list) {
4368 		if (op->func_hash == &global_ops.local_hash &&
4369 		    op->flags & FTRACE_OPS_FL_ENABLED) {
4370 			ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
4371 			/* Only need to do this once */
4372 			return;
4373 		}
4374 	} while_for_each_ftrace_op(op);
4375 }
4376 
ftrace_hash_move_and_update_ops(struct ftrace_ops * ops,struct ftrace_hash ** orig_hash,struct ftrace_hash * hash,int enable)4377 static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops,
4378 					   struct ftrace_hash **orig_hash,
4379 					   struct ftrace_hash *hash,
4380 					   int enable)
4381 {
4382 	struct ftrace_ops_hash old_hash_ops;
4383 	struct ftrace_hash *old_hash;
4384 	int ret;
4385 
4386 	old_hash = *orig_hash;
4387 	old_hash_ops.filter_hash = ops->func_hash->filter_hash;
4388 	old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
4389 	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
4390 	if (!ret) {
4391 		ftrace_ops_update_code(ops, &old_hash_ops);
4392 		free_ftrace_hash_rcu(old_hash);
4393 	}
4394 	return ret;
4395 }
4396 
module_exists(const char * module)4397 static bool module_exists(const char *module)
4398 {
4399 	/* All modules have the symbol __this_module */
4400 	static const char this_mod[] = "__this_module";
4401 	char modname[MAX_PARAM_PREFIX_LEN + sizeof(this_mod) + 2];
4402 	unsigned long val;
4403 	int n;
4404 
4405 	n = snprintf(modname, sizeof(modname), "%s:%s", module, this_mod);
4406 
4407 	if (n > sizeof(modname) - 1)
4408 		return false;
4409 
4410 	val = module_kallsyms_lookup_name(modname);
4411 	return val != 0;
4412 }
4413 
cache_mod(struct trace_array * tr,const char * func,char * module,int enable)4414 static int cache_mod(struct trace_array *tr,
4415 		     const char *func, char *module, int enable)
4416 {
4417 	struct ftrace_mod_load *ftrace_mod, *n;
4418 	struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace;
4419 	int ret;
4420 
4421 	mutex_lock(&ftrace_lock);
4422 
4423 	/* We do not cache inverse filters */
4424 	if (func[0] == '!') {
4425 		func++;
4426 		ret = -EINVAL;
4427 
4428 		/* Look to remove this hash */
4429 		list_for_each_entry_safe(ftrace_mod, n, head, list) {
4430 			if (strcmp(ftrace_mod->module, module) != 0)
4431 				continue;
4432 
4433 			/* no func matches all */
4434 			if (strcmp(func, "*") == 0 ||
4435 			    (ftrace_mod->func &&
4436 			     strcmp(ftrace_mod->func, func) == 0)) {
4437 				ret = 0;
4438 				free_ftrace_mod(ftrace_mod);
4439 				continue;
4440 			}
4441 		}
4442 		goto out;
4443 	}
4444 
4445 	ret = -EINVAL;
4446 	/* We only care about modules that have not been loaded yet */
4447 	if (module_exists(module))
4448 		goto out;
4449 
4450 	/* Save this string off, and execute it when the module is loaded */
4451 	ret = ftrace_add_mod(tr, func, module, enable);
4452  out:
4453 	mutex_unlock(&ftrace_lock);
4454 
4455 	return ret;
4456 }
4457 
4458 static int
4459 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
4460 		 int reset, int enable);
4461 
4462 #ifdef CONFIG_MODULES
process_mod_list(struct list_head * head,struct ftrace_ops * ops,char * mod,bool enable)4463 static void process_mod_list(struct list_head *head, struct ftrace_ops *ops,
4464 			     char *mod, bool enable)
4465 {
4466 	struct ftrace_mod_load *ftrace_mod, *n;
4467 	struct ftrace_hash **orig_hash, *new_hash;
4468 	LIST_HEAD(process_mods);
4469 	char *func;
4470 
4471 	mutex_lock(&ops->func_hash->regex_lock);
4472 
4473 	if (enable)
4474 		orig_hash = &ops->func_hash->filter_hash;
4475 	else
4476 		orig_hash = &ops->func_hash->notrace_hash;
4477 
4478 	new_hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS,
4479 					      *orig_hash);
4480 	if (!new_hash)
4481 		goto out; /* warn? */
4482 
4483 	mutex_lock(&ftrace_lock);
4484 
4485 	list_for_each_entry_safe(ftrace_mod, n, head, list) {
4486 
4487 		if (strcmp(ftrace_mod->module, mod) != 0)
4488 			continue;
4489 
4490 		if (ftrace_mod->func)
4491 			func = kstrdup(ftrace_mod->func, GFP_KERNEL);
4492 		else
4493 			func = kstrdup("*", GFP_KERNEL);
4494 
4495 		if (!func) /* warn? */
4496 			continue;
4497 
4498 		list_move(&ftrace_mod->list, &process_mods);
4499 
4500 		/* Use the newly allocated func, as it may be "*" */
4501 		kfree(ftrace_mod->func);
4502 		ftrace_mod->func = func;
4503 	}
4504 
4505 	mutex_unlock(&ftrace_lock);
4506 
4507 	list_for_each_entry_safe(ftrace_mod, n, &process_mods, list) {
4508 
4509 		func = ftrace_mod->func;
4510 
4511 		/* Grabs ftrace_lock, which is why we have this extra step */
4512 		match_records(new_hash, func, strlen(func), mod);
4513 		free_ftrace_mod(ftrace_mod);
4514 	}
4515 
4516 	if (enable && list_empty(head))
4517 		new_hash->flags &= ~FTRACE_HASH_FL_MOD;
4518 
4519 	mutex_lock(&ftrace_lock);
4520 
4521 	ftrace_hash_move_and_update_ops(ops, orig_hash,
4522 					      new_hash, enable);
4523 	mutex_unlock(&ftrace_lock);
4524 
4525  out:
4526 	mutex_unlock(&ops->func_hash->regex_lock);
4527 
4528 	free_ftrace_hash(new_hash);
4529 }
4530 
process_cached_mods(const char * mod_name)4531 static void process_cached_mods(const char *mod_name)
4532 {
4533 	struct trace_array *tr;
4534 	char *mod;
4535 
4536 	mod = kstrdup(mod_name, GFP_KERNEL);
4537 	if (!mod)
4538 		return;
4539 
4540 	mutex_lock(&trace_types_lock);
4541 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
4542 		if (!list_empty(&tr->mod_trace))
4543 			process_mod_list(&tr->mod_trace, tr->ops, mod, true);
4544 		if (!list_empty(&tr->mod_notrace))
4545 			process_mod_list(&tr->mod_notrace, tr->ops, mod, false);
4546 	}
4547 	mutex_unlock(&trace_types_lock);
4548 
4549 	kfree(mod);
4550 }
4551 #endif
4552 
4553 /*
4554  * We register the module command as a template to show others how
4555  * to register the a command as well.
4556  */
4557 
4558 static int
ftrace_mod_callback(struct trace_array * tr,struct ftrace_hash * hash,char * func_orig,char * cmd,char * module,int enable)4559 ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash,
4560 		    char *func_orig, char *cmd, char *module, int enable)
4561 {
4562 	char *func;
4563 	int ret;
4564 
4565 	/* match_records() modifies func, and we need the original */
4566 	func = kstrdup(func_orig, GFP_KERNEL);
4567 	if (!func)
4568 		return -ENOMEM;
4569 
4570 	/*
4571 	 * cmd == 'mod' because we only registered this func
4572 	 * for the 'mod' ftrace_func_command.
4573 	 * But if you register one func with multiple commands,
4574 	 * you can tell which command was used by the cmd
4575 	 * parameter.
4576 	 */
4577 	ret = match_records(hash, func, strlen(func), module);
4578 	kfree(func);
4579 
4580 	if (!ret)
4581 		return cache_mod(tr, func_orig, module, enable);
4582 	if (ret < 0)
4583 		return ret;
4584 	return 0;
4585 }
4586 
4587 static struct ftrace_func_command ftrace_mod_cmd = {
4588 	.name			= "mod",
4589 	.func			= ftrace_mod_callback,
4590 };
4591 
ftrace_mod_cmd_init(void)4592 static int __init ftrace_mod_cmd_init(void)
4593 {
4594 	return register_ftrace_command(&ftrace_mod_cmd);
4595 }
4596 core_initcall(ftrace_mod_cmd_init);
4597 
function_trace_probe_call(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * op,struct ftrace_regs * fregs)4598 static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
4599 				      struct ftrace_ops *op, struct ftrace_regs *fregs)
4600 {
4601 	struct ftrace_probe_ops *probe_ops;
4602 	struct ftrace_func_probe *probe;
4603 
4604 	probe = container_of(op, struct ftrace_func_probe, ops);
4605 	probe_ops = probe->probe_ops;
4606 
4607 	/*
4608 	 * Disable preemption for these calls to prevent a RCU grace
4609 	 * period. This syncs the hash iteration and freeing of items
4610 	 * on the hash. rcu_read_lock is too dangerous here.
4611 	 */
4612 	preempt_disable_notrace();
4613 	probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data);
4614 	preempt_enable_notrace();
4615 }
4616 
4617 struct ftrace_func_map {
4618 	struct ftrace_func_entry	entry;
4619 	void				*data;
4620 };
4621 
4622 struct ftrace_func_mapper {
4623 	struct ftrace_hash		hash;
4624 };
4625 
4626 /**
4627  * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
4628  *
4629  * Returns a ftrace_func_mapper descriptor that can be used to map ips to data.
4630  */
allocate_ftrace_func_mapper(void)4631 struct ftrace_func_mapper *allocate_ftrace_func_mapper(void)
4632 {
4633 	struct ftrace_hash *hash;
4634 
4635 	/*
4636 	 * The mapper is simply a ftrace_hash, but since the entries
4637 	 * in the hash are not ftrace_func_entry type, we define it
4638 	 * as a separate structure.
4639 	 */
4640 	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
4641 	return (struct ftrace_func_mapper *)hash;
4642 }
4643 
4644 /**
4645  * ftrace_func_mapper_find_ip - Find some data mapped to an ip
4646  * @mapper: The mapper that has the ip maps
4647  * @ip: the instruction pointer to find the data for
4648  *
4649  * Returns the data mapped to @ip if found otherwise NULL. The return
4650  * is actually the address of the mapper data pointer. The address is
4651  * returned for use cases where the data is no bigger than a long, and
4652  * the user can use the data pointer as its data instead of having to
4653  * allocate more memory for the reference.
4654  */
ftrace_func_mapper_find_ip(struct ftrace_func_mapper * mapper,unsigned long ip)4655 void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
4656 				  unsigned long ip)
4657 {
4658 	struct ftrace_func_entry *entry;
4659 	struct ftrace_func_map *map;
4660 
4661 	entry = ftrace_lookup_ip(&mapper->hash, ip);
4662 	if (!entry)
4663 		return NULL;
4664 
4665 	map = (struct ftrace_func_map *)entry;
4666 	return &map->data;
4667 }
4668 
4669 /**
4670  * ftrace_func_mapper_add_ip - Map some data to an ip
4671  * @mapper: The mapper that has the ip maps
4672  * @ip: The instruction pointer address to map @data to
4673  * @data: The data to map to @ip
4674  *
4675  * Returns 0 on success otherwise an error.
4676  */
ftrace_func_mapper_add_ip(struct ftrace_func_mapper * mapper,unsigned long ip,void * data)4677 int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
4678 			      unsigned long ip, void *data)
4679 {
4680 	struct ftrace_func_entry *entry;
4681 	struct ftrace_func_map *map;
4682 
4683 	entry = ftrace_lookup_ip(&mapper->hash, ip);
4684 	if (entry)
4685 		return -EBUSY;
4686 
4687 	map = kmalloc(sizeof(*map), GFP_KERNEL);
4688 	if (!map)
4689 		return -ENOMEM;
4690 
4691 	map->entry.ip = ip;
4692 	map->data = data;
4693 
4694 	__add_hash_entry(&mapper->hash, &map->entry);
4695 
4696 	return 0;
4697 }
4698 
4699 /**
4700  * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
4701  * @mapper: The mapper that has the ip maps
4702  * @ip: The instruction pointer address to remove the data from
4703  *
4704  * Returns the data if it is found, otherwise NULL.
4705  * Note, if the data pointer is used as the data itself, (see
4706  * ftrace_func_mapper_find_ip(), then the return value may be meaningless,
4707  * if the data pointer was set to zero.
4708  */
ftrace_func_mapper_remove_ip(struct ftrace_func_mapper * mapper,unsigned long ip)4709 void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
4710 				   unsigned long ip)
4711 {
4712 	struct ftrace_func_entry *entry;
4713 	struct ftrace_func_map *map;
4714 	void *data;
4715 
4716 	entry = ftrace_lookup_ip(&mapper->hash, ip);
4717 	if (!entry)
4718 		return NULL;
4719 
4720 	map = (struct ftrace_func_map *)entry;
4721 	data = map->data;
4722 
4723 	remove_hash_entry(&mapper->hash, entry);
4724 	kfree(entry);
4725 
4726 	return data;
4727 }
4728 
4729 /**
4730  * free_ftrace_func_mapper - free a mapping of ips and data
4731  * @mapper: The mapper that has the ip maps
4732  * @free_func: A function to be called on each data item.
4733  *
4734  * This is used to free the function mapper. The @free_func is optional
4735  * and can be used if the data needs to be freed as well.
4736  */
free_ftrace_func_mapper(struct ftrace_func_mapper * mapper,ftrace_mapper_func free_func)4737 void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
4738 			     ftrace_mapper_func free_func)
4739 {
4740 	struct ftrace_func_entry *entry;
4741 	struct ftrace_func_map *map;
4742 	struct hlist_head *hhd;
4743 	int size, i;
4744 
4745 	if (!mapper)
4746 		return;
4747 
4748 	if (free_func && mapper->hash.count) {
4749 		size = 1 << mapper->hash.size_bits;
4750 		for (i = 0; i < size; i++) {
4751 			hhd = &mapper->hash.buckets[i];
4752 			hlist_for_each_entry(entry, hhd, hlist) {
4753 				map = (struct ftrace_func_map *)entry;
4754 				free_func(map);
4755 			}
4756 		}
4757 	}
4758 	free_ftrace_hash(&mapper->hash);
4759 }
4760 
release_probe(struct ftrace_func_probe * probe)4761 static void release_probe(struct ftrace_func_probe *probe)
4762 {
4763 	struct ftrace_probe_ops *probe_ops;
4764 
4765 	mutex_lock(&ftrace_lock);
4766 
4767 	WARN_ON(probe->ref <= 0);
4768 
4769 	/* Subtract the ref that was used to protect this instance */
4770 	probe->ref--;
4771 
4772 	if (!probe->ref) {
4773 		probe_ops = probe->probe_ops;
4774 		/*
4775 		 * Sending zero as ip tells probe_ops to free
4776 		 * the probe->data itself
4777 		 */
4778 		if (probe_ops->free)
4779 			probe_ops->free(probe_ops, probe->tr, 0, probe->data);
4780 		list_del(&probe->list);
4781 		kfree(probe);
4782 	}
4783 	mutex_unlock(&ftrace_lock);
4784 }
4785 
acquire_probe_locked(struct ftrace_func_probe * probe)4786 static void acquire_probe_locked(struct ftrace_func_probe *probe)
4787 {
4788 	/*
4789 	 * Add one ref to keep it from being freed when releasing the
4790 	 * ftrace_lock mutex.
4791 	 */
4792 	probe->ref++;
4793 }
4794 
4795 int
register_ftrace_function_probe(char * glob,struct trace_array * tr,struct ftrace_probe_ops * probe_ops,void * data)4796 register_ftrace_function_probe(char *glob, struct trace_array *tr,
4797 			       struct ftrace_probe_ops *probe_ops,
4798 			       void *data)
4799 {
4800 	struct ftrace_func_probe *probe = NULL, *iter;
4801 	struct ftrace_func_entry *entry;
4802 	struct ftrace_hash **orig_hash;
4803 	struct ftrace_hash *old_hash;
4804 	struct ftrace_hash *hash;
4805 	int count = 0;
4806 	int size;
4807 	int ret;
4808 	int i;
4809 
4810 	if (WARN_ON(!tr))
4811 		return -EINVAL;
4812 
4813 	/* We do not support '!' for function probes */
4814 	if (WARN_ON(glob[0] == '!'))
4815 		return -EINVAL;
4816 
4817 
4818 	mutex_lock(&ftrace_lock);
4819 	/* Check if the probe_ops is already registered */
4820 	list_for_each_entry(iter, &tr->func_probes, list) {
4821 		if (iter->probe_ops == probe_ops) {
4822 			probe = iter;
4823 			break;
4824 		}
4825 	}
4826 	if (!probe) {
4827 		probe = kzalloc(sizeof(*probe), GFP_KERNEL);
4828 		if (!probe) {
4829 			mutex_unlock(&ftrace_lock);
4830 			return -ENOMEM;
4831 		}
4832 		probe->probe_ops = probe_ops;
4833 		probe->ops.func = function_trace_probe_call;
4834 		probe->tr = tr;
4835 		ftrace_ops_init(&probe->ops);
4836 		list_add(&probe->list, &tr->func_probes);
4837 	}
4838 
4839 	acquire_probe_locked(probe);
4840 
4841 	mutex_unlock(&ftrace_lock);
4842 
4843 	/*
4844 	 * Note, there's a small window here that the func_hash->filter_hash
4845 	 * may be NULL or empty. Need to be careful when reading the loop.
4846 	 */
4847 	mutex_lock(&probe->ops.func_hash->regex_lock);
4848 
4849 	orig_hash = &probe->ops.func_hash->filter_hash;
4850 	old_hash = *orig_hash;
4851 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4852 
4853 	if (!hash) {
4854 		ret = -ENOMEM;
4855 		goto out;
4856 	}
4857 
4858 	ret = ftrace_match_records(hash, glob, strlen(glob));
4859 
4860 	/* Nothing found? */
4861 	if (!ret)
4862 		ret = -EINVAL;
4863 
4864 	if (ret < 0)
4865 		goto out;
4866 
4867 	size = 1 << hash->size_bits;
4868 	for (i = 0; i < size; i++) {
4869 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
4870 			if (ftrace_lookup_ip(old_hash, entry->ip))
4871 				continue;
4872 			/*
4873 			 * The caller might want to do something special
4874 			 * for each function we find. We call the callback
4875 			 * to give the caller an opportunity to do so.
4876 			 */
4877 			if (probe_ops->init) {
4878 				ret = probe_ops->init(probe_ops, tr,
4879 						      entry->ip, data,
4880 						      &probe->data);
4881 				if (ret < 0) {
4882 					if (probe_ops->free && count)
4883 						probe_ops->free(probe_ops, tr,
4884 								0, probe->data);
4885 					probe->data = NULL;
4886 					goto out;
4887 				}
4888 			}
4889 			count++;
4890 		}
4891 	}
4892 
4893 	mutex_lock(&ftrace_lock);
4894 
4895 	if (!count) {
4896 		/* Nothing was added? */
4897 		ret = -EINVAL;
4898 		goto out_unlock;
4899 	}
4900 
4901 	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
4902 					      hash, 1);
4903 	if (ret < 0)
4904 		goto err_unlock;
4905 
4906 	/* One ref for each new function traced */
4907 	probe->ref += count;
4908 
4909 	if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED))
4910 		ret = ftrace_startup(&probe->ops, 0);
4911 
4912  out_unlock:
4913 	mutex_unlock(&ftrace_lock);
4914 
4915 	if (!ret)
4916 		ret = count;
4917  out:
4918 	mutex_unlock(&probe->ops.func_hash->regex_lock);
4919 	free_ftrace_hash(hash);
4920 
4921 	release_probe(probe);
4922 
4923 	return ret;
4924 
4925  err_unlock:
4926 	if (!probe_ops->free || !count)
4927 		goto out_unlock;
4928 
4929 	/* Failed to do the move, need to call the free functions */
4930 	for (i = 0; i < size; i++) {
4931 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
4932 			if (ftrace_lookup_ip(old_hash, entry->ip))
4933 				continue;
4934 			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
4935 		}
4936 	}
4937 	goto out_unlock;
4938 }
4939 
4940 int
unregister_ftrace_function_probe_func(char * glob,struct trace_array * tr,struct ftrace_probe_ops * probe_ops)4941 unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
4942 				      struct ftrace_probe_ops *probe_ops)
4943 {
4944 	struct ftrace_func_probe *probe = NULL, *iter;
4945 	struct ftrace_ops_hash old_hash_ops;
4946 	struct ftrace_func_entry *entry;
4947 	struct ftrace_glob func_g;
4948 	struct ftrace_hash **orig_hash;
4949 	struct ftrace_hash *old_hash;
4950 	struct ftrace_hash *hash = NULL;
4951 	struct hlist_node *tmp;
4952 	struct hlist_head hhd;
4953 	char str[KSYM_SYMBOL_LEN];
4954 	int count = 0;
4955 	int i, ret = -ENODEV;
4956 	int size;
4957 
4958 	if (!glob || !strlen(glob) || !strcmp(glob, "*"))
4959 		func_g.search = NULL;
4960 	else {
4961 		int not;
4962 
4963 		func_g.type = filter_parse_regex(glob, strlen(glob),
4964 						 &func_g.search, &not);
4965 		func_g.len = strlen(func_g.search);
4966 
4967 		/* we do not support '!' for function probes */
4968 		if (WARN_ON(not))
4969 			return -EINVAL;
4970 	}
4971 
4972 	mutex_lock(&ftrace_lock);
4973 	/* Check if the probe_ops is already registered */
4974 	list_for_each_entry(iter, &tr->func_probes, list) {
4975 		if (iter->probe_ops == probe_ops) {
4976 			probe = iter;
4977 			break;
4978 		}
4979 	}
4980 	if (!probe)
4981 		goto err_unlock_ftrace;
4982 
4983 	ret = -EINVAL;
4984 	if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED))
4985 		goto err_unlock_ftrace;
4986 
4987 	acquire_probe_locked(probe);
4988 
4989 	mutex_unlock(&ftrace_lock);
4990 
4991 	mutex_lock(&probe->ops.func_hash->regex_lock);
4992 
4993 	orig_hash = &probe->ops.func_hash->filter_hash;
4994 	old_hash = *orig_hash;
4995 
4996 	if (ftrace_hash_empty(old_hash))
4997 		goto out_unlock;
4998 
4999 	old_hash_ops.filter_hash = old_hash;
5000 	/* Probes only have filters */
5001 	old_hash_ops.notrace_hash = NULL;
5002 
5003 	ret = -ENOMEM;
5004 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
5005 	if (!hash)
5006 		goto out_unlock;
5007 
5008 	INIT_HLIST_HEAD(&hhd);
5009 
5010 	size = 1 << hash->size_bits;
5011 	for (i = 0; i < size; i++) {
5012 		hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
5013 
5014 			if (func_g.search) {
5015 				kallsyms_lookup(entry->ip, NULL, NULL,
5016 						NULL, str);
5017 				if (!ftrace_match(str, &func_g))
5018 					continue;
5019 			}
5020 			count++;
5021 			remove_hash_entry(hash, entry);
5022 			hlist_add_head(&entry->hlist, &hhd);
5023 		}
5024 	}
5025 
5026 	/* Nothing found? */
5027 	if (!count) {
5028 		ret = -EINVAL;
5029 		goto out_unlock;
5030 	}
5031 
5032 	mutex_lock(&ftrace_lock);
5033 
5034 	WARN_ON(probe->ref < count);
5035 
5036 	probe->ref -= count;
5037 
5038 	if (ftrace_hash_empty(hash))
5039 		ftrace_shutdown(&probe->ops, 0);
5040 
5041 	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
5042 					      hash, 1);
5043 
5044 	/* still need to update the function call sites */
5045 	if (ftrace_enabled && !ftrace_hash_empty(hash))
5046 		ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS,
5047 				       &old_hash_ops);
5048 	synchronize_rcu();
5049 
5050 	hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) {
5051 		hlist_del(&entry->hlist);
5052 		if (probe_ops->free)
5053 			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
5054 		kfree(entry);
5055 	}
5056 	mutex_unlock(&ftrace_lock);
5057 
5058  out_unlock:
5059 	mutex_unlock(&probe->ops.func_hash->regex_lock);
5060 	free_ftrace_hash(hash);
5061 
5062 	release_probe(probe);
5063 
5064 	return ret;
5065 
5066  err_unlock_ftrace:
5067 	mutex_unlock(&ftrace_lock);
5068 	return ret;
5069 }
5070 
clear_ftrace_function_probes(struct trace_array * tr)5071 void clear_ftrace_function_probes(struct trace_array *tr)
5072 {
5073 	struct ftrace_func_probe *probe, *n;
5074 
5075 	list_for_each_entry_safe(probe, n, &tr->func_probes, list)
5076 		unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops);
5077 }
5078 
5079 static LIST_HEAD(ftrace_commands);
5080 static DEFINE_MUTEX(ftrace_cmd_mutex);
5081 
5082 /*
5083  * Currently we only register ftrace commands from __init, so mark this
5084  * __init too.
5085  */
register_ftrace_command(struct ftrace_func_command * cmd)5086 __init int register_ftrace_command(struct ftrace_func_command *cmd)
5087 {
5088 	struct ftrace_func_command *p;
5089 	int ret = 0;
5090 
5091 	mutex_lock(&ftrace_cmd_mutex);
5092 	list_for_each_entry(p, &ftrace_commands, list) {
5093 		if (strcmp(cmd->name, p->name) == 0) {
5094 			ret = -EBUSY;
5095 			goto out_unlock;
5096 		}
5097 	}
5098 	list_add(&cmd->list, &ftrace_commands);
5099  out_unlock:
5100 	mutex_unlock(&ftrace_cmd_mutex);
5101 
5102 	return ret;
5103 }
5104 
5105 /*
5106  * Currently we only unregister ftrace commands from __init, so mark
5107  * this __init too.
5108  */
unregister_ftrace_command(struct ftrace_func_command * cmd)5109 __init int unregister_ftrace_command(struct ftrace_func_command *cmd)
5110 {
5111 	struct ftrace_func_command *p, *n;
5112 	int ret = -ENODEV;
5113 
5114 	mutex_lock(&ftrace_cmd_mutex);
5115 	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
5116 		if (strcmp(cmd->name, p->name) == 0) {
5117 			ret = 0;
5118 			list_del_init(&p->list);
5119 			goto out_unlock;
5120 		}
5121 	}
5122  out_unlock:
5123 	mutex_unlock(&ftrace_cmd_mutex);
5124 
5125 	return ret;
5126 }
5127 
ftrace_process_regex(struct ftrace_iterator * iter,char * buff,int len,int enable)5128 static int ftrace_process_regex(struct ftrace_iterator *iter,
5129 				char *buff, int len, int enable)
5130 {
5131 	struct ftrace_hash *hash = iter->hash;
5132 	struct trace_array *tr = iter->ops->private;
5133 	char *func, *command, *next = buff;
5134 	struct ftrace_func_command *p;
5135 	int ret = -EINVAL;
5136 
5137 	func = strsep(&next, ":");
5138 
5139 	if (!next) {
5140 		ret = ftrace_match_records(hash, func, len);
5141 		if (!ret)
5142 			ret = -EINVAL;
5143 		if (ret < 0)
5144 			return ret;
5145 		return 0;
5146 	}
5147 
5148 	/* command found */
5149 
5150 	command = strsep(&next, ":");
5151 
5152 	mutex_lock(&ftrace_cmd_mutex);
5153 	list_for_each_entry(p, &ftrace_commands, list) {
5154 		if (strcmp(p->name, command) == 0) {
5155 			ret = p->func(tr, hash, func, command, next, enable);
5156 			goto out_unlock;
5157 		}
5158 	}
5159  out_unlock:
5160 	mutex_unlock(&ftrace_cmd_mutex);
5161 
5162 	return ret;
5163 }
5164 
5165 static ssize_t
ftrace_regex_write(struct file * file,const char __user * ubuf,size_t cnt,loff_t * ppos,int enable)5166 ftrace_regex_write(struct file *file, const char __user *ubuf,
5167 		   size_t cnt, loff_t *ppos, int enable)
5168 {
5169 	struct ftrace_iterator *iter;
5170 	struct trace_parser *parser;
5171 	ssize_t ret, read;
5172 
5173 	if (!cnt)
5174 		return 0;
5175 
5176 	if (file->f_mode & FMODE_READ) {
5177 		struct seq_file *m = file->private_data;
5178 		iter = m->private;
5179 	} else
5180 		iter = file->private_data;
5181 
5182 	if (unlikely(ftrace_disabled))
5183 		return -ENODEV;
5184 
5185 	/* iter->hash is a local copy, so we don't need regex_lock */
5186 
5187 	parser = &iter->parser;
5188 	read = trace_get_user(parser, ubuf, cnt, ppos);
5189 
5190 	if (read >= 0 && trace_parser_loaded(parser) &&
5191 	    !trace_parser_cont(parser)) {
5192 		ret = ftrace_process_regex(iter, parser->buffer,
5193 					   parser->idx, enable);
5194 		trace_parser_clear(parser);
5195 		if (ret < 0)
5196 			goto out;
5197 	}
5198 
5199 	ret = read;
5200  out:
5201 	return ret;
5202 }
5203 
5204 ssize_t
ftrace_filter_write(struct file * file,const char __user * ubuf,size_t cnt,loff_t * ppos)5205 ftrace_filter_write(struct file *file, const char __user *ubuf,
5206 		    size_t cnt, loff_t *ppos)
5207 {
5208 	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
5209 }
5210 
5211 ssize_t
ftrace_notrace_write(struct file * file,const char __user * ubuf,size_t cnt,loff_t * ppos)5212 ftrace_notrace_write(struct file *file, const char __user *ubuf,
5213 		     size_t cnt, loff_t *ppos)
5214 {
5215 	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
5216 }
5217 
5218 static int
__ftrace_match_addr(struct ftrace_hash * hash,unsigned long ip,int remove)5219 __ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
5220 {
5221 	struct ftrace_func_entry *entry;
5222 
5223 	ip = ftrace_location(ip);
5224 	if (!ip)
5225 		return -EINVAL;
5226 
5227 	if (remove) {
5228 		entry = ftrace_lookup_ip(hash, ip);
5229 		if (!entry)
5230 			return -ENOENT;
5231 		free_hash_entry(hash, entry);
5232 		return 0;
5233 	}
5234 
5235 	entry = add_hash_entry(hash, ip);
5236 	return entry ? 0 :  -ENOMEM;
5237 }
5238 
5239 static int
ftrace_match_addr(struct ftrace_hash * hash,unsigned long * ips,unsigned int cnt,int remove)5240 ftrace_match_addr(struct ftrace_hash *hash, unsigned long *ips,
5241 		  unsigned int cnt, int remove)
5242 {
5243 	unsigned int i;
5244 	int err;
5245 
5246 	for (i = 0; i < cnt; i++) {
5247 		err = __ftrace_match_addr(hash, ips[i], remove);
5248 		if (err) {
5249 			/*
5250 			 * This expects the @hash is a temporary hash and if this
5251 			 * fails the caller must free the @hash.
5252 			 */
5253 			return err;
5254 		}
5255 	}
5256 	return 0;
5257 }
5258 
5259 static int
ftrace_set_hash(struct ftrace_ops * ops,unsigned char * buf,int len,unsigned long * ips,unsigned int cnt,int remove,int reset,int enable)5260 ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
5261 		unsigned long *ips, unsigned int cnt,
5262 		int remove, int reset, int enable)
5263 {
5264 	struct ftrace_hash **orig_hash;
5265 	struct ftrace_hash *hash;
5266 	int ret;
5267 
5268 	if (unlikely(ftrace_disabled))
5269 		return -ENODEV;
5270 
5271 	mutex_lock(&ops->func_hash->regex_lock);
5272 
5273 	if (enable)
5274 		orig_hash = &ops->func_hash->filter_hash;
5275 	else
5276 		orig_hash = &ops->func_hash->notrace_hash;
5277 
5278 	if (reset)
5279 		hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
5280 	else
5281 		hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
5282 
5283 	if (!hash) {
5284 		ret = -ENOMEM;
5285 		goto out_regex_unlock;
5286 	}
5287 
5288 	if (buf && !ftrace_match_records(hash, buf, len)) {
5289 		ret = -EINVAL;
5290 		goto out_regex_unlock;
5291 	}
5292 	if (ips) {
5293 		ret = ftrace_match_addr(hash, ips, cnt, remove);
5294 		if (ret < 0)
5295 			goto out_regex_unlock;
5296 	}
5297 
5298 	mutex_lock(&ftrace_lock);
5299 	ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable);
5300 	mutex_unlock(&ftrace_lock);
5301 
5302  out_regex_unlock:
5303 	mutex_unlock(&ops->func_hash->regex_lock);
5304 
5305 	free_ftrace_hash(hash);
5306 	return ret;
5307 }
5308 
5309 static int
ftrace_set_addr(struct ftrace_ops * ops,unsigned long * ips,unsigned int cnt,int remove,int reset,int enable)5310 ftrace_set_addr(struct ftrace_ops *ops, unsigned long *ips, unsigned int cnt,
5311 		int remove, int reset, int enable)
5312 {
5313 	return ftrace_set_hash(ops, NULL, 0, ips, cnt, remove, reset, enable);
5314 }
5315 
5316 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
5317 
5318 struct ftrace_direct_func {
5319 	struct list_head	next;
5320 	unsigned long		addr;
5321 	int			count;
5322 };
5323 
5324 static LIST_HEAD(ftrace_direct_funcs);
5325 
5326 static int register_ftrace_function_nolock(struct ftrace_ops *ops);
5327 
5328 /*
5329  * If there are multiple ftrace_ops, use SAVE_REGS by default, so that direct
5330  * call will be jumped from ftrace_regs_caller. Only if the architecture does
5331  * not support ftrace_regs_caller but direct_call, use SAVE_ARGS so that it
5332  * jumps from ftrace_caller for multiple ftrace_ops.
5333  */
5334 #ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_REGS
5335 #define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_ARGS)
5336 #else
5337 #define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS)
5338 #endif
5339 
check_direct_multi(struct ftrace_ops * ops)5340 static int check_direct_multi(struct ftrace_ops *ops)
5341 {
5342 	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED))
5343 		return -EINVAL;
5344 	if ((ops->flags & MULTI_FLAGS) != MULTI_FLAGS)
5345 		return -EINVAL;
5346 	return 0;
5347 }
5348 
remove_direct_functions_hash(struct ftrace_hash * hash,unsigned long addr)5349 static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long addr)
5350 {
5351 	struct ftrace_func_entry *entry, *del;
5352 	int size, i;
5353 
5354 	size = 1 << hash->size_bits;
5355 	for (i = 0; i < size; i++) {
5356 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
5357 			del = __ftrace_lookup_ip(direct_functions, entry->ip);
5358 			if (del && del->direct == addr) {
5359 				remove_hash_entry(direct_functions, del);
5360 				kfree(del);
5361 			}
5362 		}
5363 	}
5364 }
5365 
5366 /**
5367  * register_ftrace_direct - Call a custom trampoline directly
5368  * for multiple functions registered in @ops
5369  * @ops: The address of the struct ftrace_ops object
5370  * @addr: The address of the trampoline to call at @ops functions
5371  *
5372  * This is used to connect a direct calls to @addr from the nop locations
5373  * of the functions registered in @ops (with by ftrace_set_filter_ip
5374  * function).
5375  *
5376  * The location that it calls (@addr) must be able to handle a direct call,
5377  * and save the parameters of the function being traced, and restore them
5378  * (or inject new ones if needed), before returning.
5379  *
5380  * Returns:
5381  *  0 on success
5382  *  -EINVAL  - The @ops object was already registered with this call or
5383  *             when there are no functions in @ops object.
5384  *  -EBUSY   - Another direct function is already attached (there can be only one)
5385  *  -ENODEV  - @ip does not point to a ftrace nop location (or not supported)
5386  *  -ENOMEM  - There was an allocation failure.
5387  */
register_ftrace_direct(struct ftrace_ops * ops,unsigned long addr)5388 int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
5389 {
5390 	struct ftrace_hash *hash, *new_hash = NULL, *free_hash = NULL;
5391 	struct ftrace_func_entry *entry, *new;
5392 	int err = -EBUSY, size, i;
5393 
5394 	if (ops->func || ops->trampoline)
5395 		return -EINVAL;
5396 	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED))
5397 		return -EINVAL;
5398 	if (ops->flags & FTRACE_OPS_FL_ENABLED)
5399 		return -EINVAL;
5400 
5401 	hash = ops->func_hash->filter_hash;
5402 	if (ftrace_hash_empty(hash))
5403 		return -EINVAL;
5404 
5405 	mutex_lock(&direct_mutex);
5406 
5407 	/* Make sure requested entries are not already registered.. */
5408 	size = 1 << hash->size_bits;
5409 	for (i = 0; i < size; i++) {
5410 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
5411 			if (ftrace_find_rec_direct(entry->ip))
5412 				goto out_unlock;
5413 		}
5414 	}
5415 
5416 	err = -ENOMEM;
5417 
5418 	/* Make a copy hash to place the new and the old entries in */
5419 	size = hash->count + direct_functions->count;
5420 	if (size > 32)
5421 		size = 32;
5422 	new_hash = alloc_ftrace_hash(fls(size));
5423 	if (!new_hash)
5424 		goto out_unlock;
5425 
5426 	/* Now copy over the existing direct entries */
5427 	size = 1 << direct_functions->size_bits;
5428 	for (i = 0; i < size; i++) {
5429 		hlist_for_each_entry(entry, &direct_functions->buckets[i], hlist) {
5430 			new = add_hash_entry(new_hash, entry->ip);
5431 			if (!new)
5432 				goto out_unlock;
5433 			new->direct = entry->direct;
5434 		}
5435 	}
5436 
5437 	/* ... and add the new entries */
5438 	size = 1 << hash->size_bits;
5439 	for (i = 0; i < size; i++) {
5440 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
5441 			new = add_hash_entry(new_hash, entry->ip);
5442 			if (!new)
5443 				goto out_unlock;
5444 			/* Update both the copy and the hash entry */
5445 			new->direct = addr;
5446 			entry->direct = addr;
5447 		}
5448 	}
5449 
5450 	free_hash = direct_functions;
5451 	rcu_assign_pointer(direct_functions, new_hash);
5452 	new_hash = NULL;
5453 
5454 	ops->func = call_direct_funcs;
5455 	ops->flags = MULTI_FLAGS;
5456 	ops->trampoline = FTRACE_REGS_ADDR;
5457 	ops->direct_call = addr;
5458 
5459 	err = register_ftrace_function_nolock(ops);
5460 
5461  out_unlock:
5462 	mutex_unlock(&direct_mutex);
5463 
5464 	if (free_hash && free_hash != EMPTY_HASH) {
5465 		synchronize_rcu_tasks();
5466 		free_ftrace_hash(free_hash);
5467 	}
5468 
5469 	if (new_hash)
5470 		free_ftrace_hash(new_hash);
5471 
5472 	return err;
5473 }
5474 EXPORT_SYMBOL_GPL(register_ftrace_direct);
5475 
5476 /**
5477  * unregister_ftrace_direct - Remove calls to custom trampoline
5478  * previously registered by register_ftrace_direct for @ops object.
5479  * @ops: The address of the struct ftrace_ops object
5480  *
5481  * This is used to remove a direct calls to @addr from the nop locations
5482  * of the functions registered in @ops (with by ftrace_set_filter_ip
5483  * function).
5484  *
5485  * Returns:
5486  *  0 on success
5487  *  -EINVAL - The @ops object was not properly registered.
5488  */
unregister_ftrace_direct(struct ftrace_ops * ops,unsigned long addr,bool free_filters)5489 int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr,
5490 			     bool free_filters)
5491 {
5492 	struct ftrace_hash *hash = ops->func_hash->filter_hash;
5493 	int err;
5494 
5495 	if (check_direct_multi(ops))
5496 		return -EINVAL;
5497 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
5498 		return -EINVAL;
5499 
5500 	mutex_lock(&direct_mutex);
5501 	err = unregister_ftrace_function(ops);
5502 	remove_direct_functions_hash(hash, addr);
5503 	mutex_unlock(&direct_mutex);
5504 
5505 	/* cleanup for possible another register call */
5506 	ops->func = NULL;
5507 	ops->trampoline = 0;
5508 
5509 	if (free_filters)
5510 		ftrace_free_filter(ops);
5511 	return err;
5512 }
5513 EXPORT_SYMBOL_GPL(unregister_ftrace_direct);
5514 
5515 static int
__modify_ftrace_direct(struct ftrace_ops * ops,unsigned long addr)5516 __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
5517 {
5518 	struct ftrace_hash *hash;
5519 	struct ftrace_func_entry *entry, *iter;
5520 	static struct ftrace_ops tmp_ops = {
5521 		.func		= ftrace_stub,
5522 		.flags		= FTRACE_OPS_FL_STUB,
5523 	};
5524 	int i, size;
5525 	int err;
5526 
5527 	lockdep_assert_held_once(&direct_mutex);
5528 
5529 	/* Enable the tmp_ops to have the same functions as the direct ops */
5530 	ftrace_ops_init(&tmp_ops);
5531 	tmp_ops.func_hash = ops->func_hash;
5532 	tmp_ops.direct_call = addr;
5533 
5534 	err = register_ftrace_function_nolock(&tmp_ops);
5535 	if (err)
5536 		return err;
5537 
5538 	/*
5539 	 * Now the ftrace_ops_list_func() is called to do the direct callers.
5540 	 * We can safely change the direct functions attached to each entry.
5541 	 */
5542 	mutex_lock(&ftrace_lock);
5543 
5544 	hash = ops->func_hash->filter_hash;
5545 	size = 1 << hash->size_bits;
5546 	for (i = 0; i < size; i++) {
5547 		hlist_for_each_entry(iter, &hash->buckets[i], hlist) {
5548 			entry = __ftrace_lookup_ip(direct_functions, iter->ip);
5549 			if (!entry)
5550 				continue;
5551 			entry->direct = addr;
5552 		}
5553 	}
5554 	/* Prevent store tearing if a trampoline concurrently accesses the value */
5555 	WRITE_ONCE(ops->direct_call, addr);
5556 
5557 	mutex_unlock(&ftrace_lock);
5558 
5559 	/* Removing the tmp_ops will add the updated direct callers to the functions */
5560 	unregister_ftrace_function(&tmp_ops);
5561 
5562 	return err;
5563 }
5564 
5565 /**
5566  * modify_ftrace_direct_nolock - Modify an existing direct 'multi' call
5567  * to call something else
5568  * @ops: The address of the struct ftrace_ops object
5569  * @addr: The address of the new trampoline to call at @ops functions
5570  *
5571  * This is used to unregister currently registered direct caller and
5572  * register new one @addr on functions registered in @ops object.
5573  *
5574  * Note there's window between ftrace_shutdown and ftrace_startup calls
5575  * where there will be no callbacks called.
5576  *
5577  * Caller should already have direct_mutex locked, so we don't lock
5578  * direct_mutex here.
5579  *
5580  * Returns: zero on success. Non zero on error, which includes:
5581  *  -EINVAL - The @ops object was not properly registered.
5582  */
modify_ftrace_direct_nolock(struct ftrace_ops * ops,unsigned long addr)5583 int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr)
5584 {
5585 	if (check_direct_multi(ops))
5586 		return -EINVAL;
5587 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
5588 		return -EINVAL;
5589 
5590 	return __modify_ftrace_direct(ops, addr);
5591 }
5592 EXPORT_SYMBOL_GPL(modify_ftrace_direct_nolock);
5593 
5594 /**
5595  * modify_ftrace_direct - Modify an existing direct 'multi' call
5596  * to call something else
5597  * @ops: The address of the struct ftrace_ops object
5598  * @addr: The address of the new trampoline to call at @ops functions
5599  *
5600  * This is used to unregister currently registered direct caller and
5601  * register new one @addr on functions registered in @ops object.
5602  *
5603  * Note there's window between ftrace_shutdown and ftrace_startup calls
5604  * where there will be no callbacks called.
5605  *
5606  * Returns: zero on success. Non zero on error, which includes:
5607  *  -EINVAL - The @ops object was not properly registered.
5608  */
modify_ftrace_direct(struct ftrace_ops * ops,unsigned long addr)5609 int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
5610 {
5611 	int err;
5612 
5613 	if (check_direct_multi(ops))
5614 		return -EINVAL;
5615 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
5616 		return -EINVAL;
5617 
5618 	mutex_lock(&direct_mutex);
5619 	err = __modify_ftrace_direct(ops, addr);
5620 	mutex_unlock(&direct_mutex);
5621 	return err;
5622 }
5623 EXPORT_SYMBOL_GPL(modify_ftrace_direct);
5624 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
5625 
5626 /**
5627  * ftrace_set_filter_ip - set a function to filter on in ftrace by address
5628  * @ops - the ops to set the filter with
5629  * @ip - the address to add to or remove from the filter.
5630  * @remove - non zero to remove the ip from the filter
5631  * @reset - non zero to reset all filters before applying this filter.
5632  *
5633  * Filters denote which functions should be enabled when tracing is enabled
5634  * If @ip is NULL, it fails to update filter.
5635  *
5636  * This can allocate memory which must be freed before @ops can be freed,
5637  * either by removing each filtered addr or by using
5638  * ftrace_free_filter(@ops).
5639  */
ftrace_set_filter_ip(struct ftrace_ops * ops,unsigned long ip,int remove,int reset)5640 int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
5641 			 int remove, int reset)
5642 {
5643 	ftrace_ops_init(ops);
5644 	return ftrace_set_addr(ops, &ip, 1, remove, reset, 1);
5645 }
5646 EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
5647 
5648 /**
5649  * ftrace_set_filter_ips - set functions to filter on in ftrace by addresses
5650  * @ops - the ops to set the filter with
5651  * @ips - the array of addresses to add to or remove from the filter.
5652  * @cnt - the number of addresses in @ips
5653  * @remove - non zero to remove ips from the filter
5654  * @reset - non zero to reset all filters before applying this filter.
5655  *
5656  * Filters denote which functions should be enabled when tracing is enabled
5657  * If @ips array or any ip specified within is NULL , it fails to update filter.
5658  *
5659  * This can allocate memory which must be freed before @ops can be freed,
5660  * either by removing each filtered addr or by using
5661  * ftrace_free_filter(@ops).
5662 */
ftrace_set_filter_ips(struct ftrace_ops * ops,unsigned long * ips,unsigned int cnt,int remove,int reset)5663 int ftrace_set_filter_ips(struct ftrace_ops *ops, unsigned long *ips,
5664 			  unsigned int cnt, int remove, int reset)
5665 {
5666 	ftrace_ops_init(ops);
5667 	return ftrace_set_addr(ops, ips, cnt, remove, reset, 1);
5668 }
5669 EXPORT_SYMBOL_GPL(ftrace_set_filter_ips);
5670 
5671 /**
5672  * ftrace_ops_set_global_filter - setup ops to use global filters
5673  * @ops - the ops which will use the global filters
5674  *
5675  * ftrace users who need global function trace filtering should call this.
5676  * It can set the global filter only if ops were not initialized before.
5677  */
ftrace_ops_set_global_filter(struct ftrace_ops * ops)5678 void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
5679 {
5680 	if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
5681 		return;
5682 
5683 	ftrace_ops_init(ops);
5684 	ops->func_hash = &global_ops.local_hash;
5685 }
5686 EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);
5687 
5688 static int
ftrace_set_regex(struct ftrace_ops * ops,unsigned char * buf,int len,int reset,int enable)5689 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
5690 		 int reset, int enable)
5691 {
5692 	return ftrace_set_hash(ops, buf, len, NULL, 0, 0, reset, enable);
5693 }
5694 
5695 /**
5696  * ftrace_set_filter - set a function to filter on in ftrace
5697  * @ops - the ops to set the filter with
5698  * @buf - the string that holds the function filter text.
5699  * @len - the length of the string.
5700  * @reset - non zero to reset all filters before applying this filter.
5701  *
5702  * Filters denote which functions should be enabled when tracing is enabled.
5703  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
5704  *
5705  * This can allocate memory which must be freed before @ops can be freed,
5706  * either by removing each filtered addr or by using
5707  * ftrace_free_filter(@ops).
5708  */
ftrace_set_filter(struct ftrace_ops * ops,unsigned char * buf,int len,int reset)5709 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
5710 		       int len, int reset)
5711 {
5712 	ftrace_ops_init(ops);
5713 	return ftrace_set_regex(ops, buf, len, reset, 1);
5714 }
5715 EXPORT_SYMBOL_GPL(ftrace_set_filter);
5716 
5717 /**
5718  * ftrace_set_notrace - set a function to not trace in ftrace
5719  * @ops - the ops to set the notrace filter with
5720  * @buf - the string that holds the function notrace text.
5721  * @len - the length of the string.
5722  * @reset - non zero to reset all filters before applying this filter.
5723  *
5724  * Notrace Filters denote which functions should not be enabled when tracing
5725  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
5726  * for tracing.
5727  *
5728  * This can allocate memory which must be freed before @ops can be freed,
5729  * either by removing each filtered addr or by using
5730  * ftrace_free_filter(@ops).
5731  */
ftrace_set_notrace(struct ftrace_ops * ops,unsigned char * buf,int len,int reset)5732 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
5733 			int len, int reset)
5734 {
5735 	ftrace_ops_init(ops);
5736 	return ftrace_set_regex(ops, buf, len, reset, 0);
5737 }
5738 EXPORT_SYMBOL_GPL(ftrace_set_notrace);
5739 /**
5740  * ftrace_set_global_filter - set a function to filter on with global tracers
5741  * @buf - the string that holds the function filter text.
5742  * @len - the length of the string.
5743  * @reset - non zero to reset all filters before applying this filter.
5744  *
5745  * Filters denote which functions should be enabled when tracing is enabled.
5746  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
5747  */
ftrace_set_global_filter(unsigned char * buf,int len,int reset)5748 void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
5749 {
5750 	ftrace_set_regex(&global_ops, buf, len, reset, 1);
5751 }
5752 EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
5753 
5754 /**
5755  * ftrace_set_global_notrace - set a function to not trace with global tracers
5756  * @buf - the string that holds the function notrace text.
5757  * @len - the length of the string.
5758  * @reset - non zero to reset all filters before applying this filter.
5759  *
5760  * Notrace Filters denote which functions should not be enabled when tracing
5761  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
5762  * for tracing.
5763  */
ftrace_set_global_notrace(unsigned char * buf,int len,int reset)5764 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
5765 {
5766 	ftrace_set_regex(&global_ops, buf, len, reset, 0);
5767 }
5768 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
5769 
5770 /*
5771  * command line interface to allow users to set filters on boot up.
5772  */
5773 #define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
5774 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
5775 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
5776 
5777 /* Used by function selftest to not test if filter is set */
5778 bool ftrace_filter_param __initdata;
5779 
set_ftrace_notrace(char * str)5780 static int __init set_ftrace_notrace(char *str)
5781 {
5782 	ftrace_filter_param = true;
5783 	strscpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
5784 	return 1;
5785 }
5786 __setup("ftrace_notrace=", set_ftrace_notrace);
5787 
set_ftrace_filter(char * str)5788 static int __init set_ftrace_filter(char *str)
5789 {
5790 	ftrace_filter_param = true;
5791 	strscpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
5792 	return 1;
5793 }
5794 __setup("ftrace_filter=", set_ftrace_filter);
5795 
5796 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
5797 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
5798 static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
5799 static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer);
5800 
set_graph_function(char * str)5801 static int __init set_graph_function(char *str)
5802 {
5803 	strscpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
5804 	return 1;
5805 }
5806 __setup("ftrace_graph_filter=", set_graph_function);
5807 
set_graph_notrace_function(char * str)5808 static int __init set_graph_notrace_function(char *str)
5809 {
5810 	strscpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
5811 	return 1;
5812 }
5813 __setup("ftrace_graph_notrace=", set_graph_notrace_function);
5814 
set_graph_max_depth_function(char * str)5815 static int __init set_graph_max_depth_function(char *str)
5816 {
5817 	if (!str)
5818 		return 0;
5819 	fgraph_max_depth = simple_strtoul(str, NULL, 0);
5820 	return 1;
5821 }
5822 __setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
5823 
set_ftrace_early_graph(char * buf,int enable)5824 static void __init set_ftrace_early_graph(char *buf, int enable)
5825 {
5826 	int ret;
5827 	char *func;
5828 	struct ftrace_hash *hash;
5829 
5830 	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
5831 	if (MEM_FAIL(!hash, "Failed to allocate hash\n"))
5832 		return;
5833 
5834 	while (buf) {
5835 		func = strsep(&buf, ",");
5836 		/* we allow only one expression at a time */
5837 		ret = ftrace_graph_set_hash(hash, func);
5838 		if (ret)
5839 			printk(KERN_DEBUG "ftrace: function %s not "
5840 					  "traceable\n", func);
5841 	}
5842 
5843 	if (enable)
5844 		ftrace_graph_hash = hash;
5845 	else
5846 		ftrace_graph_notrace_hash = hash;
5847 }
5848 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
5849 
5850 void __init
ftrace_set_early_filter(struct ftrace_ops * ops,char * buf,int enable)5851 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
5852 {
5853 	char *func;
5854 
5855 	ftrace_ops_init(ops);
5856 
5857 	while (buf) {
5858 		func = strsep(&buf, ",");
5859 		ftrace_set_regex(ops, func, strlen(func), 0, enable);
5860 	}
5861 }
5862 
set_ftrace_early_filters(void)5863 static void __init set_ftrace_early_filters(void)
5864 {
5865 	if (ftrace_filter_buf[0])
5866 		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
5867 	if (ftrace_notrace_buf[0])
5868 		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
5869 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
5870 	if (ftrace_graph_buf[0])
5871 		set_ftrace_early_graph(ftrace_graph_buf, 1);
5872 	if (ftrace_graph_notrace_buf[0])
5873 		set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
5874 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
5875 }
5876 
ftrace_regex_release(struct inode * inode,struct file * file)5877 int ftrace_regex_release(struct inode *inode, struct file *file)
5878 {
5879 	struct seq_file *m = (struct seq_file *)file->private_data;
5880 	struct ftrace_iterator *iter;
5881 	struct ftrace_hash **orig_hash;
5882 	struct trace_parser *parser;
5883 	int filter_hash;
5884 
5885 	if (file->f_mode & FMODE_READ) {
5886 		iter = m->private;
5887 		seq_release(inode, file);
5888 	} else
5889 		iter = file->private_data;
5890 
5891 	parser = &iter->parser;
5892 	if (trace_parser_loaded(parser)) {
5893 		int enable = !(iter->flags & FTRACE_ITER_NOTRACE);
5894 
5895 		ftrace_process_regex(iter, parser->buffer,
5896 				     parser->idx, enable);
5897 	}
5898 
5899 	trace_parser_put(parser);
5900 
5901 	mutex_lock(&iter->ops->func_hash->regex_lock);
5902 
5903 	if (file->f_mode & FMODE_WRITE) {
5904 		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
5905 
5906 		if (filter_hash) {
5907 			orig_hash = &iter->ops->func_hash->filter_hash;
5908 			if (iter->tr) {
5909 				if (list_empty(&iter->tr->mod_trace))
5910 					iter->hash->flags &= ~FTRACE_HASH_FL_MOD;
5911 				else
5912 					iter->hash->flags |= FTRACE_HASH_FL_MOD;
5913 			}
5914 		} else
5915 			orig_hash = &iter->ops->func_hash->notrace_hash;
5916 
5917 		mutex_lock(&ftrace_lock);
5918 		ftrace_hash_move_and_update_ops(iter->ops, orig_hash,
5919 						      iter->hash, filter_hash);
5920 		mutex_unlock(&ftrace_lock);
5921 	} else {
5922 		/* For read only, the hash is the ops hash */
5923 		iter->hash = NULL;
5924 	}
5925 
5926 	mutex_unlock(&iter->ops->func_hash->regex_lock);
5927 	free_ftrace_hash(iter->hash);
5928 	if (iter->tr)
5929 		trace_array_put(iter->tr);
5930 	kfree(iter);
5931 
5932 	return 0;
5933 }
5934 
5935 static const struct file_operations ftrace_avail_fops = {
5936 	.open = ftrace_avail_open,
5937 	.read = seq_read,
5938 	.llseek = seq_lseek,
5939 	.release = seq_release_private,
5940 };
5941 
5942 static const struct file_operations ftrace_enabled_fops = {
5943 	.open = ftrace_enabled_open,
5944 	.read = seq_read,
5945 	.llseek = seq_lseek,
5946 	.release = seq_release_private,
5947 };
5948 
5949 static const struct file_operations ftrace_touched_fops = {
5950 	.open = ftrace_touched_open,
5951 	.read = seq_read,
5952 	.llseek = seq_lseek,
5953 	.release = seq_release_private,
5954 };
5955 
5956 static const struct file_operations ftrace_avail_addrs_fops = {
5957 	.open = ftrace_avail_addrs_open,
5958 	.read = seq_read,
5959 	.llseek = seq_lseek,
5960 	.release = seq_release_private,
5961 };
5962 
5963 static const struct file_operations ftrace_filter_fops = {
5964 	.open = ftrace_filter_open,
5965 	.read = seq_read,
5966 	.write = ftrace_filter_write,
5967 	.llseek = tracing_lseek,
5968 	.release = ftrace_regex_release,
5969 };
5970 
5971 static const struct file_operations ftrace_notrace_fops = {
5972 	.open = ftrace_notrace_open,
5973 	.read = seq_read,
5974 	.write = ftrace_notrace_write,
5975 	.llseek = tracing_lseek,
5976 	.release = ftrace_regex_release,
5977 };
5978 
5979 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
5980 
5981 static DEFINE_MUTEX(graph_lock);
5982 
5983 struct ftrace_hash __rcu *ftrace_graph_hash = EMPTY_HASH;
5984 struct ftrace_hash __rcu *ftrace_graph_notrace_hash = EMPTY_HASH;
5985 
5986 enum graph_filter_type {
5987 	GRAPH_FILTER_NOTRACE	= 0,
5988 	GRAPH_FILTER_FUNCTION,
5989 };
5990 
5991 #define FTRACE_GRAPH_EMPTY	((void *)1)
5992 
5993 struct ftrace_graph_data {
5994 	struct ftrace_hash		*hash;
5995 	struct ftrace_func_entry	*entry;
5996 	int				idx;   /* for hash table iteration */
5997 	enum graph_filter_type		type;
5998 	struct ftrace_hash		*new_hash;
5999 	const struct seq_operations	*seq_ops;
6000 	struct trace_parser		parser;
6001 };
6002 
6003 static void *
__g_next(struct seq_file * m,loff_t * pos)6004 __g_next(struct seq_file *m, loff_t *pos)
6005 {
6006 	struct ftrace_graph_data *fgd = m->private;
6007 	struct ftrace_func_entry *entry = fgd->entry;
6008 	struct hlist_head *head;
6009 	int i, idx = fgd->idx;
6010 
6011 	if (*pos >= fgd->hash->count)
6012 		return NULL;
6013 
6014 	if (entry) {
6015 		hlist_for_each_entry_continue(entry, hlist) {
6016 			fgd->entry = entry;
6017 			return entry;
6018 		}
6019 
6020 		idx++;
6021 	}
6022 
6023 	for (i = idx; i < 1 << fgd->hash->size_bits; i++) {
6024 		head = &fgd->hash->buckets[i];
6025 		hlist_for_each_entry(entry, head, hlist) {
6026 			fgd->entry = entry;
6027 			fgd->idx = i;
6028 			return entry;
6029 		}
6030 	}
6031 	return NULL;
6032 }
6033 
6034 static void *
g_next(struct seq_file * m,void * v,loff_t * pos)6035 g_next(struct seq_file *m, void *v, loff_t *pos)
6036 {
6037 	(*pos)++;
6038 	return __g_next(m, pos);
6039 }
6040 
g_start(struct seq_file * m,loff_t * pos)6041 static void *g_start(struct seq_file *m, loff_t *pos)
6042 {
6043 	struct ftrace_graph_data *fgd = m->private;
6044 
6045 	mutex_lock(&graph_lock);
6046 
6047 	if (fgd->type == GRAPH_FILTER_FUNCTION)
6048 		fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
6049 					lockdep_is_held(&graph_lock));
6050 	else
6051 		fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
6052 					lockdep_is_held(&graph_lock));
6053 
6054 	/* Nothing, tell g_show to print all functions are enabled */
6055 	if (ftrace_hash_empty(fgd->hash) && !*pos)
6056 		return FTRACE_GRAPH_EMPTY;
6057 
6058 	fgd->idx = 0;
6059 	fgd->entry = NULL;
6060 	return __g_next(m, pos);
6061 }
6062 
g_stop(struct seq_file * m,void * p)6063 static void g_stop(struct seq_file *m, void *p)
6064 {
6065 	mutex_unlock(&graph_lock);
6066 }
6067 
g_show(struct seq_file * m,void * v)6068 static int g_show(struct seq_file *m, void *v)
6069 {
6070 	struct ftrace_func_entry *entry = v;
6071 
6072 	if (!entry)
6073 		return 0;
6074 
6075 	if (entry == FTRACE_GRAPH_EMPTY) {
6076 		struct ftrace_graph_data *fgd = m->private;
6077 
6078 		if (fgd->type == GRAPH_FILTER_FUNCTION)
6079 			seq_puts(m, "#### all functions enabled ####\n");
6080 		else
6081 			seq_puts(m, "#### no functions disabled ####\n");
6082 		return 0;
6083 	}
6084 
6085 	seq_printf(m, "%ps\n", (void *)entry->ip);
6086 
6087 	return 0;
6088 }
6089 
6090 static const struct seq_operations ftrace_graph_seq_ops = {
6091 	.start = g_start,
6092 	.next = g_next,
6093 	.stop = g_stop,
6094 	.show = g_show,
6095 };
6096 
6097 static int
__ftrace_graph_open(struct inode * inode,struct file * file,struct ftrace_graph_data * fgd)6098 __ftrace_graph_open(struct inode *inode, struct file *file,
6099 		    struct ftrace_graph_data *fgd)
6100 {
6101 	int ret;
6102 	struct ftrace_hash *new_hash = NULL;
6103 
6104 	ret = security_locked_down(LOCKDOWN_TRACEFS);
6105 	if (ret)
6106 		return ret;
6107 
6108 	if (file->f_mode & FMODE_WRITE) {
6109 		const int size_bits = FTRACE_HASH_DEFAULT_BITS;
6110 
6111 		if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX))
6112 			return -ENOMEM;
6113 
6114 		if (file->f_flags & O_TRUNC)
6115 			new_hash = alloc_ftrace_hash(size_bits);
6116 		else
6117 			new_hash = alloc_and_copy_ftrace_hash(size_bits,
6118 							      fgd->hash);
6119 		if (!new_hash) {
6120 			ret = -ENOMEM;
6121 			goto out;
6122 		}
6123 	}
6124 
6125 	if (file->f_mode & FMODE_READ) {
6126 		ret = seq_open(file, &ftrace_graph_seq_ops);
6127 		if (!ret) {
6128 			struct seq_file *m = file->private_data;
6129 			m->private = fgd;
6130 		} else {
6131 			/* Failed */
6132 			free_ftrace_hash(new_hash);
6133 			new_hash = NULL;
6134 		}
6135 	} else
6136 		file->private_data = fgd;
6137 
6138 out:
6139 	if (ret < 0 && file->f_mode & FMODE_WRITE)
6140 		trace_parser_put(&fgd->parser);
6141 
6142 	fgd->new_hash = new_hash;
6143 
6144 	/*
6145 	 * All uses of fgd->hash must be taken with the graph_lock
6146 	 * held. The graph_lock is going to be released, so force
6147 	 * fgd->hash to be reinitialized when it is taken again.
6148 	 */
6149 	fgd->hash = NULL;
6150 
6151 	return ret;
6152 }
6153 
6154 static int
ftrace_graph_open(struct inode * inode,struct file * file)6155 ftrace_graph_open(struct inode *inode, struct file *file)
6156 {
6157 	struct ftrace_graph_data *fgd;
6158 	int ret;
6159 
6160 	if (unlikely(ftrace_disabled))
6161 		return -ENODEV;
6162 
6163 	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
6164 	if (fgd == NULL)
6165 		return -ENOMEM;
6166 
6167 	mutex_lock(&graph_lock);
6168 
6169 	fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
6170 					lockdep_is_held(&graph_lock));
6171 	fgd->type = GRAPH_FILTER_FUNCTION;
6172 	fgd->seq_ops = &ftrace_graph_seq_ops;
6173 
6174 	ret = __ftrace_graph_open(inode, file, fgd);
6175 	if (ret < 0)
6176 		kfree(fgd);
6177 
6178 	mutex_unlock(&graph_lock);
6179 	return ret;
6180 }
6181 
6182 static int
ftrace_graph_notrace_open(struct inode * inode,struct file * file)6183 ftrace_graph_notrace_open(struct inode *inode, struct file *file)
6184 {
6185 	struct ftrace_graph_data *fgd;
6186 	int ret;
6187 
6188 	if (unlikely(ftrace_disabled))
6189 		return -ENODEV;
6190 
6191 	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
6192 	if (fgd == NULL)
6193 		return -ENOMEM;
6194 
6195 	mutex_lock(&graph_lock);
6196 
6197 	fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
6198 					lockdep_is_held(&graph_lock));
6199 	fgd->type = GRAPH_FILTER_NOTRACE;
6200 	fgd->seq_ops = &ftrace_graph_seq_ops;
6201 
6202 	ret = __ftrace_graph_open(inode, file, fgd);
6203 	if (ret < 0)
6204 		kfree(fgd);
6205 
6206 	mutex_unlock(&graph_lock);
6207 	return ret;
6208 }
6209 
6210 static int
ftrace_graph_release(struct inode * inode,struct file * file)6211 ftrace_graph_release(struct inode *inode, struct file *file)
6212 {
6213 	struct ftrace_graph_data *fgd;
6214 	struct ftrace_hash *old_hash, *new_hash;
6215 	struct trace_parser *parser;
6216 	int ret = 0;
6217 
6218 	if (file->f_mode & FMODE_READ) {
6219 		struct seq_file *m = file->private_data;
6220 
6221 		fgd = m->private;
6222 		seq_release(inode, file);
6223 	} else {
6224 		fgd = file->private_data;
6225 	}
6226 
6227 
6228 	if (file->f_mode & FMODE_WRITE) {
6229 
6230 		parser = &fgd->parser;
6231 
6232 		if (trace_parser_loaded((parser))) {
6233 			ret = ftrace_graph_set_hash(fgd->new_hash,
6234 						    parser->buffer);
6235 		}
6236 
6237 		trace_parser_put(parser);
6238 
6239 		new_hash = __ftrace_hash_move(fgd->new_hash);
6240 		if (!new_hash) {
6241 			ret = -ENOMEM;
6242 			goto out;
6243 		}
6244 
6245 		mutex_lock(&graph_lock);
6246 
6247 		if (fgd->type == GRAPH_FILTER_FUNCTION) {
6248 			old_hash = rcu_dereference_protected(ftrace_graph_hash,
6249 					lockdep_is_held(&graph_lock));
6250 			rcu_assign_pointer(ftrace_graph_hash, new_hash);
6251 		} else {
6252 			old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
6253 					lockdep_is_held(&graph_lock));
6254 			rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash);
6255 		}
6256 
6257 		mutex_unlock(&graph_lock);
6258 
6259 		/*
6260 		 * We need to do a hard force of sched synchronization.
6261 		 * This is because we use preempt_disable() to do RCU, but
6262 		 * the function tracers can be called where RCU is not watching
6263 		 * (like before user_exit()). We can not rely on the RCU
6264 		 * infrastructure to do the synchronization, thus we must do it
6265 		 * ourselves.
6266 		 */
6267 		if (old_hash != EMPTY_HASH)
6268 			synchronize_rcu_tasks_rude();
6269 
6270 		free_ftrace_hash(old_hash);
6271 	}
6272 
6273  out:
6274 	free_ftrace_hash(fgd->new_hash);
6275 	kfree(fgd);
6276 
6277 	return ret;
6278 }
6279 
6280 static int
ftrace_graph_set_hash(struct ftrace_hash * hash,char * buffer)6281 ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer)
6282 {
6283 	struct ftrace_glob func_g;
6284 	struct dyn_ftrace *rec;
6285 	struct ftrace_page *pg;
6286 	struct ftrace_func_entry *entry;
6287 	int fail = 1;
6288 	int not;
6289 
6290 	/* decode regex */
6291 	func_g.type = filter_parse_regex(buffer, strlen(buffer),
6292 					 &func_g.search, &not);
6293 
6294 	func_g.len = strlen(func_g.search);
6295 
6296 	mutex_lock(&ftrace_lock);
6297 
6298 	if (unlikely(ftrace_disabled)) {
6299 		mutex_unlock(&ftrace_lock);
6300 		return -ENODEV;
6301 	}
6302 
6303 	do_for_each_ftrace_rec(pg, rec) {
6304 
6305 		if (rec->flags & FTRACE_FL_DISABLED)
6306 			continue;
6307 
6308 		if (ftrace_match_record(rec, &func_g, NULL, 0)) {
6309 			entry = ftrace_lookup_ip(hash, rec->ip);
6310 
6311 			if (!not) {
6312 				fail = 0;
6313 
6314 				if (entry)
6315 					continue;
6316 				if (add_hash_entry(hash, rec->ip) == NULL)
6317 					goto out;
6318 			} else {
6319 				if (entry) {
6320 					free_hash_entry(hash, entry);
6321 					fail = 0;
6322 				}
6323 			}
6324 		}
6325 	} while_for_each_ftrace_rec();
6326 out:
6327 	mutex_unlock(&ftrace_lock);
6328 
6329 	if (fail)
6330 		return -EINVAL;
6331 
6332 	return 0;
6333 }
6334 
6335 static ssize_t
ftrace_graph_write(struct file * file,const char __user * ubuf,size_t cnt,loff_t * ppos)6336 ftrace_graph_write(struct file *file, const char __user *ubuf,
6337 		   size_t cnt, loff_t *ppos)
6338 {
6339 	ssize_t read, ret = 0;
6340 	struct ftrace_graph_data *fgd = file->private_data;
6341 	struct trace_parser *parser;
6342 
6343 	if (!cnt)
6344 		return 0;
6345 
6346 	/* Read mode uses seq functions */
6347 	if (file->f_mode & FMODE_READ) {
6348 		struct seq_file *m = file->private_data;
6349 		fgd = m->private;
6350 	}
6351 
6352 	parser = &fgd->parser;
6353 
6354 	read = trace_get_user(parser, ubuf, cnt, ppos);
6355 
6356 	if (read >= 0 && trace_parser_loaded(parser) &&
6357 	    !trace_parser_cont(parser)) {
6358 
6359 		ret = ftrace_graph_set_hash(fgd->new_hash,
6360 					    parser->buffer);
6361 		trace_parser_clear(parser);
6362 	}
6363 
6364 	if (!ret)
6365 		ret = read;
6366 
6367 	return ret;
6368 }
6369 
6370 static const struct file_operations ftrace_graph_fops = {
6371 	.open		= ftrace_graph_open,
6372 	.read		= seq_read,
6373 	.write		= ftrace_graph_write,
6374 	.llseek		= tracing_lseek,
6375 	.release	= ftrace_graph_release,
6376 };
6377 
6378 static const struct file_operations ftrace_graph_notrace_fops = {
6379 	.open		= ftrace_graph_notrace_open,
6380 	.read		= seq_read,
6381 	.write		= ftrace_graph_write,
6382 	.llseek		= tracing_lseek,
6383 	.release	= ftrace_graph_release,
6384 };
6385 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
6386 
ftrace_create_filter_files(struct ftrace_ops * ops,struct dentry * parent)6387 void ftrace_create_filter_files(struct ftrace_ops *ops,
6388 				struct dentry *parent)
6389 {
6390 
6391 	trace_create_file("set_ftrace_filter", TRACE_MODE_WRITE, parent,
6392 			  ops, &ftrace_filter_fops);
6393 
6394 	trace_create_file("set_ftrace_notrace", TRACE_MODE_WRITE, parent,
6395 			  ops, &ftrace_notrace_fops);
6396 }
6397 
6398 /*
6399  * The name "destroy_filter_files" is really a misnomer. Although
6400  * in the future, it may actually delete the files, but this is
6401  * really intended to make sure the ops passed in are disabled
6402  * and that when this function returns, the caller is free to
6403  * free the ops.
6404  *
6405  * The "destroy" name is only to match the "create" name that this
6406  * should be paired with.
6407  */
ftrace_destroy_filter_files(struct ftrace_ops * ops)6408 void ftrace_destroy_filter_files(struct ftrace_ops *ops)
6409 {
6410 	mutex_lock(&ftrace_lock);
6411 	if (ops->flags & FTRACE_OPS_FL_ENABLED)
6412 		ftrace_shutdown(ops, 0);
6413 	ops->flags |= FTRACE_OPS_FL_DELETED;
6414 	ftrace_free_filter(ops);
6415 	mutex_unlock(&ftrace_lock);
6416 }
6417 
ftrace_init_dyn_tracefs(struct dentry * d_tracer)6418 static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
6419 {
6420 
6421 	trace_create_file("available_filter_functions", TRACE_MODE_READ,
6422 			d_tracer, NULL, &ftrace_avail_fops);
6423 
6424 	trace_create_file("available_filter_functions_addrs", TRACE_MODE_READ,
6425 			d_tracer, NULL, &ftrace_avail_addrs_fops);
6426 
6427 	trace_create_file("enabled_functions", TRACE_MODE_READ,
6428 			d_tracer, NULL, &ftrace_enabled_fops);
6429 
6430 	trace_create_file("touched_functions", TRACE_MODE_READ,
6431 			d_tracer, NULL, &ftrace_touched_fops);
6432 
6433 	ftrace_create_filter_files(&global_ops, d_tracer);
6434 
6435 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
6436 	trace_create_file("set_graph_function", TRACE_MODE_WRITE, d_tracer,
6437 				    NULL,
6438 				    &ftrace_graph_fops);
6439 	trace_create_file("set_graph_notrace", TRACE_MODE_WRITE, d_tracer,
6440 				    NULL,
6441 				    &ftrace_graph_notrace_fops);
6442 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
6443 
6444 	return 0;
6445 }
6446 
ftrace_cmp_ips(const void * a,const void * b)6447 static int ftrace_cmp_ips(const void *a, const void *b)
6448 {
6449 	const unsigned long *ipa = a;
6450 	const unsigned long *ipb = b;
6451 
6452 	if (*ipa > *ipb)
6453 		return 1;
6454 	if (*ipa < *ipb)
6455 		return -1;
6456 	return 0;
6457 }
6458 
6459 #ifdef CONFIG_FTRACE_SORT_STARTUP_TEST
test_is_sorted(unsigned long * start,unsigned long count)6460 static void test_is_sorted(unsigned long *start, unsigned long count)
6461 {
6462 	int i;
6463 
6464 	for (i = 1; i < count; i++) {
6465 		if (WARN(start[i - 1] > start[i],
6466 			 "[%d] %pS at %lx is not sorted with %pS at %lx\n", i,
6467 			 (void *)start[i - 1], start[i - 1],
6468 			 (void *)start[i], start[i]))
6469 			break;
6470 	}
6471 	if (i == count)
6472 		pr_info("ftrace section at %px sorted properly\n", start);
6473 }
6474 #else
test_is_sorted(unsigned long * start,unsigned long count)6475 static void test_is_sorted(unsigned long *start, unsigned long count)
6476 {
6477 }
6478 #endif
6479 
ftrace_process_locs(struct module * mod,unsigned long * start,unsigned long * end)6480 static int ftrace_process_locs(struct module *mod,
6481 			       unsigned long *start,
6482 			       unsigned long *end)
6483 {
6484 	struct ftrace_page *pg_unuse = NULL;
6485 	struct ftrace_page *start_pg;
6486 	struct ftrace_page *pg;
6487 	struct dyn_ftrace *rec;
6488 	unsigned long skipped = 0;
6489 	unsigned long count;
6490 	unsigned long *p;
6491 	unsigned long addr;
6492 	unsigned long flags = 0; /* Shut up gcc */
6493 	int ret = -ENOMEM;
6494 
6495 	count = end - start;
6496 
6497 	if (!count)
6498 		return 0;
6499 
6500 	/*
6501 	 * Sorting mcount in vmlinux at build time depend on
6502 	 * CONFIG_BUILDTIME_MCOUNT_SORT, while mcount loc in
6503 	 * modules can not be sorted at build time.
6504 	 */
6505 	if (!IS_ENABLED(CONFIG_BUILDTIME_MCOUNT_SORT) || mod) {
6506 		sort(start, count, sizeof(*start),
6507 		     ftrace_cmp_ips, NULL);
6508 	} else {
6509 		test_is_sorted(start, count);
6510 	}
6511 
6512 	start_pg = ftrace_allocate_pages(count);
6513 	if (!start_pg)
6514 		return -ENOMEM;
6515 
6516 	mutex_lock(&ftrace_lock);
6517 
6518 	/*
6519 	 * Core and each module needs their own pages, as
6520 	 * modules will free them when they are removed.
6521 	 * Force a new page to be allocated for modules.
6522 	 */
6523 	if (!mod) {
6524 		WARN_ON(ftrace_pages || ftrace_pages_start);
6525 		/* First initialization */
6526 		ftrace_pages = ftrace_pages_start = start_pg;
6527 	} else {
6528 		if (!ftrace_pages)
6529 			goto out;
6530 
6531 		if (WARN_ON(ftrace_pages->next)) {
6532 			/* Hmm, we have free pages? */
6533 			while (ftrace_pages->next)
6534 				ftrace_pages = ftrace_pages->next;
6535 		}
6536 
6537 		ftrace_pages->next = start_pg;
6538 	}
6539 
6540 	p = start;
6541 	pg = start_pg;
6542 	while (p < end) {
6543 		unsigned long end_offset;
6544 		addr = ftrace_call_adjust(*p++);
6545 		/*
6546 		 * Some architecture linkers will pad between
6547 		 * the different mcount_loc sections of different
6548 		 * object files to satisfy alignments.
6549 		 * Skip any NULL pointers.
6550 		 */
6551 		if (!addr) {
6552 			skipped++;
6553 			continue;
6554 		}
6555 
6556 		end_offset = (pg->index+1) * sizeof(pg->records[0]);
6557 		if (end_offset > PAGE_SIZE << pg->order) {
6558 			/* We should have allocated enough */
6559 			if (WARN_ON(!pg->next))
6560 				break;
6561 			pg = pg->next;
6562 		}
6563 
6564 		rec = &pg->records[pg->index++];
6565 		rec->ip = addr;
6566 	}
6567 
6568 	if (pg->next) {
6569 		pg_unuse = pg->next;
6570 		pg->next = NULL;
6571 	}
6572 
6573 	/* Assign the last page to ftrace_pages */
6574 	ftrace_pages = pg;
6575 
6576 	/*
6577 	 * We only need to disable interrupts on start up
6578 	 * because we are modifying code that an interrupt
6579 	 * may execute, and the modification is not atomic.
6580 	 * But for modules, nothing runs the code we modify
6581 	 * until we are finished with it, and there's no
6582 	 * reason to cause large interrupt latencies while we do it.
6583 	 */
6584 	if (!mod)
6585 		local_irq_save(flags);
6586 	ftrace_update_code(mod, start_pg);
6587 	if (!mod)
6588 		local_irq_restore(flags);
6589 	ret = 0;
6590  out:
6591 	mutex_unlock(&ftrace_lock);
6592 
6593 	/* We should have used all pages unless we skipped some */
6594 	if (pg_unuse) {
6595 		WARN_ON(!skipped);
6596 		ftrace_free_pages(pg_unuse);
6597 	}
6598 	return ret;
6599 }
6600 
6601 struct ftrace_mod_func {
6602 	struct list_head	list;
6603 	char			*name;
6604 	unsigned long		ip;
6605 	unsigned int		size;
6606 };
6607 
6608 struct ftrace_mod_map {
6609 	struct rcu_head		rcu;
6610 	struct list_head	list;
6611 	struct module		*mod;
6612 	unsigned long		start_addr;
6613 	unsigned long		end_addr;
6614 	struct list_head	funcs;
6615 	unsigned int		num_funcs;
6616 };
6617 
ftrace_get_trampoline_kallsym(unsigned int symnum,unsigned long * value,char * type,char * name,char * module_name,int * exported)6618 static int ftrace_get_trampoline_kallsym(unsigned int symnum,
6619 					 unsigned long *value, char *type,
6620 					 char *name, char *module_name,
6621 					 int *exported)
6622 {
6623 	struct ftrace_ops *op;
6624 
6625 	list_for_each_entry_rcu(op, &ftrace_ops_trampoline_list, list) {
6626 		if (!op->trampoline || symnum--)
6627 			continue;
6628 		*value = op->trampoline;
6629 		*type = 't';
6630 		strscpy(name, FTRACE_TRAMPOLINE_SYM, KSYM_NAME_LEN);
6631 		strscpy(module_name, FTRACE_TRAMPOLINE_MOD, MODULE_NAME_LEN);
6632 		*exported = 0;
6633 		return 0;
6634 	}
6635 
6636 	return -ERANGE;
6637 }
6638 
6639 #if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) || defined(CONFIG_MODULES)
6640 /*
6641  * Check if the current ops references the given ip.
6642  *
6643  * If the ops traces all functions, then it was already accounted for.
6644  * If the ops does not trace the current record function, skip it.
6645  * If the ops ignores the function via notrace filter, skip it.
6646  */
6647 static bool
ops_references_ip(struct ftrace_ops * ops,unsigned long ip)6648 ops_references_ip(struct ftrace_ops *ops, unsigned long ip)
6649 {
6650 	/* If ops isn't enabled, ignore it */
6651 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
6652 		return false;
6653 
6654 	/* If ops traces all then it includes this function */
6655 	if (ops_traces_mod(ops))
6656 		return true;
6657 
6658 	/* The function must be in the filter */
6659 	if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
6660 	    !__ftrace_lookup_ip(ops->func_hash->filter_hash, ip))
6661 		return false;
6662 
6663 	/* If in notrace hash, we ignore it too */
6664 	if (ftrace_lookup_ip(ops->func_hash->notrace_hash, ip))
6665 		return false;
6666 
6667 	return true;
6668 }
6669 #endif
6670 
6671 #ifdef CONFIG_MODULES
6672 
6673 #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
6674 
6675 static LIST_HEAD(ftrace_mod_maps);
6676 
referenced_filters(struct dyn_ftrace * rec)6677 static int referenced_filters(struct dyn_ftrace *rec)
6678 {
6679 	struct ftrace_ops *ops;
6680 	int cnt = 0;
6681 
6682 	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
6683 		if (ops_references_ip(ops, rec->ip)) {
6684 			if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_DIRECT))
6685 				continue;
6686 			if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_IPMODIFY))
6687 				continue;
6688 			cnt++;
6689 			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
6690 				rec->flags |= FTRACE_FL_REGS;
6691 			if (cnt == 1 && ops->trampoline)
6692 				rec->flags |= FTRACE_FL_TRAMP;
6693 			else
6694 				rec->flags &= ~FTRACE_FL_TRAMP;
6695 		}
6696 	}
6697 
6698 	return cnt;
6699 }
6700 
6701 static void
clear_mod_from_hash(struct ftrace_page * pg,struct ftrace_hash * hash)6702 clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash)
6703 {
6704 	struct ftrace_func_entry *entry;
6705 	struct dyn_ftrace *rec;
6706 	int i;
6707 
6708 	if (ftrace_hash_empty(hash))
6709 		return;
6710 
6711 	for (i = 0; i < pg->index; i++) {
6712 		rec = &pg->records[i];
6713 		entry = __ftrace_lookup_ip(hash, rec->ip);
6714 		/*
6715 		 * Do not allow this rec to match again.
6716 		 * Yeah, it may waste some memory, but will be removed
6717 		 * if/when the hash is modified again.
6718 		 */
6719 		if (entry)
6720 			entry->ip = 0;
6721 	}
6722 }
6723 
6724 /* Clear any records from hashes */
clear_mod_from_hashes(struct ftrace_page * pg)6725 static void clear_mod_from_hashes(struct ftrace_page *pg)
6726 {
6727 	struct trace_array *tr;
6728 
6729 	mutex_lock(&trace_types_lock);
6730 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6731 		if (!tr->ops || !tr->ops->func_hash)
6732 			continue;
6733 		mutex_lock(&tr->ops->func_hash->regex_lock);
6734 		clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash);
6735 		clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash);
6736 		mutex_unlock(&tr->ops->func_hash->regex_lock);
6737 	}
6738 	mutex_unlock(&trace_types_lock);
6739 }
6740 
ftrace_free_mod_map(struct rcu_head * rcu)6741 static void ftrace_free_mod_map(struct rcu_head *rcu)
6742 {
6743 	struct ftrace_mod_map *mod_map = container_of(rcu, struct ftrace_mod_map, rcu);
6744 	struct ftrace_mod_func *mod_func;
6745 	struct ftrace_mod_func *n;
6746 
6747 	/* All the contents of mod_map are now not visible to readers */
6748 	list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) {
6749 		kfree(mod_func->name);
6750 		list_del(&mod_func->list);
6751 		kfree(mod_func);
6752 	}
6753 
6754 	kfree(mod_map);
6755 }
6756 
ftrace_release_mod(struct module * mod)6757 void ftrace_release_mod(struct module *mod)
6758 {
6759 	struct ftrace_mod_map *mod_map;
6760 	struct ftrace_mod_map *n;
6761 	struct dyn_ftrace *rec;
6762 	struct ftrace_page **last_pg;
6763 	struct ftrace_page *tmp_page = NULL;
6764 	struct ftrace_page *pg;
6765 
6766 	mutex_lock(&ftrace_lock);
6767 
6768 	if (ftrace_disabled)
6769 		goto out_unlock;
6770 
6771 	list_for_each_entry_safe(mod_map, n, &ftrace_mod_maps, list) {
6772 		if (mod_map->mod == mod) {
6773 			list_del_rcu(&mod_map->list);
6774 			call_rcu(&mod_map->rcu, ftrace_free_mod_map);
6775 			break;
6776 		}
6777 	}
6778 
6779 	/*
6780 	 * Each module has its own ftrace_pages, remove
6781 	 * them from the list.
6782 	 */
6783 	last_pg = &ftrace_pages_start;
6784 	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
6785 		rec = &pg->records[0];
6786 		if (within_module(rec->ip, mod)) {
6787 			/*
6788 			 * As core pages are first, the first
6789 			 * page should never be a module page.
6790 			 */
6791 			if (WARN_ON(pg == ftrace_pages_start))
6792 				goto out_unlock;
6793 
6794 			/* Check if we are deleting the last page */
6795 			if (pg == ftrace_pages)
6796 				ftrace_pages = next_to_ftrace_page(last_pg);
6797 
6798 			ftrace_update_tot_cnt -= pg->index;
6799 			*last_pg = pg->next;
6800 
6801 			pg->next = tmp_page;
6802 			tmp_page = pg;
6803 		} else
6804 			last_pg = &pg->next;
6805 	}
6806  out_unlock:
6807 	mutex_unlock(&ftrace_lock);
6808 
6809 	for (pg = tmp_page; pg; pg = tmp_page) {
6810 
6811 		/* Needs to be called outside of ftrace_lock */
6812 		clear_mod_from_hashes(pg);
6813 
6814 		if (pg->records) {
6815 			free_pages((unsigned long)pg->records, pg->order);
6816 			ftrace_number_of_pages -= 1 << pg->order;
6817 		}
6818 		tmp_page = pg->next;
6819 		kfree(pg);
6820 		ftrace_number_of_groups--;
6821 	}
6822 }
6823 
ftrace_module_enable(struct module * mod)6824 void ftrace_module_enable(struct module *mod)
6825 {
6826 	struct dyn_ftrace *rec;
6827 	struct ftrace_page *pg;
6828 
6829 	mutex_lock(&ftrace_lock);
6830 
6831 	if (ftrace_disabled)
6832 		goto out_unlock;
6833 
6834 	/*
6835 	 * If the tracing is enabled, go ahead and enable the record.
6836 	 *
6837 	 * The reason not to enable the record immediately is the
6838 	 * inherent check of ftrace_make_nop/ftrace_make_call for
6839 	 * correct previous instructions.  Making first the NOP
6840 	 * conversion puts the module to the correct state, thus
6841 	 * passing the ftrace_make_call check.
6842 	 *
6843 	 * We also delay this to after the module code already set the
6844 	 * text to read-only, as we now need to set it back to read-write
6845 	 * so that we can modify the text.
6846 	 */
6847 	if (ftrace_start_up)
6848 		ftrace_arch_code_modify_prepare();
6849 
6850 	do_for_each_ftrace_rec(pg, rec) {
6851 		int cnt;
6852 		/*
6853 		 * do_for_each_ftrace_rec() is a double loop.
6854 		 * module text shares the pg. If a record is
6855 		 * not part of this module, then skip this pg,
6856 		 * which the "break" will do.
6857 		 */
6858 		if (!within_module(rec->ip, mod))
6859 			break;
6860 
6861 		/* Weak functions should still be ignored */
6862 		if (!test_for_valid_rec(rec)) {
6863 			/* Clear all other flags. Should not be enabled anyway */
6864 			rec->flags = FTRACE_FL_DISABLED;
6865 			continue;
6866 		}
6867 
6868 		cnt = 0;
6869 
6870 		/*
6871 		 * When adding a module, we need to check if tracers are
6872 		 * currently enabled and if they are, and can trace this record,
6873 		 * we need to enable the module functions as well as update the
6874 		 * reference counts for those function records.
6875 		 */
6876 		if (ftrace_start_up)
6877 			cnt += referenced_filters(rec);
6878 
6879 		rec->flags &= ~FTRACE_FL_DISABLED;
6880 		rec->flags += cnt;
6881 
6882 		if (ftrace_start_up && cnt) {
6883 			int failed = __ftrace_replace_code(rec, 1);
6884 			if (failed) {
6885 				ftrace_bug(failed, rec);
6886 				goto out_loop;
6887 			}
6888 		}
6889 
6890 	} while_for_each_ftrace_rec();
6891 
6892  out_loop:
6893 	if (ftrace_start_up)
6894 		ftrace_arch_code_modify_post_process();
6895 
6896  out_unlock:
6897 	mutex_unlock(&ftrace_lock);
6898 
6899 	process_cached_mods(mod->name);
6900 }
6901 
ftrace_module_init(struct module * mod)6902 void ftrace_module_init(struct module *mod)
6903 {
6904 	int ret;
6905 
6906 	if (ftrace_disabled || !mod->num_ftrace_callsites)
6907 		return;
6908 
6909 	ret = ftrace_process_locs(mod, mod->ftrace_callsites,
6910 				  mod->ftrace_callsites + mod->num_ftrace_callsites);
6911 	if (ret)
6912 		pr_warn("ftrace: failed to allocate entries for module '%s' functions\n",
6913 			mod->name);
6914 }
6915 
save_ftrace_mod_rec(struct ftrace_mod_map * mod_map,struct dyn_ftrace * rec)6916 static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
6917 				struct dyn_ftrace *rec)
6918 {
6919 	struct ftrace_mod_func *mod_func;
6920 	unsigned long symsize;
6921 	unsigned long offset;
6922 	char str[KSYM_SYMBOL_LEN];
6923 	char *modname;
6924 	const char *ret;
6925 
6926 	ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str);
6927 	if (!ret)
6928 		return;
6929 
6930 	mod_func = kmalloc(sizeof(*mod_func), GFP_KERNEL);
6931 	if (!mod_func)
6932 		return;
6933 
6934 	mod_func->name = kstrdup(str, GFP_KERNEL);
6935 	if (!mod_func->name) {
6936 		kfree(mod_func);
6937 		return;
6938 	}
6939 
6940 	mod_func->ip = rec->ip - offset;
6941 	mod_func->size = symsize;
6942 
6943 	mod_map->num_funcs++;
6944 
6945 	list_add_rcu(&mod_func->list, &mod_map->funcs);
6946 }
6947 
6948 static struct ftrace_mod_map *
allocate_ftrace_mod_map(struct module * mod,unsigned long start,unsigned long end)6949 allocate_ftrace_mod_map(struct module *mod,
6950 			unsigned long start, unsigned long end)
6951 {
6952 	struct ftrace_mod_map *mod_map;
6953 
6954 	mod_map = kmalloc(sizeof(*mod_map), GFP_KERNEL);
6955 	if (!mod_map)
6956 		return NULL;
6957 
6958 	mod_map->mod = mod;
6959 	mod_map->start_addr = start;
6960 	mod_map->end_addr = end;
6961 	mod_map->num_funcs = 0;
6962 
6963 	INIT_LIST_HEAD_RCU(&mod_map->funcs);
6964 
6965 	list_add_rcu(&mod_map->list, &ftrace_mod_maps);
6966 
6967 	return mod_map;
6968 }
6969 
6970 static const char *
ftrace_func_address_lookup(struct ftrace_mod_map * mod_map,unsigned long addr,unsigned long * size,unsigned long * off,char * sym)6971 ftrace_func_address_lookup(struct ftrace_mod_map *mod_map,
6972 			   unsigned long addr, unsigned long *size,
6973 			   unsigned long *off, char *sym)
6974 {
6975 	struct ftrace_mod_func *found_func =  NULL;
6976 	struct ftrace_mod_func *mod_func;
6977 
6978 	list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
6979 		if (addr >= mod_func->ip &&
6980 		    addr < mod_func->ip + mod_func->size) {
6981 			found_func = mod_func;
6982 			break;
6983 		}
6984 	}
6985 
6986 	if (found_func) {
6987 		if (size)
6988 			*size = found_func->size;
6989 		if (off)
6990 			*off = addr - found_func->ip;
6991 		if (sym)
6992 			strscpy(sym, found_func->name, KSYM_NAME_LEN);
6993 
6994 		return found_func->name;
6995 	}
6996 
6997 	return NULL;
6998 }
6999 
7000 const char *
ftrace_mod_address_lookup(unsigned long addr,unsigned long * size,unsigned long * off,char ** modname,char * sym)7001 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
7002 		   unsigned long *off, char **modname, char *sym)
7003 {
7004 	struct ftrace_mod_map *mod_map;
7005 	const char *ret = NULL;
7006 
7007 	/* mod_map is freed via call_rcu() */
7008 	preempt_disable();
7009 	list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
7010 		ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym);
7011 		if (ret) {
7012 			if (modname)
7013 				*modname = mod_map->mod->name;
7014 			break;
7015 		}
7016 	}
7017 	preempt_enable();
7018 
7019 	return ret;
7020 }
7021 
ftrace_mod_get_kallsym(unsigned int symnum,unsigned long * value,char * type,char * name,char * module_name,int * exported)7022 int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
7023 			   char *type, char *name,
7024 			   char *module_name, int *exported)
7025 {
7026 	struct ftrace_mod_map *mod_map;
7027 	struct ftrace_mod_func *mod_func;
7028 	int ret;
7029 
7030 	preempt_disable();
7031 	list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
7032 
7033 		if (symnum >= mod_map->num_funcs) {
7034 			symnum -= mod_map->num_funcs;
7035 			continue;
7036 		}
7037 
7038 		list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
7039 			if (symnum > 1) {
7040 				symnum--;
7041 				continue;
7042 			}
7043 
7044 			*value = mod_func->ip;
7045 			*type = 'T';
7046 			strscpy(name, mod_func->name, KSYM_NAME_LEN);
7047 			strscpy(module_name, mod_map->mod->name, MODULE_NAME_LEN);
7048 			*exported = 1;
7049 			preempt_enable();
7050 			return 0;
7051 		}
7052 		WARN_ON(1);
7053 		break;
7054 	}
7055 	ret = ftrace_get_trampoline_kallsym(symnum, value, type, name,
7056 					    module_name, exported);
7057 	preempt_enable();
7058 	return ret;
7059 }
7060 
7061 #else
save_ftrace_mod_rec(struct ftrace_mod_map * mod_map,struct dyn_ftrace * rec)7062 static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
7063 				struct dyn_ftrace *rec) { }
7064 static inline struct ftrace_mod_map *
allocate_ftrace_mod_map(struct module * mod,unsigned long start,unsigned long end)7065 allocate_ftrace_mod_map(struct module *mod,
7066 			unsigned long start, unsigned long end)
7067 {
7068 	return NULL;
7069 }
ftrace_mod_get_kallsym(unsigned int symnum,unsigned long * value,char * type,char * name,char * module_name,int * exported)7070 int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
7071 			   char *type, char *name, char *module_name,
7072 			   int *exported)
7073 {
7074 	int ret;
7075 
7076 	preempt_disable();
7077 	ret = ftrace_get_trampoline_kallsym(symnum, value, type, name,
7078 					    module_name, exported);
7079 	preempt_enable();
7080 	return ret;
7081 }
7082 #endif /* CONFIG_MODULES */
7083 
7084 struct ftrace_init_func {
7085 	struct list_head list;
7086 	unsigned long ip;
7087 };
7088 
7089 /* Clear any init ips from hashes */
7090 static void
clear_func_from_hash(struct ftrace_init_func * func,struct ftrace_hash * hash)7091 clear_func_from_hash(struct ftrace_init_func *func, struct ftrace_hash *hash)
7092 {
7093 	struct ftrace_func_entry *entry;
7094 
7095 	entry = ftrace_lookup_ip(hash, func->ip);
7096 	/*
7097 	 * Do not allow this rec to match again.
7098 	 * Yeah, it may waste some memory, but will be removed
7099 	 * if/when the hash is modified again.
7100 	 */
7101 	if (entry)
7102 		entry->ip = 0;
7103 }
7104 
7105 static void
clear_func_from_hashes(struct ftrace_init_func * func)7106 clear_func_from_hashes(struct ftrace_init_func *func)
7107 {
7108 	struct trace_array *tr;
7109 
7110 	mutex_lock(&trace_types_lock);
7111 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7112 		if (!tr->ops || !tr->ops->func_hash)
7113 			continue;
7114 		mutex_lock(&tr->ops->func_hash->regex_lock);
7115 		clear_func_from_hash(func, tr->ops->func_hash->filter_hash);
7116 		clear_func_from_hash(func, tr->ops->func_hash->notrace_hash);
7117 		mutex_unlock(&tr->ops->func_hash->regex_lock);
7118 	}
7119 	mutex_unlock(&trace_types_lock);
7120 }
7121 
add_to_clear_hash_list(struct list_head * clear_list,struct dyn_ftrace * rec)7122 static void add_to_clear_hash_list(struct list_head *clear_list,
7123 				   struct dyn_ftrace *rec)
7124 {
7125 	struct ftrace_init_func *func;
7126 
7127 	func = kmalloc(sizeof(*func), GFP_KERNEL);
7128 	if (!func) {
7129 		MEM_FAIL(1, "alloc failure, ftrace filter could be stale\n");
7130 		return;
7131 	}
7132 
7133 	func->ip = rec->ip;
7134 	list_add(&func->list, clear_list);
7135 }
7136 
ftrace_free_mem(struct module * mod,void * start_ptr,void * end_ptr)7137 void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr)
7138 {
7139 	unsigned long start = (unsigned long)(start_ptr);
7140 	unsigned long end = (unsigned long)(end_ptr);
7141 	struct ftrace_page **last_pg = &ftrace_pages_start;
7142 	struct ftrace_page *pg;
7143 	struct dyn_ftrace *rec;
7144 	struct dyn_ftrace key;
7145 	struct ftrace_mod_map *mod_map = NULL;
7146 	struct ftrace_init_func *func, *func_next;
7147 	LIST_HEAD(clear_hash);
7148 
7149 	key.ip = start;
7150 	key.flags = end;	/* overload flags, as it is unsigned long */
7151 
7152 	mutex_lock(&ftrace_lock);
7153 
7154 	/*
7155 	 * If we are freeing module init memory, then check if
7156 	 * any tracer is active. If so, we need to save a mapping of
7157 	 * the module functions being freed with the address.
7158 	 */
7159 	if (mod && ftrace_ops_list != &ftrace_list_end)
7160 		mod_map = allocate_ftrace_mod_map(mod, start, end);
7161 
7162 	for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {
7163 		if (end < pg->records[0].ip ||
7164 		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
7165 			continue;
7166  again:
7167 		rec = bsearch(&key, pg->records, pg->index,
7168 			      sizeof(struct dyn_ftrace),
7169 			      ftrace_cmp_recs);
7170 		if (!rec)
7171 			continue;
7172 
7173 		/* rec will be cleared from hashes after ftrace_lock unlock */
7174 		add_to_clear_hash_list(&clear_hash, rec);
7175 
7176 		if (mod_map)
7177 			save_ftrace_mod_rec(mod_map, rec);
7178 
7179 		pg->index--;
7180 		ftrace_update_tot_cnt--;
7181 		if (!pg->index) {
7182 			*last_pg = pg->next;
7183 			if (pg->records) {
7184 				free_pages((unsigned long)pg->records, pg->order);
7185 				ftrace_number_of_pages -= 1 << pg->order;
7186 			}
7187 			ftrace_number_of_groups--;
7188 			kfree(pg);
7189 			pg = container_of(last_pg, struct ftrace_page, next);
7190 			if (!(*last_pg))
7191 				ftrace_pages = pg;
7192 			continue;
7193 		}
7194 		memmove(rec, rec + 1,
7195 			(pg->index - (rec - pg->records)) * sizeof(*rec));
7196 		/* More than one function may be in this block */
7197 		goto again;
7198 	}
7199 	mutex_unlock(&ftrace_lock);
7200 
7201 	list_for_each_entry_safe(func, func_next, &clear_hash, list) {
7202 		clear_func_from_hashes(func);
7203 		kfree(func);
7204 	}
7205 }
7206 
ftrace_free_init_mem(void)7207 void __init ftrace_free_init_mem(void)
7208 {
7209 	void *start = (void *)(&__init_begin);
7210 	void *end = (void *)(&__init_end);
7211 
7212 	ftrace_boot_snapshot();
7213 
7214 	ftrace_free_mem(NULL, start, end);
7215 }
7216 
ftrace_dyn_arch_init(void)7217 int __init __weak ftrace_dyn_arch_init(void)
7218 {
7219 	return 0;
7220 }
7221 
ftrace_init(void)7222 void __init ftrace_init(void)
7223 {
7224 	extern unsigned long __start_mcount_loc[];
7225 	extern unsigned long __stop_mcount_loc[];
7226 	unsigned long count, flags;
7227 	int ret;
7228 
7229 	local_irq_save(flags);
7230 	ret = ftrace_dyn_arch_init();
7231 	local_irq_restore(flags);
7232 	if (ret)
7233 		goto failed;
7234 
7235 	count = __stop_mcount_loc - __start_mcount_loc;
7236 	if (!count) {
7237 		pr_info("ftrace: No functions to be traced?\n");
7238 		goto failed;
7239 	}
7240 
7241 	pr_info("ftrace: allocating %ld entries in %ld pages\n",
7242 		count, DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
7243 
7244 	ret = ftrace_process_locs(NULL,
7245 				  __start_mcount_loc,
7246 				  __stop_mcount_loc);
7247 	if (ret) {
7248 		pr_warn("ftrace: failed to allocate entries for functions\n");
7249 		goto failed;
7250 	}
7251 
7252 	pr_info("ftrace: allocated %ld pages with %ld groups\n",
7253 		ftrace_number_of_pages, ftrace_number_of_groups);
7254 
7255 	last_ftrace_enabled = ftrace_enabled = 1;
7256 
7257 	set_ftrace_early_filters();
7258 
7259 	return;
7260  failed:
7261 	ftrace_disabled = 1;
7262 }
7263 
7264 /* Do nothing if arch does not support this */
arch_ftrace_update_trampoline(struct ftrace_ops * ops)7265 void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
7266 {
7267 }
7268 
ftrace_update_trampoline(struct ftrace_ops * ops)7269 static void ftrace_update_trampoline(struct ftrace_ops *ops)
7270 {
7271 	unsigned long trampoline = ops->trampoline;
7272 
7273 	arch_ftrace_update_trampoline(ops);
7274 	if (ops->trampoline && ops->trampoline != trampoline &&
7275 	    (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP)) {
7276 		/* Add to kallsyms before the perf events */
7277 		ftrace_add_trampoline_to_kallsyms(ops);
7278 		perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
7279 				   ops->trampoline, ops->trampoline_size, false,
7280 				   FTRACE_TRAMPOLINE_SYM);
7281 		/*
7282 		 * Record the perf text poke event after the ksymbol register
7283 		 * event.
7284 		 */
7285 		perf_event_text_poke((void *)ops->trampoline, NULL, 0,
7286 				     (void *)ops->trampoline,
7287 				     ops->trampoline_size);
7288 	}
7289 }
7290 
ftrace_init_trace_array(struct trace_array * tr)7291 void ftrace_init_trace_array(struct trace_array *tr)
7292 {
7293 	INIT_LIST_HEAD(&tr->func_probes);
7294 	INIT_LIST_HEAD(&tr->mod_trace);
7295 	INIT_LIST_HEAD(&tr->mod_notrace);
7296 }
7297 #else
7298 
7299 struct ftrace_ops global_ops = {
7300 	.func			= ftrace_stub,
7301 	.flags			= FTRACE_OPS_FL_INITIALIZED |
7302 				  FTRACE_OPS_FL_PID,
7303 };
7304 
ftrace_nodyn_init(void)7305 static int __init ftrace_nodyn_init(void)
7306 {
7307 	ftrace_enabled = 1;
7308 	return 0;
7309 }
7310 core_initcall(ftrace_nodyn_init);
7311 
ftrace_init_dyn_tracefs(struct dentry * d_tracer)7312 static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
ftrace_startup_all(int command)7313 static inline void ftrace_startup_all(int command) { }
7314 
ftrace_update_trampoline(struct ftrace_ops * ops)7315 static void ftrace_update_trampoline(struct ftrace_ops *ops)
7316 {
7317 }
7318 
7319 #endif /* CONFIG_DYNAMIC_FTRACE */
7320 
ftrace_init_global_array_ops(struct trace_array * tr)7321 __init void ftrace_init_global_array_ops(struct trace_array *tr)
7322 {
7323 	tr->ops = &global_ops;
7324 	tr->ops->private = tr;
7325 	ftrace_init_trace_array(tr);
7326 }
7327 
ftrace_init_array_ops(struct trace_array * tr,ftrace_func_t func)7328 void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
7329 {
7330 	/* If we filter on pids, update to use the pid function */
7331 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
7332 		if (WARN_ON(tr->ops->func != ftrace_stub))
7333 			printk("ftrace ops had %pS for function\n",
7334 			       tr->ops->func);
7335 	}
7336 	tr->ops->func = func;
7337 	tr->ops->private = tr;
7338 }
7339 
ftrace_reset_array_ops(struct trace_array * tr)7340 void ftrace_reset_array_ops(struct trace_array *tr)
7341 {
7342 	tr->ops->func = ftrace_stub;
7343 }
7344 
7345 static nokprobe_inline void
__ftrace_ops_list_func(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * ignored,struct ftrace_regs * fregs)7346 __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
7347 		       struct ftrace_ops *ignored, struct ftrace_regs *fregs)
7348 {
7349 	struct pt_regs *regs = ftrace_get_regs(fregs);
7350 	struct ftrace_ops *op;
7351 	int bit;
7352 
7353 	/*
7354 	 * The ftrace_test_and_set_recursion() will disable preemption,
7355 	 * which is required since some of the ops may be dynamically
7356 	 * allocated, they must be freed after a synchronize_rcu().
7357 	 */
7358 	bit = trace_test_and_set_recursion(ip, parent_ip, TRACE_LIST_START);
7359 	if (bit < 0)
7360 		return;
7361 
7362 	do_for_each_ftrace_op(op, ftrace_ops_list) {
7363 		/* Stub functions don't need to be called nor tested */
7364 		if (op->flags & FTRACE_OPS_FL_STUB)
7365 			continue;
7366 		/*
7367 		 * Check the following for each ops before calling their func:
7368 		 *  if RCU flag is set, then rcu_is_watching() must be true
7369 		 *  Otherwise test if the ip matches the ops filter
7370 		 *
7371 		 * If any of the above fails then the op->func() is not executed.
7372 		 */
7373 		if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
7374 		    ftrace_ops_test(op, ip, regs)) {
7375 			if (FTRACE_WARN_ON(!op->func)) {
7376 				pr_warn("op=%p %pS\n", op, op);
7377 				goto out;
7378 			}
7379 			op->func(ip, parent_ip, op, fregs);
7380 		}
7381 	} while_for_each_ftrace_op(op);
7382 out:
7383 	trace_clear_recursion(bit);
7384 }
7385 
7386 /*
7387  * Some archs only support passing ip and parent_ip. Even though
7388  * the list function ignores the op parameter, we do not want any
7389  * C side effects, where a function is called without the caller
7390  * sending a third parameter.
7391  * Archs are to support both the regs and ftrace_ops at the same time.
7392  * If they support ftrace_ops, it is assumed they support regs.
7393  * If call backs want to use regs, they must either check for regs
7394  * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
7395  * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
7396  * An architecture can pass partial regs with ftrace_ops and still
7397  * set the ARCH_SUPPORTS_FTRACE_OPS.
7398  *
7399  * In vmlinux.lds.h, ftrace_ops_list_func() is defined to be
7400  * arch_ftrace_ops_list_func.
7401  */
7402 #if ARCH_SUPPORTS_FTRACE_OPS
arch_ftrace_ops_list_func(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * op,struct ftrace_regs * fregs)7403 void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
7404 			       struct ftrace_ops *op, struct ftrace_regs *fregs)
7405 {
7406 	__ftrace_ops_list_func(ip, parent_ip, NULL, fregs);
7407 }
7408 #else
arch_ftrace_ops_list_func(unsigned long ip,unsigned long parent_ip)7409 void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
7410 {
7411 	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
7412 }
7413 #endif
7414 NOKPROBE_SYMBOL(arch_ftrace_ops_list_func);
7415 
7416 /*
7417  * If there's only one function registered but it does not support
7418  * recursion, needs RCU protection, then this function will be called
7419  * by the mcount trampoline.
7420  */
ftrace_ops_assist_func(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * op,struct ftrace_regs * fregs)7421 static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
7422 				   struct ftrace_ops *op, struct ftrace_regs *fregs)
7423 {
7424 	int bit;
7425 
7426 	bit = trace_test_and_set_recursion(ip, parent_ip, TRACE_LIST_START);
7427 	if (bit < 0)
7428 		return;
7429 
7430 	if (!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching())
7431 		op->func(ip, parent_ip, op, fregs);
7432 
7433 	trace_clear_recursion(bit);
7434 }
7435 NOKPROBE_SYMBOL(ftrace_ops_assist_func);
7436 
7437 /**
7438  * ftrace_ops_get_func - get the function a trampoline should call
7439  * @ops: the ops to get the function for
7440  *
7441  * Normally the mcount trampoline will call the ops->func, but there
7442  * are times that it should not. For example, if the ops does not
7443  * have its own recursion protection, then it should call the
7444  * ftrace_ops_assist_func() instead.
7445  *
7446  * Returns the function that the trampoline should call for @ops.
7447  */
ftrace_ops_get_func(struct ftrace_ops * ops)7448 ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
7449 {
7450 	/*
7451 	 * If the function does not handle recursion or needs to be RCU safe,
7452 	 * then we need to call the assist handler.
7453 	 */
7454 	if (ops->flags & (FTRACE_OPS_FL_RECURSION |
7455 			  FTRACE_OPS_FL_RCU))
7456 		return ftrace_ops_assist_func;
7457 
7458 	return ops->func;
7459 }
7460 
7461 static void
ftrace_filter_pid_sched_switch_probe(void * data,bool preempt,struct task_struct * prev,struct task_struct * next,unsigned int prev_state)7462 ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
7463 				     struct task_struct *prev,
7464 				     struct task_struct *next,
7465 				     unsigned int prev_state)
7466 {
7467 	struct trace_array *tr = data;
7468 	struct trace_pid_list *pid_list;
7469 	struct trace_pid_list *no_pid_list;
7470 
7471 	pid_list = rcu_dereference_sched(tr->function_pids);
7472 	no_pid_list = rcu_dereference_sched(tr->function_no_pids);
7473 
7474 	if (trace_ignore_this_task(pid_list, no_pid_list, next))
7475 		this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
7476 			       FTRACE_PID_IGNORE);
7477 	else
7478 		this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
7479 			       next->pid);
7480 }
7481 
7482 static void
ftrace_pid_follow_sched_process_fork(void * data,struct task_struct * self,struct task_struct * task)7483 ftrace_pid_follow_sched_process_fork(void *data,
7484 				     struct task_struct *self,
7485 				     struct task_struct *task)
7486 {
7487 	struct trace_pid_list *pid_list;
7488 	struct trace_array *tr = data;
7489 
7490 	pid_list = rcu_dereference_sched(tr->function_pids);
7491 	trace_filter_add_remove_task(pid_list, self, task);
7492 
7493 	pid_list = rcu_dereference_sched(tr->function_no_pids);
7494 	trace_filter_add_remove_task(pid_list, self, task);
7495 }
7496 
7497 static void
ftrace_pid_follow_sched_process_exit(void * data,struct task_struct * task)7498 ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task)
7499 {
7500 	struct trace_pid_list *pid_list;
7501 	struct trace_array *tr = data;
7502 
7503 	pid_list = rcu_dereference_sched(tr->function_pids);
7504 	trace_filter_add_remove_task(pid_list, NULL, task);
7505 
7506 	pid_list = rcu_dereference_sched(tr->function_no_pids);
7507 	trace_filter_add_remove_task(pid_list, NULL, task);
7508 }
7509 
ftrace_pid_follow_fork(struct trace_array * tr,bool enable)7510 void ftrace_pid_follow_fork(struct trace_array *tr, bool enable)
7511 {
7512 	if (enable) {
7513 		register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
7514 						  tr);
7515 		register_trace_sched_process_free(ftrace_pid_follow_sched_process_exit,
7516 						  tr);
7517 	} else {
7518 		unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
7519 						    tr);
7520 		unregister_trace_sched_process_free(ftrace_pid_follow_sched_process_exit,
7521 						    tr);
7522 	}
7523 }
7524 
clear_ftrace_pids(struct trace_array * tr,int type)7525 static void clear_ftrace_pids(struct trace_array *tr, int type)
7526 {
7527 	struct trace_pid_list *pid_list;
7528 	struct trace_pid_list *no_pid_list;
7529 	int cpu;
7530 
7531 	pid_list = rcu_dereference_protected(tr->function_pids,
7532 					     lockdep_is_held(&ftrace_lock));
7533 	no_pid_list = rcu_dereference_protected(tr->function_no_pids,
7534 						lockdep_is_held(&ftrace_lock));
7535 
7536 	/* Make sure there's something to do */
7537 	if (!pid_type_enabled(type, pid_list, no_pid_list))
7538 		return;
7539 
7540 	/* See if the pids still need to be checked after this */
7541 	if (!still_need_pid_events(type, pid_list, no_pid_list)) {
7542 		unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
7543 		for_each_possible_cpu(cpu)
7544 			per_cpu_ptr(tr->array_buffer.data, cpu)->ftrace_ignore_pid = FTRACE_PID_TRACE;
7545 	}
7546 
7547 	if (type & TRACE_PIDS)
7548 		rcu_assign_pointer(tr->function_pids, NULL);
7549 
7550 	if (type & TRACE_NO_PIDS)
7551 		rcu_assign_pointer(tr->function_no_pids, NULL);
7552 
7553 	/* Wait till all users are no longer using pid filtering */
7554 	synchronize_rcu();
7555 
7556 	if ((type & TRACE_PIDS) && pid_list)
7557 		trace_pid_list_free(pid_list);
7558 
7559 	if ((type & TRACE_NO_PIDS) && no_pid_list)
7560 		trace_pid_list_free(no_pid_list);
7561 }
7562 
ftrace_clear_pids(struct trace_array * tr)7563 void ftrace_clear_pids(struct trace_array *tr)
7564 {
7565 	mutex_lock(&ftrace_lock);
7566 
7567 	clear_ftrace_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);
7568 
7569 	mutex_unlock(&ftrace_lock);
7570 }
7571 
ftrace_pid_reset(struct trace_array * tr,int type)7572 static void ftrace_pid_reset(struct trace_array *tr, int type)
7573 {
7574 	mutex_lock(&ftrace_lock);
7575 	clear_ftrace_pids(tr, type);
7576 
7577 	ftrace_update_pid_func();
7578 	ftrace_startup_all(0);
7579 
7580 	mutex_unlock(&ftrace_lock);
7581 }
7582 
7583 /* Greater than any max PID */
7584 #define FTRACE_NO_PIDS		(void *)(PID_MAX_LIMIT + 1)
7585 
fpid_start(struct seq_file * m,loff_t * pos)7586 static void *fpid_start(struct seq_file *m, loff_t *pos)
7587 	__acquires(RCU)
7588 {
7589 	struct trace_pid_list *pid_list;
7590 	struct trace_array *tr = m->private;
7591 
7592 	mutex_lock(&ftrace_lock);
7593 	rcu_read_lock_sched();
7594 
7595 	pid_list = rcu_dereference_sched(tr->function_pids);
7596 
7597 	if (!pid_list)
7598 		return !(*pos) ? FTRACE_NO_PIDS : NULL;
7599 
7600 	return trace_pid_start(pid_list, pos);
7601 }
7602 
fpid_next(struct seq_file * m,void * v,loff_t * pos)7603 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
7604 {
7605 	struct trace_array *tr = m->private;
7606 	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);
7607 
7608 	if (v == FTRACE_NO_PIDS) {
7609 		(*pos)++;
7610 		return NULL;
7611 	}
7612 	return trace_pid_next(pid_list, v, pos);
7613 }
7614 
fpid_stop(struct seq_file * m,void * p)7615 static void fpid_stop(struct seq_file *m, void *p)
7616 	__releases(RCU)
7617 {
7618 	rcu_read_unlock_sched();
7619 	mutex_unlock(&ftrace_lock);
7620 }
7621 
fpid_show(struct seq_file * m,void * v)7622 static int fpid_show(struct seq_file *m, void *v)
7623 {
7624 	if (v == FTRACE_NO_PIDS) {
7625 		seq_puts(m, "no pid\n");
7626 		return 0;
7627 	}
7628 
7629 	return trace_pid_show(m, v);
7630 }
7631 
7632 static const struct seq_operations ftrace_pid_sops = {
7633 	.start = fpid_start,
7634 	.next = fpid_next,
7635 	.stop = fpid_stop,
7636 	.show = fpid_show,
7637 };
7638 
fnpid_start(struct seq_file * m,loff_t * pos)7639 static void *fnpid_start(struct seq_file *m, loff_t *pos)
7640 	__acquires(RCU)
7641 {
7642 	struct trace_pid_list *pid_list;
7643 	struct trace_array *tr = m->private;
7644 
7645 	mutex_lock(&ftrace_lock);
7646 	rcu_read_lock_sched();
7647 
7648 	pid_list = rcu_dereference_sched(tr->function_no_pids);
7649 
7650 	if (!pid_list)
7651 		return !(*pos) ? FTRACE_NO_PIDS : NULL;
7652 
7653 	return trace_pid_start(pid_list, pos);
7654 }
7655 
fnpid_next(struct seq_file * m,void * v,loff_t * pos)7656 static void *fnpid_next(struct seq_file *m, void *v, loff_t *pos)
7657 {
7658 	struct trace_array *tr = m->private;
7659 	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_no_pids);
7660 
7661 	if (v == FTRACE_NO_PIDS) {
7662 		(*pos)++;
7663 		return NULL;
7664 	}
7665 	return trace_pid_next(pid_list, v, pos);
7666 }
7667 
7668 static const struct seq_operations ftrace_no_pid_sops = {
7669 	.start = fnpid_start,
7670 	.next = fnpid_next,
7671 	.stop = fpid_stop,
7672 	.show = fpid_show,
7673 };
7674 
pid_open(struct inode * inode,struct file * file,int type)7675 static int pid_open(struct inode *inode, struct file *file, int type)
7676 {
7677 	const struct seq_operations *seq_ops;
7678 	struct trace_array *tr = inode->i_private;
7679 	struct seq_file *m;
7680 	int ret = 0;
7681 
7682 	ret = tracing_check_open_get_tr(tr);
7683 	if (ret)
7684 		return ret;
7685 
7686 	if ((file->f_mode & FMODE_WRITE) &&
7687 	    (file->f_flags & O_TRUNC))
7688 		ftrace_pid_reset(tr, type);
7689 
7690 	switch (type) {
7691 	case TRACE_PIDS:
7692 		seq_ops = &ftrace_pid_sops;
7693 		break;
7694 	case TRACE_NO_PIDS:
7695 		seq_ops = &ftrace_no_pid_sops;
7696 		break;
7697 	default:
7698 		trace_array_put(tr);
7699 		WARN_ON_ONCE(1);
7700 		return -EINVAL;
7701 	}
7702 
7703 	ret = seq_open(file, seq_ops);
7704 	if (ret < 0) {
7705 		trace_array_put(tr);
7706 	} else {
7707 		m = file->private_data;
7708 		/* copy tr over to seq ops */
7709 		m->private = tr;
7710 	}
7711 
7712 	return ret;
7713 }
7714 
7715 static int
ftrace_pid_open(struct inode * inode,struct file * file)7716 ftrace_pid_open(struct inode *inode, struct file *file)
7717 {
7718 	return pid_open(inode, file, TRACE_PIDS);
7719 }
7720 
7721 static int
ftrace_no_pid_open(struct inode * inode,struct file * file)7722 ftrace_no_pid_open(struct inode *inode, struct file *file)
7723 {
7724 	return pid_open(inode, file, TRACE_NO_PIDS);
7725 }
7726 
ignore_task_cpu(void * data)7727 static void ignore_task_cpu(void *data)
7728 {
7729 	struct trace_array *tr = data;
7730 	struct trace_pid_list *pid_list;
7731 	struct trace_pid_list *no_pid_list;
7732 
7733 	/*
7734 	 * This function is called by on_each_cpu() while the
7735 	 * event_mutex is held.
7736 	 */
7737 	pid_list = rcu_dereference_protected(tr->function_pids,
7738 					     mutex_is_locked(&ftrace_lock));
7739 	no_pid_list = rcu_dereference_protected(tr->function_no_pids,
7740 						mutex_is_locked(&ftrace_lock));
7741 
7742 	if (trace_ignore_this_task(pid_list, no_pid_list, current))
7743 		this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
7744 			       FTRACE_PID_IGNORE);
7745 	else
7746 		this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
7747 			       current->pid);
7748 }
7749 
7750 static ssize_t
pid_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos,int type)7751 pid_write(struct file *filp, const char __user *ubuf,
7752 	  size_t cnt, loff_t *ppos, int type)
7753 {
7754 	struct seq_file *m = filp->private_data;
7755 	struct trace_array *tr = m->private;
7756 	struct trace_pid_list *filtered_pids;
7757 	struct trace_pid_list *other_pids;
7758 	struct trace_pid_list *pid_list;
7759 	ssize_t ret;
7760 
7761 	if (!cnt)
7762 		return 0;
7763 
7764 	mutex_lock(&ftrace_lock);
7765 
7766 	switch (type) {
7767 	case TRACE_PIDS:
7768 		filtered_pids = rcu_dereference_protected(tr->function_pids,
7769 					     lockdep_is_held(&ftrace_lock));
7770 		other_pids = rcu_dereference_protected(tr->function_no_pids,
7771 					     lockdep_is_held(&ftrace_lock));
7772 		break;
7773 	case TRACE_NO_PIDS:
7774 		filtered_pids = rcu_dereference_protected(tr->function_no_pids,
7775 					     lockdep_is_held(&ftrace_lock));
7776 		other_pids = rcu_dereference_protected(tr->function_pids,
7777 					     lockdep_is_held(&ftrace_lock));
7778 		break;
7779 	default:
7780 		ret = -EINVAL;
7781 		WARN_ON_ONCE(1);
7782 		goto out;
7783 	}
7784 
7785 	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
7786 	if (ret < 0)
7787 		goto out;
7788 
7789 	switch (type) {
7790 	case TRACE_PIDS:
7791 		rcu_assign_pointer(tr->function_pids, pid_list);
7792 		break;
7793 	case TRACE_NO_PIDS:
7794 		rcu_assign_pointer(tr->function_no_pids, pid_list);
7795 		break;
7796 	}
7797 
7798 
7799 	if (filtered_pids) {
7800 		synchronize_rcu();
7801 		trace_pid_list_free(filtered_pids);
7802 	} else if (pid_list && !other_pids) {
7803 		/* Register a probe to set whether to ignore the tracing of a task */
7804 		register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
7805 	}
7806 
7807 	/*
7808 	 * Ignoring of pids is done at task switch. But we have to
7809 	 * check for those tasks that are currently running.
7810 	 * Always do this in case a pid was appended or removed.
7811 	 */
7812 	on_each_cpu(ignore_task_cpu, tr, 1);
7813 
7814 	ftrace_update_pid_func();
7815 	ftrace_startup_all(0);
7816  out:
7817 	mutex_unlock(&ftrace_lock);
7818 
7819 	if (ret > 0)
7820 		*ppos += ret;
7821 
7822 	return ret;
7823 }
7824 
7825 static ssize_t
ftrace_pid_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)7826 ftrace_pid_write(struct file *filp, const char __user *ubuf,
7827 		 size_t cnt, loff_t *ppos)
7828 {
7829 	return pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS);
7830 }
7831 
7832 static ssize_t
ftrace_no_pid_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)7833 ftrace_no_pid_write(struct file *filp, const char __user *ubuf,
7834 		    size_t cnt, loff_t *ppos)
7835 {
7836 	return pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS);
7837 }
7838 
7839 static int
ftrace_pid_release(struct inode * inode,struct file * file)7840 ftrace_pid_release(struct inode *inode, struct file *file)
7841 {
7842 	struct trace_array *tr = inode->i_private;
7843 
7844 	trace_array_put(tr);
7845 
7846 	return seq_release(inode, file);
7847 }
7848 
7849 static const struct file_operations ftrace_pid_fops = {
7850 	.open		= ftrace_pid_open,
7851 	.write		= ftrace_pid_write,
7852 	.read		= seq_read,
7853 	.llseek		= tracing_lseek,
7854 	.release	= ftrace_pid_release,
7855 };
7856 
7857 static const struct file_operations ftrace_no_pid_fops = {
7858 	.open		= ftrace_no_pid_open,
7859 	.write		= ftrace_no_pid_write,
7860 	.read		= seq_read,
7861 	.llseek		= tracing_lseek,
7862 	.release	= ftrace_pid_release,
7863 };
7864 
ftrace_init_tracefs(struct trace_array * tr,struct dentry * d_tracer)7865 void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
7866 {
7867 	trace_create_file("set_ftrace_pid", TRACE_MODE_WRITE, d_tracer,
7868 			    tr, &ftrace_pid_fops);
7869 	trace_create_file("set_ftrace_notrace_pid", TRACE_MODE_WRITE,
7870 			  d_tracer, tr, &ftrace_no_pid_fops);
7871 }
7872 
ftrace_init_tracefs_toplevel(struct trace_array * tr,struct dentry * d_tracer)7873 void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
7874 					 struct dentry *d_tracer)
7875 {
7876 	/* Only the top level directory has the dyn_tracefs and profile */
7877 	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
7878 
7879 	ftrace_init_dyn_tracefs(d_tracer);
7880 	ftrace_profile_tracefs(d_tracer);
7881 }
7882 
7883 /**
7884  * ftrace_kill - kill ftrace
7885  *
7886  * This function should be used by panic code. It stops ftrace
7887  * but in a not so nice way. If you need to simply kill ftrace
7888  * from a non-atomic section, use ftrace_kill.
7889  */
ftrace_kill(void)7890 void ftrace_kill(void)
7891 {
7892 	ftrace_disabled = 1;
7893 	ftrace_enabled = 0;
7894 	ftrace_trace_function = ftrace_stub;
7895 }
7896 
7897 /**
7898  * ftrace_is_dead - Test if ftrace is dead or not.
7899  *
7900  * Returns 1 if ftrace is "dead", zero otherwise.
7901  */
ftrace_is_dead(void)7902 int ftrace_is_dead(void)
7903 {
7904 	return ftrace_disabled;
7905 }
7906 
7907 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
7908 /*
7909  * When registering ftrace_ops with IPMODIFY, it is necessary to make sure
7910  * it doesn't conflict with any direct ftrace_ops. If there is existing
7911  * direct ftrace_ops on a kernel function being patched, call
7912  * FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER on it to enable sharing.
7913  *
7914  * @ops:     ftrace_ops being registered.
7915  *
7916  * Returns:
7917  *         0 on success;
7918  *         Negative on failure.
7919  */
prepare_direct_functions_for_ipmodify(struct ftrace_ops * ops)7920 static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops)
7921 {
7922 	struct ftrace_func_entry *entry;
7923 	struct ftrace_hash *hash;
7924 	struct ftrace_ops *op;
7925 	int size, i, ret;
7926 
7927 	lockdep_assert_held_once(&direct_mutex);
7928 
7929 	if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
7930 		return 0;
7931 
7932 	hash = ops->func_hash->filter_hash;
7933 	size = 1 << hash->size_bits;
7934 	for (i = 0; i < size; i++) {
7935 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
7936 			unsigned long ip = entry->ip;
7937 			bool found_op = false;
7938 
7939 			mutex_lock(&ftrace_lock);
7940 			do_for_each_ftrace_op(op, ftrace_ops_list) {
7941 				if (!(op->flags & FTRACE_OPS_FL_DIRECT))
7942 					continue;
7943 				if (ops_references_ip(op, ip)) {
7944 					found_op = true;
7945 					break;
7946 				}
7947 			} while_for_each_ftrace_op(op);
7948 			mutex_unlock(&ftrace_lock);
7949 
7950 			if (found_op) {
7951 				if (!op->ops_func)
7952 					return -EBUSY;
7953 
7954 				ret = op->ops_func(op, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER);
7955 				if (ret)
7956 					return ret;
7957 			}
7958 		}
7959 	}
7960 
7961 	return 0;
7962 }
7963 
7964 /*
7965  * Similar to prepare_direct_functions_for_ipmodify, clean up after ops
7966  * with IPMODIFY is unregistered. The cleanup is optional for most DIRECT
7967  * ops.
7968  */
cleanup_direct_functions_after_ipmodify(struct ftrace_ops * ops)7969 static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops)
7970 {
7971 	struct ftrace_func_entry *entry;
7972 	struct ftrace_hash *hash;
7973 	struct ftrace_ops *op;
7974 	int size, i;
7975 
7976 	if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
7977 		return;
7978 
7979 	mutex_lock(&direct_mutex);
7980 
7981 	hash = ops->func_hash->filter_hash;
7982 	size = 1 << hash->size_bits;
7983 	for (i = 0; i < size; i++) {
7984 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
7985 			unsigned long ip = entry->ip;
7986 			bool found_op = false;
7987 
7988 			mutex_lock(&ftrace_lock);
7989 			do_for_each_ftrace_op(op, ftrace_ops_list) {
7990 				if (!(op->flags & FTRACE_OPS_FL_DIRECT))
7991 					continue;
7992 				if (ops_references_ip(op, ip)) {
7993 					found_op = true;
7994 					break;
7995 				}
7996 			} while_for_each_ftrace_op(op);
7997 			mutex_unlock(&ftrace_lock);
7998 
7999 			/* The cleanup is optional, ignore any errors */
8000 			if (found_op && op->ops_func)
8001 				op->ops_func(op, FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER);
8002 		}
8003 	}
8004 	mutex_unlock(&direct_mutex);
8005 }
8006 
8007 #define lock_direct_mutex()	mutex_lock(&direct_mutex)
8008 #define unlock_direct_mutex()	mutex_unlock(&direct_mutex)
8009 
8010 #else  /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
8011 
prepare_direct_functions_for_ipmodify(struct ftrace_ops * ops)8012 static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops)
8013 {
8014 	return 0;
8015 }
8016 
cleanup_direct_functions_after_ipmodify(struct ftrace_ops * ops)8017 static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops)
8018 {
8019 }
8020 
8021 #define lock_direct_mutex()	do { } while (0)
8022 #define unlock_direct_mutex()	do { } while (0)
8023 
8024 #endif  /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
8025 
8026 /*
8027  * Similar to register_ftrace_function, except we don't lock direct_mutex.
8028  */
register_ftrace_function_nolock(struct ftrace_ops * ops)8029 static int register_ftrace_function_nolock(struct ftrace_ops *ops)
8030 {
8031 	int ret;
8032 
8033 	ftrace_ops_init(ops);
8034 
8035 	mutex_lock(&ftrace_lock);
8036 
8037 	ret = ftrace_startup(ops, 0);
8038 
8039 	mutex_unlock(&ftrace_lock);
8040 
8041 	return ret;
8042 }
8043 
8044 /**
8045  * register_ftrace_function - register a function for profiling
8046  * @ops:	ops structure that holds the function for profiling.
8047  *
8048  * Register a function to be called by all functions in the
8049  * kernel.
8050  *
8051  * Note: @ops->func and all the functions it calls must be labeled
8052  *       with "notrace", otherwise it will go into a
8053  *       recursive loop.
8054  */
register_ftrace_function(struct ftrace_ops * ops)8055 int register_ftrace_function(struct ftrace_ops *ops)
8056 {
8057 	int ret;
8058 
8059 	lock_direct_mutex();
8060 	ret = prepare_direct_functions_for_ipmodify(ops);
8061 	if (ret < 0)
8062 		goto out_unlock;
8063 
8064 	ret = register_ftrace_function_nolock(ops);
8065 
8066 out_unlock:
8067 	unlock_direct_mutex();
8068 	return ret;
8069 }
8070 EXPORT_SYMBOL_GPL(register_ftrace_function);
8071 
8072 /**
8073  * unregister_ftrace_function - unregister a function for profiling.
8074  * @ops:	ops structure that holds the function to unregister
8075  *
8076  * Unregister a function that was added to be called by ftrace profiling.
8077  */
unregister_ftrace_function(struct ftrace_ops * ops)8078 int unregister_ftrace_function(struct ftrace_ops *ops)
8079 {
8080 	int ret;
8081 
8082 	mutex_lock(&ftrace_lock);
8083 	ret = ftrace_shutdown(ops, 0);
8084 	mutex_unlock(&ftrace_lock);
8085 
8086 	cleanup_direct_functions_after_ipmodify(ops);
8087 	return ret;
8088 }
8089 EXPORT_SYMBOL_GPL(unregister_ftrace_function);
8090 
symbols_cmp(const void * a,const void * b)8091 static int symbols_cmp(const void *a, const void *b)
8092 {
8093 	const char **str_a = (const char **) a;
8094 	const char **str_b = (const char **) b;
8095 
8096 	return strcmp(*str_a, *str_b);
8097 }
8098 
8099 struct kallsyms_data {
8100 	unsigned long *addrs;
8101 	const char **syms;
8102 	size_t cnt;
8103 	size_t found;
8104 };
8105 
8106 /* This function gets called for all kernel and module symbols
8107  * and returns 1 in case we resolved all the requested symbols,
8108  * 0 otherwise.
8109  */
kallsyms_callback(void * data,const char * name,unsigned long addr)8110 static int kallsyms_callback(void *data, const char *name, unsigned long addr)
8111 {
8112 	struct kallsyms_data *args = data;
8113 	const char **sym;
8114 	int idx;
8115 
8116 	sym = bsearch(&name, args->syms, args->cnt, sizeof(*args->syms), symbols_cmp);
8117 	if (!sym)
8118 		return 0;
8119 
8120 	idx = sym - args->syms;
8121 	if (args->addrs[idx])
8122 		return 0;
8123 
8124 	if (!ftrace_location(addr))
8125 		return 0;
8126 
8127 	args->addrs[idx] = addr;
8128 	args->found++;
8129 	return args->found == args->cnt ? 1 : 0;
8130 }
8131 
8132 /**
8133  * ftrace_lookup_symbols - Lookup addresses for array of symbols
8134  *
8135  * @sorted_syms: array of symbols pointers symbols to resolve,
8136  * must be alphabetically sorted
8137  * @cnt: number of symbols/addresses in @syms/@addrs arrays
8138  * @addrs: array for storing resulting addresses
8139  *
8140  * This function looks up addresses for array of symbols provided in
8141  * @syms array (must be alphabetically sorted) and stores them in
8142  * @addrs array, which needs to be big enough to store at least @cnt
8143  * addresses.
8144  *
8145  * This function returns 0 if all provided symbols are found,
8146  * -ESRCH otherwise.
8147  */
ftrace_lookup_symbols(const char ** sorted_syms,size_t cnt,unsigned long * addrs)8148 int ftrace_lookup_symbols(const char **sorted_syms, size_t cnt, unsigned long *addrs)
8149 {
8150 	struct kallsyms_data args;
8151 	int found_all;
8152 
8153 	memset(addrs, 0, sizeof(*addrs) * cnt);
8154 	args.addrs = addrs;
8155 	args.syms = sorted_syms;
8156 	args.cnt = cnt;
8157 	args.found = 0;
8158 
8159 	found_all = kallsyms_on_each_symbol(kallsyms_callback, &args);
8160 	if (found_all)
8161 		return 0;
8162 	found_all = module_kallsyms_on_each_symbol(NULL, kallsyms_callback, &args);
8163 	return found_all ? 0 : -ESRCH;
8164 }
8165 
8166 #ifdef CONFIG_SYSCTL
8167 
8168 #ifdef CONFIG_DYNAMIC_FTRACE
ftrace_startup_sysctl(void)8169 static void ftrace_startup_sysctl(void)
8170 {
8171 	int command;
8172 
8173 	if (unlikely(ftrace_disabled))
8174 		return;
8175 
8176 	/* Force update next time */
8177 	saved_ftrace_func = NULL;
8178 	/* ftrace_start_up is true if we want ftrace running */
8179 	if (ftrace_start_up) {
8180 		command = FTRACE_UPDATE_CALLS;
8181 		if (ftrace_graph_active)
8182 			command |= FTRACE_START_FUNC_RET;
8183 		ftrace_startup_enable(command);
8184 	}
8185 }
8186 
ftrace_shutdown_sysctl(void)8187 static void ftrace_shutdown_sysctl(void)
8188 {
8189 	int command;
8190 
8191 	if (unlikely(ftrace_disabled))
8192 		return;
8193 
8194 	/* ftrace_start_up is true if ftrace is running */
8195 	if (ftrace_start_up) {
8196 		command = FTRACE_DISABLE_CALLS;
8197 		if (ftrace_graph_active)
8198 			command |= FTRACE_STOP_FUNC_RET;
8199 		ftrace_run_update_code(command);
8200 	}
8201 }
8202 #else
8203 # define ftrace_startup_sysctl()       do { } while (0)
8204 # define ftrace_shutdown_sysctl()      do { } while (0)
8205 #endif /* CONFIG_DYNAMIC_FTRACE */
8206 
is_permanent_ops_registered(void)8207 static bool is_permanent_ops_registered(void)
8208 {
8209 	struct ftrace_ops *op;
8210 
8211 	do_for_each_ftrace_op(op, ftrace_ops_list) {
8212 		if (op->flags & FTRACE_OPS_FL_PERMANENT)
8213 			return true;
8214 	} while_for_each_ftrace_op(op);
8215 
8216 	return false;
8217 }
8218 
8219 static int
ftrace_enable_sysctl(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)8220 ftrace_enable_sysctl(struct ctl_table *table, int write,
8221 		     void *buffer, size_t *lenp, loff_t *ppos)
8222 {
8223 	int ret = -ENODEV;
8224 
8225 	mutex_lock(&ftrace_lock);
8226 
8227 	if (unlikely(ftrace_disabled))
8228 		goto out;
8229 
8230 	ret = proc_dointvec(table, write, buffer, lenp, ppos);
8231 
8232 	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
8233 		goto out;
8234 
8235 	if (ftrace_enabled) {
8236 
8237 		/* we are starting ftrace again */
8238 		if (rcu_dereference_protected(ftrace_ops_list,
8239 			lockdep_is_held(&ftrace_lock)) != &ftrace_list_end)
8240 			update_ftrace_function();
8241 
8242 		ftrace_startup_sysctl();
8243 
8244 	} else {
8245 		if (is_permanent_ops_registered()) {
8246 			ftrace_enabled = true;
8247 			ret = -EBUSY;
8248 			goto out;
8249 		}
8250 
8251 		/* stopping ftrace calls (just send to ftrace_stub) */
8252 		ftrace_trace_function = ftrace_stub;
8253 
8254 		ftrace_shutdown_sysctl();
8255 	}
8256 
8257 	last_ftrace_enabled = !!ftrace_enabled;
8258  out:
8259 	mutex_unlock(&ftrace_lock);
8260 	return ret;
8261 }
8262 
8263 static struct ctl_table ftrace_sysctls[] = {
8264 	{
8265 		.procname       = "ftrace_enabled",
8266 		.data           = &ftrace_enabled,
8267 		.maxlen         = sizeof(int),
8268 		.mode           = 0644,
8269 		.proc_handler   = ftrace_enable_sysctl,
8270 	},
8271 	{}
8272 };
8273 
ftrace_sysctl_init(void)8274 static int __init ftrace_sysctl_init(void)
8275 {
8276 	register_sysctl_init("kernel", ftrace_sysctls);
8277 	return 0;
8278 }
8279 late_initcall(ftrace_sysctl_init);
8280 #endif
8281