1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * ring buffer based function tracer
4  *
5  * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
6  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7  *
8  * Originally taken from the RT patch by:
9  *    Arnaldo Carvalho de Melo <acme@redhat.com>
10  *
11  * Based on code from the latency_tracer, that is:
12  *  Copyright (C) 2004-2006 Ingo Molnar
13  *  Copyright (C) 2004 Nadia Yvette Chambers
14  */
15 #include <linux/ring_buffer.h>
16 #include <generated/utsrelease.h>
17 #include <linux/stacktrace.h>
18 #include <linux/writeback.h>
19 #include <linux/kallsyms.h>
20 #include <linux/security.h>
21 #include <linux/seq_file.h>
22 #include <linux/irqflags.h>
23 #include <linux/debugfs.h>
24 #include <linux/tracefs.h>
25 #include <linux/pagemap.h>
26 #include <linux/hardirq.h>
27 #include <linux/linkage.h>
28 #include <linux/uaccess.h>
29 #include <linux/vmalloc.h>
30 #include <linux/ftrace.h>
31 #include <linux/module.h>
32 #include <linux/percpu.h>
33 #include <linux/splice.h>
34 #include <linux/kdebug.h>
35 #include <linux/string.h>
36 #include <linux/mount.h>
37 #include <linux/rwsem.h>
38 #include <linux/slab.h>
39 #include <linux/ctype.h>
40 #include <linux/init.h>
41 #include <linux/panic_notifier.h>
42 #include <linux/kmemleak.h>
43 #include <linux/poll.h>
44 #include <linux/nmi.h>
45 #include <linux/fs.h>
46 #include <linux/trace.h>
47 #include <linux/sched/clock.h>
48 #include <linux/sched/rt.h>
49 #include <linux/fsnotify.h>
50 #include <linux/irq_work.h>
51 #include <linux/workqueue.h>
52 
53 #include <asm/setup.h> /* COMMAND_LINE_SIZE */
54 
55 #include "trace.h"
56 #include "trace_output.h"
57 
58 /*
59  * On boot up, the ring buffer is set to the minimum size, so that
60  * we do not waste memory on systems that are not using tracing.
61  */
62 bool ring_buffer_expanded;
63 
64 #ifdef CONFIG_FTRACE_STARTUP_TEST
65 /*
66  * We need to change this state when a selftest is running.
67  * A selftest will lurk into the ring-buffer to count the
68  * entries inserted during the selftest although some concurrent
69  * insertions into the ring-buffer such as trace_printk could occurred
70  * at the same time, giving false positive or negative results.
71  */
72 static bool __read_mostly tracing_selftest_running;
73 
74 /*
75  * If boot-time tracing including tracers/events via kernel cmdline
76  * is running, we do not want to run SELFTEST.
77  */
78 bool __read_mostly tracing_selftest_disabled;
79 
disable_tracing_selftest(const char * reason)80 void __init disable_tracing_selftest(const char *reason)
81 {
82 	if (!tracing_selftest_disabled) {
83 		tracing_selftest_disabled = true;
84 		pr_info("Ftrace startup test is disabled due to %s\n", reason);
85 	}
86 }
87 #else
88 #define tracing_selftest_running	0
89 #define tracing_selftest_disabled	0
90 #endif
91 
92 /* Pipe tracepoints to printk */
93 static struct trace_iterator *tracepoint_print_iter;
94 int tracepoint_printk;
95 static bool tracepoint_printk_stop_on_boot __initdata;
96 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
97 
98 /* For tracers that don't implement custom flags */
99 static struct tracer_opt dummy_tracer_opt[] = {
100 	{ }
101 };
102 
103 static int
dummy_set_flag(struct trace_array * tr,u32 old_flags,u32 bit,int set)104 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
105 {
106 	return 0;
107 }
108 
109 /*
110  * To prevent the comm cache from being overwritten when no
111  * tracing is active, only save the comm when a trace event
112  * occurred.
113  */
114 static DEFINE_PER_CPU(bool, trace_taskinfo_save);
115 
116 /*
117  * Kill all tracing for good (never come back).
118  * It is initialized to 1 but will turn to zero if the initialization
119  * of the tracer is successful. But that is the only place that sets
120  * this back to zero.
121  */
122 static int tracing_disabled = 1;
123 
124 cpumask_var_t __read_mostly	tracing_buffer_mask;
125 
126 /*
127  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
128  *
129  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
130  * is set, then ftrace_dump is called. This will output the contents
131  * of the ftrace buffers to the console.  This is very useful for
132  * capturing traces that lead to crashes and outputing it to a
133  * serial console.
134  *
135  * It is default off, but you can enable it with either specifying
136  * "ftrace_dump_on_oops" in the kernel command line, or setting
137  * /proc/sys/kernel/ftrace_dump_on_oops
138  * Set 1 if you want to dump buffers of all CPUs
139  * Set 2 if you want to dump the buffer of the CPU that triggered oops
140  */
141 
142 enum ftrace_dump_mode ftrace_dump_on_oops;
143 
144 /* When set, tracing will stop when a WARN*() is hit */
145 int __disable_trace_on_warning;
146 
147 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
148 /* Map of enums to their values, for "eval_map" file */
149 struct trace_eval_map_head {
150 	struct module			*mod;
151 	unsigned long			length;
152 };
153 
154 union trace_eval_map_item;
155 
156 struct trace_eval_map_tail {
157 	/*
158 	 * "end" is first and points to NULL as it must be different
159 	 * than "mod" or "eval_string"
160 	 */
161 	union trace_eval_map_item	*next;
162 	const char			*end;	/* points to NULL */
163 };
164 
165 static DEFINE_MUTEX(trace_eval_mutex);
166 
167 /*
168  * The trace_eval_maps are saved in an array with two extra elements,
169  * one at the beginning, and one at the end. The beginning item contains
170  * the count of the saved maps (head.length), and the module they
171  * belong to if not built in (head.mod). The ending item contains a
172  * pointer to the next array of saved eval_map items.
173  */
174 union trace_eval_map_item {
175 	struct trace_eval_map		map;
176 	struct trace_eval_map_head	head;
177 	struct trace_eval_map_tail	tail;
178 };
179 
180 static union trace_eval_map_item *trace_eval_maps;
181 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
182 
183 int tracing_set_tracer(struct trace_array *tr, const char *buf);
184 static void ftrace_trace_userstack(struct trace_array *tr,
185 				   struct trace_buffer *buffer,
186 				   unsigned int trace_ctx);
187 
188 #define MAX_TRACER_SIZE		100
189 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
190 static char *default_bootup_tracer;
191 
192 static bool allocate_snapshot;
193 static bool snapshot_at_boot;
194 
195 static char boot_instance_info[COMMAND_LINE_SIZE] __initdata;
196 static int boot_instance_index;
197 
198 static char boot_snapshot_info[COMMAND_LINE_SIZE] __initdata;
199 static int boot_snapshot_index;
200 
set_cmdline_ftrace(char * str)201 static int __init set_cmdline_ftrace(char *str)
202 {
203 	strscpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
204 	default_bootup_tracer = bootup_tracer_buf;
205 	/* We are using ftrace early, expand it */
206 	ring_buffer_expanded = true;
207 	return 1;
208 }
209 __setup("ftrace=", set_cmdline_ftrace);
210 
set_ftrace_dump_on_oops(char * str)211 static int __init set_ftrace_dump_on_oops(char *str)
212 {
213 	if (*str++ != '=' || !*str || !strcmp("1", str)) {
214 		ftrace_dump_on_oops = DUMP_ALL;
215 		return 1;
216 	}
217 
218 	if (!strcmp("orig_cpu", str) || !strcmp("2", str)) {
219 		ftrace_dump_on_oops = DUMP_ORIG;
220                 return 1;
221         }
222 
223         return 0;
224 }
225 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
226 
stop_trace_on_warning(char * str)227 static int __init stop_trace_on_warning(char *str)
228 {
229 	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
230 		__disable_trace_on_warning = 1;
231 	return 1;
232 }
233 __setup("traceoff_on_warning", stop_trace_on_warning);
234 
boot_alloc_snapshot(char * str)235 static int __init boot_alloc_snapshot(char *str)
236 {
237 	char *slot = boot_snapshot_info + boot_snapshot_index;
238 	int left = sizeof(boot_snapshot_info) - boot_snapshot_index;
239 	int ret;
240 
241 	if (str[0] == '=') {
242 		str++;
243 		if (strlen(str) >= left)
244 			return -1;
245 
246 		ret = snprintf(slot, left, "%s\t", str);
247 		boot_snapshot_index += ret;
248 	} else {
249 		allocate_snapshot = true;
250 		/* We also need the main ring buffer expanded */
251 		ring_buffer_expanded = true;
252 	}
253 	return 1;
254 }
255 __setup("alloc_snapshot", boot_alloc_snapshot);
256 
257 
boot_snapshot(char * str)258 static int __init boot_snapshot(char *str)
259 {
260 	snapshot_at_boot = true;
261 	boot_alloc_snapshot(str);
262 	return 1;
263 }
264 __setup("ftrace_boot_snapshot", boot_snapshot);
265 
266 
boot_instance(char * str)267 static int __init boot_instance(char *str)
268 {
269 	char *slot = boot_instance_info + boot_instance_index;
270 	int left = sizeof(boot_instance_info) - boot_instance_index;
271 	int ret;
272 
273 	if (strlen(str) >= left)
274 		return -1;
275 
276 	ret = snprintf(slot, left, "%s\t", str);
277 	boot_instance_index += ret;
278 
279 	return 1;
280 }
281 __setup("trace_instance=", boot_instance);
282 
283 
284 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
285 
set_trace_boot_options(char * str)286 static int __init set_trace_boot_options(char *str)
287 {
288 	strscpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
289 	return 1;
290 }
291 __setup("trace_options=", set_trace_boot_options);
292 
293 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
294 static char *trace_boot_clock __initdata;
295 
set_trace_boot_clock(char * str)296 static int __init set_trace_boot_clock(char *str)
297 {
298 	strscpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
299 	trace_boot_clock = trace_boot_clock_buf;
300 	return 1;
301 }
302 __setup("trace_clock=", set_trace_boot_clock);
303 
set_tracepoint_printk(char * str)304 static int __init set_tracepoint_printk(char *str)
305 {
306 	/* Ignore the "tp_printk_stop_on_boot" param */
307 	if (*str == '_')
308 		return 0;
309 
310 	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
311 		tracepoint_printk = 1;
312 	return 1;
313 }
314 __setup("tp_printk", set_tracepoint_printk);
315 
set_tracepoint_printk_stop(char * str)316 static int __init set_tracepoint_printk_stop(char *str)
317 {
318 	tracepoint_printk_stop_on_boot = true;
319 	return 1;
320 }
321 __setup("tp_printk_stop_on_boot", set_tracepoint_printk_stop);
322 
ns2usecs(u64 nsec)323 unsigned long long ns2usecs(u64 nsec)
324 {
325 	nsec += 500;
326 	do_div(nsec, 1000);
327 	return nsec;
328 }
329 
330 static void
trace_process_export(struct trace_export * export,struct ring_buffer_event * event,int flag)331 trace_process_export(struct trace_export *export,
332 	       struct ring_buffer_event *event, int flag)
333 {
334 	struct trace_entry *entry;
335 	unsigned int size = 0;
336 
337 	if (export->flags & flag) {
338 		entry = ring_buffer_event_data(event);
339 		size = ring_buffer_event_length(event);
340 		export->write(export, entry, size);
341 	}
342 }
343 
344 static DEFINE_MUTEX(ftrace_export_lock);
345 
346 static struct trace_export __rcu *ftrace_exports_list __read_mostly;
347 
348 static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled);
349 static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled);
350 static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled);
351 
ftrace_exports_enable(struct trace_export * export)352 static inline void ftrace_exports_enable(struct trace_export *export)
353 {
354 	if (export->flags & TRACE_EXPORT_FUNCTION)
355 		static_branch_inc(&trace_function_exports_enabled);
356 
357 	if (export->flags & TRACE_EXPORT_EVENT)
358 		static_branch_inc(&trace_event_exports_enabled);
359 
360 	if (export->flags & TRACE_EXPORT_MARKER)
361 		static_branch_inc(&trace_marker_exports_enabled);
362 }
363 
ftrace_exports_disable(struct trace_export * export)364 static inline void ftrace_exports_disable(struct trace_export *export)
365 {
366 	if (export->flags & TRACE_EXPORT_FUNCTION)
367 		static_branch_dec(&trace_function_exports_enabled);
368 
369 	if (export->flags & TRACE_EXPORT_EVENT)
370 		static_branch_dec(&trace_event_exports_enabled);
371 
372 	if (export->flags & TRACE_EXPORT_MARKER)
373 		static_branch_dec(&trace_marker_exports_enabled);
374 }
375 
ftrace_exports(struct ring_buffer_event * event,int flag)376 static void ftrace_exports(struct ring_buffer_event *event, int flag)
377 {
378 	struct trace_export *export;
379 
380 	preempt_disable_notrace();
381 
382 	export = rcu_dereference_raw_check(ftrace_exports_list);
383 	while (export) {
384 		trace_process_export(export, event, flag);
385 		export = rcu_dereference_raw_check(export->next);
386 	}
387 
388 	preempt_enable_notrace();
389 }
390 
391 static inline void
add_trace_export(struct trace_export ** list,struct trace_export * export)392 add_trace_export(struct trace_export **list, struct trace_export *export)
393 {
394 	rcu_assign_pointer(export->next, *list);
395 	/*
396 	 * We are entering export into the list but another
397 	 * CPU might be walking that list. We need to make sure
398 	 * the export->next pointer is valid before another CPU sees
399 	 * the export pointer included into the list.
400 	 */
401 	rcu_assign_pointer(*list, export);
402 }
403 
404 static inline int
rm_trace_export(struct trace_export ** list,struct trace_export * export)405 rm_trace_export(struct trace_export **list, struct trace_export *export)
406 {
407 	struct trace_export **p;
408 
409 	for (p = list; *p != NULL; p = &(*p)->next)
410 		if (*p == export)
411 			break;
412 
413 	if (*p != export)
414 		return -1;
415 
416 	rcu_assign_pointer(*p, (*p)->next);
417 
418 	return 0;
419 }
420 
421 static inline void
add_ftrace_export(struct trace_export ** list,struct trace_export * export)422 add_ftrace_export(struct trace_export **list, struct trace_export *export)
423 {
424 	ftrace_exports_enable(export);
425 
426 	add_trace_export(list, export);
427 }
428 
429 static inline int
rm_ftrace_export(struct trace_export ** list,struct trace_export * export)430 rm_ftrace_export(struct trace_export **list, struct trace_export *export)
431 {
432 	int ret;
433 
434 	ret = rm_trace_export(list, export);
435 	ftrace_exports_disable(export);
436 
437 	return ret;
438 }
439 
register_ftrace_export(struct trace_export * export)440 int register_ftrace_export(struct trace_export *export)
441 {
442 	if (WARN_ON_ONCE(!export->write))
443 		return -1;
444 
445 	mutex_lock(&ftrace_export_lock);
446 
447 	add_ftrace_export(&ftrace_exports_list, export);
448 
449 	mutex_unlock(&ftrace_export_lock);
450 
451 	return 0;
452 }
453 EXPORT_SYMBOL_GPL(register_ftrace_export);
454 
unregister_ftrace_export(struct trace_export * export)455 int unregister_ftrace_export(struct trace_export *export)
456 {
457 	int ret;
458 
459 	mutex_lock(&ftrace_export_lock);
460 
461 	ret = rm_ftrace_export(&ftrace_exports_list, export);
462 
463 	mutex_unlock(&ftrace_export_lock);
464 
465 	return ret;
466 }
467 EXPORT_SYMBOL_GPL(unregister_ftrace_export);
468 
469 /* trace_flags holds trace_options default values */
470 #define TRACE_DEFAULT_FLAGS						\
471 	(FUNCTION_DEFAULT_FLAGS |					\
472 	 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |			\
473 	 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO |		\
474 	 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |			\
475 	 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS |			\
476 	 TRACE_ITER_HASH_PTR)
477 
478 /* trace_options that are only supported by global_trace */
479 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK |			\
480 	       TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
481 
482 /* trace_flags that are default zero for instances */
483 #define ZEROED_TRACE_FLAGS \
484 	(TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
485 
486 /*
487  * The global_trace is the descriptor that holds the top-level tracing
488  * buffers for the live tracing.
489  */
490 static struct trace_array global_trace = {
491 	.trace_flags = TRACE_DEFAULT_FLAGS,
492 };
493 
494 LIST_HEAD(ftrace_trace_arrays);
495 
trace_array_get(struct trace_array * this_tr)496 int trace_array_get(struct trace_array *this_tr)
497 {
498 	struct trace_array *tr;
499 	int ret = -ENODEV;
500 
501 	mutex_lock(&trace_types_lock);
502 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
503 		if (tr == this_tr) {
504 			tr->ref++;
505 			ret = 0;
506 			break;
507 		}
508 	}
509 	mutex_unlock(&trace_types_lock);
510 
511 	return ret;
512 }
513 
__trace_array_put(struct trace_array * this_tr)514 static void __trace_array_put(struct trace_array *this_tr)
515 {
516 	WARN_ON(!this_tr->ref);
517 	this_tr->ref--;
518 }
519 
520 /**
521  * trace_array_put - Decrement the reference counter for this trace array.
522  * @this_tr : pointer to the trace array
523  *
524  * NOTE: Use this when we no longer need the trace array returned by
525  * trace_array_get_by_name(). This ensures the trace array can be later
526  * destroyed.
527  *
528  */
trace_array_put(struct trace_array * this_tr)529 void trace_array_put(struct trace_array *this_tr)
530 {
531 	if (!this_tr)
532 		return;
533 
534 	mutex_lock(&trace_types_lock);
535 	__trace_array_put(this_tr);
536 	mutex_unlock(&trace_types_lock);
537 }
538 EXPORT_SYMBOL_GPL(trace_array_put);
539 
tracing_check_open_get_tr(struct trace_array * tr)540 int tracing_check_open_get_tr(struct trace_array *tr)
541 {
542 	int ret;
543 
544 	ret = security_locked_down(LOCKDOWN_TRACEFS);
545 	if (ret)
546 		return ret;
547 
548 	if (tracing_disabled)
549 		return -ENODEV;
550 
551 	if (tr && trace_array_get(tr) < 0)
552 		return -ENODEV;
553 
554 	return 0;
555 }
556 
call_filter_check_discard(struct trace_event_call * call,void * rec,struct trace_buffer * buffer,struct ring_buffer_event * event)557 int call_filter_check_discard(struct trace_event_call *call, void *rec,
558 			      struct trace_buffer *buffer,
559 			      struct ring_buffer_event *event)
560 {
561 	if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
562 	    !filter_match_preds(call->filter, rec)) {
563 		__trace_event_discard_commit(buffer, event);
564 		return 1;
565 	}
566 
567 	return 0;
568 }
569 
570 /**
571  * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
572  * @filtered_pids: The list of pids to check
573  * @search_pid: The PID to find in @filtered_pids
574  *
575  * Returns true if @search_pid is found in @filtered_pids, and false otherwise.
576  */
577 bool
trace_find_filtered_pid(struct trace_pid_list * filtered_pids,pid_t search_pid)578 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
579 {
580 	return trace_pid_list_is_set(filtered_pids, search_pid);
581 }
582 
583 /**
584  * trace_ignore_this_task - should a task be ignored for tracing
585  * @filtered_pids: The list of pids to check
586  * @filtered_no_pids: The list of pids not to be traced
587  * @task: The task that should be ignored if not filtered
588  *
589  * Checks if @task should be traced or not from @filtered_pids.
590  * Returns true if @task should *NOT* be traced.
591  * Returns false if @task should be traced.
592  */
593 bool
trace_ignore_this_task(struct trace_pid_list * filtered_pids,struct trace_pid_list * filtered_no_pids,struct task_struct * task)594 trace_ignore_this_task(struct trace_pid_list *filtered_pids,
595 		       struct trace_pid_list *filtered_no_pids,
596 		       struct task_struct *task)
597 {
598 	/*
599 	 * If filtered_no_pids is not empty, and the task's pid is listed
600 	 * in filtered_no_pids, then return true.
601 	 * Otherwise, if filtered_pids is empty, that means we can
602 	 * trace all tasks. If it has content, then only trace pids
603 	 * within filtered_pids.
604 	 */
605 
606 	return (filtered_pids &&
607 		!trace_find_filtered_pid(filtered_pids, task->pid)) ||
608 		(filtered_no_pids &&
609 		 trace_find_filtered_pid(filtered_no_pids, task->pid));
610 }
611 
612 /**
613  * trace_filter_add_remove_task - Add or remove a task from a pid_list
614  * @pid_list: The list to modify
615  * @self: The current task for fork or NULL for exit
616  * @task: The task to add or remove
617  *
618  * If adding a task, if @self is defined, the task is only added if @self
619  * is also included in @pid_list. This happens on fork and tasks should
620  * only be added when the parent is listed. If @self is NULL, then the
621  * @task pid will be removed from the list, which would happen on exit
622  * of a task.
623  */
trace_filter_add_remove_task(struct trace_pid_list * pid_list,struct task_struct * self,struct task_struct * task)624 void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
625 				  struct task_struct *self,
626 				  struct task_struct *task)
627 {
628 	if (!pid_list)
629 		return;
630 
631 	/* For forks, we only add if the forking task is listed */
632 	if (self) {
633 		if (!trace_find_filtered_pid(pid_list, self->pid))
634 			return;
635 	}
636 
637 	/* "self" is set for forks, and NULL for exits */
638 	if (self)
639 		trace_pid_list_set(pid_list, task->pid);
640 	else
641 		trace_pid_list_clear(pid_list, task->pid);
642 }
643 
644 /**
645  * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
646  * @pid_list: The pid list to show
647  * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
648  * @pos: The position of the file
649  *
650  * This is used by the seq_file "next" operation to iterate the pids
651  * listed in a trace_pid_list structure.
652  *
653  * Returns the pid+1 as we want to display pid of zero, but NULL would
654  * stop the iteration.
655  */
trace_pid_next(struct trace_pid_list * pid_list,void * v,loff_t * pos)656 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
657 {
658 	long pid = (unsigned long)v;
659 	unsigned int next;
660 
661 	(*pos)++;
662 
663 	/* pid already is +1 of the actual previous bit */
664 	if (trace_pid_list_next(pid_list, pid, &next) < 0)
665 		return NULL;
666 
667 	pid = next;
668 
669 	/* Return pid + 1 to allow zero to be represented */
670 	return (void *)(pid + 1);
671 }
672 
673 /**
674  * trace_pid_start - Used for seq_file to start reading pid lists
675  * @pid_list: The pid list to show
676  * @pos: The position of the file
677  *
678  * This is used by seq_file "start" operation to start the iteration
679  * of listing pids.
680  *
681  * Returns the pid+1 as we want to display pid of zero, but NULL would
682  * stop the iteration.
683  */
trace_pid_start(struct trace_pid_list * pid_list,loff_t * pos)684 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
685 {
686 	unsigned long pid;
687 	unsigned int first;
688 	loff_t l = 0;
689 
690 	if (trace_pid_list_first(pid_list, &first) < 0)
691 		return NULL;
692 
693 	pid = first;
694 
695 	/* Return pid + 1 so that zero can be the exit value */
696 	for (pid++; pid && l < *pos;
697 	     pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
698 		;
699 	return (void *)pid;
700 }
701 
702 /**
703  * trace_pid_show - show the current pid in seq_file processing
704  * @m: The seq_file structure to write into
705  * @v: A void pointer of the pid (+1) value to display
706  *
707  * Can be directly used by seq_file operations to display the current
708  * pid value.
709  */
trace_pid_show(struct seq_file * m,void * v)710 int trace_pid_show(struct seq_file *m, void *v)
711 {
712 	unsigned long pid = (unsigned long)v - 1;
713 
714 	seq_printf(m, "%lu\n", pid);
715 	return 0;
716 }
717 
718 /* 128 should be much more than enough */
719 #define PID_BUF_SIZE		127
720 
trace_pid_write(struct trace_pid_list * filtered_pids,struct trace_pid_list ** new_pid_list,const char __user * ubuf,size_t cnt)721 int trace_pid_write(struct trace_pid_list *filtered_pids,
722 		    struct trace_pid_list **new_pid_list,
723 		    const char __user *ubuf, size_t cnt)
724 {
725 	struct trace_pid_list *pid_list;
726 	struct trace_parser parser;
727 	unsigned long val;
728 	int nr_pids = 0;
729 	ssize_t read = 0;
730 	ssize_t ret;
731 	loff_t pos;
732 	pid_t pid;
733 
734 	if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
735 		return -ENOMEM;
736 
737 	/*
738 	 * Always recreate a new array. The write is an all or nothing
739 	 * operation. Always create a new array when adding new pids by
740 	 * the user. If the operation fails, then the current list is
741 	 * not modified.
742 	 */
743 	pid_list = trace_pid_list_alloc();
744 	if (!pid_list) {
745 		trace_parser_put(&parser);
746 		return -ENOMEM;
747 	}
748 
749 	if (filtered_pids) {
750 		/* copy the current bits to the new max */
751 		ret = trace_pid_list_first(filtered_pids, &pid);
752 		while (!ret) {
753 			trace_pid_list_set(pid_list, pid);
754 			ret = trace_pid_list_next(filtered_pids, pid + 1, &pid);
755 			nr_pids++;
756 		}
757 	}
758 
759 	ret = 0;
760 	while (cnt > 0) {
761 
762 		pos = 0;
763 
764 		ret = trace_get_user(&parser, ubuf, cnt, &pos);
765 		if (ret < 0)
766 			break;
767 
768 		read += ret;
769 		ubuf += ret;
770 		cnt -= ret;
771 
772 		if (!trace_parser_loaded(&parser))
773 			break;
774 
775 		ret = -EINVAL;
776 		if (kstrtoul(parser.buffer, 0, &val))
777 			break;
778 
779 		pid = (pid_t)val;
780 
781 		if (trace_pid_list_set(pid_list, pid) < 0) {
782 			ret = -1;
783 			break;
784 		}
785 		nr_pids++;
786 
787 		trace_parser_clear(&parser);
788 		ret = 0;
789 	}
790 	trace_parser_put(&parser);
791 
792 	if (ret < 0) {
793 		trace_pid_list_free(pid_list);
794 		return ret;
795 	}
796 
797 	if (!nr_pids) {
798 		/* Cleared the list of pids */
799 		trace_pid_list_free(pid_list);
800 		pid_list = NULL;
801 	}
802 
803 	*new_pid_list = pid_list;
804 
805 	return read;
806 }
807 
buffer_ftrace_now(struct array_buffer * buf,int cpu)808 static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
809 {
810 	u64 ts;
811 
812 	/* Early boot up does not have a buffer yet */
813 	if (!buf->buffer)
814 		return trace_clock_local();
815 
816 	ts = ring_buffer_time_stamp(buf->buffer);
817 	ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
818 
819 	return ts;
820 }
821 
ftrace_now(int cpu)822 u64 ftrace_now(int cpu)
823 {
824 	return buffer_ftrace_now(&global_trace.array_buffer, cpu);
825 }
826 
827 /**
828  * tracing_is_enabled - Show if global_trace has been enabled
829  *
830  * Shows if the global trace has been enabled or not. It uses the
831  * mirror flag "buffer_disabled" to be used in fast paths such as for
832  * the irqsoff tracer. But it may be inaccurate due to races. If you
833  * need to know the accurate state, use tracing_is_on() which is a little
834  * slower, but accurate.
835  */
tracing_is_enabled(void)836 int tracing_is_enabled(void)
837 {
838 	/*
839 	 * For quick access (irqsoff uses this in fast path), just
840 	 * return the mirror variable of the state of the ring buffer.
841 	 * It's a little racy, but we don't really care.
842 	 */
843 	smp_rmb();
844 	return !global_trace.buffer_disabled;
845 }
846 
847 /*
848  * trace_buf_size is the size in bytes that is allocated
849  * for a buffer. Note, the number of bytes is always rounded
850  * to page size.
851  *
852  * This number is purposely set to a low number of 16384.
853  * If the dump on oops happens, it will be much appreciated
854  * to not have to wait for all that output. Anyway this can be
855  * boot time and run time configurable.
856  */
857 #define TRACE_BUF_SIZE_DEFAULT	1441792UL /* 16384 * 88 (sizeof(entry)) */
858 
859 static unsigned long		trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
860 
861 /* trace_types holds a link list of available tracers. */
862 static struct tracer		*trace_types __read_mostly;
863 
864 /*
865  * trace_types_lock is used to protect the trace_types list.
866  */
867 DEFINE_MUTEX(trace_types_lock);
868 
869 /*
870  * serialize the access of the ring buffer
871  *
872  * ring buffer serializes readers, but it is low level protection.
873  * The validity of the events (which returns by ring_buffer_peek() ..etc)
874  * are not protected by ring buffer.
875  *
876  * The content of events may become garbage if we allow other process consumes
877  * these events concurrently:
878  *   A) the page of the consumed events may become a normal page
879  *      (not reader page) in ring buffer, and this page will be rewritten
880  *      by events producer.
881  *   B) The page of the consumed events may become a page for splice_read,
882  *      and this page will be returned to system.
883  *
884  * These primitives allow multi process access to different cpu ring buffer
885  * concurrently.
886  *
887  * These primitives don't distinguish read-only and read-consume access.
888  * Multi read-only access are also serialized.
889  */
890 
891 #ifdef CONFIG_SMP
892 static DECLARE_RWSEM(all_cpu_access_lock);
893 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
894 
trace_access_lock(int cpu)895 static inline void trace_access_lock(int cpu)
896 {
897 	if (cpu == RING_BUFFER_ALL_CPUS) {
898 		/* gain it for accessing the whole ring buffer. */
899 		down_write(&all_cpu_access_lock);
900 	} else {
901 		/* gain it for accessing a cpu ring buffer. */
902 
903 		/* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
904 		down_read(&all_cpu_access_lock);
905 
906 		/* Secondly block other access to this @cpu ring buffer. */
907 		mutex_lock(&per_cpu(cpu_access_lock, cpu));
908 	}
909 }
910 
trace_access_unlock(int cpu)911 static inline void trace_access_unlock(int cpu)
912 {
913 	if (cpu == RING_BUFFER_ALL_CPUS) {
914 		up_write(&all_cpu_access_lock);
915 	} else {
916 		mutex_unlock(&per_cpu(cpu_access_lock, cpu));
917 		up_read(&all_cpu_access_lock);
918 	}
919 }
920 
trace_access_lock_init(void)921 static inline void trace_access_lock_init(void)
922 {
923 	int cpu;
924 
925 	for_each_possible_cpu(cpu)
926 		mutex_init(&per_cpu(cpu_access_lock, cpu));
927 }
928 
929 #else
930 
931 static DEFINE_MUTEX(access_lock);
932 
trace_access_lock(int cpu)933 static inline void trace_access_lock(int cpu)
934 {
935 	(void)cpu;
936 	mutex_lock(&access_lock);
937 }
938 
trace_access_unlock(int cpu)939 static inline void trace_access_unlock(int cpu)
940 {
941 	(void)cpu;
942 	mutex_unlock(&access_lock);
943 }
944 
trace_access_lock_init(void)945 static inline void trace_access_lock_init(void)
946 {
947 }
948 
949 #endif
950 
951 #ifdef CONFIG_STACKTRACE
952 static void __ftrace_trace_stack(struct trace_buffer *buffer,
953 				 unsigned int trace_ctx,
954 				 int skip, struct pt_regs *regs);
955 static inline void ftrace_trace_stack(struct trace_array *tr,
956 				      struct trace_buffer *buffer,
957 				      unsigned int trace_ctx,
958 				      int skip, struct pt_regs *regs);
959 
960 #else
__ftrace_trace_stack(struct trace_buffer * buffer,unsigned int trace_ctx,int skip,struct pt_regs * regs)961 static inline void __ftrace_trace_stack(struct trace_buffer *buffer,
962 					unsigned int trace_ctx,
963 					int skip, struct pt_regs *regs)
964 {
965 }
ftrace_trace_stack(struct trace_array * tr,struct trace_buffer * buffer,unsigned long trace_ctx,int skip,struct pt_regs * regs)966 static inline void ftrace_trace_stack(struct trace_array *tr,
967 				      struct trace_buffer *buffer,
968 				      unsigned long trace_ctx,
969 				      int skip, struct pt_regs *regs)
970 {
971 }
972 
973 #endif
974 
975 static __always_inline void
trace_event_setup(struct ring_buffer_event * event,int type,unsigned int trace_ctx)976 trace_event_setup(struct ring_buffer_event *event,
977 		  int type, unsigned int trace_ctx)
978 {
979 	struct trace_entry *ent = ring_buffer_event_data(event);
980 
981 	tracing_generic_entry_update(ent, type, trace_ctx);
982 }
983 
984 static __always_inline struct ring_buffer_event *
__trace_buffer_lock_reserve(struct trace_buffer * buffer,int type,unsigned long len,unsigned int trace_ctx)985 __trace_buffer_lock_reserve(struct trace_buffer *buffer,
986 			  int type,
987 			  unsigned long len,
988 			  unsigned int trace_ctx)
989 {
990 	struct ring_buffer_event *event;
991 
992 	event = ring_buffer_lock_reserve(buffer, len);
993 	if (event != NULL)
994 		trace_event_setup(event, type, trace_ctx);
995 
996 	return event;
997 }
998 
tracer_tracing_on(struct trace_array * tr)999 void tracer_tracing_on(struct trace_array *tr)
1000 {
1001 	if (tr->array_buffer.buffer)
1002 		ring_buffer_record_on(tr->array_buffer.buffer);
1003 	/*
1004 	 * This flag is looked at when buffers haven't been allocated
1005 	 * yet, or by some tracers (like irqsoff), that just want to
1006 	 * know if the ring buffer has been disabled, but it can handle
1007 	 * races of where it gets disabled but we still do a record.
1008 	 * As the check is in the fast path of the tracers, it is more
1009 	 * important to be fast than accurate.
1010 	 */
1011 	tr->buffer_disabled = 0;
1012 	/* Make the flag seen by readers */
1013 	smp_wmb();
1014 }
1015 
1016 /**
1017  * tracing_on - enable tracing buffers
1018  *
1019  * This function enables tracing buffers that may have been
1020  * disabled with tracing_off.
1021  */
tracing_on(void)1022 void tracing_on(void)
1023 {
1024 	tracer_tracing_on(&global_trace);
1025 }
1026 EXPORT_SYMBOL_GPL(tracing_on);
1027 
1028 
1029 static __always_inline void
__buffer_unlock_commit(struct trace_buffer * buffer,struct ring_buffer_event * event)1030 __buffer_unlock_commit(struct trace_buffer *buffer, struct ring_buffer_event *event)
1031 {
1032 	__this_cpu_write(trace_taskinfo_save, true);
1033 
1034 	/* If this is the temp buffer, we need to commit fully */
1035 	if (this_cpu_read(trace_buffered_event) == event) {
1036 		/* Length is in event->array[0] */
1037 		ring_buffer_write(buffer, event->array[0], &event->array[1]);
1038 		/* Release the temp buffer */
1039 		this_cpu_dec(trace_buffered_event_cnt);
1040 		/* ring_buffer_unlock_commit() enables preemption */
1041 		preempt_enable_notrace();
1042 	} else
1043 		ring_buffer_unlock_commit(buffer);
1044 }
1045 
__trace_array_puts(struct trace_array * tr,unsigned long ip,const char * str,int size)1046 int __trace_array_puts(struct trace_array *tr, unsigned long ip,
1047 		       const char *str, int size)
1048 {
1049 	struct ring_buffer_event *event;
1050 	struct trace_buffer *buffer;
1051 	struct print_entry *entry;
1052 	unsigned int trace_ctx;
1053 	int alloc;
1054 
1055 	if (!(tr->trace_flags & TRACE_ITER_PRINTK))
1056 		return 0;
1057 
1058 	if (unlikely(tracing_selftest_running && tr == &global_trace))
1059 		return 0;
1060 
1061 	if (unlikely(tracing_disabled))
1062 		return 0;
1063 
1064 	alloc = sizeof(*entry) + size + 2; /* possible \n added */
1065 
1066 	trace_ctx = tracing_gen_ctx();
1067 	buffer = tr->array_buffer.buffer;
1068 	ring_buffer_nest_start(buffer);
1069 	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
1070 					    trace_ctx);
1071 	if (!event) {
1072 		size = 0;
1073 		goto out;
1074 	}
1075 
1076 	entry = ring_buffer_event_data(event);
1077 	entry->ip = ip;
1078 
1079 	memcpy(&entry->buf, str, size);
1080 
1081 	/* Add a newline if necessary */
1082 	if (entry->buf[size - 1] != '\n') {
1083 		entry->buf[size] = '\n';
1084 		entry->buf[size + 1] = '\0';
1085 	} else
1086 		entry->buf[size] = '\0';
1087 
1088 	__buffer_unlock_commit(buffer, event);
1089 	ftrace_trace_stack(tr, buffer, trace_ctx, 4, NULL);
1090  out:
1091 	ring_buffer_nest_end(buffer);
1092 	return size;
1093 }
1094 EXPORT_SYMBOL_GPL(__trace_array_puts);
1095 
1096 /**
1097  * __trace_puts - write a constant string into the trace buffer.
1098  * @ip:	   The address of the caller
1099  * @str:   The constant string to write
1100  * @size:  The size of the string.
1101  */
__trace_puts(unsigned long ip,const char * str,int size)1102 int __trace_puts(unsigned long ip, const char *str, int size)
1103 {
1104 	return __trace_array_puts(&global_trace, ip, str, size);
1105 }
1106 EXPORT_SYMBOL_GPL(__trace_puts);
1107 
1108 /**
1109  * __trace_bputs - write the pointer to a constant string into trace buffer
1110  * @ip:	   The address of the caller
1111  * @str:   The constant string to write to the buffer to
1112  */
__trace_bputs(unsigned long ip,const char * str)1113 int __trace_bputs(unsigned long ip, const char *str)
1114 {
1115 	struct ring_buffer_event *event;
1116 	struct trace_buffer *buffer;
1117 	struct bputs_entry *entry;
1118 	unsigned int trace_ctx;
1119 	int size = sizeof(struct bputs_entry);
1120 	int ret = 0;
1121 
1122 	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
1123 		return 0;
1124 
1125 	if (unlikely(tracing_selftest_running || tracing_disabled))
1126 		return 0;
1127 
1128 	trace_ctx = tracing_gen_ctx();
1129 	buffer = global_trace.array_buffer.buffer;
1130 
1131 	ring_buffer_nest_start(buffer);
1132 	event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
1133 					    trace_ctx);
1134 	if (!event)
1135 		goto out;
1136 
1137 	entry = ring_buffer_event_data(event);
1138 	entry->ip			= ip;
1139 	entry->str			= str;
1140 
1141 	__buffer_unlock_commit(buffer, event);
1142 	ftrace_trace_stack(&global_trace, buffer, trace_ctx, 4, NULL);
1143 
1144 	ret = 1;
1145  out:
1146 	ring_buffer_nest_end(buffer);
1147 	return ret;
1148 }
1149 EXPORT_SYMBOL_GPL(__trace_bputs);
1150 
1151 #ifdef CONFIG_TRACER_SNAPSHOT
tracing_snapshot_instance_cond(struct trace_array * tr,void * cond_data)1152 static void tracing_snapshot_instance_cond(struct trace_array *tr,
1153 					   void *cond_data)
1154 {
1155 	struct tracer *tracer = tr->current_trace;
1156 	unsigned long flags;
1157 
1158 	if (in_nmi()) {
1159 		trace_array_puts(tr, "*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
1160 		trace_array_puts(tr, "*** snapshot is being ignored        ***\n");
1161 		return;
1162 	}
1163 
1164 	if (!tr->allocated_snapshot) {
1165 		trace_array_puts(tr, "*** SNAPSHOT NOT ALLOCATED ***\n");
1166 		trace_array_puts(tr, "*** stopping trace here!   ***\n");
1167 		tracer_tracing_off(tr);
1168 		return;
1169 	}
1170 
1171 	/* Note, snapshot can not be used when the tracer uses it */
1172 	if (tracer->use_max_tr) {
1173 		trace_array_puts(tr, "*** LATENCY TRACER ACTIVE ***\n");
1174 		trace_array_puts(tr, "*** Can not use snapshot (sorry) ***\n");
1175 		return;
1176 	}
1177 
1178 	local_irq_save(flags);
1179 	update_max_tr(tr, current, smp_processor_id(), cond_data);
1180 	local_irq_restore(flags);
1181 }
1182 
tracing_snapshot_instance(struct trace_array * tr)1183 void tracing_snapshot_instance(struct trace_array *tr)
1184 {
1185 	tracing_snapshot_instance_cond(tr, NULL);
1186 }
1187 
1188 /**
1189  * tracing_snapshot - take a snapshot of the current buffer.
1190  *
1191  * This causes a swap between the snapshot buffer and the current live
1192  * tracing buffer. You can use this to take snapshots of the live
1193  * trace when some condition is triggered, but continue to trace.
1194  *
1195  * Note, make sure to allocate the snapshot with either
1196  * a tracing_snapshot_alloc(), or by doing it manually
1197  * with: echo 1 > /sys/kernel/tracing/snapshot
1198  *
1199  * If the snapshot buffer is not allocated, it will stop tracing.
1200  * Basically making a permanent snapshot.
1201  */
tracing_snapshot(void)1202 void tracing_snapshot(void)
1203 {
1204 	struct trace_array *tr = &global_trace;
1205 
1206 	tracing_snapshot_instance(tr);
1207 }
1208 EXPORT_SYMBOL_GPL(tracing_snapshot);
1209 
1210 /**
1211  * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
1212  * @tr:		The tracing instance to snapshot
1213  * @cond_data:	The data to be tested conditionally, and possibly saved
1214  *
1215  * This is the same as tracing_snapshot() except that the snapshot is
1216  * conditional - the snapshot will only happen if the
1217  * cond_snapshot.update() implementation receiving the cond_data
1218  * returns true, which means that the trace array's cond_snapshot
1219  * update() operation used the cond_data to determine whether the
1220  * snapshot should be taken, and if it was, presumably saved it along
1221  * with the snapshot.
1222  */
tracing_snapshot_cond(struct trace_array * tr,void * cond_data)1223 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1224 {
1225 	tracing_snapshot_instance_cond(tr, cond_data);
1226 }
1227 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1228 
1229 /**
1230  * tracing_cond_snapshot_data - get the user data associated with a snapshot
1231  * @tr:		The tracing instance
1232  *
1233  * When the user enables a conditional snapshot using
1234  * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1235  * with the snapshot.  This accessor is used to retrieve it.
1236  *
1237  * Should not be called from cond_snapshot.update(), since it takes
1238  * the tr->max_lock lock, which the code calling
1239  * cond_snapshot.update() has already done.
1240  *
1241  * Returns the cond_data associated with the trace array's snapshot.
1242  */
tracing_cond_snapshot_data(struct trace_array * tr)1243 void *tracing_cond_snapshot_data(struct trace_array *tr)
1244 {
1245 	void *cond_data = NULL;
1246 
1247 	local_irq_disable();
1248 	arch_spin_lock(&tr->max_lock);
1249 
1250 	if (tr->cond_snapshot)
1251 		cond_data = tr->cond_snapshot->cond_data;
1252 
1253 	arch_spin_unlock(&tr->max_lock);
1254 	local_irq_enable();
1255 
1256 	return cond_data;
1257 }
1258 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1259 
1260 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
1261 					struct array_buffer *size_buf, int cpu_id);
1262 static void set_buffer_entries(struct array_buffer *buf, unsigned long val);
1263 
tracing_alloc_snapshot_instance(struct trace_array * tr)1264 int tracing_alloc_snapshot_instance(struct trace_array *tr)
1265 {
1266 	int ret;
1267 
1268 	if (!tr->allocated_snapshot) {
1269 
1270 		/* allocate spare buffer */
1271 		ret = resize_buffer_duplicate_size(&tr->max_buffer,
1272 				   &tr->array_buffer, RING_BUFFER_ALL_CPUS);
1273 		if (ret < 0)
1274 			return ret;
1275 
1276 		tr->allocated_snapshot = true;
1277 	}
1278 
1279 	return 0;
1280 }
1281 
free_snapshot(struct trace_array * tr)1282 static void free_snapshot(struct trace_array *tr)
1283 {
1284 	/*
1285 	 * We don't free the ring buffer. instead, resize it because
1286 	 * The max_tr ring buffer has some state (e.g. ring->clock) and
1287 	 * we want preserve it.
1288 	 */
1289 	ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1290 	set_buffer_entries(&tr->max_buffer, 1);
1291 	tracing_reset_online_cpus(&tr->max_buffer);
1292 	tr->allocated_snapshot = false;
1293 }
1294 
1295 /**
1296  * tracing_alloc_snapshot - allocate snapshot buffer.
1297  *
1298  * This only allocates the snapshot buffer if it isn't already
1299  * allocated - it doesn't also take a snapshot.
1300  *
1301  * This is meant to be used in cases where the snapshot buffer needs
1302  * to be set up for events that can't sleep but need to be able to
1303  * trigger a snapshot.
1304  */
tracing_alloc_snapshot(void)1305 int tracing_alloc_snapshot(void)
1306 {
1307 	struct trace_array *tr = &global_trace;
1308 	int ret;
1309 
1310 	ret = tracing_alloc_snapshot_instance(tr);
1311 	WARN_ON(ret < 0);
1312 
1313 	return ret;
1314 }
1315 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1316 
1317 /**
1318  * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1319  *
1320  * This is similar to tracing_snapshot(), but it will allocate the
1321  * snapshot buffer if it isn't already allocated. Use this only
1322  * where it is safe to sleep, as the allocation may sleep.
1323  *
1324  * This causes a swap between the snapshot buffer and the current live
1325  * tracing buffer. You can use this to take snapshots of the live
1326  * trace when some condition is triggered, but continue to trace.
1327  */
tracing_snapshot_alloc(void)1328 void tracing_snapshot_alloc(void)
1329 {
1330 	int ret;
1331 
1332 	ret = tracing_alloc_snapshot();
1333 	if (ret < 0)
1334 		return;
1335 
1336 	tracing_snapshot();
1337 }
1338 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1339 
1340 /**
1341  * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1342  * @tr:		The tracing instance
1343  * @cond_data:	User data to associate with the snapshot
1344  * @update:	Implementation of the cond_snapshot update function
1345  *
1346  * Check whether the conditional snapshot for the given instance has
1347  * already been enabled, or if the current tracer is already using a
1348  * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1349  * save the cond_data and update function inside.
1350  *
1351  * Returns 0 if successful, error otherwise.
1352  */
tracing_snapshot_cond_enable(struct trace_array * tr,void * cond_data,cond_update_fn_t update)1353 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data,
1354 				 cond_update_fn_t update)
1355 {
1356 	struct cond_snapshot *cond_snapshot;
1357 	int ret = 0;
1358 
1359 	cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
1360 	if (!cond_snapshot)
1361 		return -ENOMEM;
1362 
1363 	cond_snapshot->cond_data = cond_data;
1364 	cond_snapshot->update = update;
1365 
1366 	mutex_lock(&trace_types_lock);
1367 
1368 	ret = tracing_alloc_snapshot_instance(tr);
1369 	if (ret)
1370 		goto fail_unlock;
1371 
1372 	if (tr->current_trace->use_max_tr) {
1373 		ret = -EBUSY;
1374 		goto fail_unlock;
1375 	}
1376 
1377 	/*
1378 	 * The cond_snapshot can only change to NULL without the
1379 	 * trace_types_lock. We don't care if we race with it going
1380 	 * to NULL, but we want to make sure that it's not set to
1381 	 * something other than NULL when we get here, which we can
1382 	 * do safely with only holding the trace_types_lock and not
1383 	 * having to take the max_lock.
1384 	 */
1385 	if (tr->cond_snapshot) {
1386 		ret = -EBUSY;
1387 		goto fail_unlock;
1388 	}
1389 
1390 	local_irq_disable();
1391 	arch_spin_lock(&tr->max_lock);
1392 	tr->cond_snapshot = cond_snapshot;
1393 	arch_spin_unlock(&tr->max_lock);
1394 	local_irq_enable();
1395 
1396 	mutex_unlock(&trace_types_lock);
1397 
1398 	return ret;
1399 
1400  fail_unlock:
1401 	mutex_unlock(&trace_types_lock);
1402 	kfree(cond_snapshot);
1403 	return ret;
1404 }
1405 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1406 
1407 /**
1408  * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1409  * @tr:		The tracing instance
1410  *
1411  * Check whether the conditional snapshot for the given instance is
1412  * enabled; if so, free the cond_snapshot associated with it,
1413  * otherwise return -EINVAL.
1414  *
1415  * Returns 0 if successful, error otherwise.
1416  */
tracing_snapshot_cond_disable(struct trace_array * tr)1417 int tracing_snapshot_cond_disable(struct trace_array *tr)
1418 {
1419 	int ret = 0;
1420 
1421 	local_irq_disable();
1422 	arch_spin_lock(&tr->max_lock);
1423 
1424 	if (!tr->cond_snapshot)
1425 		ret = -EINVAL;
1426 	else {
1427 		kfree(tr->cond_snapshot);
1428 		tr->cond_snapshot = NULL;
1429 	}
1430 
1431 	arch_spin_unlock(&tr->max_lock);
1432 	local_irq_enable();
1433 
1434 	return ret;
1435 }
1436 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1437 #else
tracing_snapshot(void)1438 void tracing_snapshot(void)
1439 {
1440 	WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1441 }
1442 EXPORT_SYMBOL_GPL(tracing_snapshot);
tracing_snapshot_cond(struct trace_array * tr,void * cond_data)1443 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1444 {
1445 	WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1446 }
1447 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
tracing_alloc_snapshot(void)1448 int tracing_alloc_snapshot(void)
1449 {
1450 	WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1451 	return -ENODEV;
1452 }
1453 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
tracing_snapshot_alloc(void)1454 void tracing_snapshot_alloc(void)
1455 {
1456 	/* Give warning */
1457 	tracing_snapshot();
1458 }
1459 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
tracing_cond_snapshot_data(struct trace_array * tr)1460 void *tracing_cond_snapshot_data(struct trace_array *tr)
1461 {
1462 	return NULL;
1463 }
1464 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
tracing_snapshot_cond_enable(struct trace_array * tr,void * cond_data,cond_update_fn_t update)1465 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update)
1466 {
1467 	return -ENODEV;
1468 }
1469 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
tracing_snapshot_cond_disable(struct trace_array * tr)1470 int tracing_snapshot_cond_disable(struct trace_array *tr)
1471 {
1472 	return false;
1473 }
1474 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1475 #define free_snapshot(tr)	do { } while (0)
1476 #endif /* CONFIG_TRACER_SNAPSHOT */
1477 
tracer_tracing_off(struct trace_array * tr)1478 void tracer_tracing_off(struct trace_array *tr)
1479 {
1480 	if (tr->array_buffer.buffer)
1481 		ring_buffer_record_off(tr->array_buffer.buffer);
1482 	/*
1483 	 * This flag is looked at when buffers haven't been allocated
1484 	 * yet, or by some tracers (like irqsoff), that just want to
1485 	 * know if the ring buffer has been disabled, but it can handle
1486 	 * races of where it gets disabled but we still do a record.
1487 	 * As the check is in the fast path of the tracers, it is more
1488 	 * important to be fast than accurate.
1489 	 */
1490 	tr->buffer_disabled = 1;
1491 	/* Make the flag seen by readers */
1492 	smp_wmb();
1493 }
1494 
1495 /**
1496  * tracing_off - turn off tracing buffers
1497  *
1498  * This function stops the tracing buffers from recording data.
1499  * It does not disable any overhead the tracers themselves may
1500  * be causing. This function simply causes all recording to
1501  * the ring buffers to fail.
1502  */
tracing_off(void)1503 void tracing_off(void)
1504 {
1505 	tracer_tracing_off(&global_trace);
1506 }
1507 EXPORT_SYMBOL_GPL(tracing_off);
1508 
disable_trace_on_warning(void)1509 void disable_trace_on_warning(void)
1510 {
1511 	if (__disable_trace_on_warning) {
1512 		trace_array_printk_buf(global_trace.array_buffer.buffer, _THIS_IP_,
1513 			"Disabling tracing due to warning\n");
1514 		tracing_off();
1515 	}
1516 }
1517 
1518 /**
1519  * tracer_tracing_is_on - show real state of ring buffer enabled
1520  * @tr : the trace array to know if ring buffer is enabled
1521  *
1522  * Shows real state of the ring buffer if it is enabled or not.
1523  */
tracer_tracing_is_on(struct trace_array * tr)1524 bool tracer_tracing_is_on(struct trace_array *tr)
1525 {
1526 	if (tr->array_buffer.buffer)
1527 		return ring_buffer_record_is_on(tr->array_buffer.buffer);
1528 	return !tr->buffer_disabled;
1529 }
1530 
1531 /**
1532  * tracing_is_on - show state of ring buffers enabled
1533  */
tracing_is_on(void)1534 int tracing_is_on(void)
1535 {
1536 	return tracer_tracing_is_on(&global_trace);
1537 }
1538 EXPORT_SYMBOL_GPL(tracing_is_on);
1539 
set_buf_size(char * str)1540 static int __init set_buf_size(char *str)
1541 {
1542 	unsigned long buf_size;
1543 
1544 	if (!str)
1545 		return 0;
1546 	buf_size = memparse(str, &str);
1547 	/*
1548 	 * nr_entries can not be zero and the startup
1549 	 * tests require some buffer space. Therefore
1550 	 * ensure we have at least 4096 bytes of buffer.
1551 	 */
1552 	trace_buf_size = max(4096UL, buf_size);
1553 	return 1;
1554 }
1555 __setup("trace_buf_size=", set_buf_size);
1556 
set_tracing_thresh(char * str)1557 static int __init set_tracing_thresh(char *str)
1558 {
1559 	unsigned long threshold;
1560 	int ret;
1561 
1562 	if (!str)
1563 		return 0;
1564 	ret = kstrtoul(str, 0, &threshold);
1565 	if (ret < 0)
1566 		return 0;
1567 	tracing_thresh = threshold * 1000;
1568 	return 1;
1569 }
1570 __setup("tracing_thresh=", set_tracing_thresh);
1571 
nsecs_to_usecs(unsigned long nsecs)1572 unsigned long nsecs_to_usecs(unsigned long nsecs)
1573 {
1574 	return nsecs / 1000;
1575 }
1576 
1577 /*
1578  * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1579  * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1580  * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1581  * of strings in the order that the evals (enum) were defined.
1582  */
1583 #undef C
1584 #define C(a, b) b
1585 
1586 /* These must match the bit positions in trace_iterator_flags */
1587 static const char *trace_options[] = {
1588 	TRACE_FLAGS
1589 	NULL
1590 };
1591 
1592 static struct {
1593 	u64 (*func)(void);
1594 	const char *name;
1595 	int in_ns;		/* is this clock in nanoseconds? */
1596 } trace_clocks[] = {
1597 	{ trace_clock_local,		"local",	1 },
1598 	{ trace_clock_global,		"global",	1 },
1599 	{ trace_clock_counter,		"counter",	0 },
1600 	{ trace_clock_jiffies,		"uptime",	0 },
1601 	{ trace_clock,			"perf",		1 },
1602 	{ ktime_get_mono_fast_ns,	"mono",		1 },
1603 	{ ktime_get_raw_fast_ns,	"mono_raw",	1 },
1604 	{ ktime_get_boot_fast_ns,	"boot",		1 },
1605 	{ ktime_get_tai_fast_ns,	"tai",		1 },
1606 	ARCH_TRACE_CLOCKS
1607 };
1608 
trace_clock_in_ns(struct trace_array * tr)1609 bool trace_clock_in_ns(struct trace_array *tr)
1610 {
1611 	if (trace_clocks[tr->clock_id].in_ns)
1612 		return true;
1613 
1614 	return false;
1615 }
1616 
1617 /*
1618  * trace_parser_get_init - gets the buffer for trace parser
1619  */
trace_parser_get_init(struct trace_parser * parser,int size)1620 int trace_parser_get_init(struct trace_parser *parser, int size)
1621 {
1622 	memset(parser, 0, sizeof(*parser));
1623 
1624 	parser->buffer = kmalloc(size, GFP_KERNEL);
1625 	if (!parser->buffer)
1626 		return 1;
1627 
1628 	parser->size = size;
1629 	return 0;
1630 }
1631 
1632 /*
1633  * trace_parser_put - frees the buffer for trace parser
1634  */
trace_parser_put(struct trace_parser * parser)1635 void trace_parser_put(struct trace_parser *parser)
1636 {
1637 	kfree(parser->buffer);
1638 	parser->buffer = NULL;
1639 }
1640 
1641 /*
1642  * trace_get_user - reads the user input string separated by  space
1643  * (matched by isspace(ch))
1644  *
1645  * For each string found the 'struct trace_parser' is updated,
1646  * and the function returns.
1647  *
1648  * Returns number of bytes read.
1649  *
1650  * See kernel/trace/trace.h for 'struct trace_parser' details.
1651  */
trace_get_user(struct trace_parser * parser,const char __user * ubuf,size_t cnt,loff_t * ppos)1652 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1653 	size_t cnt, loff_t *ppos)
1654 {
1655 	char ch;
1656 	size_t read = 0;
1657 	ssize_t ret;
1658 
1659 	if (!*ppos)
1660 		trace_parser_clear(parser);
1661 
1662 	ret = get_user(ch, ubuf++);
1663 	if (ret)
1664 		goto out;
1665 
1666 	read++;
1667 	cnt--;
1668 
1669 	/*
1670 	 * The parser is not finished with the last write,
1671 	 * continue reading the user input without skipping spaces.
1672 	 */
1673 	if (!parser->cont) {
1674 		/* skip white space */
1675 		while (cnt && isspace(ch)) {
1676 			ret = get_user(ch, ubuf++);
1677 			if (ret)
1678 				goto out;
1679 			read++;
1680 			cnt--;
1681 		}
1682 
1683 		parser->idx = 0;
1684 
1685 		/* only spaces were written */
1686 		if (isspace(ch) || !ch) {
1687 			*ppos += read;
1688 			ret = read;
1689 			goto out;
1690 		}
1691 	}
1692 
1693 	/* read the non-space input */
1694 	while (cnt && !isspace(ch) && ch) {
1695 		if (parser->idx < parser->size - 1)
1696 			parser->buffer[parser->idx++] = ch;
1697 		else {
1698 			ret = -EINVAL;
1699 			goto out;
1700 		}
1701 		ret = get_user(ch, ubuf++);
1702 		if (ret)
1703 			goto out;
1704 		read++;
1705 		cnt--;
1706 	}
1707 
1708 	/* We either got finished input or we have to wait for another call. */
1709 	if (isspace(ch) || !ch) {
1710 		parser->buffer[parser->idx] = 0;
1711 		parser->cont = false;
1712 	} else if (parser->idx < parser->size - 1) {
1713 		parser->cont = true;
1714 		parser->buffer[parser->idx++] = ch;
1715 		/* Make sure the parsed string always terminates with '\0'. */
1716 		parser->buffer[parser->idx] = 0;
1717 	} else {
1718 		ret = -EINVAL;
1719 		goto out;
1720 	}
1721 
1722 	*ppos += read;
1723 	ret = read;
1724 
1725 out:
1726 	return ret;
1727 }
1728 
1729 /* TODO add a seq_buf_to_buffer() */
trace_seq_to_buffer(struct trace_seq * s,void * buf,size_t cnt)1730 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1731 {
1732 	int len;
1733 
1734 	if (trace_seq_used(s) <= s->seq.readpos)
1735 		return -EBUSY;
1736 
1737 	len = trace_seq_used(s) - s->seq.readpos;
1738 	if (cnt > len)
1739 		cnt = len;
1740 	memcpy(buf, s->buffer + s->seq.readpos, cnt);
1741 
1742 	s->seq.readpos += cnt;
1743 	return cnt;
1744 }
1745 
1746 unsigned long __read_mostly	tracing_thresh;
1747 
1748 #ifdef CONFIG_TRACER_MAX_TRACE
1749 static const struct file_operations tracing_max_lat_fops;
1750 
1751 #ifdef LATENCY_FS_NOTIFY
1752 
1753 static struct workqueue_struct *fsnotify_wq;
1754 
latency_fsnotify_workfn(struct work_struct * work)1755 static void latency_fsnotify_workfn(struct work_struct *work)
1756 {
1757 	struct trace_array *tr = container_of(work, struct trace_array,
1758 					      fsnotify_work);
1759 	fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY);
1760 }
1761 
latency_fsnotify_workfn_irq(struct irq_work * iwork)1762 static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
1763 {
1764 	struct trace_array *tr = container_of(iwork, struct trace_array,
1765 					      fsnotify_irqwork);
1766 	queue_work(fsnotify_wq, &tr->fsnotify_work);
1767 }
1768 
trace_create_maxlat_file(struct trace_array * tr,struct dentry * d_tracer)1769 static void trace_create_maxlat_file(struct trace_array *tr,
1770 				     struct dentry *d_tracer)
1771 {
1772 	INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
1773 	init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
1774 	tr->d_max_latency = trace_create_file("tracing_max_latency",
1775 					      TRACE_MODE_WRITE,
1776 					      d_tracer, tr,
1777 					      &tracing_max_lat_fops);
1778 }
1779 
latency_fsnotify_init(void)1780 __init static int latency_fsnotify_init(void)
1781 {
1782 	fsnotify_wq = alloc_workqueue("tr_max_lat_wq",
1783 				      WQ_UNBOUND | WQ_HIGHPRI, 0);
1784 	if (!fsnotify_wq) {
1785 		pr_err("Unable to allocate tr_max_lat_wq\n");
1786 		return -ENOMEM;
1787 	}
1788 	return 0;
1789 }
1790 
1791 late_initcall_sync(latency_fsnotify_init);
1792 
latency_fsnotify(struct trace_array * tr)1793 void latency_fsnotify(struct trace_array *tr)
1794 {
1795 	if (!fsnotify_wq)
1796 		return;
1797 	/*
1798 	 * We cannot call queue_work(&tr->fsnotify_work) from here because it's
1799 	 * possible that we are called from __schedule() or do_idle(), which
1800 	 * could cause a deadlock.
1801 	 */
1802 	irq_work_queue(&tr->fsnotify_irqwork);
1803 }
1804 
1805 #else /* !LATENCY_FS_NOTIFY */
1806 
1807 #define trace_create_maxlat_file(tr, d_tracer)				\
1808 	trace_create_file("tracing_max_latency", TRACE_MODE_WRITE,	\
1809 			  d_tracer, tr, &tracing_max_lat_fops)
1810 
1811 #endif
1812 
1813 /*
1814  * Copy the new maximum trace into the separate maximum-trace
1815  * structure. (this way the maximum trace is permanently saved,
1816  * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1817  */
1818 static void
__update_max_tr(struct trace_array * tr,struct task_struct * tsk,int cpu)1819 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1820 {
1821 	struct array_buffer *trace_buf = &tr->array_buffer;
1822 	struct array_buffer *max_buf = &tr->max_buffer;
1823 	struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1824 	struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1825 
1826 	max_buf->cpu = cpu;
1827 	max_buf->time_start = data->preempt_timestamp;
1828 
1829 	max_data->saved_latency = tr->max_latency;
1830 	max_data->critical_start = data->critical_start;
1831 	max_data->critical_end = data->critical_end;
1832 
1833 	strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1834 	max_data->pid = tsk->pid;
1835 	/*
1836 	 * If tsk == current, then use current_uid(), as that does not use
1837 	 * RCU. The irq tracer can be called out of RCU scope.
1838 	 */
1839 	if (tsk == current)
1840 		max_data->uid = current_uid();
1841 	else
1842 		max_data->uid = task_uid(tsk);
1843 
1844 	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1845 	max_data->policy = tsk->policy;
1846 	max_data->rt_priority = tsk->rt_priority;
1847 
1848 	/* record this tasks comm */
1849 	tracing_record_cmdline(tsk);
1850 	latency_fsnotify(tr);
1851 }
1852 
1853 /**
1854  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1855  * @tr: tracer
1856  * @tsk: the task with the latency
1857  * @cpu: The cpu that initiated the trace.
1858  * @cond_data: User data associated with a conditional snapshot
1859  *
1860  * Flip the buffers between the @tr and the max_tr and record information
1861  * about which task was the cause of this latency.
1862  */
1863 void
update_max_tr(struct trace_array * tr,struct task_struct * tsk,int cpu,void * cond_data)1864 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
1865 	      void *cond_data)
1866 {
1867 	if (tr->stop_count)
1868 		return;
1869 
1870 	WARN_ON_ONCE(!irqs_disabled());
1871 
1872 	if (!tr->allocated_snapshot) {
1873 		/* Only the nop tracer should hit this when disabling */
1874 		WARN_ON_ONCE(tr->current_trace != &nop_trace);
1875 		return;
1876 	}
1877 
1878 	arch_spin_lock(&tr->max_lock);
1879 
1880 	/* Inherit the recordable setting from array_buffer */
1881 	if (ring_buffer_record_is_set_on(tr->array_buffer.buffer))
1882 		ring_buffer_record_on(tr->max_buffer.buffer);
1883 	else
1884 		ring_buffer_record_off(tr->max_buffer.buffer);
1885 
1886 #ifdef CONFIG_TRACER_SNAPSHOT
1887 	if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) {
1888 		arch_spin_unlock(&tr->max_lock);
1889 		return;
1890 	}
1891 #endif
1892 	swap(tr->array_buffer.buffer, tr->max_buffer.buffer);
1893 
1894 	__update_max_tr(tr, tsk, cpu);
1895 
1896 	arch_spin_unlock(&tr->max_lock);
1897 
1898 	/* Any waiters on the old snapshot buffer need to wake up */
1899 	ring_buffer_wake_waiters(tr->array_buffer.buffer, RING_BUFFER_ALL_CPUS);
1900 }
1901 
1902 /**
1903  * update_max_tr_single - only copy one trace over, and reset the rest
1904  * @tr: tracer
1905  * @tsk: task with the latency
1906  * @cpu: the cpu of the buffer to copy.
1907  *
1908  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1909  */
1910 void
update_max_tr_single(struct trace_array * tr,struct task_struct * tsk,int cpu)1911 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1912 {
1913 	int ret;
1914 
1915 	if (tr->stop_count)
1916 		return;
1917 
1918 	WARN_ON_ONCE(!irqs_disabled());
1919 	if (!tr->allocated_snapshot) {
1920 		/* Only the nop tracer should hit this when disabling */
1921 		WARN_ON_ONCE(tr->current_trace != &nop_trace);
1922 		return;
1923 	}
1924 
1925 	arch_spin_lock(&tr->max_lock);
1926 
1927 	ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu);
1928 
1929 	if (ret == -EBUSY) {
1930 		/*
1931 		 * We failed to swap the buffer due to a commit taking
1932 		 * place on this CPU. We fail to record, but we reset
1933 		 * the max trace buffer (no one writes directly to it)
1934 		 * and flag that it failed.
1935 		 * Another reason is resize is in progress.
1936 		 */
1937 		trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1938 			"Failed to swap buffers due to commit or resize in progress\n");
1939 	}
1940 
1941 	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1942 
1943 	__update_max_tr(tr, tsk, cpu);
1944 	arch_spin_unlock(&tr->max_lock);
1945 }
1946 
1947 #endif /* CONFIG_TRACER_MAX_TRACE */
1948 
wait_on_pipe(struct trace_iterator * iter,int full)1949 static int wait_on_pipe(struct trace_iterator *iter, int full)
1950 {
1951 	int ret;
1952 
1953 	/* Iterators are static, they should be filled or empty */
1954 	if (trace_buffer_iter(iter, iter->cpu_file))
1955 		return 0;
1956 
1957 	ret = ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file, full);
1958 
1959 #ifdef CONFIG_TRACER_MAX_TRACE
1960 	/*
1961 	 * Make sure this is still the snapshot buffer, as if a snapshot were
1962 	 * to happen, this would now be the main buffer.
1963 	 */
1964 	if (iter->snapshot)
1965 		iter->array_buffer = &iter->tr->max_buffer;
1966 #endif
1967 	return ret;
1968 }
1969 
1970 #ifdef CONFIG_FTRACE_STARTUP_TEST
1971 static bool selftests_can_run;
1972 
1973 struct trace_selftests {
1974 	struct list_head		list;
1975 	struct tracer			*type;
1976 };
1977 
1978 static LIST_HEAD(postponed_selftests);
1979 
save_selftest(struct tracer * type)1980 static int save_selftest(struct tracer *type)
1981 {
1982 	struct trace_selftests *selftest;
1983 
1984 	selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1985 	if (!selftest)
1986 		return -ENOMEM;
1987 
1988 	selftest->type = type;
1989 	list_add(&selftest->list, &postponed_selftests);
1990 	return 0;
1991 }
1992 
run_tracer_selftest(struct tracer * type)1993 static int run_tracer_selftest(struct tracer *type)
1994 {
1995 	struct trace_array *tr = &global_trace;
1996 	struct tracer *saved_tracer = tr->current_trace;
1997 	int ret;
1998 
1999 	if (!type->selftest || tracing_selftest_disabled)
2000 		return 0;
2001 
2002 	/*
2003 	 * If a tracer registers early in boot up (before scheduling is
2004 	 * initialized and such), then do not run its selftests yet.
2005 	 * Instead, run it a little later in the boot process.
2006 	 */
2007 	if (!selftests_can_run)
2008 		return save_selftest(type);
2009 
2010 	if (!tracing_is_on()) {
2011 		pr_warn("Selftest for tracer %s skipped due to tracing disabled\n",
2012 			type->name);
2013 		return 0;
2014 	}
2015 
2016 	/*
2017 	 * Run a selftest on this tracer.
2018 	 * Here we reset the trace buffer, and set the current
2019 	 * tracer to be this tracer. The tracer can then run some
2020 	 * internal tracing to verify that everything is in order.
2021 	 * If we fail, we do not register this tracer.
2022 	 */
2023 	tracing_reset_online_cpus(&tr->array_buffer);
2024 
2025 	tr->current_trace = type;
2026 
2027 #ifdef CONFIG_TRACER_MAX_TRACE
2028 	if (type->use_max_tr) {
2029 		/* If we expanded the buffers, make sure the max is expanded too */
2030 		if (ring_buffer_expanded)
2031 			ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
2032 					   RING_BUFFER_ALL_CPUS);
2033 		tr->allocated_snapshot = true;
2034 	}
2035 #endif
2036 
2037 	/* the test is responsible for initializing and enabling */
2038 	pr_info("Testing tracer %s: ", type->name);
2039 	ret = type->selftest(type, tr);
2040 	/* the test is responsible for resetting too */
2041 	tr->current_trace = saved_tracer;
2042 	if (ret) {
2043 		printk(KERN_CONT "FAILED!\n");
2044 		/* Add the warning after printing 'FAILED' */
2045 		WARN_ON(1);
2046 		return -1;
2047 	}
2048 	/* Only reset on passing, to avoid touching corrupted buffers */
2049 	tracing_reset_online_cpus(&tr->array_buffer);
2050 
2051 #ifdef CONFIG_TRACER_MAX_TRACE
2052 	if (type->use_max_tr) {
2053 		tr->allocated_snapshot = false;
2054 
2055 		/* Shrink the max buffer again */
2056 		if (ring_buffer_expanded)
2057 			ring_buffer_resize(tr->max_buffer.buffer, 1,
2058 					   RING_BUFFER_ALL_CPUS);
2059 	}
2060 #endif
2061 
2062 	printk(KERN_CONT "PASSED\n");
2063 	return 0;
2064 }
2065 
do_run_tracer_selftest(struct tracer * type)2066 static int do_run_tracer_selftest(struct tracer *type)
2067 {
2068 	int ret;
2069 
2070 	/*
2071 	 * Tests can take a long time, especially if they are run one after the
2072 	 * other, as does happen during bootup when all the tracers are
2073 	 * registered. This could cause the soft lockup watchdog to trigger.
2074 	 */
2075 	cond_resched();
2076 
2077 	tracing_selftest_running = true;
2078 	ret = run_tracer_selftest(type);
2079 	tracing_selftest_running = false;
2080 
2081 	return ret;
2082 }
2083 
init_trace_selftests(void)2084 static __init int init_trace_selftests(void)
2085 {
2086 	struct trace_selftests *p, *n;
2087 	struct tracer *t, **last;
2088 	int ret;
2089 
2090 	selftests_can_run = true;
2091 
2092 	mutex_lock(&trace_types_lock);
2093 
2094 	if (list_empty(&postponed_selftests))
2095 		goto out;
2096 
2097 	pr_info("Running postponed tracer tests:\n");
2098 
2099 	tracing_selftest_running = true;
2100 	list_for_each_entry_safe(p, n, &postponed_selftests, list) {
2101 		/* This loop can take minutes when sanitizers are enabled, so
2102 		 * lets make sure we allow RCU processing.
2103 		 */
2104 		cond_resched();
2105 		ret = run_tracer_selftest(p->type);
2106 		/* If the test fails, then warn and remove from available_tracers */
2107 		if (ret < 0) {
2108 			WARN(1, "tracer: %s failed selftest, disabling\n",
2109 			     p->type->name);
2110 			last = &trace_types;
2111 			for (t = trace_types; t; t = t->next) {
2112 				if (t == p->type) {
2113 					*last = t->next;
2114 					break;
2115 				}
2116 				last = &t->next;
2117 			}
2118 		}
2119 		list_del(&p->list);
2120 		kfree(p);
2121 	}
2122 	tracing_selftest_running = false;
2123 
2124  out:
2125 	mutex_unlock(&trace_types_lock);
2126 
2127 	return 0;
2128 }
2129 core_initcall(init_trace_selftests);
2130 #else
run_tracer_selftest(struct tracer * type)2131 static inline int run_tracer_selftest(struct tracer *type)
2132 {
2133 	return 0;
2134 }
do_run_tracer_selftest(struct tracer * type)2135 static inline int do_run_tracer_selftest(struct tracer *type)
2136 {
2137 	return 0;
2138 }
2139 #endif /* CONFIG_FTRACE_STARTUP_TEST */
2140 
2141 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
2142 
2143 static void __init apply_trace_boot_options(void);
2144 
2145 /**
2146  * register_tracer - register a tracer with the ftrace system.
2147  * @type: the plugin for the tracer
2148  *
2149  * Register a new plugin tracer.
2150  */
register_tracer(struct tracer * type)2151 int __init register_tracer(struct tracer *type)
2152 {
2153 	struct tracer *t;
2154 	int ret = 0;
2155 
2156 	if (!type->name) {
2157 		pr_info("Tracer must have a name\n");
2158 		return -1;
2159 	}
2160 
2161 	if (strlen(type->name) >= MAX_TRACER_SIZE) {
2162 		pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
2163 		return -1;
2164 	}
2165 
2166 	if (security_locked_down(LOCKDOWN_TRACEFS)) {
2167 		pr_warn("Can not register tracer %s due to lockdown\n",
2168 			   type->name);
2169 		return -EPERM;
2170 	}
2171 
2172 	mutex_lock(&trace_types_lock);
2173 
2174 	for (t = trace_types; t; t = t->next) {
2175 		if (strcmp(type->name, t->name) == 0) {
2176 			/* already found */
2177 			pr_info("Tracer %s already registered\n",
2178 				type->name);
2179 			ret = -1;
2180 			goto out;
2181 		}
2182 	}
2183 
2184 	if (!type->set_flag)
2185 		type->set_flag = &dummy_set_flag;
2186 	if (!type->flags) {
2187 		/*allocate a dummy tracer_flags*/
2188 		type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
2189 		if (!type->flags) {
2190 			ret = -ENOMEM;
2191 			goto out;
2192 		}
2193 		type->flags->val = 0;
2194 		type->flags->opts = dummy_tracer_opt;
2195 	} else
2196 		if (!type->flags->opts)
2197 			type->flags->opts = dummy_tracer_opt;
2198 
2199 	/* store the tracer for __set_tracer_option */
2200 	type->flags->trace = type;
2201 
2202 	ret = do_run_tracer_selftest(type);
2203 	if (ret < 0)
2204 		goto out;
2205 
2206 	type->next = trace_types;
2207 	trace_types = type;
2208 	add_tracer_options(&global_trace, type);
2209 
2210  out:
2211 	mutex_unlock(&trace_types_lock);
2212 
2213 	if (ret || !default_bootup_tracer)
2214 		goto out_unlock;
2215 
2216 	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
2217 		goto out_unlock;
2218 
2219 	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
2220 	/* Do we want this tracer to start on bootup? */
2221 	tracing_set_tracer(&global_trace, type->name);
2222 	default_bootup_tracer = NULL;
2223 
2224 	apply_trace_boot_options();
2225 
2226 	/* disable other selftests, since this will break it. */
2227 	disable_tracing_selftest("running a tracer");
2228 
2229  out_unlock:
2230 	return ret;
2231 }
2232 
tracing_reset_cpu(struct array_buffer * buf,int cpu)2233 static void tracing_reset_cpu(struct array_buffer *buf, int cpu)
2234 {
2235 	struct trace_buffer *buffer = buf->buffer;
2236 
2237 	if (!buffer)
2238 		return;
2239 
2240 	ring_buffer_record_disable(buffer);
2241 
2242 	/* Make sure all commits have finished */
2243 	synchronize_rcu();
2244 	ring_buffer_reset_cpu(buffer, cpu);
2245 
2246 	ring_buffer_record_enable(buffer);
2247 }
2248 
tracing_reset_online_cpus(struct array_buffer * buf)2249 void tracing_reset_online_cpus(struct array_buffer *buf)
2250 {
2251 	struct trace_buffer *buffer = buf->buffer;
2252 
2253 	if (!buffer)
2254 		return;
2255 
2256 	ring_buffer_record_disable(buffer);
2257 
2258 	/* Make sure all commits have finished */
2259 	synchronize_rcu();
2260 
2261 	buf->time_start = buffer_ftrace_now(buf, buf->cpu);
2262 
2263 	ring_buffer_reset_online_cpus(buffer);
2264 
2265 	ring_buffer_record_enable(buffer);
2266 }
2267 
2268 /* Must have trace_types_lock held */
tracing_reset_all_online_cpus_unlocked(void)2269 void tracing_reset_all_online_cpus_unlocked(void)
2270 {
2271 	struct trace_array *tr;
2272 
2273 	lockdep_assert_held(&trace_types_lock);
2274 
2275 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
2276 		if (!tr->clear_trace)
2277 			continue;
2278 		tr->clear_trace = false;
2279 		tracing_reset_online_cpus(&tr->array_buffer);
2280 #ifdef CONFIG_TRACER_MAX_TRACE
2281 		tracing_reset_online_cpus(&tr->max_buffer);
2282 #endif
2283 	}
2284 }
2285 
tracing_reset_all_online_cpus(void)2286 void tracing_reset_all_online_cpus(void)
2287 {
2288 	mutex_lock(&trace_types_lock);
2289 	tracing_reset_all_online_cpus_unlocked();
2290 	mutex_unlock(&trace_types_lock);
2291 }
2292 
2293 /*
2294  * The tgid_map array maps from pid to tgid; i.e. the value stored at index i
2295  * is the tgid last observed corresponding to pid=i.
2296  */
2297 static int *tgid_map;
2298 
2299 /* The maximum valid index into tgid_map. */
2300 static size_t tgid_map_max;
2301 
2302 #define SAVED_CMDLINES_DEFAULT 128
2303 #define NO_CMDLINE_MAP UINT_MAX
2304 /*
2305  * Preemption must be disabled before acquiring trace_cmdline_lock.
2306  * The various trace_arrays' max_lock must be acquired in a context
2307  * where interrupt is disabled.
2308  */
2309 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
2310 struct saved_cmdlines_buffer {
2311 	unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
2312 	unsigned *map_cmdline_to_pid;
2313 	unsigned cmdline_num;
2314 	int cmdline_idx;
2315 	char saved_cmdlines[];
2316 };
2317 static struct saved_cmdlines_buffer *savedcmd;
2318 
get_saved_cmdlines(int idx)2319 static inline char *get_saved_cmdlines(int idx)
2320 {
2321 	return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
2322 }
2323 
set_cmdline(int idx,const char * cmdline)2324 static inline void set_cmdline(int idx, const char *cmdline)
2325 {
2326 	strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
2327 }
2328 
free_saved_cmdlines_buffer(struct saved_cmdlines_buffer * s)2329 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
2330 {
2331 	int order = get_order(sizeof(*s) + s->cmdline_num * TASK_COMM_LEN);
2332 
2333 	kfree(s->map_cmdline_to_pid);
2334 	kmemleak_free(s);
2335 	free_pages((unsigned long)s, order);
2336 }
2337 
allocate_cmdlines_buffer(unsigned int val)2338 static struct saved_cmdlines_buffer *allocate_cmdlines_buffer(unsigned int val)
2339 {
2340 	struct saved_cmdlines_buffer *s;
2341 	struct page *page;
2342 	int orig_size, size;
2343 	int order;
2344 
2345 	/* Figure out how much is needed to hold the given number of cmdlines */
2346 	orig_size = sizeof(*s) + val * TASK_COMM_LEN;
2347 	order = get_order(orig_size);
2348 	size = 1 << (order + PAGE_SHIFT);
2349 	page = alloc_pages(GFP_KERNEL, order);
2350 	if (!page)
2351 		return NULL;
2352 
2353 	s = page_address(page);
2354 	kmemleak_alloc(s, size, 1, GFP_KERNEL);
2355 	memset(s, 0, sizeof(*s));
2356 
2357 	/* Round up to actual allocation */
2358 	val = (size - sizeof(*s)) / TASK_COMM_LEN;
2359 	s->cmdline_num = val;
2360 
2361 	s->map_cmdline_to_pid = kmalloc_array(val,
2362 					      sizeof(*s->map_cmdline_to_pid),
2363 					      GFP_KERNEL);
2364 	if (!s->map_cmdline_to_pid) {
2365 		free_saved_cmdlines_buffer(s);
2366 		return NULL;
2367 	}
2368 
2369 	s->cmdline_idx = 0;
2370 	memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
2371 	       sizeof(s->map_pid_to_cmdline));
2372 	memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
2373 	       val * sizeof(*s->map_cmdline_to_pid));
2374 
2375 	return s;
2376 }
2377 
trace_create_savedcmd(void)2378 static int trace_create_savedcmd(void)
2379 {
2380 	savedcmd = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT);
2381 
2382 	return savedcmd ? 0 : -ENOMEM;
2383 }
2384 
is_tracing_stopped(void)2385 int is_tracing_stopped(void)
2386 {
2387 	return global_trace.stop_count;
2388 }
2389 
tracing_start_tr(struct trace_array * tr)2390 static void tracing_start_tr(struct trace_array *tr)
2391 {
2392 	struct trace_buffer *buffer;
2393 	unsigned long flags;
2394 
2395 	if (tracing_disabled)
2396 		return;
2397 
2398 	raw_spin_lock_irqsave(&tr->start_lock, flags);
2399 	if (--tr->stop_count) {
2400 		if (WARN_ON_ONCE(tr->stop_count < 0)) {
2401 			/* Someone screwed up their debugging */
2402 			tr->stop_count = 0;
2403 		}
2404 		goto out;
2405 	}
2406 
2407 	/* Prevent the buffers from switching */
2408 	arch_spin_lock(&tr->max_lock);
2409 
2410 	buffer = tr->array_buffer.buffer;
2411 	if (buffer)
2412 		ring_buffer_record_enable(buffer);
2413 
2414 #ifdef CONFIG_TRACER_MAX_TRACE
2415 	buffer = tr->max_buffer.buffer;
2416 	if (buffer)
2417 		ring_buffer_record_enable(buffer);
2418 #endif
2419 
2420 	arch_spin_unlock(&tr->max_lock);
2421 
2422  out:
2423 	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2424 }
2425 
2426 /**
2427  * tracing_start - quick start of the tracer
2428  *
2429  * If tracing is enabled but was stopped by tracing_stop,
2430  * this will start the tracer back up.
2431  */
tracing_start(void)2432 void tracing_start(void)
2433 
2434 {
2435 	return tracing_start_tr(&global_trace);
2436 }
2437 
tracing_stop_tr(struct trace_array * tr)2438 static void tracing_stop_tr(struct trace_array *tr)
2439 {
2440 	struct trace_buffer *buffer;
2441 	unsigned long flags;
2442 
2443 	raw_spin_lock_irqsave(&tr->start_lock, flags);
2444 	if (tr->stop_count++)
2445 		goto out;
2446 
2447 	/* Prevent the buffers from switching */
2448 	arch_spin_lock(&tr->max_lock);
2449 
2450 	buffer = tr->array_buffer.buffer;
2451 	if (buffer)
2452 		ring_buffer_record_disable(buffer);
2453 
2454 #ifdef CONFIG_TRACER_MAX_TRACE
2455 	buffer = tr->max_buffer.buffer;
2456 	if (buffer)
2457 		ring_buffer_record_disable(buffer);
2458 #endif
2459 
2460 	arch_spin_unlock(&tr->max_lock);
2461 
2462  out:
2463 	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2464 }
2465 
2466 /**
2467  * tracing_stop - quick stop of the tracer
2468  *
2469  * Light weight way to stop tracing. Use in conjunction with
2470  * tracing_start.
2471  */
tracing_stop(void)2472 void tracing_stop(void)
2473 {
2474 	return tracing_stop_tr(&global_trace);
2475 }
2476 
trace_save_cmdline(struct task_struct * tsk)2477 static int trace_save_cmdline(struct task_struct *tsk)
2478 {
2479 	unsigned tpid, idx;
2480 
2481 	/* treat recording of idle task as a success */
2482 	if (!tsk->pid)
2483 		return 1;
2484 
2485 	tpid = tsk->pid & (PID_MAX_DEFAULT - 1);
2486 
2487 	/*
2488 	 * It's not the end of the world if we don't get
2489 	 * the lock, but we also don't want to spin
2490 	 * nor do we want to disable interrupts,
2491 	 * so if we miss here, then better luck next time.
2492 	 *
2493 	 * This is called within the scheduler and wake up, so interrupts
2494 	 * had better been disabled and run queue lock been held.
2495 	 */
2496 	lockdep_assert_preemption_disabled();
2497 	if (!arch_spin_trylock(&trace_cmdline_lock))
2498 		return 0;
2499 
2500 	idx = savedcmd->map_pid_to_cmdline[tpid];
2501 	if (idx == NO_CMDLINE_MAP) {
2502 		idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
2503 
2504 		savedcmd->map_pid_to_cmdline[tpid] = idx;
2505 		savedcmd->cmdline_idx = idx;
2506 	}
2507 
2508 	savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
2509 	set_cmdline(idx, tsk->comm);
2510 
2511 	arch_spin_unlock(&trace_cmdline_lock);
2512 
2513 	return 1;
2514 }
2515 
__trace_find_cmdline(int pid,char comm[])2516 static void __trace_find_cmdline(int pid, char comm[])
2517 {
2518 	unsigned map;
2519 	int tpid;
2520 
2521 	if (!pid) {
2522 		strcpy(comm, "<idle>");
2523 		return;
2524 	}
2525 
2526 	if (WARN_ON_ONCE(pid < 0)) {
2527 		strcpy(comm, "<XXX>");
2528 		return;
2529 	}
2530 
2531 	tpid = pid & (PID_MAX_DEFAULT - 1);
2532 	map = savedcmd->map_pid_to_cmdline[tpid];
2533 	if (map != NO_CMDLINE_MAP) {
2534 		tpid = savedcmd->map_cmdline_to_pid[map];
2535 		if (tpid == pid) {
2536 			strscpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
2537 			return;
2538 		}
2539 	}
2540 	strcpy(comm, "<...>");
2541 }
2542 
trace_find_cmdline(int pid,char comm[])2543 void trace_find_cmdline(int pid, char comm[])
2544 {
2545 	preempt_disable();
2546 	arch_spin_lock(&trace_cmdline_lock);
2547 
2548 	__trace_find_cmdline(pid, comm);
2549 
2550 	arch_spin_unlock(&trace_cmdline_lock);
2551 	preempt_enable();
2552 }
2553 
trace_find_tgid_ptr(int pid)2554 static int *trace_find_tgid_ptr(int pid)
2555 {
2556 	/*
2557 	 * Pairs with the smp_store_release in set_tracer_flag() to ensure that
2558 	 * if we observe a non-NULL tgid_map then we also observe the correct
2559 	 * tgid_map_max.
2560 	 */
2561 	int *map = smp_load_acquire(&tgid_map);
2562 
2563 	if (unlikely(!map || pid > tgid_map_max))
2564 		return NULL;
2565 
2566 	return &map[pid];
2567 }
2568 
trace_find_tgid(int pid)2569 int trace_find_tgid(int pid)
2570 {
2571 	int *ptr = trace_find_tgid_ptr(pid);
2572 
2573 	return ptr ? *ptr : 0;
2574 }
2575 
trace_save_tgid(struct task_struct * tsk)2576 static int trace_save_tgid(struct task_struct *tsk)
2577 {
2578 	int *ptr;
2579 
2580 	/* treat recording of idle task as a success */
2581 	if (!tsk->pid)
2582 		return 1;
2583 
2584 	ptr = trace_find_tgid_ptr(tsk->pid);
2585 	if (!ptr)
2586 		return 0;
2587 
2588 	*ptr = tsk->tgid;
2589 	return 1;
2590 }
2591 
tracing_record_taskinfo_skip(int flags)2592 static bool tracing_record_taskinfo_skip(int flags)
2593 {
2594 	if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2595 		return true;
2596 	if (!__this_cpu_read(trace_taskinfo_save))
2597 		return true;
2598 	return false;
2599 }
2600 
2601 /**
2602  * tracing_record_taskinfo - record the task info of a task
2603  *
2604  * @task:  task to record
2605  * @flags: TRACE_RECORD_CMDLINE for recording comm
2606  *         TRACE_RECORD_TGID for recording tgid
2607  */
tracing_record_taskinfo(struct task_struct * task,int flags)2608 void tracing_record_taskinfo(struct task_struct *task, int flags)
2609 {
2610 	bool done;
2611 
2612 	if (tracing_record_taskinfo_skip(flags))
2613 		return;
2614 
2615 	/*
2616 	 * Record as much task information as possible. If some fail, continue
2617 	 * to try to record the others.
2618 	 */
2619 	done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2620 	done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2621 
2622 	/* If recording any information failed, retry again soon. */
2623 	if (!done)
2624 		return;
2625 
2626 	__this_cpu_write(trace_taskinfo_save, false);
2627 }
2628 
2629 /**
2630  * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2631  *
2632  * @prev: previous task during sched_switch
2633  * @next: next task during sched_switch
2634  * @flags: TRACE_RECORD_CMDLINE for recording comm
2635  *         TRACE_RECORD_TGID for recording tgid
2636  */
tracing_record_taskinfo_sched_switch(struct task_struct * prev,struct task_struct * next,int flags)2637 void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2638 					  struct task_struct *next, int flags)
2639 {
2640 	bool done;
2641 
2642 	if (tracing_record_taskinfo_skip(flags))
2643 		return;
2644 
2645 	/*
2646 	 * Record as much task information as possible. If some fail, continue
2647 	 * to try to record the others.
2648 	 */
2649 	done  = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2650 	done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2651 	done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2652 	done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
2653 
2654 	/* If recording any information failed, retry again soon. */
2655 	if (!done)
2656 		return;
2657 
2658 	__this_cpu_write(trace_taskinfo_save, false);
2659 }
2660 
2661 /* Helpers to record a specific task information */
tracing_record_cmdline(struct task_struct * task)2662 void tracing_record_cmdline(struct task_struct *task)
2663 {
2664 	tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2665 }
2666 
tracing_record_tgid(struct task_struct * task)2667 void tracing_record_tgid(struct task_struct *task)
2668 {
2669 	tracing_record_taskinfo(task, TRACE_RECORD_TGID);
2670 }
2671 
2672 /*
2673  * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2674  * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2675  * simplifies those functions and keeps them in sync.
2676  */
trace_handle_return(struct trace_seq * s)2677 enum print_line_t trace_handle_return(struct trace_seq *s)
2678 {
2679 	return trace_seq_has_overflowed(s) ?
2680 		TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2681 }
2682 EXPORT_SYMBOL_GPL(trace_handle_return);
2683 
migration_disable_value(void)2684 static unsigned short migration_disable_value(void)
2685 {
2686 #if defined(CONFIG_SMP)
2687 	return current->migration_disabled;
2688 #else
2689 	return 0;
2690 #endif
2691 }
2692 
tracing_gen_ctx_irq_test(unsigned int irqs_status)2693 unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status)
2694 {
2695 	unsigned int trace_flags = irqs_status;
2696 	unsigned int pc;
2697 
2698 	pc = preempt_count();
2699 
2700 	if (pc & NMI_MASK)
2701 		trace_flags |= TRACE_FLAG_NMI;
2702 	if (pc & HARDIRQ_MASK)
2703 		trace_flags |= TRACE_FLAG_HARDIRQ;
2704 	if (in_serving_softirq())
2705 		trace_flags |= TRACE_FLAG_SOFTIRQ;
2706 	if (softirq_count() >> (SOFTIRQ_SHIFT + 1))
2707 		trace_flags |= TRACE_FLAG_BH_OFF;
2708 
2709 	if (tif_need_resched())
2710 		trace_flags |= TRACE_FLAG_NEED_RESCHED;
2711 	if (test_preempt_need_resched())
2712 		trace_flags |= TRACE_FLAG_PREEMPT_RESCHED;
2713 	return (trace_flags << 16) | (min_t(unsigned int, pc & 0xff, 0xf)) |
2714 		(min_t(unsigned int, migration_disable_value(), 0xf)) << 4;
2715 }
2716 
2717 struct ring_buffer_event *
trace_buffer_lock_reserve(struct trace_buffer * buffer,int type,unsigned long len,unsigned int trace_ctx)2718 trace_buffer_lock_reserve(struct trace_buffer *buffer,
2719 			  int type,
2720 			  unsigned long len,
2721 			  unsigned int trace_ctx)
2722 {
2723 	return __trace_buffer_lock_reserve(buffer, type, len, trace_ctx);
2724 }
2725 
2726 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2727 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2728 static int trace_buffered_event_ref;
2729 
2730 /**
2731  * trace_buffered_event_enable - enable buffering events
2732  *
2733  * When events are being filtered, it is quicker to use a temporary
2734  * buffer to write the event data into if there's a likely chance
2735  * that it will not be committed. The discard of the ring buffer
2736  * is not as fast as committing, and is much slower than copying
2737  * a commit.
2738  *
2739  * When an event is to be filtered, allocate per cpu buffers to
2740  * write the event data into, and if the event is filtered and discarded
2741  * it is simply dropped, otherwise, the entire data is to be committed
2742  * in one shot.
2743  */
trace_buffered_event_enable(void)2744 void trace_buffered_event_enable(void)
2745 {
2746 	struct ring_buffer_event *event;
2747 	struct page *page;
2748 	int cpu;
2749 
2750 	WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2751 
2752 	if (trace_buffered_event_ref++)
2753 		return;
2754 
2755 	for_each_tracing_cpu(cpu) {
2756 		page = alloc_pages_node(cpu_to_node(cpu),
2757 					GFP_KERNEL | __GFP_NORETRY, 0);
2758 		/* This is just an optimization and can handle failures */
2759 		if (!page) {
2760 			pr_err("Failed to allocate event buffer\n");
2761 			break;
2762 		}
2763 
2764 		event = page_address(page);
2765 		memset(event, 0, sizeof(*event));
2766 
2767 		per_cpu(trace_buffered_event, cpu) = event;
2768 
2769 		preempt_disable();
2770 		if (cpu == smp_processor_id() &&
2771 		    __this_cpu_read(trace_buffered_event) !=
2772 		    per_cpu(trace_buffered_event, cpu))
2773 			WARN_ON_ONCE(1);
2774 		preempt_enable();
2775 	}
2776 }
2777 
enable_trace_buffered_event(void * data)2778 static void enable_trace_buffered_event(void *data)
2779 {
2780 	/* Probably not needed, but do it anyway */
2781 	smp_rmb();
2782 	this_cpu_dec(trace_buffered_event_cnt);
2783 }
2784 
disable_trace_buffered_event(void * data)2785 static void disable_trace_buffered_event(void *data)
2786 {
2787 	this_cpu_inc(trace_buffered_event_cnt);
2788 }
2789 
2790 /**
2791  * trace_buffered_event_disable - disable buffering events
2792  *
2793  * When a filter is removed, it is faster to not use the buffered
2794  * events, and to commit directly into the ring buffer. Free up
2795  * the temp buffers when there are no more users. This requires
2796  * special synchronization with current events.
2797  */
trace_buffered_event_disable(void)2798 void trace_buffered_event_disable(void)
2799 {
2800 	int cpu;
2801 
2802 	WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2803 
2804 	if (WARN_ON_ONCE(!trace_buffered_event_ref))
2805 		return;
2806 
2807 	if (--trace_buffered_event_ref)
2808 		return;
2809 
2810 	/* For each CPU, set the buffer as used. */
2811 	on_each_cpu_mask(tracing_buffer_mask, disable_trace_buffered_event,
2812 			 NULL, true);
2813 
2814 	/* Wait for all current users to finish */
2815 	synchronize_rcu();
2816 
2817 	for_each_tracing_cpu(cpu) {
2818 		free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2819 		per_cpu(trace_buffered_event, cpu) = NULL;
2820 	}
2821 
2822 	/*
2823 	 * Wait for all CPUs that potentially started checking if they can use
2824 	 * their event buffer only after the previous synchronize_rcu() call and
2825 	 * they still read a valid pointer from trace_buffered_event. It must be
2826 	 * ensured they don't see cleared trace_buffered_event_cnt else they
2827 	 * could wrongly decide to use the pointed-to buffer which is now freed.
2828 	 */
2829 	synchronize_rcu();
2830 
2831 	/* For each CPU, relinquish the buffer */
2832 	on_each_cpu_mask(tracing_buffer_mask, enable_trace_buffered_event, NULL,
2833 			 true);
2834 }
2835 
2836 static struct trace_buffer *temp_buffer;
2837 
2838 struct ring_buffer_event *
trace_event_buffer_lock_reserve(struct trace_buffer ** current_rb,struct trace_event_file * trace_file,int type,unsigned long len,unsigned int trace_ctx)2839 trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
2840 			  struct trace_event_file *trace_file,
2841 			  int type, unsigned long len,
2842 			  unsigned int trace_ctx)
2843 {
2844 	struct ring_buffer_event *entry;
2845 	struct trace_array *tr = trace_file->tr;
2846 	int val;
2847 
2848 	*current_rb = tr->array_buffer.buffer;
2849 
2850 	if (!tr->no_filter_buffering_ref &&
2851 	    (trace_file->flags & (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED))) {
2852 		preempt_disable_notrace();
2853 		/*
2854 		 * Filtering is on, so try to use the per cpu buffer first.
2855 		 * This buffer will simulate a ring_buffer_event,
2856 		 * where the type_len is zero and the array[0] will
2857 		 * hold the full length.
2858 		 * (see include/linux/ring-buffer.h for details on
2859 		 *  how the ring_buffer_event is structured).
2860 		 *
2861 		 * Using a temp buffer during filtering and copying it
2862 		 * on a matched filter is quicker than writing directly
2863 		 * into the ring buffer and then discarding it when
2864 		 * it doesn't match. That is because the discard
2865 		 * requires several atomic operations to get right.
2866 		 * Copying on match and doing nothing on a failed match
2867 		 * is still quicker than no copy on match, but having
2868 		 * to discard out of the ring buffer on a failed match.
2869 		 */
2870 		if ((entry = __this_cpu_read(trace_buffered_event))) {
2871 			int max_len = PAGE_SIZE - struct_size(entry, array, 1);
2872 
2873 			val = this_cpu_inc_return(trace_buffered_event_cnt);
2874 
2875 			/*
2876 			 * Preemption is disabled, but interrupts and NMIs
2877 			 * can still come in now. If that happens after
2878 			 * the above increment, then it will have to go
2879 			 * back to the old method of allocating the event
2880 			 * on the ring buffer, and if the filter fails, it
2881 			 * will have to call ring_buffer_discard_commit()
2882 			 * to remove it.
2883 			 *
2884 			 * Need to also check the unlikely case that the
2885 			 * length is bigger than the temp buffer size.
2886 			 * If that happens, then the reserve is pretty much
2887 			 * guaranteed to fail, as the ring buffer currently
2888 			 * only allows events less than a page. But that may
2889 			 * change in the future, so let the ring buffer reserve
2890 			 * handle the failure in that case.
2891 			 */
2892 			if (val == 1 && likely(len <= max_len)) {
2893 				trace_event_setup(entry, type, trace_ctx);
2894 				entry->array[0] = len;
2895 				/* Return with preemption disabled */
2896 				return entry;
2897 			}
2898 			this_cpu_dec(trace_buffered_event_cnt);
2899 		}
2900 		/* __trace_buffer_lock_reserve() disables preemption */
2901 		preempt_enable_notrace();
2902 	}
2903 
2904 	entry = __trace_buffer_lock_reserve(*current_rb, type, len,
2905 					    trace_ctx);
2906 	/*
2907 	 * If tracing is off, but we have triggers enabled
2908 	 * we still need to look at the event data. Use the temp_buffer
2909 	 * to store the trace event for the trigger to use. It's recursive
2910 	 * safe and will not be recorded anywhere.
2911 	 */
2912 	if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2913 		*current_rb = temp_buffer;
2914 		entry = __trace_buffer_lock_reserve(*current_rb, type, len,
2915 						    trace_ctx);
2916 	}
2917 	return entry;
2918 }
2919 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2920 
2921 static DEFINE_RAW_SPINLOCK(tracepoint_iter_lock);
2922 static DEFINE_MUTEX(tracepoint_printk_mutex);
2923 
output_printk(struct trace_event_buffer * fbuffer)2924 static void output_printk(struct trace_event_buffer *fbuffer)
2925 {
2926 	struct trace_event_call *event_call;
2927 	struct trace_event_file *file;
2928 	struct trace_event *event;
2929 	unsigned long flags;
2930 	struct trace_iterator *iter = tracepoint_print_iter;
2931 
2932 	/* We should never get here if iter is NULL */
2933 	if (WARN_ON_ONCE(!iter))
2934 		return;
2935 
2936 	event_call = fbuffer->trace_file->event_call;
2937 	if (!event_call || !event_call->event.funcs ||
2938 	    !event_call->event.funcs->trace)
2939 		return;
2940 
2941 	file = fbuffer->trace_file;
2942 	if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
2943 	    (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
2944 	     !filter_match_preds(file->filter, fbuffer->entry)))
2945 		return;
2946 
2947 	event = &fbuffer->trace_file->event_call->event;
2948 
2949 	raw_spin_lock_irqsave(&tracepoint_iter_lock, flags);
2950 	trace_seq_init(&iter->seq);
2951 	iter->ent = fbuffer->entry;
2952 	event_call->event.funcs->trace(iter, 0, event);
2953 	trace_seq_putc(&iter->seq, 0);
2954 	printk("%s", iter->seq.buffer);
2955 
2956 	raw_spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2957 }
2958 
tracepoint_printk_sysctl(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)2959 int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2960 			     void *buffer, size_t *lenp,
2961 			     loff_t *ppos)
2962 {
2963 	int save_tracepoint_printk;
2964 	int ret;
2965 
2966 	mutex_lock(&tracepoint_printk_mutex);
2967 	save_tracepoint_printk = tracepoint_printk;
2968 
2969 	ret = proc_dointvec(table, write, buffer, lenp, ppos);
2970 
2971 	/*
2972 	 * This will force exiting early, as tracepoint_printk
2973 	 * is always zero when tracepoint_printk_iter is not allocated
2974 	 */
2975 	if (!tracepoint_print_iter)
2976 		tracepoint_printk = 0;
2977 
2978 	if (save_tracepoint_printk == tracepoint_printk)
2979 		goto out;
2980 
2981 	if (tracepoint_printk)
2982 		static_key_enable(&tracepoint_printk_key.key);
2983 	else
2984 		static_key_disable(&tracepoint_printk_key.key);
2985 
2986  out:
2987 	mutex_unlock(&tracepoint_printk_mutex);
2988 
2989 	return ret;
2990 }
2991 
trace_event_buffer_commit(struct trace_event_buffer * fbuffer)2992 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2993 {
2994 	enum event_trigger_type tt = ETT_NONE;
2995 	struct trace_event_file *file = fbuffer->trace_file;
2996 
2997 	if (__event_trigger_test_discard(file, fbuffer->buffer, fbuffer->event,
2998 			fbuffer->entry, &tt))
2999 		goto discard;
3000 
3001 	if (static_key_false(&tracepoint_printk_key.key))
3002 		output_printk(fbuffer);
3003 
3004 	if (static_branch_unlikely(&trace_event_exports_enabled))
3005 		ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT);
3006 
3007 	trace_buffer_unlock_commit_regs(file->tr, fbuffer->buffer,
3008 			fbuffer->event, fbuffer->trace_ctx, fbuffer->regs);
3009 
3010 discard:
3011 	if (tt)
3012 		event_triggers_post_call(file, tt);
3013 
3014 }
3015 EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
3016 
3017 /*
3018  * Skip 3:
3019  *
3020  *   trace_buffer_unlock_commit_regs()
3021  *   trace_event_buffer_commit()
3022  *   trace_event_raw_event_xxx()
3023  */
3024 # define STACK_SKIP 3
3025 
trace_buffer_unlock_commit_regs(struct trace_array * tr,struct trace_buffer * buffer,struct ring_buffer_event * event,unsigned int trace_ctx,struct pt_regs * regs)3026 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
3027 				     struct trace_buffer *buffer,
3028 				     struct ring_buffer_event *event,
3029 				     unsigned int trace_ctx,
3030 				     struct pt_regs *regs)
3031 {
3032 	__buffer_unlock_commit(buffer, event);
3033 
3034 	/*
3035 	 * If regs is not set, then skip the necessary functions.
3036 	 * Note, we can still get here via blktrace, wakeup tracer
3037 	 * and mmiotrace, but that's ok if they lose a function or
3038 	 * two. They are not that meaningful.
3039 	 */
3040 	ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs);
3041 	ftrace_trace_userstack(tr, buffer, trace_ctx);
3042 }
3043 
3044 /*
3045  * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
3046  */
3047 void
trace_buffer_unlock_commit_nostack(struct trace_buffer * buffer,struct ring_buffer_event * event)3048 trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
3049 				   struct ring_buffer_event *event)
3050 {
3051 	__buffer_unlock_commit(buffer, event);
3052 }
3053 
3054 void
trace_function(struct trace_array * tr,unsigned long ip,unsigned long parent_ip,unsigned int trace_ctx)3055 trace_function(struct trace_array *tr, unsigned long ip, unsigned long
3056 	       parent_ip, unsigned int trace_ctx)
3057 {
3058 	struct trace_event_call *call = &event_function;
3059 	struct trace_buffer *buffer = tr->array_buffer.buffer;
3060 	struct ring_buffer_event *event;
3061 	struct ftrace_entry *entry;
3062 
3063 	event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
3064 					    trace_ctx);
3065 	if (!event)
3066 		return;
3067 	entry	= ring_buffer_event_data(event);
3068 	entry->ip			= ip;
3069 	entry->parent_ip		= parent_ip;
3070 
3071 	if (!call_filter_check_discard(call, entry, buffer, event)) {
3072 		if (static_branch_unlikely(&trace_function_exports_enabled))
3073 			ftrace_exports(event, TRACE_EXPORT_FUNCTION);
3074 		__buffer_unlock_commit(buffer, event);
3075 	}
3076 }
3077 
3078 #ifdef CONFIG_STACKTRACE
3079 
3080 /* Allow 4 levels of nesting: normal, softirq, irq, NMI */
3081 #define FTRACE_KSTACK_NESTING	4
3082 
3083 #define FTRACE_KSTACK_ENTRIES	(PAGE_SIZE / FTRACE_KSTACK_NESTING)
3084 
3085 struct ftrace_stack {
3086 	unsigned long		calls[FTRACE_KSTACK_ENTRIES];
3087 };
3088 
3089 
3090 struct ftrace_stacks {
3091 	struct ftrace_stack	stacks[FTRACE_KSTACK_NESTING];
3092 };
3093 
3094 static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
3095 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
3096 
__ftrace_trace_stack(struct trace_buffer * buffer,unsigned int trace_ctx,int skip,struct pt_regs * regs)3097 static void __ftrace_trace_stack(struct trace_buffer *buffer,
3098 				 unsigned int trace_ctx,
3099 				 int skip, struct pt_regs *regs)
3100 {
3101 	struct trace_event_call *call = &event_kernel_stack;
3102 	struct ring_buffer_event *event;
3103 	unsigned int size, nr_entries;
3104 	struct ftrace_stack *fstack;
3105 	struct stack_entry *entry;
3106 	int stackidx;
3107 
3108 	/*
3109 	 * Add one, for this function and the call to save_stack_trace()
3110 	 * If regs is set, then these functions will not be in the way.
3111 	 */
3112 #ifndef CONFIG_UNWINDER_ORC
3113 	if (!regs)
3114 		skip++;
3115 #endif
3116 
3117 	preempt_disable_notrace();
3118 
3119 	stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
3120 
3121 	/* This should never happen. If it does, yell once and skip */
3122 	if (WARN_ON_ONCE(stackidx >= FTRACE_KSTACK_NESTING))
3123 		goto out;
3124 
3125 	/*
3126 	 * The above __this_cpu_inc_return() is 'atomic' cpu local. An
3127 	 * interrupt will either see the value pre increment or post
3128 	 * increment. If the interrupt happens pre increment it will have
3129 	 * restored the counter when it returns.  We just need a barrier to
3130 	 * keep gcc from moving things around.
3131 	 */
3132 	barrier();
3133 
3134 	fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
3135 	size = ARRAY_SIZE(fstack->calls);
3136 
3137 	if (regs) {
3138 		nr_entries = stack_trace_save_regs(regs, fstack->calls,
3139 						   size, skip);
3140 	} else {
3141 		nr_entries = stack_trace_save(fstack->calls, size, skip);
3142 	}
3143 
3144 	event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
3145 				    struct_size(entry, caller, nr_entries),
3146 				    trace_ctx);
3147 	if (!event)
3148 		goto out;
3149 	entry = ring_buffer_event_data(event);
3150 
3151 	entry->size = nr_entries;
3152 	memcpy(&entry->caller, fstack->calls,
3153 	       flex_array_size(entry, caller, nr_entries));
3154 
3155 	if (!call_filter_check_discard(call, entry, buffer, event))
3156 		__buffer_unlock_commit(buffer, event);
3157 
3158  out:
3159 	/* Again, don't let gcc optimize things here */
3160 	barrier();
3161 	__this_cpu_dec(ftrace_stack_reserve);
3162 	preempt_enable_notrace();
3163 
3164 }
3165 
ftrace_trace_stack(struct trace_array * tr,struct trace_buffer * buffer,unsigned int trace_ctx,int skip,struct pt_regs * regs)3166 static inline void ftrace_trace_stack(struct trace_array *tr,
3167 				      struct trace_buffer *buffer,
3168 				      unsigned int trace_ctx,
3169 				      int skip, struct pt_regs *regs)
3170 {
3171 	if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
3172 		return;
3173 
3174 	__ftrace_trace_stack(buffer, trace_ctx, skip, regs);
3175 }
3176 
__trace_stack(struct trace_array * tr,unsigned int trace_ctx,int skip)3177 void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,
3178 		   int skip)
3179 {
3180 	struct trace_buffer *buffer = tr->array_buffer.buffer;
3181 
3182 	if (rcu_is_watching()) {
3183 		__ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
3184 		return;
3185 	}
3186 
3187 	if (WARN_ON_ONCE(IS_ENABLED(CONFIG_GENERIC_ENTRY)))
3188 		return;
3189 
3190 	/*
3191 	 * When an NMI triggers, RCU is enabled via ct_nmi_enter(),
3192 	 * but if the above rcu_is_watching() failed, then the NMI
3193 	 * triggered someplace critical, and ct_irq_enter() should
3194 	 * not be called from NMI.
3195 	 */
3196 	if (unlikely(in_nmi()))
3197 		return;
3198 
3199 	ct_irq_enter_irqson();
3200 	__ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
3201 	ct_irq_exit_irqson();
3202 }
3203 
3204 /**
3205  * trace_dump_stack - record a stack back trace in the trace buffer
3206  * @skip: Number of functions to skip (helper handlers)
3207  */
trace_dump_stack(int skip)3208 void trace_dump_stack(int skip)
3209 {
3210 	if (tracing_disabled || tracing_selftest_running)
3211 		return;
3212 
3213 #ifndef CONFIG_UNWINDER_ORC
3214 	/* Skip 1 to skip this function. */
3215 	skip++;
3216 #endif
3217 	__ftrace_trace_stack(global_trace.array_buffer.buffer,
3218 			     tracing_gen_ctx(), skip, NULL);
3219 }
3220 EXPORT_SYMBOL_GPL(trace_dump_stack);
3221 
3222 #ifdef CONFIG_USER_STACKTRACE_SUPPORT
3223 static DEFINE_PER_CPU(int, user_stack_count);
3224 
3225 static void
ftrace_trace_userstack(struct trace_array * tr,struct trace_buffer * buffer,unsigned int trace_ctx)3226 ftrace_trace_userstack(struct trace_array *tr,
3227 		       struct trace_buffer *buffer, unsigned int trace_ctx)
3228 {
3229 	struct trace_event_call *call = &event_user_stack;
3230 	struct ring_buffer_event *event;
3231 	struct userstack_entry *entry;
3232 
3233 	if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE))
3234 		return;
3235 
3236 	/*
3237 	 * NMIs can not handle page faults, even with fix ups.
3238 	 * The save user stack can (and often does) fault.
3239 	 */
3240 	if (unlikely(in_nmi()))
3241 		return;
3242 
3243 	/*
3244 	 * prevent recursion, since the user stack tracing may
3245 	 * trigger other kernel events.
3246 	 */
3247 	preempt_disable();
3248 	if (__this_cpu_read(user_stack_count))
3249 		goto out;
3250 
3251 	__this_cpu_inc(user_stack_count);
3252 
3253 	event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
3254 					    sizeof(*entry), trace_ctx);
3255 	if (!event)
3256 		goto out_drop_count;
3257 	entry	= ring_buffer_event_data(event);
3258 
3259 	entry->tgid		= current->tgid;
3260 	memset(&entry->caller, 0, sizeof(entry->caller));
3261 
3262 	stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
3263 	if (!call_filter_check_discard(call, entry, buffer, event))
3264 		__buffer_unlock_commit(buffer, event);
3265 
3266  out_drop_count:
3267 	__this_cpu_dec(user_stack_count);
3268  out:
3269 	preempt_enable();
3270 }
3271 #else /* CONFIG_USER_STACKTRACE_SUPPORT */
ftrace_trace_userstack(struct trace_array * tr,struct trace_buffer * buffer,unsigned int trace_ctx)3272 static void ftrace_trace_userstack(struct trace_array *tr,
3273 				   struct trace_buffer *buffer,
3274 				   unsigned int trace_ctx)
3275 {
3276 }
3277 #endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
3278 
3279 #endif /* CONFIG_STACKTRACE */
3280 
3281 static inline void
func_repeats_set_delta_ts(struct func_repeats_entry * entry,unsigned long long delta)3282 func_repeats_set_delta_ts(struct func_repeats_entry *entry,
3283 			  unsigned long long delta)
3284 {
3285 	entry->bottom_delta_ts = delta & U32_MAX;
3286 	entry->top_delta_ts = (delta >> 32);
3287 }
3288 
trace_last_func_repeats(struct trace_array * tr,struct trace_func_repeats * last_info,unsigned int trace_ctx)3289 void trace_last_func_repeats(struct trace_array *tr,
3290 			     struct trace_func_repeats *last_info,
3291 			     unsigned int trace_ctx)
3292 {
3293 	struct trace_buffer *buffer = tr->array_buffer.buffer;
3294 	struct func_repeats_entry *entry;
3295 	struct ring_buffer_event *event;
3296 	u64 delta;
3297 
3298 	event = __trace_buffer_lock_reserve(buffer, TRACE_FUNC_REPEATS,
3299 					    sizeof(*entry), trace_ctx);
3300 	if (!event)
3301 		return;
3302 
3303 	delta = ring_buffer_event_time_stamp(buffer, event) -
3304 		last_info->ts_last_call;
3305 
3306 	entry = ring_buffer_event_data(event);
3307 	entry->ip = last_info->ip;
3308 	entry->parent_ip = last_info->parent_ip;
3309 	entry->count = last_info->count;
3310 	func_repeats_set_delta_ts(entry, delta);
3311 
3312 	__buffer_unlock_commit(buffer, event);
3313 }
3314 
3315 /* created for use with alloc_percpu */
3316 struct trace_buffer_struct {
3317 	int nesting;
3318 	char buffer[4][TRACE_BUF_SIZE];
3319 };
3320 
3321 static struct trace_buffer_struct __percpu *trace_percpu_buffer;
3322 
3323 /*
3324  * This allows for lockless recording.  If we're nested too deeply, then
3325  * this returns NULL.
3326  */
get_trace_buf(void)3327 static char *get_trace_buf(void)
3328 {
3329 	struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
3330 
3331 	if (!trace_percpu_buffer || buffer->nesting >= 4)
3332 		return NULL;
3333 
3334 	buffer->nesting++;
3335 
3336 	/* Interrupts must see nesting incremented before we use the buffer */
3337 	barrier();
3338 	return &buffer->buffer[buffer->nesting - 1][0];
3339 }
3340 
put_trace_buf(void)3341 static void put_trace_buf(void)
3342 {
3343 	/* Don't let the decrement of nesting leak before this */
3344 	barrier();
3345 	this_cpu_dec(trace_percpu_buffer->nesting);
3346 }
3347 
alloc_percpu_trace_buffer(void)3348 static int alloc_percpu_trace_buffer(void)
3349 {
3350 	struct trace_buffer_struct __percpu *buffers;
3351 
3352 	if (trace_percpu_buffer)
3353 		return 0;
3354 
3355 	buffers = alloc_percpu(struct trace_buffer_struct);
3356 	if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer"))
3357 		return -ENOMEM;
3358 
3359 	trace_percpu_buffer = buffers;
3360 	return 0;
3361 }
3362 
3363 static int buffers_allocated;
3364 
trace_printk_init_buffers(void)3365 void trace_printk_init_buffers(void)
3366 {
3367 	if (buffers_allocated)
3368 		return;
3369 
3370 	if (alloc_percpu_trace_buffer())
3371 		return;
3372 
3373 	/* trace_printk() is for debug use only. Don't use it in production. */
3374 
3375 	pr_warn("\n");
3376 	pr_warn("**********************************************************\n");
3377 	pr_warn("**   NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE   **\n");
3378 	pr_warn("**                                                      **\n");
3379 	pr_warn("** trace_printk() being used. Allocating extra memory.  **\n");
3380 	pr_warn("**                                                      **\n");
3381 	pr_warn("** This means that this is a DEBUG kernel and it is     **\n");
3382 	pr_warn("** unsafe for production use.                           **\n");
3383 	pr_warn("**                                                      **\n");
3384 	pr_warn("** If you see this message and you are not debugging    **\n");
3385 	pr_warn("** the kernel, report this immediately to your vendor!  **\n");
3386 	pr_warn("**                                                      **\n");
3387 	pr_warn("**   NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE   **\n");
3388 	pr_warn("**********************************************************\n");
3389 
3390 	/* Expand the buffers to set size */
3391 	tracing_update_buffers();
3392 
3393 	buffers_allocated = 1;
3394 
3395 	/*
3396 	 * trace_printk_init_buffers() can be called by modules.
3397 	 * If that happens, then we need to start cmdline recording
3398 	 * directly here. If the global_trace.buffer is already
3399 	 * allocated here, then this was called by module code.
3400 	 */
3401 	if (global_trace.array_buffer.buffer)
3402 		tracing_start_cmdline_record();
3403 }
3404 EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
3405 
trace_printk_start_comm(void)3406 void trace_printk_start_comm(void)
3407 {
3408 	/* Start tracing comms if trace printk is set */
3409 	if (!buffers_allocated)
3410 		return;
3411 	tracing_start_cmdline_record();
3412 }
3413 
trace_printk_start_stop_comm(int enabled)3414 static void trace_printk_start_stop_comm(int enabled)
3415 {
3416 	if (!buffers_allocated)
3417 		return;
3418 
3419 	if (enabled)
3420 		tracing_start_cmdline_record();
3421 	else
3422 		tracing_stop_cmdline_record();
3423 }
3424 
3425 /**
3426  * trace_vbprintk - write binary msg to tracing buffer
3427  * @ip:    The address of the caller
3428  * @fmt:   The string format to write to the buffer
3429  * @args:  Arguments for @fmt
3430  */
trace_vbprintk(unsigned long ip,const char * fmt,va_list args)3431 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
3432 {
3433 	struct trace_event_call *call = &event_bprint;
3434 	struct ring_buffer_event *event;
3435 	struct trace_buffer *buffer;
3436 	struct trace_array *tr = &global_trace;
3437 	struct bprint_entry *entry;
3438 	unsigned int trace_ctx;
3439 	char *tbuffer;
3440 	int len = 0, size;
3441 
3442 	if (unlikely(tracing_selftest_running || tracing_disabled))
3443 		return 0;
3444 
3445 	/* Don't pollute graph traces with trace_vprintk internals */
3446 	pause_graph_tracing();
3447 
3448 	trace_ctx = tracing_gen_ctx();
3449 	preempt_disable_notrace();
3450 
3451 	tbuffer = get_trace_buf();
3452 	if (!tbuffer) {
3453 		len = 0;
3454 		goto out_nobuffer;
3455 	}
3456 
3457 	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
3458 
3459 	if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
3460 		goto out_put;
3461 
3462 	size = sizeof(*entry) + sizeof(u32) * len;
3463 	buffer = tr->array_buffer.buffer;
3464 	ring_buffer_nest_start(buffer);
3465 	event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
3466 					    trace_ctx);
3467 	if (!event)
3468 		goto out;
3469 	entry = ring_buffer_event_data(event);
3470 	entry->ip			= ip;
3471 	entry->fmt			= fmt;
3472 
3473 	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
3474 	if (!call_filter_check_discard(call, entry, buffer, event)) {
3475 		__buffer_unlock_commit(buffer, event);
3476 		ftrace_trace_stack(tr, buffer, trace_ctx, 6, NULL);
3477 	}
3478 
3479 out:
3480 	ring_buffer_nest_end(buffer);
3481 out_put:
3482 	put_trace_buf();
3483 
3484 out_nobuffer:
3485 	preempt_enable_notrace();
3486 	unpause_graph_tracing();
3487 
3488 	return len;
3489 }
3490 EXPORT_SYMBOL_GPL(trace_vbprintk);
3491 
3492 __printf(3, 0)
3493 static int
__trace_array_vprintk(struct trace_buffer * buffer,unsigned long ip,const char * fmt,va_list args)3494 __trace_array_vprintk(struct trace_buffer *buffer,
3495 		      unsigned long ip, const char *fmt, va_list args)
3496 {
3497 	struct trace_event_call *call = &event_print;
3498 	struct ring_buffer_event *event;
3499 	int len = 0, size;
3500 	struct print_entry *entry;
3501 	unsigned int trace_ctx;
3502 	char *tbuffer;
3503 
3504 	if (tracing_disabled)
3505 		return 0;
3506 
3507 	/* Don't pollute graph traces with trace_vprintk internals */
3508 	pause_graph_tracing();
3509 
3510 	trace_ctx = tracing_gen_ctx();
3511 	preempt_disable_notrace();
3512 
3513 
3514 	tbuffer = get_trace_buf();
3515 	if (!tbuffer) {
3516 		len = 0;
3517 		goto out_nobuffer;
3518 	}
3519 
3520 	len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3521 
3522 	size = sizeof(*entry) + len + 1;
3523 	ring_buffer_nest_start(buffer);
3524 	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3525 					    trace_ctx);
3526 	if (!event)
3527 		goto out;
3528 	entry = ring_buffer_event_data(event);
3529 	entry->ip = ip;
3530 
3531 	memcpy(&entry->buf, tbuffer, len + 1);
3532 	if (!call_filter_check_discard(call, entry, buffer, event)) {
3533 		__buffer_unlock_commit(buffer, event);
3534 		ftrace_trace_stack(&global_trace, buffer, trace_ctx, 6, NULL);
3535 	}
3536 
3537 out:
3538 	ring_buffer_nest_end(buffer);
3539 	put_trace_buf();
3540 
3541 out_nobuffer:
3542 	preempt_enable_notrace();
3543 	unpause_graph_tracing();
3544 
3545 	return len;
3546 }
3547 
3548 __printf(3, 0)
trace_array_vprintk(struct trace_array * tr,unsigned long ip,const char * fmt,va_list args)3549 int trace_array_vprintk(struct trace_array *tr,
3550 			unsigned long ip, const char *fmt, va_list args)
3551 {
3552 	if (tracing_selftest_running && tr == &global_trace)
3553 		return 0;
3554 
3555 	return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args);
3556 }
3557 
3558 /**
3559  * trace_array_printk - Print a message to a specific instance
3560  * @tr: The instance trace_array descriptor
3561  * @ip: The instruction pointer that this is called from.
3562  * @fmt: The format to print (printf format)
3563  *
3564  * If a subsystem sets up its own instance, they have the right to
3565  * printk strings into their tracing instance buffer using this
3566  * function. Note, this function will not write into the top level
3567  * buffer (use trace_printk() for that), as writing into the top level
3568  * buffer should only have events that can be individually disabled.
3569  * trace_printk() is only used for debugging a kernel, and should not
3570  * be ever incorporated in normal use.
3571  *
3572  * trace_array_printk() can be used, as it will not add noise to the
3573  * top level tracing buffer.
3574  *
3575  * Note, trace_array_init_printk() must be called on @tr before this
3576  * can be used.
3577  */
3578 __printf(3, 0)
trace_array_printk(struct trace_array * tr,unsigned long ip,const char * fmt,...)3579 int trace_array_printk(struct trace_array *tr,
3580 		       unsigned long ip, const char *fmt, ...)
3581 {
3582 	int ret;
3583 	va_list ap;
3584 
3585 	if (!tr)
3586 		return -ENOENT;
3587 
3588 	/* This is only allowed for created instances */
3589 	if (tr == &global_trace)
3590 		return 0;
3591 
3592 	if (!(tr->trace_flags & TRACE_ITER_PRINTK))
3593 		return 0;
3594 
3595 	va_start(ap, fmt);
3596 	ret = trace_array_vprintk(tr, ip, fmt, ap);
3597 	va_end(ap);
3598 	return ret;
3599 }
3600 EXPORT_SYMBOL_GPL(trace_array_printk);
3601 
3602 /**
3603  * trace_array_init_printk - Initialize buffers for trace_array_printk()
3604  * @tr: The trace array to initialize the buffers for
3605  *
3606  * As trace_array_printk() only writes into instances, they are OK to
3607  * have in the kernel (unlike trace_printk()). This needs to be called
3608  * before trace_array_printk() can be used on a trace_array.
3609  */
trace_array_init_printk(struct trace_array * tr)3610 int trace_array_init_printk(struct trace_array *tr)
3611 {
3612 	if (!tr)
3613 		return -ENOENT;
3614 
3615 	/* This is only allowed for created instances */
3616 	if (tr == &global_trace)
3617 		return -EINVAL;
3618 
3619 	return alloc_percpu_trace_buffer();
3620 }
3621 EXPORT_SYMBOL_GPL(trace_array_init_printk);
3622 
3623 __printf(3, 4)
trace_array_printk_buf(struct trace_buffer * buffer,unsigned long ip,const char * fmt,...)3624 int trace_array_printk_buf(struct trace_buffer *buffer,
3625 			   unsigned long ip, const char *fmt, ...)
3626 {
3627 	int ret;
3628 	va_list ap;
3629 
3630 	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3631 		return 0;
3632 
3633 	va_start(ap, fmt);
3634 	ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3635 	va_end(ap);
3636 	return ret;
3637 }
3638 
3639 __printf(2, 0)
trace_vprintk(unsigned long ip,const char * fmt,va_list args)3640 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3641 {
3642 	return trace_array_vprintk(&global_trace, ip, fmt, args);
3643 }
3644 EXPORT_SYMBOL_GPL(trace_vprintk);
3645 
trace_iterator_increment(struct trace_iterator * iter)3646 static void trace_iterator_increment(struct trace_iterator *iter)
3647 {
3648 	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3649 
3650 	iter->idx++;
3651 	if (buf_iter)
3652 		ring_buffer_iter_advance(buf_iter);
3653 }
3654 
3655 static struct trace_entry *
peek_next_entry(struct trace_iterator * iter,int cpu,u64 * ts,unsigned long * lost_events)3656 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3657 		unsigned long *lost_events)
3658 {
3659 	struct ring_buffer_event *event;
3660 	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3661 
3662 	if (buf_iter) {
3663 		event = ring_buffer_iter_peek(buf_iter, ts);
3664 		if (lost_events)
3665 			*lost_events = ring_buffer_iter_dropped(buf_iter) ?
3666 				(unsigned long)-1 : 0;
3667 	} else {
3668 		event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,
3669 					 lost_events);
3670 	}
3671 
3672 	if (event) {
3673 		iter->ent_size = ring_buffer_event_length(event);
3674 		return ring_buffer_event_data(event);
3675 	}
3676 	iter->ent_size = 0;
3677 	return NULL;
3678 }
3679 
3680 static struct trace_entry *
__find_next_entry(struct trace_iterator * iter,int * ent_cpu,unsigned long * missing_events,u64 * ent_ts)3681 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3682 		  unsigned long *missing_events, u64 *ent_ts)
3683 {
3684 	struct trace_buffer *buffer = iter->array_buffer->buffer;
3685 	struct trace_entry *ent, *next = NULL;
3686 	unsigned long lost_events = 0, next_lost = 0;
3687 	int cpu_file = iter->cpu_file;
3688 	u64 next_ts = 0, ts;
3689 	int next_cpu = -1;
3690 	int next_size = 0;
3691 	int cpu;
3692 
3693 	/*
3694 	 * If we are in a per_cpu trace file, don't bother by iterating over
3695 	 * all cpu and peek directly.
3696 	 */
3697 	if (cpu_file > RING_BUFFER_ALL_CPUS) {
3698 		if (ring_buffer_empty_cpu(buffer, cpu_file))
3699 			return NULL;
3700 		ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3701 		if (ent_cpu)
3702 			*ent_cpu = cpu_file;
3703 
3704 		return ent;
3705 	}
3706 
3707 	for_each_tracing_cpu(cpu) {
3708 
3709 		if (ring_buffer_empty_cpu(buffer, cpu))
3710 			continue;
3711 
3712 		ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3713 
3714 		/*
3715 		 * Pick the entry with the smallest timestamp:
3716 		 */
3717 		if (ent && (!next || ts < next_ts)) {
3718 			next = ent;
3719 			next_cpu = cpu;
3720 			next_ts = ts;
3721 			next_lost = lost_events;
3722 			next_size = iter->ent_size;
3723 		}
3724 	}
3725 
3726 	iter->ent_size = next_size;
3727 
3728 	if (ent_cpu)
3729 		*ent_cpu = next_cpu;
3730 
3731 	if (ent_ts)
3732 		*ent_ts = next_ts;
3733 
3734 	if (missing_events)
3735 		*missing_events = next_lost;
3736 
3737 	return next;
3738 }
3739 
3740 #define STATIC_FMT_BUF_SIZE	128
3741 static char static_fmt_buf[STATIC_FMT_BUF_SIZE];
3742 
trace_iter_expand_format(struct trace_iterator * iter)3743 char *trace_iter_expand_format(struct trace_iterator *iter)
3744 {
3745 	char *tmp;
3746 
3747 	/*
3748 	 * iter->tr is NULL when used with tp_printk, which makes
3749 	 * this get called where it is not safe to call krealloc().
3750 	 */
3751 	if (!iter->tr || iter->fmt == static_fmt_buf)
3752 		return NULL;
3753 
3754 	tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE,
3755 		       GFP_KERNEL);
3756 	if (tmp) {
3757 		iter->fmt_size += STATIC_FMT_BUF_SIZE;
3758 		iter->fmt = tmp;
3759 	}
3760 
3761 	return tmp;
3762 }
3763 
3764 /* Returns true if the string is safe to dereference from an event */
trace_safe_str(struct trace_iterator * iter,const char * str,bool star,int len)3765 static bool trace_safe_str(struct trace_iterator *iter, const char *str,
3766 			   bool star, int len)
3767 {
3768 	unsigned long addr = (unsigned long)str;
3769 	struct trace_event *trace_event;
3770 	struct trace_event_call *event;
3771 
3772 	/* Ignore strings with no length */
3773 	if (star && !len)
3774 		return true;
3775 
3776 	/* OK if part of the event data */
3777 	if ((addr >= (unsigned long)iter->ent) &&
3778 	    (addr < (unsigned long)iter->ent + iter->ent_size))
3779 		return true;
3780 
3781 	/* OK if part of the temp seq buffer */
3782 	if ((addr >= (unsigned long)iter->tmp_seq.buffer) &&
3783 	    (addr < (unsigned long)iter->tmp_seq.buffer + PAGE_SIZE))
3784 		return true;
3785 
3786 	/* Core rodata can not be freed */
3787 	if (is_kernel_rodata(addr))
3788 		return true;
3789 
3790 	if (trace_is_tracepoint_string(str))
3791 		return true;
3792 
3793 	/*
3794 	 * Now this could be a module event, referencing core module
3795 	 * data, which is OK.
3796 	 */
3797 	if (!iter->ent)
3798 		return false;
3799 
3800 	trace_event = ftrace_find_event(iter->ent->type);
3801 	if (!trace_event)
3802 		return false;
3803 
3804 	event = container_of(trace_event, struct trace_event_call, event);
3805 	if ((event->flags & TRACE_EVENT_FL_DYNAMIC) || !event->module)
3806 		return false;
3807 
3808 	/* Would rather have rodata, but this will suffice */
3809 	if (within_module_core(addr, event->module))
3810 		return true;
3811 
3812 	return false;
3813 }
3814 
show_buffer(struct trace_seq * s)3815 static const char *show_buffer(struct trace_seq *s)
3816 {
3817 	struct seq_buf *seq = &s->seq;
3818 
3819 	seq_buf_terminate(seq);
3820 
3821 	return seq->buffer;
3822 }
3823 
3824 static DEFINE_STATIC_KEY_FALSE(trace_no_verify);
3825 
test_can_verify_check(const char * fmt,...)3826 static int test_can_verify_check(const char *fmt, ...)
3827 {
3828 	char buf[16];
3829 	va_list ap;
3830 	int ret;
3831 
3832 	/*
3833 	 * The verifier is dependent on vsnprintf() modifies the va_list
3834 	 * passed to it, where it is sent as a reference. Some architectures
3835 	 * (like x86_32) passes it by value, which means that vsnprintf()
3836 	 * does not modify the va_list passed to it, and the verifier
3837 	 * would then need to be able to understand all the values that
3838 	 * vsnprintf can use. If it is passed by value, then the verifier
3839 	 * is disabled.
3840 	 */
3841 	va_start(ap, fmt);
3842 	vsnprintf(buf, 16, "%d", ap);
3843 	ret = va_arg(ap, int);
3844 	va_end(ap);
3845 
3846 	return ret;
3847 }
3848 
test_can_verify(void)3849 static void test_can_verify(void)
3850 {
3851 	if (!test_can_verify_check("%d %d", 0, 1)) {
3852 		pr_info("trace event string verifier disabled\n");
3853 		static_branch_inc(&trace_no_verify);
3854 	}
3855 }
3856 
3857 /**
3858  * trace_check_vprintf - Check dereferenced strings while writing to the seq buffer
3859  * @iter: The iterator that holds the seq buffer and the event being printed
3860  * @fmt: The format used to print the event
3861  * @ap: The va_list holding the data to print from @fmt.
3862  *
3863  * This writes the data into the @iter->seq buffer using the data from
3864  * @fmt and @ap. If the format has a %s, then the source of the string
3865  * is examined to make sure it is safe to print, otherwise it will
3866  * warn and print "[UNSAFE MEMORY]" in place of the dereferenced string
3867  * pointer.
3868  */
trace_check_vprintf(struct trace_iterator * iter,const char * fmt,va_list ap)3869 void trace_check_vprintf(struct trace_iterator *iter, const char *fmt,
3870 			 va_list ap)
3871 {
3872 	const char *p = fmt;
3873 	const char *str;
3874 	int i, j;
3875 
3876 	if (WARN_ON_ONCE(!fmt))
3877 		return;
3878 
3879 	if (static_branch_unlikely(&trace_no_verify))
3880 		goto print;
3881 
3882 	/* Don't bother checking when doing a ftrace_dump() */
3883 	if (iter->fmt == static_fmt_buf)
3884 		goto print;
3885 
3886 	while (*p) {
3887 		bool star = false;
3888 		int len = 0;
3889 
3890 		j = 0;
3891 
3892 		/* We only care about %s and variants */
3893 		for (i = 0; p[i]; i++) {
3894 			if (i + 1 >= iter->fmt_size) {
3895 				/*
3896 				 * If we can't expand the copy buffer,
3897 				 * just print it.
3898 				 */
3899 				if (!trace_iter_expand_format(iter))
3900 					goto print;
3901 			}
3902 
3903 			if (p[i] == '\\' && p[i+1]) {
3904 				i++;
3905 				continue;
3906 			}
3907 			if (p[i] == '%') {
3908 				/* Need to test cases like %08.*s */
3909 				for (j = 1; p[i+j]; j++) {
3910 					if (isdigit(p[i+j]) ||
3911 					    p[i+j] == '.')
3912 						continue;
3913 					if (p[i+j] == '*') {
3914 						star = true;
3915 						continue;
3916 					}
3917 					break;
3918 				}
3919 				if (p[i+j] == 's')
3920 					break;
3921 				star = false;
3922 			}
3923 			j = 0;
3924 		}
3925 		/* If no %s found then just print normally */
3926 		if (!p[i])
3927 			break;
3928 
3929 		/* Copy up to the %s, and print that */
3930 		strncpy(iter->fmt, p, i);
3931 		iter->fmt[i] = '\0';
3932 		trace_seq_vprintf(&iter->seq, iter->fmt, ap);
3933 
3934 		/*
3935 		 * If iter->seq is full, the above call no longer guarantees
3936 		 * that ap is in sync with fmt processing, and further calls
3937 		 * to va_arg() can return wrong positional arguments.
3938 		 *
3939 		 * Ensure that ap is no longer used in this case.
3940 		 */
3941 		if (iter->seq.full) {
3942 			p = "";
3943 			break;
3944 		}
3945 
3946 		if (star)
3947 			len = va_arg(ap, int);
3948 
3949 		/* The ap now points to the string data of the %s */
3950 		str = va_arg(ap, const char *);
3951 
3952 		/*
3953 		 * If you hit this warning, it is likely that the
3954 		 * trace event in question used %s on a string that
3955 		 * was saved at the time of the event, but may not be
3956 		 * around when the trace is read. Use __string(),
3957 		 * __assign_str() and __get_str() helpers in the TRACE_EVENT()
3958 		 * instead. See samples/trace_events/trace-events-sample.h
3959 		 * for reference.
3960 		 */
3961 		if (WARN_ONCE(!trace_safe_str(iter, str, star, len),
3962 			      "fmt: '%s' current_buffer: '%s'",
3963 			      fmt, show_buffer(&iter->seq))) {
3964 			int ret;
3965 
3966 			/* Try to safely read the string */
3967 			if (star) {
3968 				if (len + 1 > iter->fmt_size)
3969 					len = iter->fmt_size - 1;
3970 				if (len < 0)
3971 					len = 0;
3972 				ret = copy_from_kernel_nofault(iter->fmt, str, len);
3973 				iter->fmt[len] = 0;
3974 				star = false;
3975 			} else {
3976 				ret = strncpy_from_kernel_nofault(iter->fmt, str,
3977 								  iter->fmt_size);
3978 			}
3979 			if (ret < 0)
3980 				trace_seq_printf(&iter->seq, "(0x%px)", str);
3981 			else
3982 				trace_seq_printf(&iter->seq, "(0x%px:%s)",
3983 						 str, iter->fmt);
3984 			str = "[UNSAFE-MEMORY]";
3985 			strcpy(iter->fmt, "%s");
3986 		} else {
3987 			strncpy(iter->fmt, p + i, j + 1);
3988 			iter->fmt[j+1] = '\0';
3989 		}
3990 		if (star)
3991 			trace_seq_printf(&iter->seq, iter->fmt, len, str);
3992 		else
3993 			trace_seq_printf(&iter->seq, iter->fmt, str);
3994 
3995 		p += i + j + 1;
3996 	}
3997  print:
3998 	if (*p)
3999 		trace_seq_vprintf(&iter->seq, p, ap);
4000 }
4001 
trace_event_format(struct trace_iterator * iter,const char * fmt)4002 const char *trace_event_format(struct trace_iterator *iter, const char *fmt)
4003 {
4004 	const char *p, *new_fmt;
4005 	char *q;
4006 
4007 	if (WARN_ON_ONCE(!fmt))
4008 		return fmt;
4009 
4010 	if (!iter->tr || iter->tr->trace_flags & TRACE_ITER_HASH_PTR)
4011 		return fmt;
4012 
4013 	p = fmt;
4014 	new_fmt = q = iter->fmt;
4015 	while (*p) {
4016 		if (unlikely(q - new_fmt + 3 > iter->fmt_size)) {
4017 			if (!trace_iter_expand_format(iter))
4018 				return fmt;
4019 
4020 			q += iter->fmt - new_fmt;
4021 			new_fmt = iter->fmt;
4022 		}
4023 
4024 		*q++ = *p++;
4025 
4026 		/* Replace %p with %px */
4027 		if (p[-1] == '%') {
4028 			if (p[0] == '%') {
4029 				*q++ = *p++;
4030 			} else if (p[0] == 'p' && !isalnum(p[1])) {
4031 				*q++ = *p++;
4032 				*q++ = 'x';
4033 			}
4034 		}
4035 	}
4036 	*q = '\0';
4037 
4038 	return new_fmt;
4039 }
4040 
4041 #define STATIC_TEMP_BUF_SIZE	128
4042 static char static_temp_buf[STATIC_TEMP_BUF_SIZE] __aligned(4);
4043 
4044 /* Find the next real entry, without updating the iterator itself */
trace_find_next_entry(struct trace_iterator * iter,int * ent_cpu,u64 * ent_ts)4045 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
4046 					  int *ent_cpu, u64 *ent_ts)
4047 {
4048 	/* __find_next_entry will reset ent_size */
4049 	int ent_size = iter->ent_size;
4050 	struct trace_entry *entry;
4051 
4052 	/*
4053 	 * If called from ftrace_dump(), then the iter->temp buffer
4054 	 * will be the static_temp_buf and not created from kmalloc.
4055 	 * If the entry size is greater than the buffer, we can
4056 	 * not save it. Just return NULL in that case. This is only
4057 	 * used to add markers when two consecutive events' time
4058 	 * stamps have a large delta. See trace_print_lat_context()
4059 	 */
4060 	if (iter->temp == static_temp_buf &&
4061 	    STATIC_TEMP_BUF_SIZE < ent_size)
4062 		return NULL;
4063 
4064 	/*
4065 	 * The __find_next_entry() may call peek_next_entry(), which may
4066 	 * call ring_buffer_peek() that may make the contents of iter->ent
4067 	 * undefined. Need to copy iter->ent now.
4068 	 */
4069 	if (iter->ent && iter->ent != iter->temp) {
4070 		if ((!iter->temp || iter->temp_size < iter->ent_size) &&
4071 		    !WARN_ON_ONCE(iter->temp == static_temp_buf)) {
4072 			void *temp;
4073 			temp = kmalloc(iter->ent_size, GFP_KERNEL);
4074 			if (!temp)
4075 				return NULL;
4076 			kfree(iter->temp);
4077 			iter->temp = temp;
4078 			iter->temp_size = iter->ent_size;
4079 		}
4080 		memcpy(iter->temp, iter->ent, iter->ent_size);
4081 		iter->ent = iter->temp;
4082 	}
4083 	entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
4084 	/* Put back the original ent_size */
4085 	iter->ent_size = ent_size;
4086 
4087 	return entry;
4088 }
4089 
4090 /* Find the next real entry, and increment the iterator to the next entry */
trace_find_next_entry_inc(struct trace_iterator * iter)4091 void *trace_find_next_entry_inc(struct trace_iterator *iter)
4092 {
4093 	iter->ent = __find_next_entry(iter, &iter->cpu,
4094 				      &iter->lost_events, &iter->ts);
4095 
4096 	if (iter->ent)
4097 		trace_iterator_increment(iter);
4098 
4099 	return iter->ent ? iter : NULL;
4100 }
4101 
trace_consume(struct trace_iterator * iter)4102 static void trace_consume(struct trace_iterator *iter)
4103 {
4104 	ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,
4105 			    &iter->lost_events);
4106 }
4107 
s_next(struct seq_file * m,void * v,loff_t * pos)4108 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
4109 {
4110 	struct trace_iterator *iter = m->private;
4111 	int i = (int)*pos;
4112 	void *ent;
4113 
4114 	WARN_ON_ONCE(iter->leftover);
4115 
4116 	(*pos)++;
4117 
4118 	/* can't go backwards */
4119 	if (iter->idx > i)
4120 		return NULL;
4121 
4122 	if (iter->idx < 0)
4123 		ent = trace_find_next_entry_inc(iter);
4124 	else
4125 		ent = iter;
4126 
4127 	while (ent && iter->idx < i)
4128 		ent = trace_find_next_entry_inc(iter);
4129 
4130 	iter->pos = *pos;
4131 
4132 	return ent;
4133 }
4134 
tracing_iter_reset(struct trace_iterator * iter,int cpu)4135 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
4136 {
4137 	struct ring_buffer_iter *buf_iter;
4138 	unsigned long entries = 0;
4139 	u64 ts;
4140 
4141 	per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
4142 
4143 	buf_iter = trace_buffer_iter(iter, cpu);
4144 	if (!buf_iter)
4145 		return;
4146 
4147 	ring_buffer_iter_reset(buf_iter);
4148 
4149 	/*
4150 	 * We could have the case with the max latency tracers
4151 	 * that a reset never took place on a cpu. This is evident
4152 	 * by the timestamp being before the start of the buffer.
4153 	 */
4154 	while (ring_buffer_iter_peek(buf_iter, &ts)) {
4155 		if (ts >= iter->array_buffer->time_start)
4156 			break;
4157 		entries++;
4158 		ring_buffer_iter_advance(buf_iter);
4159 	}
4160 
4161 	per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
4162 }
4163 
4164 /*
4165  * The current tracer is copied to avoid a global locking
4166  * all around.
4167  */
s_start(struct seq_file * m,loff_t * pos)4168 static void *s_start(struct seq_file *m, loff_t *pos)
4169 {
4170 	struct trace_iterator *iter = m->private;
4171 	struct trace_array *tr = iter->tr;
4172 	int cpu_file = iter->cpu_file;
4173 	void *p = NULL;
4174 	loff_t l = 0;
4175 	int cpu;
4176 
4177 	mutex_lock(&trace_types_lock);
4178 	if (unlikely(tr->current_trace != iter->trace)) {
4179 		/* Close iter->trace before switching to the new current tracer */
4180 		if (iter->trace->close)
4181 			iter->trace->close(iter);
4182 		iter->trace = tr->current_trace;
4183 		/* Reopen the new current tracer */
4184 		if (iter->trace->open)
4185 			iter->trace->open(iter);
4186 	}
4187 	mutex_unlock(&trace_types_lock);
4188 
4189 #ifdef CONFIG_TRACER_MAX_TRACE
4190 	if (iter->snapshot && iter->trace->use_max_tr)
4191 		return ERR_PTR(-EBUSY);
4192 #endif
4193 
4194 	if (*pos != iter->pos) {
4195 		iter->ent = NULL;
4196 		iter->cpu = 0;
4197 		iter->idx = -1;
4198 
4199 		if (cpu_file == RING_BUFFER_ALL_CPUS) {
4200 			for_each_tracing_cpu(cpu)
4201 				tracing_iter_reset(iter, cpu);
4202 		} else
4203 			tracing_iter_reset(iter, cpu_file);
4204 
4205 		iter->leftover = 0;
4206 		for (p = iter; p && l < *pos; p = s_next(m, p, &l))
4207 			;
4208 
4209 	} else {
4210 		/*
4211 		 * If we overflowed the seq_file before, then we want
4212 		 * to just reuse the trace_seq buffer again.
4213 		 */
4214 		if (iter->leftover)
4215 			p = iter;
4216 		else {
4217 			l = *pos - 1;
4218 			p = s_next(m, p, &l);
4219 		}
4220 	}
4221 
4222 	trace_event_read_lock();
4223 	trace_access_lock(cpu_file);
4224 	return p;
4225 }
4226 
s_stop(struct seq_file * m,void * p)4227 static void s_stop(struct seq_file *m, void *p)
4228 {
4229 	struct trace_iterator *iter = m->private;
4230 
4231 #ifdef CONFIG_TRACER_MAX_TRACE
4232 	if (iter->snapshot && iter->trace->use_max_tr)
4233 		return;
4234 #endif
4235 
4236 	trace_access_unlock(iter->cpu_file);
4237 	trace_event_read_unlock();
4238 }
4239 
4240 static void
get_total_entries_cpu(struct array_buffer * buf,unsigned long * total,unsigned long * entries,int cpu)4241 get_total_entries_cpu(struct array_buffer *buf, unsigned long *total,
4242 		      unsigned long *entries, int cpu)
4243 {
4244 	unsigned long count;
4245 
4246 	count = ring_buffer_entries_cpu(buf->buffer, cpu);
4247 	/*
4248 	 * If this buffer has skipped entries, then we hold all
4249 	 * entries for the trace and we need to ignore the
4250 	 * ones before the time stamp.
4251 	 */
4252 	if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
4253 		count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
4254 		/* total is the same as the entries */
4255 		*total = count;
4256 	} else
4257 		*total = count +
4258 			ring_buffer_overrun_cpu(buf->buffer, cpu);
4259 	*entries = count;
4260 }
4261 
4262 static void
get_total_entries(struct array_buffer * buf,unsigned long * total,unsigned long * entries)4263 get_total_entries(struct array_buffer *buf,
4264 		  unsigned long *total, unsigned long *entries)
4265 {
4266 	unsigned long t, e;
4267 	int cpu;
4268 
4269 	*total = 0;
4270 	*entries = 0;
4271 
4272 	for_each_tracing_cpu(cpu) {
4273 		get_total_entries_cpu(buf, &t, &e, cpu);
4274 		*total += t;
4275 		*entries += e;
4276 	}
4277 }
4278 
trace_total_entries_cpu(struct trace_array * tr,int cpu)4279 unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
4280 {
4281 	unsigned long total, entries;
4282 
4283 	if (!tr)
4284 		tr = &global_trace;
4285 
4286 	get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);
4287 
4288 	return entries;
4289 }
4290 
trace_total_entries(struct trace_array * tr)4291 unsigned long trace_total_entries(struct trace_array *tr)
4292 {
4293 	unsigned long total, entries;
4294 
4295 	if (!tr)
4296 		tr = &global_trace;
4297 
4298 	get_total_entries(&tr->array_buffer, &total, &entries);
4299 
4300 	return entries;
4301 }
4302 
print_lat_help_header(struct seq_file * m)4303 static void print_lat_help_header(struct seq_file *m)
4304 {
4305 	seq_puts(m, "#                    _------=> CPU#            \n"
4306 		    "#                   / _-----=> irqs-off/BH-disabled\n"
4307 		    "#                  | / _----=> need-resched    \n"
4308 		    "#                  || / _---=> hardirq/softirq \n"
4309 		    "#                  ||| / _--=> preempt-depth   \n"
4310 		    "#                  |||| / _-=> migrate-disable \n"
4311 		    "#                  ||||| /     delay           \n"
4312 		    "#  cmd     pid     |||||| time  |   caller     \n"
4313 		    "#     \\   /        ||||||  \\    |    /       \n");
4314 }
4315 
print_event_info(struct array_buffer * buf,struct seq_file * m)4316 static void print_event_info(struct array_buffer *buf, struct seq_file *m)
4317 {
4318 	unsigned long total;
4319 	unsigned long entries;
4320 
4321 	get_total_entries(buf, &total, &entries);
4322 	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
4323 		   entries, total, num_online_cpus());
4324 	seq_puts(m, "#\n");
4325 }
4326 
print_func_help_header(struct array_buffer * buf,struct seq_file * m,unsigned int flags)4327 static void print_func_help_header(struct array_buffer *buf, struct seq_file *m,
4328 				   unsigned int flags)
4329 {
4330 	bool tgid = flags & TRACE_ITER_RECORD_TGID;
4331 
4332 	print_event_info(buf, m);
4333 
4334 	seq_printf(m, "#           TASK-PID    %s CPU#     TIMESTAMP  FUNCTION\n", tgid ? "   TGID   " : "");
4335 	seq_printf(m, "#              | |      %s   |         |         |\n",      tgid ? "     |    " : "");
4336 }
4337 
print_func_help_header_irq(struct array_buffer * buf,struct seq_file * m,unsigned int flags)4338 static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m,
4339 				       unsigned int flags)
4340 {
4341 	bool tgid = flags & TRACE_ITER_RECORD_TGID;
4342 	static const char space[] = "            ";
4343 	int prec = tgid ? 12 : 2;
4344 
4345 	print_event_info(buf, m);
4346 
4347 	seq_printf(m, "#                            %.*s  _-----=> irqs-off/BH-disabled\n", prec, space);
4348 	seq_printf(m, "#                            %.*s / _----=> need-resched\n", prec, space);
4349 	seq_printf(m, "#                            %.*s| / _---=> hardirq/softirq\n", prec, space);
4350 	seq_printf(m, "#                            %.*s|| / _--=> preempt-depth\n", prec, space);
4351 	seq_printf(m, "#                            %.*s||| / _-=> migrate-disable\n", prec, space);
4352 	seq_printf(m, "#                            %.*s|||| /     delay\n", prec, space);
4353 	seq_printf(m, "#           TASK-PID  %.*s CPU#  |||||  TIMESTAMP  FUNCTION\n", prec, "     TGID   ");
4354 	seq_printf(m, "#              | |    %.*s   |   |||||     |         |\n", prec, "       |    ");
4355 }
4356 
4357 void
print_trace_header(struct seq_file * m,struct trace_iterator * iter)4358 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
4359 {
4360 	unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
4361 	struct array_buffer *buf = iter->array_buffer;
4362 	struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
4363 	struct tracer *type = iter->trace;
4364 	unsigned long entries;
4365 	unsigned long total;
4366 	const char *name = type->name;
4367 
4368 	get_total_entries(buf, &total, &entries);
4369 
4370 	seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
4371 		   name, UTS_RELEASE);
4372 	seq_puts(m, "# -----------------------------------"
4373 		 "---------------------------------\n");
4374 	seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
4375 		   " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
4376 		   nsecs_to_usecs(data->saved_latency),
4377 		   entries,
4378 		   total,
4379 		   buf->cpu,
4380 		   preempt_model_none()      ? "server" :
4381 		   preempt_model_voluntary() ? "desktop" :
4382 		   preempt_model_full()      ? "preempt" :
4383 		   preempt_model_rt()        ? "preempt_rt" :
4384 		   "unknown",
4385 		   /* These are reserved for later use */
4386 		   0, 0, 0, 0);
4387 #ifdef CONFIG_SMP
4388 	seq_printf(m, " #P:%d)\n", num_online_cpus());
4389 #else
4390 	seq_puts(m, ")\n");
4391 #endif
4392 	seq_puts(m, "#    -----------------\n");
4393 	seq_printf(m, "#    | task: %.16s-%d "
4394 		   "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
4395 		   data->comm, data->pid,
4396 		   from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
4397 		   data->policy, data->rt_priority);
4398 	seq_puts(m, "#    -----------------\n");
4399 
4400 	if (data->critical_start) {
4401 		seq_puts(m, "#  => started at: ");
4402 		seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
4403 		trace_print_seq(m, &iter->seq);
4404 		seq_puts(m, "\n#  => ended at:   ");
4405 		seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
4406 		trace_print_seq(m, &iter->seq);
4407 		seq_puts(m, "\n#\n");
4408 	}
4409 
4410 	seq_puts(m, "#\n");
4411 }
4412 
test_cpu_buff_start(struct trace_iterator * iter)4413 static void test_cpu_buff_start(struct trace_iterator *iter)
4414 {
4415 	struct trace_seq *s = &iter->seq;
4416 	struct trace_array *tr = iter->tr;
4417 
4418 	if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
4419 		return;
4420 
4421 	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
4422 		return;
4423 
4424 	if (cpumask_available(iter->started) &&
4425 	    cpumask_test_cpu(iter->cpu, iter->started))
4426 		return;
4427 
4428 	if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
4429 		return;
4430 
4431 	if (cpumask_available(iter->started))
4432 		cpumask_set_cpu(iter->cpu, iter->started);
4433 
4434 	/* Don't print started cpu buffer for the first entry of the trace */
4435 	if (iter->idx > 1)
4436 		trace_seq_printf(s, "##### CPU %u buffer started ####\n",
4437 				iter->cpu);
4438 }
4439 
print_trace_fmt(struct trace_iterator * iter)4440 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
4441 {
4442 	struct trace_array *tr = iter->tr;
4443 	struct trace_seq *s = &iter->seq;
4444 	unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
4445 	struct trace_entry *entry;
4446 	struct trace_event *event;
4447 
4448 	entry = iter->ent;
4449 
4450 	test_cpu_buff_start(iter);
4451 
4452 	event = ftrace_find_event(entry->type);
4453 
4454 	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4455 		if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4456 			trace_print_lat_context(iter);
4457 		else
4458 			trace_print_context(iter);
4459 	}
4460 
4461 	if (trace_seq_has_overflowed(s))
4462 		return TRACE_TYPE_PARTIAL_LINE;
4463 
4464 	if (event) {
4465 		if (tr->trace_flags & TRACE_ITER_FIELDS)
4466 			return print_event_fields(iter, event);
4467 		return event->funcs->trace(iter, sym_flags, event);
4468 	}
4469 
4470 	trace_seq_printf(s, "Unknown type %d\n", entry->type);
4471 
4472 	return trace_handle_return(s);
4473 }
4474 
print_raw_fmt(struct trace_iterator * iter)4475 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
4476 {
4477 	struct trace_array *tr = iter->tr;
4478 	struct trace_seq *s = &iter->seq;
4479 	struct trace_entry *entry;
4480 	struct trace_event *event;
4481 
4482 	entry = iter->ent;
4483 
4484 	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
4485 		trace_seq_printf(s, "%d %d %llu ",
4486 				 entry->pid, iter->cpu, iter->ts);
4487 
4488 	if (trace_seq_has_overflowed(s))
4489 		return TRACE_TYPE_PARTIAL_LINE;
4490 
4491 	event = ftrace_find_event(entry->type);
4492 	if (event)
4493 		return event->funcs->raw(iter, 0, event);
4494 
4495 	trace_seq_printf(s, "%d ?\n", entry->type);
4496 
4497 	return trace_handle_return(s);
4498 }
4499 
print_hex_fmt(struct trace_iterator * iter)4500 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
4501 {
4502 	struct trace_array *tr = iter->tr;
4503 	struct trace_seq *s = &iter->seq;
4504 	unsigned char newline = '\n';
4505 	struct trace_entry *entry;
4506 	struct trace_event *event;
4507 
4508 	entry = iter->ent;
4509 
4510 	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4511 		SEQ_PUT_HEX_FIELD(s, entry->pid);
4512 		SEQ_PUT_HEX_FIELD(s, iter->cpu);
4513 		SEQ_PUT_HEX_FIELD(s, iter->ts);
4514 		if (trace_seq_has_overflowed(s))
4515 			return TRACE_TYPE_PARTIAL_LINE;
4516 	}
4517 
4518 	event = ftrace_find_event(entry->type);
4519 	if (event) {
4520 		enum print_line_t ret = event->funcs->hex(iter, 0, event);
4521 		if (ret != TRACE_TYPE_HANDLED)
4522 			return ret;
4523 	}
4524 
4525 	SEQ_PUT_FIELD(s, newline);
4526 
4527 	return trace_handle_return(s);
4528 }
4529 
print_bin_fmt(struct trace_iterator * iter)4530 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
4531 {
4532 	struct trace_array *tr = iter->tr;
4533 	struct trace_seq *s = &iter->seq;
4534 	struct trace_entry *entry;
4535 	struct trace_event *event;
4536 
4537 	entry = iter->ent;
4538 
4539 	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4540 		SEQ_PUT_FIELD(s, entry->pid);
4541 		SEQ_PUT_FIELD(s, iter->cpu);
4542 		SEQ_PUT_FIELD(s, iter->ts);
4543 		if (trace_seq_has_overflowed(s))
4544 			return TRACE_TYPE_PARTIAL_LINE;
4545 	}
4546 
4547 	event = ftrace_find_event(entry->type);
4548 	return event ? event->funcs->binary(iter, 0, event) :
4549 		TRACE_TYPE_HANDLED;
4550 }
4551 
trace_empty(struct trace_iterator * iter)4552 int trace_empty(struct trace_iterator *iter)
4553 {
4554 	struct ring_buffer_iter *buf_iter;
4555 	int cpu;
4556 
4557 	/* If we are looking at one CPU buffer, only check that one */
4558 	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4559 		cpu = iter->cpu_file;
4560 		buf_iter = trace_buffer_iter(iter, cpu);
4561 		if (buf_iter) {
4562 			if (!ring_buffer_iter_empty(buf_iter))
4563 				return 0;
4564 		} else {
4565 			if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4566 				return 0;
4567 		}
4568 		return 1;
4569 	}
4570 
4571 	for_each_tracing_cpu(cpu) {
4572 		buf_iter = trace_buffer_iter(iter, cpu);
4573 		if (buf_iter) {
4574 			if (!ring_buffer_iter_empty(buf_iter))
4575 				return 0;
4576 		} else {
4577 			if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4578 				return 0;
4579 		}
4580 	}
4581 
4582 	return 1;
4583 }
4584 
4585 /*  Called with trace_event_read_lock() held. */
print_trace_line(struct trace_iterator * iter)4586 enum print_line_t print_trace_line(struct trace_iterator *iter)
4587 {
4588 	struct trace_array *tr = iter->tr;
4589 	unsigned long trace_flags = tr->trace_flags;
4590 	enum print_line_t ret;
4591 
4592 	if (iter->lost_events) {
4593 		if (iter->lost_events == (unsigned long)-1)
4594 			trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",
4595 					 iter->cpu);
4596 		else
4597 			trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
4598 					 iter->cpu, iter->lost_events);
4599 		if (trace_seq_has_overflowed(&iter->seq))
4600 			return TRACE_TYPE_PARTIAL_LINE;
4601 	}
4602 
4603 	if (iter->trace && iter->trace->print_line) {
4604 		ret = iter->trace->print_line(iter);
4605 		if (ret != TRACE_TYPE_UNHANDLED)
4606 			return ret;
4607 	}
4608 
4609 	if (iter->ent->type == TRACE_BPUTS &&
4610 			trace_flags & TRACE_ITER_PRINTK &&
4611 			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4612 		return trace_print_bputs_msg_only(iter);
4613 
4614 	if (iter->ent->type == TRACE_BPRINT &&
4615 			trace_flags & TRACE_ITER_PRINTK &&
4616 			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4617 		return trace_print_bprintk_msg_only(iter);
4618 
4619 	if (iter->ent->type == TRACE_PRINT &&
4620 			trace_flags & TRACE_ITER_PRINTK &&
4621 			trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4622 		return trace_print_printk_msg_only(iter);
4623 
4624 	if (trace_flags & TRACE_ITER_BIN)
4625 		return print_bin_fmt(iter);
4626 
4627 	if (trace_flags & TRACE_ITER_HEX)
4628 		return print_hex_fmt(iter);
4629 
4630 	if (trace_flags & TRACE_ITER_RAW)
4631 		return print_raw_fmt(iter);
4632 
4633 	return print_trace_fmt(iter);
4634 }
4635 
trace_latency_header(struct seq_file * m)4636 void trace_latency_header(struct seq_file *m)
4637 {
4638 	struct trace_iterator *iter = m->private;
4639 	struct trace_array *tr = iter->tr;
4640 
4641 	/* print nothing if the buffers are empty */
4642 	if (trace_empty(iter))
4643 		return;
4644 
4645 	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4646 		print_trace_header(m, iter);
4647 
4648 	if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
4649 		print_lat_help_header(m);
4650 }
4651 
trace_default_header(struct seq_file * m)4652 void trace_default_header(struct seq_file *m)
4653 {
4654 	struct trace_iterator *iter = m->private;
4655 	struct trace_array *tr = iter->tr;
4656 	unsigned long trace_flags = tr->trace_flags;
4657 
4658 	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
4659 		return;
4660 
4661 	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
4662 		/* print nothing if the buffers are empty */
4663 		if (trace_empty(iter))
4664 			return;
4665 		print_trace_header(m, iter);
4666 		if (!(trace_flags & TRACE_ITER_VERBOSE))
4667 			print_lat_help_header(m);
4668 	} else {
4669 		if (!(trace_flags & TRACE_ITER_VERBOSE)) {
4670 			if (trace_flags & TRACE_ITER_IRQ_INFO)
4671 				print_func_help_header_irq(iter->array_buffer,
4672 							   m, trace_flags);
4673 			else
4674 				print_func_help_header(iter->array_buffer, m,
4675 						       trace_flags);
4676 		}
4677 	}
4678 }
4679 
test_ftrace_alive(struct seq_file * m)4680 static void test_ftrace_alive(struct seq_file *m)
4681 {
4682 	if (!ftrace_is_dead())
4683 		return;
4684 	seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
4685 		    "#          MAY BE MISSING FUNCTION EVENTS\n");
4686 }
4687 
4688 #ifdef CONFIG_TRACER_MAX_TRACE
show_snapshot_main_help(struct seq_file * m)4689 static void show_snapshot_main_help(struct seq_file *m)
4690 {
4691 	seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
4692 		    "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4693 		    "#                      Takes a snapshot of the main buffer.\n"
4694 		    "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
4695 		    "#                      (Doesn't have to be '2' works with any number that\n"
4696 		    "#                       is not a '0' or '1')\n");
4697 }
4698 
show_snapshot_percpu_help(struct seq_file * m)4699 static void show_snapshot_percpu_help(struct seq_file *m)
4700 {
4701 	seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
4702 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
4703 	seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4704 		    "#                      Takes a snapshot of the main buffer for this cpu.\n");
4705 #else
4706 	seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
4707 		    "#                     Must use main snapshot file to allocate.\n");
4708 #endif
4709 	seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
4710 		    "#                      (Doesn't have to be '2' works with any number that\n"
4711 		    "#                       is not a '0' or '1')\n");
4712 }
4713 
print_snapshot_help(struct seq_file * m,struct trace_iterator * iter)4714 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
4715 {
4716 	if (iter->tr->allocated_snapshot)
4717 		seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
4718 	else
4719 		seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
4720 
4721 	seq_puts(m, "# Snapshot commands:\n");
4722 	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4723 		show_snapshot_main_help(m);
4724 	else
4725 		show_snapshot_percpu_help(m);
4726 }
4727 #else
4728 /* Should never be called */
print_snapshot_help(struct seq_file * m,struct trace_iterator * iter)4729 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
4730 #endif
4731 
s_show(struct seq_file * m,void * v)4732 static int s_show(struct seq_file *m, void *v)
4733 {
4734 	struct trace_iterator *iter = v;
4735 	int ret;
4736 
4737 	if (iter->ent == NULL) {
4738 		if (iter->tr) {
4739 			seq_printf(m, "# tracer: %s\n", iter->trace->name);
4740 			seq_puts(m, "#\n");
4741 			test_ftrace_alive(m);
4742 		}
4743 		if (iter->snapshot && trace_empty(iter))
4744 			print_snapshot_help(m, iter);
4745 		else if (iter->trace && iter->trace->print_header)
4746 			iter->trace->print_header(m);
4747 		else
4748 			trace_default_header(m);
4749 
4750 	} else if (iter->leftover) {
4751 		/*
4752 		 * If we filled the seq_file buffer earlier, we
4753 		 * want to just show it now.
4754 		 */
4755 		ret = trace_print_seq(m, &iter->seq);
4756 
4757 		/* ret should this time be zero, but you never know */
4758 		iter->leftover = ret;
4759 
4760 	} else {
4761 		ret = print_trace_line(iter);
4762 		if (ret == TRACE_TYPE_PARTIAL_LINE) {
4763 			iter->seq.full = 0;
4764 			trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n");
4765 		}
4766 		ret = trace_print_seq(m, &iter->seq);
4767 		/*
4768 		 * If we overflow the seq_file buffer, then it will
4769 		 * ask us for this data again at start up.
4770 		 * Use that instead.
4771 		 *  ret is 0 if seq_file write succeeded.
4772 		 *        -1 otherwise.
4773 		 */
4774 		iter->leftover = ret;
4775 	}
4776 
4777 	return 0;
4778 }
4779 
4780 /*
4781  * Should be used after trace_array_get(), trace_types_lock
4782  * ensures that i_cdev was already initialized.
4783  */
tracing_get_cpu(struct inode * inode)4784 static inline int tracing_get_cpu(struct inode *inode)
4785 {
4786 	if (inode->i_cdev) /* See trace_create_cpu_file() */
4787 		return (long)inode->i_cdev - 1;
4788 	return RING_BUFFER_ALL_CPUS;
4789 }
4790 
4791 static const struct seq_operations tracer_seq_ops = {
4792 	.start		= s_start,
4793 	.next		= s_next,
4794 	.stop		= s_stop,
4795 	.show		= s_show,
4796 };
4797 
4798 /*
4799  * Note, as iter itself can be allocated and freed in different
4800  * ways, this function is only used to free its content, and not
4801  * the iterator itself. The only requirement to all the allocations
4802  * is that it must zero all fields (kzalloc), as freeing works with
4803  * ethier allocated content or NULL.
4804  */
free_trace_iter_content(struct trace_iterator * iter)4805 static void free_trace_iter_content(struct trace_iterator *iter)
4806 {
4807 	/* The fmt is either NULL, allocated or points to static_fmt_buf */
4808 	if (iter->fmt != static_fmt_buf)
4809 		kfree(iter->fmt);
4810 
4811 	kfree(iter->temp);
4812 	kfree(iter->buffer_iter);
4813 	mutex_destroy(&iter->mutex);
4814 	free_cpumask_var(iter->started);
4815 }
4816 
4817 static struct trace_iterator *
__tracing_open(struct inode * inode,struct file * file,bool snapshot)4818 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
4819 {
4820 	struct trace_array *tr = inode->i_private;
4821 	struct trace_iterator *iter;
4822 	int cpu;
4823 
4824 	if (tracing_disabled)
4825 		return ERR_PTR(-ENODEV);
4826 
4827 	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
4828 	if (!iter)
4829 		return ERR_PTR(-ENOMEM);
4830 
4831 	iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
4832 				    GFP_KERNEL);
4833 	if (!iter->buffer_iter)
4834 		goto release;
4835 
4836 	/*
4837 	 * trace_find_next_entry() may need to save off iter->ent.
4838 	 * It will place it into the iter->temp buffer. As most
4839 	 * events are less than 128, allocate a buffer of that size.
4840 	 * If one is greater, then trace_find_next_entry() will
4841 	 * allocate a new buffer to adjust for the bigger iter->ent.
4842 	 * It's not critical if it fails to get allocated here.
4843 	 */
4844 	iter->temp = kmalloc(128, GFP_KERNEL);
4845 	if (iter->temp)
4846 		iter->temp_size = 128;
4847 
4848 	/*
4849 	 * trace_event_printf() may need to modify given format
4850 	 * string to replace %p with %px so that it shows real address
4851 	 * instead of hash value. However, that is only for the event
4852 	 * tracing, other tracer may not need. Defer the allocation
4853 	 * until it is needed.
4854 	 */
4855 	iter->fmt = NULL;
4856 	iter->fmt_size = 0;
4857 
4858 	mutex_lock(&trace_types_lock);
4859 	iter->trace = tr->current_trace;
4860 
4861 	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
4862 		goto fail;
4863 
4864 	iter->tr = tr;
4865 
4866 #ifdef CONFIG_TRACER_MAX_TRACE
4867 	/* Currently only the top directory has a snapshot */
4868 	if (tr->current_trace->print_max || snapshot)
4869 		iter->array_buffer = &tr->max_buffer;
4870 	else
4871 #endif
4872 		iter->array_buffer = &tr->array_buffer;
4873 	iter->snapshot = snapshot;
4874 	iter->pos = -1;
4875 	iter->cpu_file = tracing_get_cpu(inode);
4876 	mutex_init(&iter->mutex);
4877 
4878 	/* Notify the tracer early; before we stop tracing. */
4879 	if (iter->trace->open)
4880 		iter->trace->open(iter);
4881 
4882 	/* Annotate start of buffers if we had overruns */
4883 	if (ring_buffer_overruns(iter->array_buffer->buffer))
4884 		iter->iter_flags |= TRACE_FILE_ANNOTATE;
4885 
4886 	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
4887 	if (trace_clocks[tr->clock_id].in_ns)
4888 		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4889 
4890 	/*
4891 	 * If pause-on-trace is enabled, then stop the trace while
4892 	 * dumping, unless this is the "snapshot" file
4893 	 */
4894 	if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE))
4895 		tracing_stop_tr(tr);
4896 
4897 	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
4898 		for_each_tracing_cpu(cpu) {
4899 			iter->buffer_iter[cpu] =
4900 				ring_buffer_read_prepare(iter->array_buffer->buffer,
4901 							 cpu, GFP_KERNEL);
4902 		}
4903 		ring_buffer_read_prepare_sync();
4904 		for_each_tracing_cpu(cpu) {
4905 			ring_buffer_read_start(iter->buffer_iter[cpu]);
4906 			tracing_iter_reset(iter, cpu);
4907 		}
4908 	} else {
4909 		cpu = iter->cpu_file;
4910 		iter->buffer_iter[cpu] =
4911 			ring_buffer_read_prepare(iter->array_buffer->buffer,
4912 						 cpu, GFP_KERNEL);
4913 		ring_buffer_read_prepare_sync();
4914 		ring_buffer_read_start(iter->buffer_iter[cpu]);
4915 		tracing_iter_reset(iter, cpu);
4916 	}
4917 
4918 	mutex_unlock(&trace_types_lock);
4919 
4920 	return iter;
4921 
4922  fail:
4923 	mutex_unlock(&trace_types_lock);
4924 	free_trace_iter_content(iter);
4925 release:
4926 	seq_release_private(inode, file);
4927 	return ERR_PTR(-ENOMEM);
4928 }
4929 
tracing_open_generic(struct inode * inode,struct file * filp)4930 int tracing_open_generic(struct inode *inode, struct file *filp)
4931 {
4932 	int ret;
4933 
4934 	ret = tracing_check_open_get_tr(NULL);
4935 	if (ret)
4936 		return ret;
4937 
4938 	filp->private_data = inode->i_private;
4939 	return 0;
4940 }
4941 
tracing_is_disabled(void)4942 bool tracing_is_disabled(void)
4943 {
4944 	return (tracing_disabled) ? true: false;
4945 }
4946 
4947 /*
4948  * Open and update trace_array ref count.
4949  * Must have the current trace_array passed to it.
4950  */
tracing_open_generic_tr(struct inode * inode,struct file * filp)4951 int tracing_open_generic_tr(struct inode *inode, struct file *filp)
4952 {
4953 	struct trace_array *tr = inode->i_private;
4954 	int ret;
4955 
4956 	ret = tracing_check_open_get_tr(tr);
4957 	if (ret)
4958 		return ret;
4959 
4960 	filp->private_data = inode->i_private;
4961 
4962 	return 0;
4963 }
4964 
4965 /*
4966  * The private pointer of the inode is the trace_event_file.
4967  * Update the tr ref count associated to it.
4968  */
tracing_open_file_tr(struct inode * inode,struct file * filp)4969 int tracing_open_file_tr(struct inode *inode, struct file *filp)
4970 {
4971 	struct trace_event_file *file = inode->i_private;
4972 	int ret;
4973 
4974 	ret = tracing_check_open_get_tr(file->tr);
4975 	if (ret)
4976 		return ret;
4977 
4978 	mutex_lock(&event_mutex);
4979 
4980 	/* Fail if the file is marked for removal */
4981 	if (file->flags & EVENT_FILE_FL_FREED) {
4982 		trace_array_put(file->tr);
4983 		ret = -ENODEV;
4984 	} else {
4985 		event_file_get(file);
4986 	}
4987 
4988 	mutex_unlock(&event_mutex);
4989 	if (ret)
4990 		return ret;
4991 
4992 	filp->private_data = inode->i_private;
4993 
4994 	return 0;
4995 }
4996 
tracing_release_file_tr(struct inode * inode,struct file * filp)4997 int tracing_release_file_tr(struct inode *inode, struct file *filp)
4998 {
4999 	struct trace_event_file *file = inode->i_private;
5000 
5001 	trace_array_put(file->tr);
5002 	event_file_put(file);
5003 
5004 	return 0;
5005 }
5006 
tracing_single_release_file_tr(struct inode * inode,struct file * filp)5007 int tracing_single_release_file_tr(struct inode *inode, struct file *filp)
5008 {
5009 	tracing_release_file_tr(inode, filp);
5010 	return single_release(inode, filp);
5011 }
5012 
tracing_mark_open(struct inode * inode,struct file * filp)5013 static int tracing_mark_open(struct inode *inode, struct file *filp)
5014 {
5015 	stream_open(inode, filp);
5016 	return tracing_open_generic_tr(inode, filp);
5017 }
5018 
tracing_release(struct inode * inode,struct file * file)5019 static int tracing_release(struct inode *inode, struct file *file)
5020 {
5021 	struct trace_array *tr = inode->i_private;
5022 	struct seq_file *m = file->private_data;
5023 	struct trace_iterator *iter;
5024 	int cpu;
5025 
5026 	if (!(file->f_mode & FMODE_READ)) {
5027 		trace_array_put(tr);
5028 		return 0;
5029 	}
5030 
5031 	/* Writes do not use seq_file */
5032 	iter = m->private;
5033 	mutex_lock(&trace_types_lock);
5034 
5035 	for_each_tracing_cpu(cpu) {
5036 		if (iter->buffer_iter[cpu])
5037 			ring_buffer_read_finish(iter->buffer_iter[cpu]);
5038 	}
5039 
5040 	if (iter->trace && iter->trace->close)
5041 		iter->trace->close(iter);
5042 
5043 	if (!iter->snapshot && tr->stop_count)
5044 		/* reenable tracing if it was previously enabled */
5045 		tracing_start_tr(tr);
5046 
5047 	__trace_array_put(tr);
5048 
5049 	mutex_unlock(&trace_types_lock);
5050 
5051 	free_trace_iter_content(iter);
5052 	seq_release_private(inode, file);
5053 
5054 	return 0;
5055 }
5056 
tracing_release_generic_tr(struct inode * inode,struct file * file)5057 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
5058 {
5059 	struct trace_array *tr = inode->i_private;
5060 
5061 	trace_array_put(tr);
5062 	return 0;
5063 }
5064 
tracing_single_release_tr(struct inode * inode,struct file * file)5065 static int tracing_single_release_tr(struct inode *inode, struct file *file)
5066 {
5067 	struct trace_array *tr = inode->i_private;
5068 
5069 	trace_array_put(tr);
5070 
5071 	return single_release(inode, file);
5072 }
5073 
tracing_open(struct inode * inode,struct file * file)5074 static int tracing_open(struct inode *inode, struct file *file)
5075 {
5076 	struct trace_array *tr = inode->i_private;
5077 	struct trace_iterator *iter;
5078 	int ret;
5079 
5080 	ret = tracing_check_open_get_tr(tr);
5081 	if (ret)
5082 		return ret;
5083 
5084 	/* If this file was open for write, then erase contents */
5085 	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
5086 		int cpu = tracing_get_cpu(inode);
5087 		struct array_buffer *trace_buf = &tr->array_buffer;
5088 
5089 #ifdef CONFIG_TRACER_MAX_TRACE
5090 		if (tr->current_trace->print_max)
5091 			trace_buf = &tr->max_buffer;
5092 #endif
5093 
5094 		if (cpu == RING_BUFFER_ALL_CPUS)
5095 			tracing_reset_online_cpus(trace_buf);
5096 		else
5097 			tracing_reset_cpu(trace_buf, cpu);
5098 	}
5099 
5100 	if (file->f_mode & FMODE_READ) {
5101 		iter = __tracing_open(inode, file, false);
5102 		if (IS_ERR(iter))
5103 			ret = PTR_ERR(iter);
5104 		else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
5105 			iter->iter_flags |= TRACE_FILE_LAT_FMT;
5106 	}
5107 
5108 	if (ret < 0)
5109 		trace_array_put(tr);
5110 
5111 	return ret;
5112 }
5113 
5114 /*
5115  * Some tracers are not suitable for instance buffers.
5116  * A tracer is always available for the global array (toplevel)
5117  * or if it explicitly states that it is.
5118  */
5119 static bool
trace_ok_for_array(struct tracer * t,struct trace_array * tr)5120 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
5121 {
5122 	return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
5123 }
5124 
5125 /* Find the next tracer that this trace array may use */
5126 static struct tracer *
get_tracer_for_array(struct trace_array * tr,struct tracer * t)5127 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
5128 {
5129 	while (t && !trace_ok_for_array(t, tr))
5130 		t = t->next;
5131 
5132 	return t;
5133 }
5134 
5135 static void *
t_next(struct seq_file * m,void * v,loff_t * pos)5136 t_next(struct seq_file *m, void *v, loff_t *pos)
5137 {
5138 	struct trace_array *tr = m->private;
5139 	struct tracer *t = v;
5140 
5141 	(*pos)++;
5142 
5143 	if (t)
5144 		t = get_tracer_for_array(tr, t->next);
5145 
5146 	return t;
5147 }
5148 
t_start(struct seq_file * m,loff_t * pos)5149 static void *t_start(struct seq_file *m, loff_t *pos)
5150 {
5151 	struct trace_array *tr = m->private;
5152 	struct tracer *t;
5153 	loff_t l = 0;
5154 
5155 	mutex_lock(&trace_types_lock);
5156 
5157 	t = get_tracer_for_array(tr, trace_types);
5158 	for (; t && l < *pos; t = t_next(m, t, &l))
5159 			;
5160 
5161 	return t;
5162 }
5163 
t_stop(struct seq_file * m,void * p)5164 static void t_stop(struct seq_file *m, void *p)
5165 {
5166 	mutex_unlock(&trace_types_lock);
5167 }
5168 
t_show(struct seq_file * m,void * v)5169 static int t_show(struct seq_file *m, void *v)
5170 {
5171 	struct tracer *t = v;
5172 
5173 	if (!t)
5174 		return 0;
5175 
5176 	seq_puts(m, t->name);
5177 	if (t->next)
5178 		seq_putc(m, ' ');
5179 	else
5180 		seq_putc(m, '\n');
5181 
5182 	return 0;
5183 }
5184 
5185 static const struct seq_operations show_traces_seq_ops = {
5186 	.start		= t_start,
5187 	.next		= t_next,
5188 	.stop		= t_stop,
5189 	.show		= t_show,
5190 };
5191 
show_traces_open(struct inode * inode,struct file * file)5192 static int show_traces_open(struct inode *inode, struct file *file)
5193 {
5194 	struct trace_array *tr = inode->i_private;
5195 	struct seq_file *m;
5196 	int ret;
5197 
5198 	ret = tracing_check_open_get_tr(tr);
5199 	if (ret)
5200 		return ret;
5201 
5202 	ret = seq_open(file, &show_traces_seq_ops);
5203 	if (ret) {
5204 		trace_array_put(tr);
5205 		return ret;
5206 	}
5207 
5208 	m = file->private_data;
5209 	m->private = tr;
5210 
5211 	return 0;
5212 }
5213 
show_traces_release(struct inode * inode,struct file * file)5214 static int show_traces_release(struct inode *inode, struct file *file)
5215 {
5216 	struct trace_array *tr = inode->i_private;
5217 
5218 	trace_array_put(tr);
5219 	return seq_release(inode, file);
5220 }
5221 
5222 static ssize_t
tracing_write_stub(struct file * filp,const char __user * ubuf,size_t count,loff_t * ppos)5223 tracing_write_stub(struct file *filp, const char __user *ubuf,
5224 		   size_t count, loff_t *ppos)
5225 {
5226 	return count;
5227 }
5228 
tracing_lseek(struct file * file,loff_t offset,int whence)5229 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
5230 {
5231 	int ret;
5232 
5233 	if (file->f_mode & FMODE_READ)
5234 		ret = seq_lseek(file, offset, whence);
5235 	else
5236 		file->f_pos = ret = 0;
5237 
5238 	return ret;
5239 }
5240 
5241 static const struct file_operations tracing_fops = {
5242 	.open		= tracing_open,
5243 	.read		= seq_read,
5244 	.read_iter	= seq_read_iter,
5245 	.splice_read	= copy_splice_read,
5246 	.write		= tracing_write_stub,
5247 	.llseek		= tracing_lseek,
5248 	.release	= tracing_release,
5249 };
5250 
5251 static const struct file_operations show_traces_fops = {
5252 	.open		= show_traces_open,
5253 	.read		= seq_read,
5254 	.llseek		= seq_lseek,
5255 	.release	= show_traces_release,
5256 };
5257 
5258 static ssize_t
tracing_cpumask_read(struct file * filp,char __user * ubuf,size_t count,loff_t * ppos)5259 tracing_cpumask_read(struct file *filp, char __user *ubuf,
5260 		     size_t count, loff_t *ppos)
5261 {
5262 	struct trace_array *tr = file_inode(filp)->i_private;
5263 	char *mask_str;
5264 	int len;
5265 
5266 	len = snprintf(NULL, 0, "%*pb\n",
5267 		       cpumask_pr_args(tr->tracing_cpumask)) + 1;
5268 	mask_str = kmalloc(len, GFP_KERNEL);
5269 	if (!mask_str)
5270 		return -ENOMEM;
5271 
5272 	len = snprintf(mask_str, len, "%*pb\n",
5273 		       cpumask_pr_args(tr->tracing_cpumask));
5274 	if (len >= count) {
5275 		count = -EINVAL;
5276 		goto out_err;
5277 	}
5278 	count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
5279 
5280 out_err:
5281 	kfree(mask_str);
5282 
5283 	return count;
5284 }
5285 
tracing_set_cpumask(struct trace_array * tr,cpumask_var_t tracing_cpumask_new)5286 int tracing_set_cpumask(struct trace_array *tr,
5287 			cpumask_var_t tracing_cpumask_new)
5288 {
5289 	int cpu;
5290 
5291 	if (!tr)
5292 		return -EINVAL;
5293 
5294 	local_irq_disable();
5295 	arch_spin_lock(&tr->max_lock);
5296 	for_each_tracing_cpu(cpu) {
5297 		/*
5298 		 * Increase/decrease the disabled counter if we are
5299 		 * about to flip a bit in the cpumask:
5300 		 */
5301 		if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
5302 				!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
5303 			atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
5304 			ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
5305 #ifdef CONFIG_TRACER_MAX_TRACE
5306 			ring_buffer_record_disable_cpu(tr->max_buffer.buffer, cpu);
5307 #endif
5308 		}
5309 		if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
5310 				cpumask_test_cpu(cpu, tracing_cpumask_new)) {
5311 			atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
5312 			ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
5313 #ifdef CONFIG_TRACER_MAX_TRACE
5314 			ring_buffer_record_enable_cpu(tr->max_buffer.buffer, cpu);
5315 #endif
5316 		}
5317 	}
5318 	arch_spin_unlock(&tr->max_lock);
5319 	local_irq_enable();
5320 
5321 	cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
5322 
5323 	return 0;
5324 }
5325 
5326 static ssize_t
tracing_cpumask_write(struct file * filp,const char __user * ubuf,size_t count,loff_t * ppos)5327 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
5328 		      size_t count, loff_t *ppos)
5329 {
5330 	struct trace_array *tr = file_inode(filp)->i_private;
5331 	cpumask_var_t tracing_cpumask_new;
5332 	int err;
5333 
5334 	if (!zalloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
5335 		return -ENOMEM;
5336 
5337 	err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
5338 	if (err)
5339 		goto err_free;
5340 
5341 	err = tracing_set_cpumask(tr, tracing_cpumask_new);
5342 	if (err)
5343 		goto err_free;
5344 
5345 	free_cpumask_var(tracing_cpumask_new);
5346 
5347 	return count;
5348 
5349 err_free:
5350 	free_cpumask_var(tracing_cpumask_new);
5351 
5352 	return err;
5353 }
5354 
5355 static const struct file_operations tracing_cpumask_fops = {
5356 	.open		= tracing_open_generic_tr,
5357 	.read		= tracing_cpumask_read,
5358 	.write		= tracing_cpumask_write,
5359 	.release	= tracing_release_generic_tr,
5360 	.llseek		= generic_file_llseek,
5361 };
5362 
tracing_trace_options_show(struct seq_file * m,void * v)5363 static int tracing_trace_options_show(struct seq_file *m, void *v)
5364 {
5365 	struct tracer_opt *trace_opts;
5366 	struct trace_array *tr = m->private;
5367 	u32 tracer_flags;
5368 	int i;
5369 
5370 	mutex_lock(&trace_types_lock);
5371 	tracer_flags = tr->current_trace->flags->val;
5372 	trace_opts = tr->current_trace->flags->opts;
5373 
5374 	for (i = 0; trace_options[i]; i++) {
5375 		if (tr->trace_flags & (1 << i))
5376 			seq_printf(m, "%s\n", trace_options[i]);
5377 		else
5378 			seq_printf(m, "no%s\n", trace_options[i]);
5379 	}
5380 
5381 	for (i = 0; trace_opts[i].name; i++) {
5382 		if (tracer_flags & trace_opts[i].bit)
5383 			seq_printf(m, "%s\n", trace_opts[i].name);
5384 		else
5385 			seq_printf(m, "no%s\n", trace_opts[i].name);
5386 	}
5387 	mutex_unlock(&trace_types_lock);
5388 
5389 	return 0;
5390 }
5391 
__set_tracer_option(struct trace_array * tr,struct tracer_flags * tracer_flags,struct tracer_opt * opts,int neg)5392 static int __set_tracer_option(struct trace_array *tr,
5393 			       struct tracer_flags *tracer_flags,
5394 			       struct tracer_opt *opts, int neg)
5395 {
5396 	struct tracer *trace = tracer_flags->trace;
5397 	int ret;
5398 
5399 	ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
5400 	if (ret)
5401 		return ret;
5402 
5403 	if (neg)
5404 		tracer_flags->val &= ~opts->bit;
5405 	else
5406 		tracer_flags->val |= opts->bit;
5407 	return 0;
5408 }
5409 
5410 /* Try to assign a tracer specific option */
set_tracer_option(struct trace_array * tr,char * cmp,int neg)5411 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
5412 {
5413 	struct tracer *trace = tr->current_trace;
5414 	struct tracer_flags *tracer_flags = trace->flags;
5415 	struct tracer_opt *opts = NULL;
5416 	int i;
5417 
5418 	for (i = 0; tracer_flags->opts[i].name; i++) {
5419 		opts = &tracer_flags->opts[i];
5420 
5421 		if (strcmp(cmp, opts->name) == 0)
5422 			return __set_tracer_option(tr, trace->flags, opts, neg);
5423 	}
5424 
5425 	return -EINVAL;
5426 }
5427 
5428 /* Some tracers require overwrite to stay enabled */
trace_keep_overwrite(struct tracer * tracer,u32 mask,int set)5429 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
5430 {
5431 	if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
5432 		return -1;
5433 
5434 	return 0;
5435 }
5436 
set_tracer_flag(struct trace_array * tr,unsigned int mask,int enabled)5437 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
5438 {
5439 	int *map;
5440 
5441 	if ((mask == TRACE_ITER_RECORD_TGID) ||
5442 	    (mask == TRACE_ITER_RECORD_CMD))
5443 		lockdep_assert_held(&event_mutex);
5444 
5445 	/* do nothing if flag is already set */
5446 	if (!!(tr->trace_flags & mask) == !!enabled)
5447 		return 0;
5448 
5449 	/* Give the tracer a chance to approve the change */
5450 	if (tr->current_trace->flag_changed)
5451 		if (tr->current_trace->flag_changed(tr, mask, !!enabled))
5452 			return -EINVAL;
5453 
5454 	if (enabled)
5455 		tr->trace_flags |= mask;
5456 	else
5457 		tr->trace_flags &= ~mask;
5458 
5459 	if (mask == TRACE_ITER_RECORD_CMD)
5460 		trace_event_enable_cmd_record(enabled);
5461 
5462 	if (mask == TRACE_ITER_RECORD_TGID) {
5463 		if (!tgid_map) {
5464 			tgid_map_max = pid_max;
5465 			map = kvcalloc(tgid_map_max + 1, sizeof(*tgid_map),
5466 				       GFP_KERNEL);
5467 
5468 			/*
5469 			 * Pairs with smp_load_acquire() in
5470 			 * trace_find_tgid_ptr() to ensure that if it observes
5471 			 * the tgid_map we just allocated then it also observes
5472 			 * the corresponding tgid_map_max value.
5473 			 */
5474 			smp_store_release(&tgid_map, map);
5475 		}
5476 		if (!tgid_map) {
5477 			tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
5478 			return -ENOMEM;
5479 		}
5480 
5481 		trace_event_enable_tgid_record(enabled);
5482 	}
5483 
5484 	if (mask == TRACE_ITER_EVENT_FORK)
5485 		trace_event_follow_fork(tr, enabled);
5486 
5487 	if (mask == TRACE_ITER_FUNC_FORK)
5488 		ftrace_pid_follow_fork(tr, enabled);
5489 
5490 	if (mask == TRACE_ITER_OVERWRITE) {
5491 		ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);
5492 #ifdef CONFIG_TRACER_MAX_TRACE
5493 		ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
5494 #endif
5495 	}
5496 
5497 	if (mask == TRACE_ITER_PRINTK) {
5498 		trace_printk_start_stop_comm(enabled);
5499 		trace_printk_control(enabled);
5500 	}
5501 
5502 	return 0;
5503 }
5504 
trace_set_options(struct trace_array * tr,char * option)5505 int trace_set_options(struct trace_array *tr, char *option)
5506 {
5507 	char *cmp;
5508 	int neg = 0;
5509 	int ret;
5510 	size_t orig_len = strlen(option);
5511 	int len;
5512 
5513 	cmp = strstrip(option);
5514 
5515 	len = str_has_prefix(cmp, "no");
5516 	if (len)
5517 		neg = 1;
5518 
5519 	cmp += len;
5520 
5521 	mutex_lock(&event_mutex);
5522 	mutex_lock(&trace_types_lock);
5523 
5524 	ret = match_string(trace_options, -1, cmp);
5525 	/* If no option could be set, test the specific tracer options */
5526 	if (ret < 0)
5527 		ret = set_tracer_option(tr, cmp, neg);
5528 	else
5529 		ret = set_tracer_flag(tr, 1 << ret, !neg);
5530 
5531 	mutex_unlock(&trace_types_lock);
5532 	mutex_unlock(&event_mutex);
5533 
5534 	/*
5535 	 * If the first trailing whitespace is replaced with '\0' by strstrip,
5536 	 * turn it back into a space.
5537 	 */
5538 	if (orig_len > strlen(option))
5539 		option[strlen(option)] = ' ';
5540 
5541 	return ret;
5542 }
5543 
apply_trace_boot_options(void)5544 static void __init apply_trace_boot_options(void)
5545 {
5546 	char *buf = trace_boot_options_buf;
5547 	char *option;
5548 
5549 	while (true) {
5550 		option = strsep(&buf, ",");
5551 
5552 		if (!option)
5553 			break;
5554 
5555 		if (*option)
5556 			trace_set_options(&global_trace, option);
5557 
5558 		/* Put back the comma to allow this to be called again */
5559 		if (buf)
5560 			*(buf - 1) = ',';
5561 	}
5562 }
5563 
5564 static ssize_t
tracing_trace_options_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)5565 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
5566 			size_t cnt, loff_t *ppos)
5567 {
5568 	struct seq_file *m = filp->private_data;
5569 	struct trace_array *tr = m->private;
5570 	char buf[64];
5571 	int ret;
5572 
5573 	if (cnt >= sizeof(buf))
5574 		return -EINVAL;
5575 
5576 	if (copy_from_user(buf, ubuf, cnt))
5577 		return -EFAULT;
5578 
5579 	buf[cnt] = 0;
5580 
5581 	ret = trace_set_options(tr, buf);
5582 	if (ret < 0)
5583 		return ret;
5584 
5585 	*ppos += cnt;
5586 
5587 	return cnt;
5588 }
5589 
tracing_trace_options_open(struct inode * inode,struct file * file)5590 static int tracing_trace_options_open(struct inode *inode, struct file *file)
5591 {
5592 	struct trace_array *tr = inode->i_private;
5593 	int ret;
5594 
5595 	ret = tracing_check_open_get_tr(tr);
5596 	if (ret)
5597 		return ret;
5598 
5599 	ret = single_open(file, tracing_trace_options_show, inode->i_private);
5600 	if (ret < 0)
5601 		trace_array_put(tr);
5602 
5603 	return ret;
5604 }
5605 
5606 static const struct file_operations tracing_iter_fops = {
5607 	.open		= tracing_trace_options_open,
5608 	.read		= seq_read,
5609 	.llseek		= seq_lseek,
5610 	.release	= tracing_single_release_tr,
5611 	.write		= tracing_trace_options_write,
5612 };
5613 
5614 static const char readme_msg[] =
5615 	"tracing mini-HOWTO:\n\n"
5616 	"# echo 0 > tracing_on : quick way to disable tracing\n"
5617 	"# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
5618 	" Important files:\n"
5619 	"  trace\t\t\t- The static contents of the buffer\n"
5620 	"\t\t\t  To clear the buffer write into this file: echo > trace\n"
5621 	"  trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
5622 	"  current_tracer\t- function and latency tracers\n"
5623 	"  available_tracers\t- list of configured tracers for current_tracer\n"
5624 	"  error_log\t- error log for failed commands (that support it)\n"
5625 	"  buffer_size_kb\t- view and modify size of per cpu buffer\n"
5626 	"  buffer_total_size_kb  - view total size of all cpu buffers\n\n"
5627 	"  trace_clock\t\t- change the clock used to order events\n"
5628 	"       local:   Per cpu clock but may not be synced across CPUs\n"
5629 	"      global:   Synced across CPUs but slows tracing down.\n"
5630 	"     counter:   Not a clock, but just an increment\n"
5631 	"      uptime:   Jiffy counter from time of boot\n"
5632 	"        perf:   Same clock that perf events use\n"
5633 #ifdef CONFIG_X86_64
5634 	"     x86-tsc:   TSC cycle counter\n"
5635 #endif
5636 	"\n  timestamp_mode\t- view the mode used to timestamp events\n"
5637 	"       delta:   Delta difference against a buffer-wide timestamp\n"
5638 	"    absolute:   Absolute (standalone) timestamp\n"
5639 	"\n  trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
5640 	"\n  trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
5641 	"  tracing_cpumask\t- Limit which CPUs to trace\n"
5642 	"  instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
5643 	"\t\t\t  Remove sub-buffer with rmdir\n"
5644 	"  trace_options\t\t- Set format or modify how tracing happens\n"
5645 	"\t\t\t  Disable an option by prefixing 'no' to the\n"
5646 	"\t\t\t  option name\n"
5647 	"  saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
5648 #ifdef CONFIG_DYNAMIC_FTRACE
5649 	"\n  available_filter_functions - list of functions that can be filtered on\n"
5650 	"  set_ftrace_filter\t- echo function name in here to only trace these\n"
5651 	"\t\t\t  functions\n"
5652 	"\t     accepts: func_full_name or glob-matching-pattern\n"
5653 	"\t     modules: Can select a group via module\n"
5654 	"\t      Format: :mod:<module-name>\n"
5655 	"\t     example: echo :mod:ext3 > set_ftrace_filter\n"
5656 	"\t    triggers: a command to perform when function is hit\n"
5657 	"\t      Format: <function>:<trigger>[:count]\n"
5658 	"\t     trigger: traceon, traceoff\n"
5659 	"\t\t      enable_event:<system>:<event>\n"
5660 	"\t\t      disable_event:<system>:<event>\n"
5661 #ifdef CONFIG_STACKTRACE
5662 	"\t\t      stacktrace\n"
5663 #endif
5664 #ifdef CONFIG_TRACER_SNAPSHOT
5665 	"\t\t      snapshot\n"
5666 #endif
5667 	"\t\t      dump\n"
5668 	"\t\t      cpudump\n"
5669 	"\t     example: echo do_fault:traceoff > set_ftrace_filter\n"
5670 	"\t              echo do_trap:traceoff:3 > set_ftrace_filter\n"
5671 	"\t     The first one will disable tracing every time do_fault is hit\n"
5672 	"\t     The second will disable tracing at most 3 times when do_trap is hit\n"
5673 	"\t       The first time do trap is hit and it disables tracing, the\n"
5674 	"\t       counter will decrement to 2. If tracing is already disabled,\n"
5675 	"\t       the counter will not decrement. It only decrements when the\n"
5676 	"\t       trigger did work\n"
5677 	"\t     To remove trigger without count:\n"
5678 	"\t       echo '!<function>:<trigger> > set_ftrace_filter\n"
5679 	"\t     To remove trigger with a count:\n"
5680 	"\t       echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
5681 	"  set_ftrace_notrace\t- echo function name in here to never trace.\n"
5682 	"\t    accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
5683 	"\t    modules: Can select a group via module command :mod:\n"
5684 	"\t    Does not accept triggers\n"
5685 #endif /* CONFIG_DYNAMIC_FTRACE */
5686 #ifdef CONFIG_FUNCTION_TRACER
5687 	"  set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
5688 	"\t\t    (function)\n"
5689 	"  set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
5690 	"\t\t    (function)\n"
5691 #endif
5692 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
5693 	"  set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
5694 	"  set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
5695 	"  max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
5696 #endif
5697 #ifdef CONFIG_TRACER_SNAPSHOT
5698 	"\n  snapshot\t\t- Like 'trace' but shows the content of the static\n"
5699 	"\t\t\t  snapshot buffer. Read the contents for more\n"
5700 	"\t\t\t  information\n"
5701 #endif
5702 #ifdef CONFIG_STACK_TRACER
5703 	"  stack_trace\t\t- Shows the max stack trace when active\n"
5704 	"  stack_max_size\t- Shows current max stack size that was traced\n"
5705 	"\t\t\t  Write into this file to reset the max size (trigger a\n"
5706 	"\t\t\t  new trace)\n"
5707 #ifdef CONFIG_DYNAMIC_FTRACE
5708 	"  stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
5709 	"\t\t\t  traces\n"
5710 #endif
5711 #endif /* CONFIG_STACK_TRACER */
5712 #ifdef CONFIG_DYNAMIC_EVENTS
5713 	"  dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
5714 	"\t\t\t  Write into this file to define/undefine new trace events.\n"
5715 #endif
5716 #ifdef CONFIG_KPROBE_EVENTS
5717 	"  kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
5718 	"\t\t\t  Write into this file to define/undefine new trace events.\n"
5719 #endif
5720 #ifdef CONFIG_UPROBE_EVENTS
5721 	"  uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
5722 	"\t\t\t  Write into this file to define/undefine new trace events.\n"
5723 #endif
5724 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) || \
5725     defined(CONFIG_FPROBE_EVENTS)
5726 	"\t  accepts: event-definitions (one definition per line)\n"
5727 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
5728 	"\t   Format: p[:[<group>/][<event>]] <place> [<args>]\n"
5729 	"\t           r[maxactive][:[<group>/][<event>]] <place> [<args>]\n"
5730 #endif
5731 #ifdef CONFIG_FPROBE_EVENTS
5732 	"\t           f[:[<group>/][<event>]] <func-name>[%return] [<args>]\n"
5733 	"\t           t[:[<group>/][<event>]] <tracepoint> [<args>]\n"
5734 #endif
5735 #ifdef CONFIG_HIST_TRIGGERS
5736 	"\t           s:[synthetic/]<event> <field> [<field>]\n"
5737 #endif
5738 	"\t           e[:[<group>/][<event>]] <attached-group>.<attached-event> [<args>] [if <filter>]\n"
5739 	"\t           -:[<group>/][<event>]\n"
5740 #ifdef CONFIG_KPROBE_EVENTS
5741 	"\t    place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
5742   "place (kretprobe): [<module>:]<symbol>[+<offset>]%return|<memaddr>\n"
5743 #endif
5744 #ifdef CONFIG_UPROBE_EVENTS
5745   "   place (uprobe): <path>:<offset>[%return][(ref_ctr_offset)]\n"
5746 #endif
5747 	"\t     args: <name>=fetcharg[:type]\n"
5748 	"\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n"
5749 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
5750 #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
5751 	"\t           $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
5752 	"\t           <argname>[->field[->field|.field...]],\n"
5753 #else
5754 	"\t           $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
5755 #endif
5756 #else
5757 	"\t           $stack<index>, $stack, $retval, $comm,\n"
5758 #endif
5759 	"\t           +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
5760 	"\t     type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n"
5761 	"\t           b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
5762 	"\t           symstr, <type>\\[<array-size>\\]\n"
5763 #ifdef CONFIG_HIST_TRIGGERS
5764 	"\t    field: <stype> <name>;\n"
5765 	"\t    stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
5766 	"\t           [unsigned] char/int/long\n"
5767 #endif
5768 	"\t    efield: For event probes ('e' types), the field is on of the fields\n"
5769 	"\t            of the <attached-group>/<attached-event>.\n"
5770 #endif
5771 	"  events/\t\t- Directory containing all trace event subsystems:\n"
5772 	"      enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
5773 	"  events/<system>/\t- Directory containing all trace events for <system>:\n"
5774 	"      enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
5775 	"\t\t\t  events\n"
5776 	"      filter\t\t- If set, only events passing filter are traced\n"
5777 	"  events/<system>/<event>/\t- Directory containing control files for\n"
5778 	"\t\t\t  <event>:\n"
5779 	"      enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
5780 	"      filter\t\t- If set, only events passing filter are traced\n"
5781 	"      trigger\t\t- If set, a command to perform when event is hit\n"
5782 	"\t    Format: <trigger>[:count][if <filter>]\n"
5783 	"\t   trigger: traceon, traceoff\n"
5784 	"\t            enable_event:<system>:<event>\n"
5785 	"\t            disable_event:<system>:<event>\n"
5786 #ifdef CONFIG_HIST_TRIGGERS
5787 	"\t            enable_hist:<system>:<event>\n"
5788 	"\t            disable_hist:<system>:<event>\n"
5789 #endif
5790 #ifdef CONFIG_STACKTRACE
5791 	"\t\t    stacktrace\n"
5792 #endif
5793 #ifdef CONFIG_TRACER_SNAPSHOT
5794 	"\t\t    snapshot\n"
5795 #endif
5796 #ifdef CONFIG_HIST_TRIGGERS
5797 	"\t\t    hist (see below)\n"
5798 #endif
5799 	"\t   example: echo traceoff > events/block/block_unplug/trigger\n"
5800 	"\t            echo traceoff:3 > events/block/block_unplug/trigger\n"
5801 	"\t            echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
5802 	"\t                  events/block/block_unplug/trigger\n"
5803 	"\t   The first disables tracing every time block_unplug is hit.\n"
5804 	"\t   The second disables tracing the first 3 times block_unplug is hit.\n"
5805 	"\t   The third enables the kmalloc event the first 3 times block_unplug\n"
5806 	"\t     is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
5807 	"\t   Like function triggers, the counter is only decremented if it\n"
5808 	"\t    enabled or disabled tracing.\n"
5809 	"\t   To remove a trigger without a count:\n"
5810 	"\t     echo '!<trigger> > <system>/<event>/trigger\n"
5811 	"\t   To remove a trigger with a count:\n"
5812 	"\t     echo '!<trigger>:0 > <system>/<event>/trigger\n"
5813 	"\t   Filters can be ignored when removing a trigger.\n"
5814 #ifdef CONFIG_HIST_TRIGGERS
5815 	"      hist trigger\t- If set, event hits are aggregated into a hash table\n"
5816 	"\t    Format: hist:keys=<field1[,field2,...]>\n"
5817 	"\t            [:<var1>=<field|var_ref|numeric_literal>[,<var2>=...]]\n"
5818 	"\t            [:values=<field1[,field2,...]>]\n"
5819 	"\t            [:sort=<field1[,field2,...]>]\n"
5820 	"\t            [:size=#entries]\n"
5821 	"\t            [:pause][:continue][:clear]\n"
5822 	"\t            [:name=histname1]\n"
5823 	"\t            [:nohitcount]\n"
5824 	"\t            [:<handler>.<action>]\n"
5825 	"\t            [if <filter>]\n\n"
5826 	"\t    Note, special fields can be used as well:\n"
5827 	"\t            common_timestamp - to record current timestamp\n"
5828 	"\t            common_cpu - to record the CPU the event happened on\n"
5829 	"\n"
5830 	"\t    A hist trigger variable can be:\n"
5831 	"\t        - a reference to a field e.g. x=current_timestamp,\n"
5832 	"\t        - a reference to another variable e.g. y=$x,\n"
5833 	"\t        - a numeric literal: e.g. ms_per_sec=1000,\n"
5834 	"\t        - an arithmetic expression: e.g. time_secs=current_timestamp/1000\n"
5835 	"\n"
5836 	"\t    hist trigger arithmetic expressions support addition(+), subtraction(-),\n"
5837 	"\t    multiplication(*) and division(/) operators. An operand can be either a\n"
5838 	"\t    variable reference, field or numeric literal.\n"
5839 	"\n"
5840 	"\t    When a matching event is hit, an entry is added to a hash\n"
5841 	"\t    table using the key(s) and value(s) named, and the value of a\n"
5842 	"\t    sum called 'hitcount' is incremented.  Keys and values\n"
5843 	"\t    correspond to fields in the event's format description.  Keys\n"
5844 	"\t    can be any field, or the special string 'common_stacktrace'.\n"
5845 	"\t    Compound keys consisting of up to two fields can be specified\n"
5846 	"\t    by the 'keys' keyword.  Values must correspond to numeric\n"
5847 	"\t    fields.  Sort keys consisting of up to two fields can be\n"
5848 	"\t    specified using the 'sort' keyword.  The sort direction can\n"
5849 	"\t    be modified by appending '.descending' or '.ascending' to a\n"
5850 	"\t    sort field.  The 'size' parameter can be used to specify more\n"
5851 	"\t    or fewer than the default 2048 entries for the hashtable size.\n"
5852 	"\t    If a hist trigger is given a name using the 'name' parameter,\n"
5853 	"\t    its histogram data will be shared with other triggers of the\n"
5854 	"\t    same name, and trigger hits will update this common data.\n\n"
5855 	"\t    Reading the 'hist' file for the event will dump the hash\n"
5856 	"\t    table in its entirety to stdout.  If there are multiple hist\n"
5857 	"\t    triggers attached to an event, there will be a table for each\n"
5858 	"\t    trigger in the output.  The table displayed for a named\n"
5859 	"\t    trigger will be the same as any other instance having the\n"
5860 	"\t    same name.  The default format used to display a given field\n"
5861 	"\t    can be modified by appending any of the following modifiers\n"
5862 	"\t    to the field name, as applicable:\n\n"
5863 	"\t            .hex        display a number as a hex value\n"
5864 	"\t            .sym        display an address as a symbol\n"
5865 	"\t            .sym-offset display an address as a symbol and offset\n"
5866 	"\t            .execname   display a common_pid as a program name\n"
5867 	"\t            .syscall    display a syscall id as a syscall name\n"
5868 	"\t            .log2       display log2 value rather than raw number\n"
5869 	"\t            .buckets=size  display values in groups of size rather than raw number\n"
5870 	"\t            .usecs      display a common_timestamp in microseconds\n"
5871 	"\t            .percent    display a number of percentage value\n"
5872 	"\t            .graph      display a bar-graph of a value\n\n"
5873 	"\t    The 'pause' parameter can be used to pause an existing hist\n"
5874 	"\t    trigger or to start a hist trigger but not log any events\n"
5875 	"\t    until told to do so.  'continue' can be used to start or\n"
5876 	"\t    restart a paused hist trigger.\n\n"
5877 	"\t    The 'clear' parameter will clear the contents of a running\n"
5878 	"\t    hist trigger and leave its current paused/active state\n"
5879 	"\t    unchanged.\n\n"
5880 	"\t    The 'nohitcount' (or NOHC) parameter will suppress display of\n"
5881 	"\t    raw hitcount in the histogram.\n\n"
5882 	"\t    The enable_hist and disable_hist triggers can be used to\n"
5883 	"\t    have one event conditionally start and stop another event's\n"
5884 	"\t    already-attached hist trigger.  The syntax is analogous to\n"
5885 	"\t    the enable_event and disable_event triggers.\n\n"
5886 	"\t    Hist trigger handlers and actions are executed whenever a\n"
5887 	"\t    a histogram entry is added or updated.  They take the form:\n\n"
5888 	"\t        <handler>.<action>\n\n"
5889 	"\t    The available handlers are:\n\n"
5890 	"\t        onmatch(matching.event)  - invoke on addition or update\n"
5891 	"\t        onmax(var)               - invoke if var exceeds current max\n"
5892 	"\t        onchange(var)            - invoke action if var changes\n\n"
5893 	"\t    The available actions are:\n\n"
5894 	"\t        trace(<synthetic_event>,param list)  - generate synthetic event\n"
5895 	"\t        save(field,...)                      - save current event fields\n"
5896 #ifdef CONFIG_TRACER_SNAPSHOT
5897 	"\t        snapshot()                           - snapshot the trace buffer\n\n"
5898 #endif
5899 #ifdef CONFIG_SYNTH_EVENTS
5900 	"  events/synthetic_events\t- Create/append/remove/show synthetic events\n"
5901 	"\t  Write into this file to define/undefine new synthetic events.\n"
5902 	"\t     example: echo 'myevent u64 lat; char name[]; long[] stack' >> synthetic_events\n"
5903 #endif
5904 #endif
5905 ;
5906 
5907 static ssize_t
tracing_readme_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)5908 tracing_readme_read(struct file *filp, char __user *ubuf,
5909 		       size_t cnt, loff_t *ppos)
5910 {
5911 	return simple_read_from_buffer(ubuf, cnt, ppos,
5912 					readme_msg, strlen(readme_msg));
5913 }
5914 
5915 static const struct file_operations tracing_readme_fops = {
5916 	.open		= tracing_open_generic,
5917 	.read		= tracing_readme_read,
5918 	.llseek		= generic_file_llseek,
5919 };
5920 
saved_tgids_next(struct seq_file * m,void * v,loff_t * pos)5921 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
5922 {
5923 	int pid = ++(*pos);
5924 
5925 	return trace_find_tgid_ptr(pid);
5926 }
5927 
saved_tgids_start(struct seq_file * m,loff_t * pos)5928 static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
5929 {
5930 	int pid = *pos;
5931 
5932 	return trace_find_tgid_ptr(pid);
5933 }
5934 
saved_tgids_stop(struct seq_file * m,void * v)5935 static void saved_tgids_stop(struct seq_file *m, void *v)
5936 {
5937 }
5938 
saved_tgids_show(struct seq_file * m,void * v)5939 static int saved_tgids_show(struct seq_file *m, void *v)
5940 {
5941 	int *entry = (int *)v;
5942 	int pid = entry - tgid_map;
5943 	int tgid = *entry;
5944 
5945 	if (tgid == 0)
5946 		return SEQ_SKIP;
5947 
5948 	seq_printf(m, "%d %d\n", pid, tgid);
5949 	return 0;
5950 }
5951 
5952 static const struct seq_operations tracing_saved_tgids_seq_ops = {
5953 	.start		= saved_tgids_start,
5954 	.stop		= saved_tgids_stop,
5955 	.next		= saved_tgids_next,
5956 	.show		= saved_tgids_show,
5957 };
5958 
tracing_saved_tgids_open(struct inode * inode,struct file * filp)5959 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
5960 {
5961 	int ret;
5962 
5963 	ret = tracing_check_open_get_tr(NULL);
5964 	if (ret)
5965 		return ret;
5966 
5967 	return seq_open(filp, &tracing_saved_tgids_seq_ops);
5968 }
5969 
5970 
5971 static const struct file_operations tracing_saved_tgids_fops = {
5972 	.open		= tracing_saved_tgids_open,
5973 	.read		= seq_read,
5974 	.llseek		= seq_lseek,
5975 	.release	= seq_release,
5976 };
5977 
saved_cmdlines_next(struct seq_file * m,void * v,loff_t * pos)5978 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
5979 {
5980 	unsigned int *ptr = v;
5981 
5982 	if (*pos || m->count)
5983 		ptr++;
5984 
5985 	(*pos)++;
5986 
5987 	for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
5988 	     ptr++) {
5989 		if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
5990 			continue;
5991 
5992 		return ptr;
5993 	}
5994 
5995 	return NULL;
5996 }
5997 
saved_cmdlines_start(struct seq_file * m,loff_t * pos)5998 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
5999 {
6000 	void *v;
6001 	loff_t l = 0;
6002 
6003 	preempt_disable();
6004 	arch_spin_lock(&trace_cmdline_lock);
6005 
6006 	v = &savedcmd->map_cmdline_to_pid[0];
6007 	while (l <= *pos) {
6008 		v = saved_cmdlines_next(m, v, &l);
6009 		if (!v)
6010 			return NULL;
6011 	}
6012 
6013 	return v;
6014 }
6015 
saved_cmdlines_stop(struct seq_file * m,void * v)6016 static void saved_cmdlines_stop(struct seq_file *m, void *v)
6017 {
6018 	arch_spin_unlock(&trace_cmdline_lock);
6019 	preempt_enable();
6020 }
6021 
saved_cmdlines_show(struct seq_file * m,void * v)6022 static int saved_cmdlines_show(struct seq_file *m, void *v)
6023 {
6024 	char buf[TASK_COMM_LEN];
6025 	unsigned int *pid = v;
6026 
6027 	__trace_find_cmdline(*pid, buf);
6028 	seq_printf(m, "%d %s\n", *pid, buf);
6029 	return 0;
6030 }
6031 
6032 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
6033 	.start		= saved_cmdlines_start,
6034 	.next		= saved_cmdlines_next,
6035 	.stop		= saved_cmdlines_stop,
6036 	.show		= saved_cmdlines_show,
6037 };
6038 
tracing_saved_cmdlines_open(struct inode * inode,struct file * filp)6039 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
6040 {
6041 	int ret;
6042 
6043 	ret = tracing_check_open_get_tr(NULL);
6044 	if (ret)
6045 		return ret;
6046 
6047 	return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
6048 }
6049 
6050 static const struct file_operations tracing_saved_cmdlines_fops = {
6051 	.open		= tracing_saved_cmdlines_open,
6052 	.read		= seq_read,
6053 	.llseek		= seq_lseek,
6054 	.release	= seq_release,
6055 };
6056 
6057 static ssize_t
tracing_saved_cmdlines_size_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)6058 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
6059 				 size_t cnt, loff_t *ppos)
6060 {
6061 	char buf[64];
6062 	int r;
6063 
6064 	preempt_disable();
6065 	arch_spin_lock(&trace_cmdline_lock);
6066 	r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
6067 	arch_spin_unlock(&trace_cmdline_lock);
6068 	preempt_enable();
6069 
6070 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6071 }
6072 
tracing_resize_saved_cmdlines(unsigned int val)6073 static int tracing_resize_saved_cmdlines(unsigned int val)
6074 {
6075 	struct saved_cmdlines_buffer *s, *savedcmd_temp;
6076 
6077 	s = allocate_cmdlines_buffer(val);
6078 	if (!s)
6079 		return -ENOMEM;
6080 
6081 	preempt_disable();
6082 	arch_spin_lock(&trace_cmdline_lock);
6083 	savedcmd_temp = savedcmd;
6084 	savedcmd = s;
6085 	arch_spin_unlock(&trace_cmdline_lock);
6086 	preempt_enable();
6087 	free_saved_cmdlines_buffer(savedcmd_temp);
6088 
6089 	return 0;
6090 }
6091 
6092 static ssize_t
tracing_saved_cmdlines_size_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)6093 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
6094 				  size_t cnt, loff_t *ppos)
6095 {
6096 	unsigned long val;
6097 	int ret;
6098 
6099 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6100 	if (ret)
6101 		return ret;
6102 
6103 	/* must have at least 1 entry or less than PID_MAX_DEFAULT */
6104 	if (!val || val > PID_MAX_DEFAULT)
6105 		return -EINVAL;
6106 
6107 	ret = tracing_resize_saved_cmdlines((unsigned int)val);
6108 	if (ret < 0)
6109 		return ret;
6110 
6111 	*ppos += cnt;
6112 
6113 	return cnt;
6114 }
6115 
6116 static const struct file_operations tracing_saved_cmdlines_size_fops = {
6117 	.open		= tracing_open_generic,
6118 	.read		= tracing_saved_cmdlines_size_read,
6119 	.write		= tracing_saved_cmdlines_size_write,
6120 };
6121 
6122 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
6123 static union trace_eval_map_item *
update_eval_map(union trace_eval_map_item * ptr)6124 update_eval_map(union trace_eval_map_item *ptr)
6125 {
6126 	if (!ptr->map.eval_string) {
6127 		if (ptr->tail.next) {
6128 			ptr = ptr->tail.next;
6129 			/* Set ptr to the next real item (skip head) */
6130 			ptr++;
6131 		} else
6132 			return NULL;
6133 	}
6134 	return ptr;
6135 }
6136 
eval_map_next(struct seq_file * m,void * v,loff_t * pos)6137 static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
6138 {
6139 	union trace_eval_map_item *ptr = v;
6140 
6141 	/*
6142 	 * Paranoid! If ptr points to end, we don't want to increment past it.
6143 	 * This really should never happen.
6144 	 */
6145 	(*pos)++;
6146 	ptr = update_eval_map(ptr);
6147 	if (WARN_ON_ONCE(!ptr))
6148 		return NULL;
6149 
6150 	ptr++;
6151 	ptr = update_eval_map(ptr);
6152 
6153 	return ptr;
6154 }
6155 
eval_map_start(struct seq_file * m,loff_t * pos)6156 static void *eval_map_start(struct seq_file *m, loff_t *pos)
6157 {
6158 	union trace_eval_map_item *v;
6159 	loff_t l = 0;
6160 
6161 	mutex_lock(&trace_eval_mutex);
6162 
6163 	v = trace_eval_maps;
6164 	if (v)
6165 		v++;
6166 
6167 	while (v && l < *pos) {
6168 		v = eval_map_next(m, v, &l);
6169 	}
6170 
6171 	return v;
6172 }
6173 
eval_map_stop(struct seq_file * m,void * v)6174 static void eval_map_stop(struct seq_file *m, void *v)
6175 {
6176 	mutex_unlock(&trace_eval_mutex);
6177 }
6178 
eval_map_show(struct seq_file * m,void * v)6179 static int eval_map_show(struct seq_file *m, void *v)
6180 {
6181 	union trace_eval_map_item *ptr = v;
6182 
6183 	seq_printf(m, "%s %ld (%s)\n",
6184 		   ptr->map.eval_string, ptr->map.eval_value,
6185 		   ptr->map.system);
6186 
6187 	return 0;
6188 }
6189 
6190 static const struct seq_operations tracing_eval_map_seq_ops = {
6191 	.start		= eval_map_start,
6192 	.next		= eval_map_next,
6193 	.stop		= eval_map_stop,
6194 	.show		= eval_map_show,
6195 };
6196 
tracing_eval_map_open(struct inode * inode,struct file * filp)6197 static int tracing_eval_map_open(struct inode *inode, struct file *filp)
6198 {
6199 	int ret;
6200 
6201 	ret = tracing_check_open_get_tr(NULL);
6202 	if (ret)
6203 		return ret;
6204 
6205 	return seq_open(filp, &tracing_eval_map_seq_ops);
6206 }
6207 
6208 static const struct file_operations tracing_eval_map_fops = {
6209 	.open		= tracing_eval_map_open,
6210 	.read		= seq_read,
6211 	.llseek		= seq_lseek,
6212 	.release	= seq_release,
6213 };
6214 
6215 static inline union trace_eval_map_item *
trace_eval_jmp_to_tail(union trace_eval_map_item * ptr)6216 trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
6217 {
6218 	/* Return tail of array given the head */
6219 	return ptr + ptr->head.length + 1;
6220 }
6221 
6222 static void
trace_insert_eval_map_file(struct module * mod,struct trace_eval_map ** start,int len)6223 trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
6224 			   int len)
6225 {
6226 	struct trace_eval_map **stop;
6227 	struct trace_eval_map **map;
6228 	union trace_eval_map_item *map_array;
6229 	union trace_eval_map_item *ptr;
6230 
6231 	stop = start + len;
6232 
6233 	/*
6234 	 * The trace_eval_maps contains the map plus a head and tail item,
6235 	 * where the head holds the module and length of array, and the
6236 	 * tail holds a pointer to the next list.
6237 	 */
6238 	map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
6239 	if (!map_array) {
6240 		pr_warn("Unable to allocate trace eval mapping\n");
6241 		return;
6242 	}
6243 
6244 	mutex_lock(&trace_eval_mutex);
6245 
6246 	if (!trace_eval_maps)
6247 		trace_eval_maps = map_array;
6248 	else {
6249 		ptr = trace_eval_maps;
6250 		for (;;) {
6251 			ptr = trace_eval_jmp_to_tail(ptr);
6252 			if (!ptr->tail.next)
6253 				break;
6254 			ptr = ptr->tail.next;
6255 
6256 		}
6257 		ptr->tail.next = map_array;
6258 	}
6259 	map_array->head.mod = mod;
6260 	map_array->head.length = len;
6261 	map_array++;
6262 
6263 	for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
6264 		map_array->map = **map;
6265 		map_array++;
6266 	}
6267 	memset(map_array, 0, sizeof(*map_array));
6268 
6269 	mutex_unlock(&trace_eval_mutex);
6270 }
6271 
trace_create_eval_file(struct dentry * d_tracer)6272 static void trace_create_eval_file(struct dentry *d_tracer)
6273 {
6274 	trace_create_file("eval_map", TRACE_MODE_READ, d_tracer,
6275 			  NULL, &tracing_eval_map_fops);
6276 }
6277 
6278 #else /* CONFIG_TRACE_EVAL_MAP_FILE */
trace_create_eval_file(struct dentry * d_tracer)6279 static inline void trace_create_eval_file(struct dentry *d_tracer) { }
trace_insert_eval_map_file(struct module * mod,struct trace_eval_map ** start,int len)6280 static inline void trace_insert_eval_map_file(struct module *mod,
6281 			      struct trace_eval_map **start, int len) { }
6282 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
6283 
trace_insert_eval_map(struct module * mod,struct trace_eval_map ** start,int len)6284 static void trace_insert_eval_map(struct module *mod,
6285 				  struct trace_eval_map **start, int len)
6286 {
6287 	struct trace_eval_map **map;
6288 
6289 	if (len <= 0)
6290 		return;
6291 
6292 	map = start;
6293 
6294 	trace_event_eval_update(map, len);
6295 
6296 	trace_insert_eval_map_file(mod, start, len);
6297 }
6298 
6299 static ssize_t
tracing_set_trace_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)6300 tracing_set_trace_read(struct file *filp, char __user *ubuf,
6301 		       size_t cnt, loff_t *ppos)
6302 {
6303 	struct trace_array *tr = filp->private_data;
6304 	char buf[MAX_TRACER_SIZE+2];
6305 	int r;
6306 
6307 	mutex_lock(&trace_types_lock);
6308 	r = sprintf(buf, "%s\n", tr->current_trace->name);
6309 	mutex_unlock(&trace_types_lock);
6310 
6311 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6312 }
6313 
tracer_init(struct tracer * t,struct trace_array * tr)6314 int tracer_init(struct tracer *t, struct trace_array *tr)
6315 {
6316 	tracing_reset_online_cpus(&tr->array_buffer);
6317 	return t->init(tr);
6318 }
6319 
set_buffer_entries(struct array_buffer * buf,unsigned long val)6320 static void set_buffer_entries(struct array_buffer *buf, unsigned long val)
6321 {
6322 	int cpu;
6323 
6324 	for_each_tracing_cpu(cpu)
6325 		per_cpu_ptr(buf->data, cpu)->entries = val;
6326 }
6327 
update_buffer_entries(struct array_buffer * buf,int cpu)6328 static void update_buffer_entries(struct array_buffer *buf, int cpu)
6329 {
6330 	if (cpu == RING_BUFFER_ALL_CPUS) {
6331 		set_buffer_entries(buf, ring_buffer_size(buf->buffer, 0));
6332 	} else {
6333 		per_cpu_ptr(buf->data, cpu)->entries = ring_buffer_size(buf->buffer, cpu);
6334 	}
6335 }
6336 
6337 #ifdef CONFIG_TRACER_MAX_TRACE
6338 /* resize @tr's buffer to the size of @size_tr's entries */
resize_buffer_duplicate_size(struct array_buffer * trace_buf,struct array_buffer * size_buf,int cpu_id)6339 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
6340 					struct array_buffer *size_buf, int cpu_id)
6341 {
6342 	int cpu, ret = 0;
6343 
6344 	if (cpu_id == RING_BUFFER_ALL_CPUS) {
6345 		for_each_tracing_cpu(cpu) {
6346 			ret = ring_buffer_resize(trace_buf->buffer,
6347 				 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
6348 			if (ret < 0)
6349 				break;
6350 			per_cpu_ptr(trace_buf->data, cpu)->entries =
6351 				per_cpu_ptr(size_buf->data, cpu)->entries;
6352 		}
6353 	} else {
6354 		ret = ring_buffer_resize(trace_buf->buffer,
6355 				 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
6356 		if (ret == 0)
6357 			per_cpu_ptr(trace_buf->data, cpu_id)->entries =
6358 				per_cpu_ptr(size_buf->data, cpu_id)->entries;
6359 	}
6360 
6361 	return ret;
6362 }
6363 #endif /* CONFIG_TRACER_MAX_TRACE */
6364 
__tracing_resize_ring_buffer(struct trace_array * tr,unsigned long size,int cpu)6365 static int __tracing_resize_ring_buffer(struct trace_array *tr,
6366 					unsigned long size, int cpu)
6367 {
6368 	int ret;
6369 
6370 	/*
6371 	 * If kernel or user changes the size of the ring buffer
6372 	 * we use the size that was given, and we can forget about
6373 	 * expanding it later.
6374 	 */
6375 	ring_buffer_expanded = true;
6376 
6377 	/* May be called before buffers are initialized */
6378 	if (!tr->array_buffer.buffer)
6379 		return 0;
6380 
6381 	/* Do not allow tracing while resizing ring buffer */
6382 	tracing_stop_tr(tr);
6383 
6384 	ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);
6385 	if (ret < 0)
6386 		goto out_start;
6387 
6388 #ifdef CONFIG_TRACER_MAX_TRACE
6389 	if (!tr->allocated_snapshot)
6390 		goto out;
6391 
6392 	ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
6393 	if (ret < 0) {
6394 		int r = resize_buffer_duplicate_size(&tr->array_buffer,
6395 						     &tr->array_buffer, cpu);
6396 		if (r < 0) {
6397 			/*
6398 			 * AARGH! We are left with different
6399 			 * size max buffer!!!!
6400 			 * The max buffer is our "snapshot" buffer.
6401 			 * When a tracer needs a snapshot (one of the
6402 			 * latency tracers), it swaps the max buffer
6403 			 * with the saved snap shot. We succeeded to
6404 			 * update the size of the main buffer, but failed to
6405 			 * update the size of the max buffer. But when we tried
6406 			 * to reset the main buffer to the original size, we
6407 			 * failed there too. This is very unlikely to
6408 			 * happen, but if it does, warn and kill all
6409 			 * tracing.
6410 			 */
6411 			WARN_ON(1);
6412 			tracing_disabled = 1;
6413 		}
6414 		goto out_start;
6415 	}
6416 
6417 	update_buffer_entries(&tr->max_buffer, cpu);
6418 
6419  out:
6420 #endif /* CONFIG_TRACER_MAX_TRACE */
6421 
6422 	update_buffer_entries(&tr->array_buffer, cpu);
6423  out_start:
6424 	tracing_start_tr(tr);
6425 	return ret;
6426 }
6427 
tracing_resize_ring_buffer(struct trace_array * tr,unsigned long size,int cpu_id)6428 ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
6429 				  unsigned long size, int cpu_id)
6430 {
6431 	int ret;
6432 
6433 	mutex_lock(&trace_types_lock);
6434 
6435 	if (cpu_id != RING_BUFFER_ALL_CPUS) {
6436 		/* make sure, this cpu is enabled in the mask */
6437 		if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
6438 			ret = -EINVAL;
6439 			goto out;
6440 		}
6441 	}
6442 
6443 	ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
6444 	if (ret < 0)
6445 		ret = -ENOMEM;
6446 
6447 out:
6448 	mutex_unlock(&trace_types_lock);
6449 
6450 	return ret;
6451 }
6452 
6453 
6454 /**
6455  * tracing_update_buffers - used by tracing facility to expand ring buffers
6456  *
6457  * To save on memory when the tracing is never used on a system with it
6458  * configured in. The ring buffers are set to a minimum size. But once
6459  * a user starts to use the tracing facility, then they need to grow
6460  * to their default size.
6461  *
6462  * This function is to be called when a tracer is about to be used.
6463  */
tracing_update_buffers(void)6464 int tracing_update_buffers(void)
6465 {
6466 	int ret = 0;
6467 
6468 	mutex_lock(&trace_types_lock);
6469 	if (!ring_buffer_expanded)
6470 		ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
6471 						RING_BUFFER_ALL_CPUS);
6472 	mutex_unlock(&trace_types_lock);
6473 
6474 	return ret;
6475 }
6476 
6477 struct trace_option_dentry;
6478 
6479 static void
6480 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
6481 
6482 /*
6483  * Used to clear out the tracer before deletion of an instance.
6484  * Must have trace_types_lock held.
6485  */
tracing_set_nop(struct trace_array * tr)6486 static void tracing_set_nop(struct trace_array *tr)
6487 {
6488 	if (tr->current_trace == &nop_trace)
6489 		return;
6490 
6491 	tr->current_trace->enabled--;
6492 
6493 	if (tr->current_trace->reset)
6494 		tr->current_trace->reset(tr);
6495 
6496 	tr->current_trace = &nop_trace;
6497 }
6498 
6499 static bool tracer_options_updated;
6500 
add_tracer_options(struct trace_array * tr,struct tracer * t)6501 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
6502 {
6503 	/* Only enable if the directory has been created already. */
6504 	if (!tr->dir)
6505 		return;
6506 
6507 	/* Only create trace option files after update_tracer_options finish */
6508 	if (!tracer_options_updated)
6509 		return;
6510 
6511 	create_trace_option_files(tr, t);
6512 }
6513 
tracing_set_tracer(struct trace_array * tr,const char * buf)6514 int tracing_set_tracer(struct trace_array *tr, const char *buf)
6515 {
6516 	struct tracer *t;
6517 #ifdef CONFIG_TRACER_MAX_TRACE
6518 	bool had_max_tr;
6519 #endif
6520 	int ret = 0;
6521 
6522 	mutex_lock(&trace_types_lock);
6523 
6524 	if (!ring_buffer_expanded) {
6525 		ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
6526 						RING_BUFFER_ALL_CPUS);
6527 		if (ret < 0)
6528 			goto out;
6529 		ret = 0;
6530 	}
6531 
6532 	for (t = trace_types; t; t = t->next) {
6533 		if (strcmp(t->name, buf) == 0)
6534 			break;
6535 	}
6536 	if (!t) {
6537 		ret = -EINVAL;
6538 		goto out;
6539 	}
6540 	if (t == tr->current_trace)
6541 		goto out;
6542 
6543 #ifdef CONFIG_TRACER_SNAPSHOT
6544 	if (t->use_max_tr) {
6545 		local_irq_disable();
6546 		arch_spin_lock(&tr->max_lock);
6547 		if (tr->cond_snapshot)
6548 			ret = -EBUSY;
6549 		arch_spin_unlock(&tr->max_lock);
6550 		local_irq_enable();
6551 		if (ret)
6552 			goto out;
6553 	}
6554 #endif
6555 	/* Some tracers won't work on kernel command line */
6556 	if (system_state < SYSTEM_RUNNING && t->noboot) {
6557 		pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
6558 			t->name);
6559 		goto out;
6560 	}
6561 
6562 	/* Some tracers are only allowed for the top level buffer */
6563 	if (!trace_ok_for_array(t, tr)) {
6564 		ret = -EINVAL;
6565 		goto out;
6566 	}
6567 
6568 	/* If trace pipe files are being read, we can't change the tracer */
6569 	if (tr->trace_ref) {
6570 		ret = -EBUSY;
6571 		goto out;
6572 	}
6573 
6574 	trace_branch_disable();
6575 
6576 	tr->current_trace->enabled--;
6577 
6578 	if (tr->current_trace->reset)
6579 		tr->current_trace->reset(tr);
6580 
6581 #ifdef CONFIG_TRACER_MAX_TRACE
6582 	had_max_tr = tr->current_trace->use_max_tr;
6583 
6584 	/* Current trace needs to be nop_trace before synchronize_rcu */
6585 	tr->current_trace = &nop_trace;
6586 
6587 	if (had_max_tr && !t->use_max_tr) {
6588 		/*
6589 		 * We need to make sure that the update_max_tr sees that
6590 		 * current_trace changed to nop_trace to keep it from
6591 		 * swapping the buffers after we resize it.
6592 		 * The update_max_tr is called from interrupts disabled
6593 		 * so a synchronized_sched() is sufficient.
6594 		 */
6595 		synchronize_rcu();
6596 		free_snapshot(tr);
6597 	}
6598 
6599 	if (t->use_max_tr && !tr->allocated_snapshot) {
6600 		ret = tracing_alloc_snapshot_instance(tr);
6601 		if (ret < 0)
6602 			goto out;
6603 	}
6604 #else
6605 	tr->current_trace = &nop_trace;
6606 #endif
6607 
6608 	if (t->init) {
6609 		ret = tracer_init(t, tr);
6610 		if (ret)
6611 			goto out;
6612 	}
6613 
6614 	tr->current_trace = t;
6615 	tr->current_trace->enabled++;
6616 	trace_branch_enable(tr);
6617  out:
6618 	mutex_unlock(&trace_types_lock);
6619 
6620 	return ret;
6621 }
6622 
6623 static ssize_t
tracing_set_trace_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)6624 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
6625 			size_t cnt, loff_t *ppos)
6626 {
6627 	struct trace_array *tr = filp->private_data;
6628 	char buf[MAX_TRACER_SIZE+1];
6629 	char *name;
6630 	size_t ret;
6631 	int err;
6632 
6633 	ret = cnt;
6634 
6635 	if (cnt > MAX_TRACER_SIZE)
6636 		cnt = MAX_TRACER_SIZE;
6637 
6638 	if (copy_from_user(buf, ubuf, cnt))
6639 		return -EFAULT;
6640 
6641 	buf[cnt] = 0;
6642 
6643 	name = strim(buf);
6644 
6645 	err = tracing_set_tracer(tr, name);
6646 	if (err)
6647 		return err;
6648 
6649 	*ppos += ret;
6650 
6651 	return ret;
6652 }
6653 
6654 static ssize_t
tracing_nsecs_read(unsigned long * ptr,char __user * ubuf,size_t cnt,loff_t * ppos)6655 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
6656 		   size_t cnt, loff_t *ppos)
6657 {
6658 	char buf[64];
6659 	int r;
6660 
6661 	r = snprintf(buf, sizeof(buf), "%ld\n",
6662 		     *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
6663 	if (r > sizeof(buf))
6664 		r = sizeof(buf);
6665 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6666 }
6667 
6668 static ssize_t
tracing_nsecs_write(unsigned long * ptr,const char __user * ubuf,size_t cnt,loff_t * ppos)6669 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
6670 		    size_t cnt, loff_t *ppos)
6671 {
6672 	unsigned long val;
6673 	int ret;
6674 
6675 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6676 	if (ret)
6677 		return ret;
6678 
6679 	*ptr = val * 1000;
6680 
6681 	return cnt;
6682 }
6683 
6684 static ssize_t
tracing_thresh_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)6685 tracing_thresh_read(struct file *filp, char __user *ubuf,
6686 		    size_t cnt, loff_t *ppos)
6687 {
6688 	return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
6689 }
6690 
6691 static ssize_t
tracing_thresh_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)6692 tracing_thresh_write(struct file *filp, const char __user *ubuf,
6693 		     size_t cnt, loff_t *ppos)
6694 {
6695 	struct trace_array *tr = filp->private_data;
6696 	int ret;
6697 
6698 	mutex_lock(&trace_types_lock);
6699 	ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
6700 	if (ret < 0)
6701 		goto out;
6702 
6703 	if (tr->current_trace->update_thresh) {
6704 		ret = tr->current_trace->update_thresh(tr);
6705 		if (ret < 0)
6706 			goto out;
6707 	}
6708 
6709 	ret = cnt;
6710 out:
6711 	mutex_unlock(&trace_types_lock);
6712 
6713 	return ret;
6714 }
6715 
6716 #ifdef CONFIG_TRACER_MAX_TRACE
6717 
6718 static ssize_t
tracing_max_lat_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)6719 tracing_max_lat_read(struct file *filp, char __user *ubuf,
6720 		     size_t cnt, loff_t *ppos)
6721 {
6722 	struct trace_array *tr = filp->private_data;
6723 
6724 	return tracing_nsecs_read(&tr->max_latency, ubuf, cnt, ppos);
6725 }
6726 
6727 static ssize_t
tracing_max_lat_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)6728 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
6729 		      size_t cnt, loff_t *ppos)
6730 {
6731 	struct trace_array *tr = filp->private_data;
6732 
6733 	return tracing_nsecs_write(&tr->max_latency, ubuf, cnt, ppos);
6734 }
6735 
6736 #endif
6737 
open_pipe_on_cpu(struct trace_array * tr,int cpu)6738 static int open_pipe_on_cpu(struct trace_array *tr, int cpu)
6739 {
6740 	if (cpu == RING_BUFFER_ALL_CPUS) {
6741 		if (cpumask_empty(tr->pipe_cpumask)) {
6742 			cpumask_setall(tr->pipe_cpumask);
6743 			return 0;
6744 		}
6745 	} else if (!cpumask_test_cpu(cpu, tr->pipe_cpumask)) {
6746 		cpumask_set_cpu(cpu, tr->pipe_cpumask);
6747 		return 0;
6748 	}
6749 	return -EBUSY;
6750 }
6751 
close_pipe_on_cpu(struct trace_array * tr,int cpu)6752 static void close_pipe_on_cpu(struct trace_array *tr, int cpu)
6753 {
6754 	if (cpu == RING_BUFFER_ALL_CPUS) {
6755 		WARN_ON(!cpumask_full(tr->pipe_cpumask));
6756 		cpumask_clear(tr->pipe_cpumask);
6757 	} else {
6758 		WARN_ON(!cpumask_test_cpu(cpu, tr->pipe_cpumask));
6759 		cpumask_clear_cpu(cpu, tr->pipe_cpumask);
6760 	}
6761 }
6762 
tracing_open_pipe(struct inode * inode,struct file * filp)6763 static int tracing_open_pipe(struct inode *inode, struct file *filp)
6764 {
6765 	struct trace_array *tr = inode->i_private;
6766 	struct trace_iterator *iter;
6767 	int cpu;
6768 	int ret;
6769 
6770 	ret = tracing_check_open_get_tr(tr);
6771 	if (ret)
6772 		return ret;
6773 
6774 	mutex_lock(&trace_types_lock);
6775 	cpu = tracing_get_cpu(inode);
6776 	ret = open_pipe_on_cpu(tr, cpu);
6777 	if (ret)
6778 		goto fail_pipe_on_cpu;
6779 
6780 	/* create a buffer to store the information to pass to userspace */
6781 	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6782 	if (!iter) {
6783 		ret = -ENOMEM;
6784 		goto fail_alloc_iter;
6785 	}
6786 
6787 	trace_seq_init(&iter->seq);
6788 	iter->trace = tr->current_trace;
6789 
6790 	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
6791 		ret = -ENOMEM;
6792 		goto fail;
6793 	}
6794 
6795 	/* trace pipe does not show start of buffer */
6796 	cpumask_setall(iter->started);
6797 
6798 	if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
6799 		iter->iter_flags |= TRACE_FILE_LAT_FMT;
6800 
6801 	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
6802 	if (trace_clocks[tr->clock_id].in_ns)
6803 		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6804 
6805 	iter->tr = tr;
6806 	iter->array_buffer = &tr->array_buffer;
6807 	iter->cpu_file = cpu;
6808 	mutex_init(&iter->mutex);
6809 	filp->private_data = iter;
6810 
6811 	if (iter->trace->pipe_open)
6812 		iter->trace->pipe_open(iter);
6813 
6814 	nonseekable_open(inode, filp);
6815 
6816 	tr->trace_ref++;
6817 
6818 	mutex_unlock(&trace_types_lock);
6819 	return ret;
6820 
6821 fail:
6822 	kfree(iter);
6823 fail_alloc_iter:
6824 	close_pipe_on_cpu(tr, cpu);
6825 fail_pipe_on_cpu:
6826 	__trace_array_put(tr);
6827 	mutex_unlock(&trace_types_lock);
6828 	return ret;
6829 }
6830 
tracing_release_pipe(struct inode * inode,struct file * file)6831 static int tracing_release_pipe(struct inode *inode, struct file *file)
6832 {
6833 	struct trace_iterator *iter = file->private_data;
6834 	struct trace_array *tr = inode->i_private;
6835 
6836 	mutex_lock(&trace_types_lock);
6837 
6838 	tr->trace_ref--;
6839 
6840 	if (iter->trace->pipe_close)
6841 		iter->trace->pipe_close(iter);
6842 	close_pipe_on_cpu(tr, iter->cpu_file);
6843 	mutex_unlock(&trace_types_lock);
6844 
6845 	free_trace_iter_content(iter);
6846 	kfree(iter);
6847 
6848 	trace_array_put(tr);
6849 
6850 	return 0;
6851 }
6852 
6853 static __poll_t
trace_poll(struct trace_iterator * iter,struct file * filp,poll_table * poll_table)6854 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
6855 {
6856 	struct trace_array *tr = iter->tr;
6857 
6858 	/* Iterators are static, they should be filled or empty */
6859 	if (trace_buffer_iter(iter, iter->cpu_file))
6860 		return EPOLLIN | EPOLLRDNORM;
6861 
6862 	if (tr->trace_flags & TRACE_ITER_BLOCK)
6863 		/*
6864 		 * Always select as readable when in blocking mode
6865 		 */
6866 		return EPOLLIN | EPOLLRDNORM;
6867 	else
6868 		return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,
6869 					     filp, poll_table, iter->tr->buffer_percent);
6870 }
6871 
6872 static __poll_t
tracing_poll_pipe(struct file * filp,poll_table * poll_table)6873 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
6874 {
6875 	struct trace_iterator *iter = filp->private_data;
6876 
6877 	return trace_poll(iter, filp, poll_table);
6878 }
6879 
6880 /* Must be called with iter->mutex held. */
tracing_wait_pipe(struct file * filp)6881 static int tracing_wait_pipe(struct file *filp)
6882 {
6883 	struct trace_iterator *iter = filp->private_data;
6884 	int ret;
6885 
6886 	while (trace_empty(iter)) {
6887 
6888 		if ((filp->f_flags & O_NONBLOCK)) {
6889 			return -EAGAIN;
6890 		}
6891 
6892 		/*
6893 		 * We block until we read something and tracing is disabled.
6894 		 * We still block if tracing is disabled, but we have never
6895 		 * read anything. This allows a user to cat this file, and
6896 		 * then enable tracing. But after we have read something,
6897 		 * we give an EOF when tracing is again disabled.
6898 		 *
6899 		 * iter->pos will be 0 if we haven't read anything.
6900 		 */
6901 		if (!tracer_tracing_is_on(iter->tr) && iter->pos)
6902 			break;
6903 
6904 		mutex_unlock(&iter->mutex);
6905 
6906 		ret = wait_on_pipe(iter, 0);
6907 
6908 		mutex_lock(&iter->mutex);
6909 
6910 		if (ret)
6911 			return ret;
6912 	}
6913 
6914 	return 1;
6915 }
6916 
6917 /*
6918  * Consumer reader.
6919  */
6920 static ssize_t
tracing_read_pipe(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)6921 tracing_read_pipe(struct file *filp, char __user *ubuf,
6922 		  size_t cnt, loff_t *ppos)
6923 {
6924 	struct trace_iterator *iter = filp->private_data;
6925 	ssize_t sret;
6926 
6927 	/*
6928 	 * Avoid more than one consumer on a single file descriptor
6929 	 * This is just a matter of traces coherency, the ring buffer itself
6930 	 * is protected.
6931 	 */
6932 	mutex_lock(&iter->mutex);
6933 
6934 	/* return any leftover data */
6935 	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6936 	if (sret != -EBUSY)
6937 		goto out;
6938 
6939 	trace_seq_init(&iter->seq);
6940 
6941 	if (iter->trace->read) {
6942 		sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
6943 		if (sret)
6944 			goto out;
6945 	}
6946 
6947 waitagain:
6948 	sret = tracing_wait_pipe(filp);
6949 	if (sret <= 0)
6950 		goto out;
6951 
6952 	/* stop when tracing is finished */
6953 	if (trace_empty(iter)) {
6954 		sret = 0;
6955 		goto out;
6956 	}
6957 
6958 	if (cnt >= PAGE_SIZE)
6959 		cnt = PAGE_SIZE - 1;
6960 
6961 	/* reset all but tr, trace, and overruns */
6962 	trace_iterator_reset(iter);
6963 	cpumask_clear(iter->started);
6964 	trace_seq_init(&iter->seq);
6965 
6966 	trace_event_read_lock();
6967 	trace_access_lock(iter->cpu_file);
6968 	while (trace_find_next_entry_inc(iter) != NULL) {
6969 		enum print_line_t ret;
6970 		int save_len = iter->seq.seq.len;
6971 
6972 		ret = print_trace_line(iter);
6973 		if (ret == TRACE_TYPE_PARTIAL_LINE) {
6974 			/*
6975 			 * If one print_trace_line() fills entire trace_seq in one shot,
6976 			 * trace_seq_to_user() will returns -EBUSY because save_len == 0,
6977 			 * In this case, we need to consume it, otherwise, loop will peek
6978 			 * this event next time, resulting in an infinite loop.
6979 			 */
6980 			if (save_len == 0) {
6981 				iter->seq.full = 0;
6982 				trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n");
6983 				trace_consume(iter);
6984 				break;
6985 			}
6986 
6987 			/* In other cases, don't print partial lines */
6988 			iter->seq.seq.len = save_len;
6989 			break;
6990 		}
6991 		if (ret != TRACE_TYPE_NO_CONSUME)
6992 			trace_consume(iter);
6993 
6994 		if (trace_seq_used(&iter->seq) >= cnt)
6995 			break;
6996 
6997 		/*
6998 		 * Setting the full flag means we reached the trace_seq buffer
6999 		 * size and we should leave by partial output condition above.
7000 		 * One of the trace_seq_* functions is not used properly.
7001 		 */
7002 		WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
7003 			  iter->ent->type);
7004 	}
7005 	trace_access_unlock(iter->cpu_file);
7006 	trace_event_read_unlock();
7007 
7008 	/* Now copy what we have to the user */
7009 	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
7010 	if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
7011 		trace_seq_init(&iter->seq);
7012 
7013 	/*
7014 	 * If there was nothing to send to user, in spite of consuming trace
7015 	 * entries, go back to wait for more entries.
7016 	 */
7017 	if (sret == -EBUSY)
7018 		goto waitagain;
7019 
7020 out:
7021 	mutex_unlock(&iter->mutex);
7022 
7023 	return sret;
7024 }
7025 
tracing_spd_release_pipe(struct splice_pipe_desc * spd,unsigned int idx)7026 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
7027 				     unsigned int idx)
7028 {
7029 	__free_page(spd->pages[idx]);
7030 }
7031 
7032 static size_t
tracing_fill_pipe_page(size_t rem,struct trace_iterator * iter)7033 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
7034 {
7035 	size_t count;
7036 	int save_len;
7037 	int ret;
7038 
7039 	/* Seq buffer is page-sized, exactly what we need. */
7040 	for (;;) {
7041 		save_len = iter->seq.seq.len;
7042 		ret = print_trace_line(iter);
7043 
7044 		if (trace_seq_has_overflowed(&iter->seq)) {
7045 			iter->seq.seq.len = save_len;
7046 			break;
7047 		}
7048 
7049 		/*
7050 		 * This should not be hit, because it should only
7051 		 * be set if the iter->seq overflowed. But check it
7052 		 * anyway to be safe.
7053 		 */
7054 		if (ret == TRACE_TYPE_PARTIAL_LINE) {
7055 			iter->seq.seq.len = save_len;
7056 			break;
7057 		}
7058 
7059 		count = trace_seq_used(&iter->seq) - save_len;
7060 		if (rem < count) {
7061 			rem = 0;
7062 			iter->seq.seq.len = save_len;
7063 			break;
7064 		}
7065 
7066 		if (ret != TRACE_TYPE_NO_CONSUME)
7067 			trace_consume(iter);
7068 		rem -= count;
7069 		if (!trace_find_next_entry_inc(iter))	{
7070 			rem = 0;
7071 			iter->ent = NULL;
7072 			break;
7073 		}
7074 	}
7075 
7076 	return rem;
7077 }
7078 
tracing_splice_read_pipe(struct file * filp,loff_t * ppos,struct pipe_inode_info * pipe,size_t len,unsigned int flags)7079 static ssize_t tracing_splice_read_pipe(struct file *filp,
7080 					loff_t *ppos,
7081 					struct pipe_inode_info *pipe,
7082 					size_t len,
7083 					unsigned int flags)
7084 {
7085 	struct page *pages_def[PIPE_DEF_BUFFERS];
7086 	struct partial_page partial_def[PIPE_DEF_BUFFERS];
7087 	struct trace_iterator *iter = filp->private_data;
7088 	struct splice_pipe_desc spd = {
7089 		.pages		= pages_def,
7090 		.partial	= partial_def,
7091 		.nr_pages	= 0, /* This gets updated below. */
7092 		.nr_pages_max	= PIPE_DEF_BUFFERS,
7093 		.ops		= &default_pipe_buf_ops,
7094 		.spd_release	= tracing_spd_release_pipe,
7095 	};
7096 	ssize_t ret;
7097 	size_t rem;
7098 	unsigned int i;
7099 
7100 	if (splice_grow_spd(pipe, &spd))
7101 		return -ENOMEM;
7102 
7103 	mutex_lock(&iter->mutex);
7104 
7105 	if (iter->trace->splice_read) {
7106 		ret = iter->trace->splice_read(iter, filp,
7107 					       ppos, pipe, len, flags);
7108 		if (ret)
7109 			goto out_err;
7110 	}
7111 
7112 	ret = tracing_wait_pipe(filp);
7113 	if (ret <= 0)
7114 		goto out_err;
7115 
7116 	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
7117 		ret = -EFAULT;
7118 		goto out_err;
7119 	}
7120 
7121 	trace_event_read_lock();
7122 	trace_access_lock(iter->cpu_file);
7123 
7124 	/* Fill as many pages as possible. */
7125 	for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
7126 		spd.pages[i] = alloc_page(GFP_KERNEL);
7127 		if (!spd.pages[i])
7128 			break;
7129 
7130 		rem = tracing_fill_pipe_page(rem, iter);
7131 
7132 		/* Copy the data into the page, so we can start over. */
7133 		ret = trace_seq_to_buffer(&iter->seq,
7134 					  page_address(spd.pages[i]),
7135 					  trace_seq_used(&iter->seq));
7136 		if (ret < 0) {
7137 			__free_page(spd.pages[i]);
7138 			break;
7139 		}
7140 		spd.partial[i].offset = 0;
7141 		spd.partial[i].len = trace_seq_used(&iter->seq);
7142 
7143 		trace_seq_init(&iter->seq);
7144 	}
7145 
7146 	trace_access_unlock(iter->cpu_file);
7147 	trace_event_read_unlock();
7148 	mutex_unlock(&iter->mutex);
7149 
7150 	spd.nr_pages = i;
7151 
7152 	if (i)
7153 		ret = splice_to_pipe(pipe, &spd);
7154 	else
7155 		ret = 0;
7156 out:
7157 	splice_shrink_spd(&spd);
7158 	return ret;
7159 
7160 out_err:
7161 	mutex_unlock(&iter->mutex);
7162 	goto out;
7163 }
7164 
7165 static ssize_t
tracing_entries_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)7166 tracing_entries_read(struct file *filp, char __user *ubuf,
7167 		     size_t cnt, loff_t *ppos)
7168 {
7169 	struct inode *inode = file_inode(filp);
7170 	struct trace_array *tr = inode->i_private;
7171 	int cpu = tracing_get_cpu(inode);
7172 	char buf[64];
7173 	int r = 0;
7174 	ssize_t ret;
7175 
7176 	mutex_lock(&trace_types_lock);
7177 
7178 	if (cpu == RING_BUFFER_ALL_CPUS) {
7179 		int cpu, buf_size_same;
7180 		unsigned long size;
7181 
7182 		size = 0;
7183 		buf_size_same = 1;
7184 		/* check if all cpu sizes are same */
7185 		for_each_tracing_cpu(cpu) {
7186 			/* fill in the size from first enabled cpu */
7187 			if (size == 0)
7188 				size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
7189 			if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
7190 				buf_size_same = 0;
7191 				break;
7192 			}
7193 		}
7194 
7195 		if (buf_size_same) {
7196 			if (!ring_buffer_expanded)
7197 				r = sprintf(buf, "%lu (expanded: %lu)\n",
7198 					    size >> 10,
7199 					    trace_buf_size >> 10);
7200 			else
7201 				r = sprintf(buf, "%lu\n", size >> 10);
7202 		} else
7203 			r = sprintf(buf, "X\n");
7204 	} else
7205 		r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
7206 
7207 	mutex_unlock(&trace_types_lock);
7208 
7209 	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7210 	return ret;
7211 }
7212 
7213 static ssize_t
tracing_entries_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)7214 tracing_entries_write(struct file *filp, const char __user *ubuf,
7215 		      size_t cnt, loff_t *ppos)
7216 {
7217 	struct inode *inode = file_inode(filp);
7218 	struct trace_array *tr = inode->i_private;
7219 	unsigned long val;
7220 	int ret;
7221 
7222 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7223 	if (ret)
7224 		return ret;
7225 
7226 	/* must have at least 1 entry */
7227 	if (!val)
7228 		return -EINVAL;
7229 
7230 	/* value is in KB */
7231 	val <<= 10;
7232 	ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
7233 	if (ret < 0)
7234 		return ret;
7235 
7236 	*ppos += cnt;
7237 
7238 	return cnt;
7239 }
7240 
7241 static ssize_t
tracing_total_entries_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)7242 tracing_total_entries_read(struct file *filp, char __user *ubuf,
7243 				size_t cnt, loff_t *ppos)
7244 {
7245 	struct trace_array *tr = filp->private_data;
7246 	char buf[64];
7247 	int r, cpu;
7248 	unsigned long size = 0, expanded_size = 0;
7249 
7250 	mutex_lock(&trace_types_lock);
7251 	for_each_tracing_cpu(cpu) {
7252 		size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
7253 		if (!ring_buffer_expanded)
7254 			expanded_size += trace_buf_size >> 10;
7255 	}
7256 	if (ring_buffer_expanded)
7257 		r = sprintf(buf, "%lu\n", size);
7258 	else
7259 		r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
7260 	mutex_unlock(&trace_types_lock);
7261 
7262 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7263 }
7264 
7265 static ssize_t
tracing_free_buffer_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)7266 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
7267 			  size_t cnt, loff_t *ppos)
7268 {
7269 	/*
7270 	 * There is no need to read what the user has written, this function
7271 	 * is just to make sure that there is no error when "echo" is used
7272 	 */
7273 
7274 	*ppos += cnt;
7275 
7276 	return cnt;
7277 }
7278 
7279 static int
tracing_free_buffer_release(struct inode * inode,struct file * filp)7280 tracing_free_buffer_release(struct inode *inode, struct file *filp)
7281 {
7282 	struct trace_array *tr = inode->i_private;
7283 
7284 	/* disable tracing ? */
7285 	if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
7286 		tracer_tracing_off(tr);
7287 	/* resize the ring buffer to 0 */
7288 	tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
7289 
7290 	trace_array_put(tr);
7291 
7292 	return 0;
7293 }
7294 
7295 static ssize_t
tracing_mark_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * fpos)7296 tracing_mark_write(struct file *filp, const char __user *ubuf,
7297 					size_t cnt, loff_t *fpos)
7298 {
7299 	struct trace_array *tr = filp->private_data;
7300 	struct ring_buffer_event *event;
7301 	enum event_trigger_type tt = ETT_NONE;
7302 	struct trace_buffer *buffer;
7303 	struct print_entry *entry;
7304 	ssize_t written;
7305 	int size;
7306 	int len;
7307 
7308 /* Used in tracing_mark_raw_write() as well */
7309 #define FAULTED_STR "<faulted>"
7310 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
7311 
7312 	if (tracing_disabled)
7313 		return -EINVAL;
7314 
7315 	if (!(tr->trace_flags & TRACE_ITER_MARKERS))
7316 		return -EINVAL;
7317 
7318 	if (cnt > TRACE_BUF_SIZE)
7319 		cnt = TRACE_BUF_SIZE;
7320 
7321 	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
7322 
7323 	size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
7324 
7325 	/* If less than "<faulted>", then make sure we can still add that */
7326 	if (cnt < FAULTED_SIZE)
7327 		size += FAULTED_SIZE - cnt;
7328 
7329 	buffer = tr->array_buffer.buffer;
7330 	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
7331 					    tracing_gen_ctx());
7332 	if (unlikely(!event))
7333 		/* Ring buffer disabled, return as if not open for write */
7334 		return -EBADF;
7335 
7336 	entry = ring_buffer_event_data(event);
7337 	entry->ip = _THIS_IP_;
7338 
7339 	len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
7340 	if (len) {
7341 		memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
7342 		cnt = FAULTED_SIZE;
7343 		written = -EFAULT;
7344 	} else
7345 		written = cnt;
7346 
7347 	if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
7348 		/* do not add \n before testing triggers, but add \0 */
7349 		entry->buf[cnt] = '\0';
7350 		tt = event_triggers_call(tr->trace_marker_file, buffer, entry, event);
7351 	}
7352 
7353 	if (entry->buf[cnt - 1] != '\n') {
7354 		entry->buf[cnt] = '\n';
7355 		entry->buf[cnt + 1] = '\0';
7356 	} else
7357 		entry->buf[cnt] = '\0';
7358 
7359 	if (static_branch_unlikely(&trace_marker_exports_enabled))
7360 		ftrace_exports(event, TRACE_EXPORT_MARKER);
7361 	__buffer_unlock_commit(buffer, event);
7362 
7363 	if (tt)
7364 		event_triggers_post_call(tr->trace_marker_file, tt);
7365 
7366 	return written;
7367 }
7368 
7369 /* Limit it for now to 3K (including tag) */
7370 #define RAW_DATA_MAX_SIZE (1024*3)
7371 
7372 static ssize_t
tracing_mark_raw_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * fpos)7373 tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
7374 					size_t cnt, loff_t *fpos)
7375 {
7376 	struct trace_array *tr = filp->private_data;
7377 	struct ring_buffer_event *event;
7378 	struct trace_buffer *buffer;
7379 	struct raw_data_entry *entry;
7380 	ssize_t written;
7381 	int size;
7382 	int len;
7383 
7384 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
7385 
7386 	if (tracing_disabled)
7387 		return -EINVAL;
7388 
7389 	if (!(tr->trace_flags & TRACE_ITER_MARKERS))
7390 		return -EINVAL;
7391 
7392 	/* The marker must at least have a tag id */
7393 	if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
7394 		return -EINVAL;
7395 
7396 	if (cnt > TRACE_BUF_SIZE)
7397 		cnt = TRACE_BUF_SIZE;
7398 
7399 	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
7400 
7401 	size = sizeof(*entry) + cnt;
7402 	if (cnt < FAULT_SIZE_ID)
7403 		size += FAULT_SIZE_ID - cnt;
7404 
7405 	buffer = tr->array_buffer.buffer;
7406 	event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
7407 					    tracing_gen_ctx());
7408 	if (!event)
7409 		/* Ring buffer disabled, return as if not open for write */
7410 		return -EBADF;
7411 
7412 	entry = ring_buffer_event_data(event);
7413 
7414 	len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
7415 	if (len) {
7416 		entry->id = -1;
7417 		memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
7418 		written = -EFAULT;
7419 	} else
7420 		written = cnt;
7421 
7422 	__buffer_unlock_commit(buffer, event);
7423 
7424 	return written;
7425 }
7426 
tracing_clock_show(struct seq_file * m,void * v)7427 static int tracing_clock_show(struct seq_file *m, void *v)
7428 {
7429 	struct trace_array *tr = m->private;
7430 	int i;
7431 
7432 	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
7433 		seq_printf(m,
7434 			"%s%s%s%s", i ? " " : "",
7435 			i == tr->clock_id ? "[" : "", trace_clocks[i].name,
7436 			i == tr->clock_id ? "]" : "");
7437 	seq_putc(m, '\n');
7438 
7439 	return 0;
7440 }
7441 
tracing_set_clock(struct trace_array * tr,const char * clockstr)7442 int tracing_set_clock(struct trace_array *tr, const char *clockstr)
7443 {
7444 	int i;
7445 
7446 	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
7447 		if (strcmp(trace_clocks[i].name, clockstr) == 0)
7448 			break;
7449 	}
7450 	if (i == ARRAY_SIZE(trace_clocks))
7451 		return -EINVAL;
7452 
7453 	mutex_lock(&trace_types_lock);
7454 
7455 	tr->clock_id = i;
7456 
7457 	ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);
7458 
7459 	/*
7460 	 * New clock may not be consistent with the previous clock.
7461 	 * Reset the buffer so that it doesn't have incomparable timestamps.
7462 	 */
7463 	tracing_reset_online_cpus(&tr->array_buffer);
7464 
7465 #ifdef CONFIG_TRACER_MAX_TRACE
7466 	if (tr->max_buffer.buffer)
7467 		ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
7468 	tracing_reset_online_cpus(&tr->max_buffer);
7469 #endif
7470 
7471 	mutex_unlock(&trace_types_lock);
7472 
7473 	return 0;
7474 }
7475 
tracing_clock_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * fpos)7476 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
7477 				   size_t cnt, loff_t *fpos)
7478 {
7479 	struct seq_file *m = filp->private_data;
7480 	struct trace_array *tr = m->private;
7481 	char buf[64];
7482 	const char *clockstr;
7483 	int ret;
7484 
7485 	if (cnt >= sizeof(buf))
7486 		return -EINVAL;
7487 
7488 	if (copy_from_user(buf, ubuf, cnt))
7489 		return -EFAULT;
7490 
7491 	buf[cnt] = 0;
7492 
7493 	clockstr = strstrip(buf);
7494 
7495 	ret = tracing_set_clock(tr, clockstr);
7496 	if (ret)
7497 		return ret;
7498 
7499 	*fpos += cnt;
7500 
7501 	return cnt;
7502 }
7503 
tracing_clock_open(struct inode * inode,struct file * file)7504 static int tracing_clock_open(struct inode *inode, struct file *file)
7505 {
7506 	struct trace_array *tr = inode->i_private;
7507 	int ret;
7508 
7509 	ret = tracing_check_open_get_tr(tr);
7510 	if (ret)
7511 		return ret;
7512 
7513 	ret = single_open(file, tracing_clock_show, inode->i_private);
7514 	if (ret < 0)
7515 		trace_array_put(tr);
7516 
7517 	return ret;
7518 }
7519 
tracing_time_stamp_mode_show(struct seq_file * m,void * v)7520 static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
7521 {
7522 	struct trace_array *tr = m->private;
7523 
7524 	mutex_lock(&trace_types_lock);
7525 
7526 	if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))
7527 		seq_puts(m, "delta [absolute]\n");
7528 	else
7529 		seq_puts(m, "[delta] absolute\n");
7530 
7531 	mutex_unlock(&trace_types_lock);
7532 
7533 	return 0;
7534 }
7535 
tracing_time_stamp_mode_open(struct inode * inode,struct file * file)7536 static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
7537 {
7538 	struct trace_array *tr = inode->i_private;
7539 	int ret;
7540 
7541 	ret = tracing_check_open_get_tr(tr);
7542 	if (ret)
7543 		return ret;
7544 
7545 	ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
7546 	if (ret < 0)
7547 		trace_array_put(tr);
7548 
7549 	return ret;
7550 }
7551 
tracing_event_time_stamp(struct trace_buffer * buffer,struct ring_buffer_event * rbe)7552 u64 tracing_event_time_stamp(struct trace_buffer *buffer, struct ring_buffer_event *rbe)
7553 {
7554 	if (rbe == this_cpu_read(trace_buffered_event))
7555 		return ring_buffer_time_stamp(buffer);
7556 
7557 	return ring_buffer_event_time_stamp(buffer, rbe);
7558 }
7559 
7560 /*
7561  * Set or disable using the per CPU trace_buffer_event when possible.
7562  */
tracing_set_filter_buffering(struct trace_array * tr,bool set)7563 int tracing_set_filter_buffering(struct trace_array *tr, bool set)
7564 {
7565 	int ret = 0;
7566 
7567 	mutex_lock(&trace_types_lock);
7568 
7569 	if (set && tr->no_filter_buffering_ref++)
7570 		goto out;
7571 
7572 	if (!set) {
7573 		if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) {
7574 			ret = -EINVAL;
7575 			goto out;
7576 		}
7577 
7578 		--tr->no_filter_buffering_ref;
7579 	}
7580  out:
7581 	mutex_unlock(&trace_types_lock);
7582 
7583 	return ret;
7584 }
7585 
7586 struct ftrace_buffer_info {
7587 	struct trace_iterator	iter;
7588 	void			*spare;
7589 	unsigned int		spare_cpu;
7590 	unsigned int		read;
7591 };
7592 
7593 #ifdef CONFIG_TRACER_SNAPSHOT
tracing_snapshot_open(struct inode * inode,struct file * file)7594 static int tracing_snapshot_open(struct inode *inode, struct file *file)
7595 {
7596 	struct trace_array *tr = inode->i_private;
7597 	struct trace_iterator *iter;
7598 	struct seq_file *m;
7599 	int ret;
7600 
7601 	ret = tracing_check_open_get_tr(tr);
7602 	if (ret)
7603 		return ret;
7604 
7605 	if (file->f_mode & FMODE_READ) {
7606 		iter = __tracing_open(inode, file, true);
7607 		if (IS_ERR(iter))
7608 			ret = PTR_ERR(iter);
7609 	} else {
7610 		/* Writes still need the seq_file to hold the private data */
7611 		ret = -ENOMEM;
7612 		m = kzalloc(sizeof(*m), GFP_KERNEL);
7613 		if (!m)
7614 			goto out;
7615 		iter = kzalloc(sizeof(*iter), GFP_KERNEL);
7616 		if (!iter) {
7617 			kfree(m);
7618 			goto out;
7619 		}
7620 		ret = 0;
7621 
7622 		iter->tr = tr;
7623 		iter->array_buffer = &tr->max_buffer;
7624 		iter->cpu_file = tracing_get_cpu(inode);
7625 		m->private = iter;
7626 		file->private_data = m;
7627 	}
7628 out:
7629 	if (ret < 0)
7630 		trace_array_put(tr);
7631 
7632 	return ret;
7633 }
7634 
tracing_swap_cpu_buffer(void * tr)7635 static void tracing_swap_cpu_buffer(void *tr)
7636 {
7637 	update_max_tr_single((struct trace_array *)tr, current, smp_processor_id());
7638 }
7639 
7640 static ssize_t
tracing_snapshot_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)7641 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
7642 		       loff_t *ppos)
7643 {
7644 	struct seq_file *m = filp->private_data;
7645 	struct trace_iterator *iter = m->private;
7646 	struct trace_array *tr = iter->tr;
7647 	unsigned long val;
7648 	int ret;
7649 
7650 	ret = tracing_update_buffers();
7651 	if (ret < 0)
7652 		return ret;
7653 
7654 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7655 	if (ret)
7656 		return ret;
7657 
7658 	mutex_lock(&trace_types_lock);
7659 
7660 	if (tr->current_trace->use_max_tr) {
7661 		ret = -EBUSY;
7662 		goto out;
7663 	}
7664 
7665 	local_irq_disable();
7666 	arch_spin_lock(&tr->max_lock);
7667 	if (tr->cond_snapshot)
7668 		ret = -EBUSY;
7669 	arch_spin_unlock(&tr->max_lock);
7670 	local_irq_enable();
7671 	if (ret)
7672 		goto out;
7673 
7674 	switch (val) {
7675 	case 0:
7676 		if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7677 			ret = -EINVAL;
7678 			break;
7679 		}
7680 		if (tr->allocated_snapshot)
7681 			free_snapshot(tr);
7682 		break;
7683 	case 1:
7684 /* Only allow per-cpu swap if the ring buffer supports it */
7685 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
7686 		if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7687 			ret = -EINVAL;
7688 			break;
7689 		}
7690 #endif
7691 		if (tr->allocated_snapshot)
7692 			ret = resize_buffer_duplicate_size(&tr->max_buffer,
7693 					&tr->array_buffer, iter->cpu_file);
7694 		else
7695 			ret = tracing_alloc_snapshot_instance(tr);
7696 		if (ret < 0)
7697 			break;
7698 		/* Now, we're going to swap */
7699 		if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
7700 			local_irq_disable();
7701 			update_max_tr(tr, current, smp_processor_id(), NULL);
7702 			local_irq_enable();
7703 		} else {
7704 			smp_call_function_single(iter->cpu_file, tracing_swap_cpu_buffer,
7705 						 (void *)tr, 1);
7706 		}
7707 		break;
7708 	default:
7709 		if (tr->allocated_snapshot) {
7710 			if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
7711 				tracing_reset_online_cpus(&tr->max_buffer);
7712 			else
7713 				tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);
7714 		}
7715 		break;
7716 	}
7717 
7718 	if (ret >= 0) {
7719 		*ppos += cnt;
7720 		ret = cnt;
7721 	}
7722 out:
7723 	mutex_unlock(&trace_types_lock);
7724 	return ret;
7725 }
7726 
tracing_snapshot_release(struct inode * inode,struct file * file)7727 static int tracing_snapshot_release(struct inode *inode, struct file *file)
7728 {
7729 	struct seq_file *m = file->private_data;
7730 	int ret;
7731 
7732 	ret = tracing_release(inode, file);
7733 
7734 	if (file->f_mode & FMODE_READ)
7735 		return ret;
7736 
7737 	/* If write only, the seq_file is just a stub */
7738 	if (m)
7739 		kfree(m->private);
7740 	kfree(m);
7741 
7742 	return 0;
7743 }
7744 
7745 static int tracing_buffers_open(struct inode *inode, struct file *filp);
7746 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
7747 				    size_t count, loff_t *ppos);
7748 static int tracing_buffers_release(struct inode *inode, struct file *file);
7749 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7750 		   struct pipe_inode_info *pipe, size_t len, unsigned int flags);
7751 
snapshot_raw_open(struct inode * inode,struct file * filp)7752 static int snapshot_raw_open(struct inode *inode, struct file *filp)
7753 {
7754 	struct ftrace_buffer_info *info;
7755 	int ret;
7756 
7757 	/* The following checks for tracefs lockdown */
7758 	ret = tracing_buffers_open(inode, filp);
7759 	if (ret < 0)
7760 		return ret;
7761 
7762 	info = filp->private_data;
7763 
7764 	if (info->iter.trace->use_max_tr) {
7765 		tracing_buffers_release(inode, filp);
7766 		return -EBUSY;
7767 	}
7768 
7769 	info->iter.snapshot = true;
7770 	info->iter.array_buffer = &info->iter.tr->max_buffer;
7771 
7772 	return ret;
7773 }
7774 
7775 #endif /* CONFIG_TRACER_SNAPSHOT */
7776 
7777 
7778 static const struct file_operations tracing_thresh_fops = {
7779 	.open		= tracing_open_generic,
7780 	.read		= tracing_thresh_read,
7781 	.write		= tracing_thresh_write,
7782 	.llseek		= generic_file_llseek,
7783 };
7784 
7785 #ifdef CONFIG_TRACER_MAX_TRACE
7786 static const struct file_operations tracing_max_lat_fops = {
7787 	.open		= tracing_open_generic_tr,
7788 	.read		= tracing_max_lat_read,
7789 	.write		= tracing_max_lat_write,
7790 	.llseek		= generic_file_llseek,
7791 	.release	= tracing_release_generic_tr,
7792 };
7793 #endif
7794 
7795 static const struct file_operations set_tracer_fops = {
7796 	.open		= tracing_open_generic_tr,
7797 	.read		= tracing_set_trace_read,
7798 	.write		= tracing_set_trace_write,
7799 	.llseek		= generic_file_llseek,
7800 	.release	= tracing_release_generic_tr,
7801 };
7802 
7803 static const struct file_operations tracing_pipe_fops = {
7804 	.open		= tracing_open_pipe,
7805 	.poll		= tracing_poll_pipe,
7806 	.read		= tracing_read_pipe,
7807 	.splice_read	= tracing_splice_read_pipe,
7808 	.release	= tracing_release_pipe,
7809 	.llseek		= no_llseek,
7810 };
7811 
7812 static const struct file_operations tracing_entries_fops = {
7813 	.open		= tracing_open_generic_tr,
7814 	.read		= tracing_entries_read,
7815 	.write		= tracing_entries_write,
7816 	.llseek		= generic_file_llseek,
7817 	.release	= tracing_release_generic_tr,
7818 };
7819 
7820 static const struct file_operations tracing_total_entries_fops = {
7821 	.open		= tracing_open_generic_tr,
7822 	.read		= tracing_total_entries_read,
7823 	.llseek		= generic_file_llseek,
7824 	.release	= tracing_release_generic_tr,
7825 };
7826 
7827 static const struct file_operations tracing_free_buffer_fops = {
7828 	.open		= tracing_open_generic_tr,
7829 	.write		= tracing_free_buffer_write,
7830 	.release	= tracing_free_buffer_release,
7831 };
7832 
7833 static const struct file_operations tracing_mark_fops = {
7834 	.open		= tracing_mark_open,
7835 	.write		= tracing_mark_write,
7836 	.release	= tracing_release_generic_tr,
7837 };
7838 
7839 static const struct file_operations tracing_mark_raw_fops = {
7840 	.open		= tracing_mark_open,
7841 	.write		= tracing_mark_raw_write,
7842 	.release	= tracing_release_generic_tr,
7843 };
7844 
7845 static const struct file_operations trace_clock_fops = {
7846 	.open		= tracing_clock_open,
7847 	.read		= seq_read,
7848 	.llseek		= seq_lseek,
7849 	.release	= tracing_single_release_tr,
7850 	.write		= tracing_clock_write,
7851 };
7852 
7853 static const struct file_operations trace_time_stamp_mode_fops = {
7854 	.open		= tracing_time_stamp_mode_open,
7855 	.read		= seq_read,
7856 	.llseek		= seq_lseek,
7857 	.release	= tracing_single_release_tr,
7858 };
7859 
7860 #ifdef CONFIG_TRACER_SNAPSHOT
7861 static const struct file_operations snapshot_fops = {
7862 	.open		= tracing_snapshot_open,
7863 	.read		= seq_read,
7864 	.write		= tracing_snapshot_write,
7865 	.llseek		= tracing_lseek,
7866 	.release	= tracing_snapshot_release,
7867 };
7868 
7869 static const struct file_operations snapshot_raw_fops = {
7870 	.open		= snapshot_raw_open,
7871 	.read		= tracing_buffers_read,
7872 	.release	= tracing_buffers_release,
7873 	.splice_read	= tracing_buffers_splice_read,
7874 	.llseek		= no_llseek,
7875 };
7876 
7877 #endif /* CONFIG_TRACER_SNAPSHOT */
7878 
7879 /*
7880  * trace_min_max_write - Write a u64 value to a trace_min_max_param struct
7881  * @filp: The active open file structure
7882  * @ubuf: The userspace provided buffer to read value into
7883  * @cnt: The maximum number of bytes to read
7884  * @ppos: The current "file" position
7885  *
7886  * This function implements the write interface for a struct trace_min_max_param.
7887  * The filp->private_data must point to a trace_min_max_param structure that
7888  * defines where to write the value, the min and the max acceptable values,
7889  * and a lock to protect the write.
7890  */
7891 static ssize_t
trace_min_max_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)7892 trace_min_max_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
7893 {
7894 	struct trace_min_max_param *param = filp->private_data;
7895 	u64 val;
7896 	int err;
7897 
7898 	if (!param)
7899 		return -EFAULT;
7900 
7901 	err = kstrtoull_from_user(ubuf, cnt, 10, &val);
7902 	if (err)
7903 		return err;
7904 
7905 	if (param->lock)
7906 		mutex_lock(param->lock);
7907 
7908 	if (param->min && val < *param->min)
7909 		err = -EINVAL;
7910 
7911 	if (param->max && val > *param->max)
7912 		err = -EINVAL;
7913 
7914 	if (!err)
7915 		*param->val = val;
7916 
7917 	if (param->lock)
7918 		mutex_unlock(param->lock);
7919 
7920 	if (err)
7921 		return err;
7922 
7923 	return cnt;
7924 }
7925 
7926 /*
7927  * trace_min_max_read - Read a u64 value from a trace_min_max_param struct
7928  * @filp: The active open file structure
7929  * @ubuf: The userspace provided buffer to read value into
7930  * @cnt: The maximum number of bytes to read
7931  * @ppos: The current "file" position
7932  *
7933  * This function implements the read interface for a struct trace_min_max_param.
7934  * The filp->private_data must point to a trace_min_max_param struct with valid
7935  * data.
7936  */
7937 static ssize_t
trace_min_max_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)7938 trace_min_max_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
7939 {
7940 	struct trace_min_max_param *param = filp->private_data;
7941 	char buf[U64_STR_SIZE];
7942 	int len;
7943 	u64 val;
7944 
7945 	if (!param)
7946 		return -EFAULT;
7947 
7948 	val = *param->val;
7949 
7950 	if (cnt > sizeof(buf))
7951 		cnt = sizeof(buf);
7952 
7953 	len = snprintf(buf, sizeof(buf), "%llu\n", val);
7954 
7955 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
7956 }
7957 
7958 const struct file_operations trace_min_max_fops = {
7959 	.open		= tracing_open_generic,
7960 	.read		= trace_min_max_read,
7961 	.write		= trace_min_max_write,
7962 };
7963 
7964 #define TRACING_LOG_ERRS_MAX	8
7965 #define TRACING_LOG_LOC_MAX	128
7966 
7967 #define CMD_PREFIX "  Command: "
7968 
7969 struct err_info {
7970 	const char	**errs;	/* ptr to loc-specific array of err strings */
7971 	u8		type;	/* index into errs -> specific err string */
7972 	u16		pos;	/* caret position */
7973 	u64		ts;
7974 };
7975 
7976 struct tracing_log_err {
7977 	struct list_head	list;
7978 	struct err_info		info;
7979 	char			loc[TRACING_LOG_LOC_MAX]; /* err location */
7980 	char			*cmd;                     /* what caused err */
7981 };
7982 
7983 static DEFINE_MUTEX(tracing_err_log_lock);
7984 
alloc_tracing_log_err(int len)7985 static struct tracing_log_err *alloc_tracing_log_err(int len)
7986 {
7987 	struct tracing_log_err *err;
7988 
7989 	err = kzalloc(sizeof(*err), GFP_KERNEL);
7990 	if (!err)
7991 		return ERR_PTR(-ENOMEM);
7992 
7993 	err->cmd = kzalloc(len, GFP_KERNEL);
7994 	if (!err->cmd) {
7995 		kfree(err);
7996 		return ERR_PTR(-ENOMEM);
7997 	}
7998 
7999 	return err;
8000 }
8001 
free_tracing_log_err(struct tracing_log_err * err)8002 static void free_tracing_log_err(struct tracing_log_err *err)
8003 {
8004 	kfree(err->cmd);
8005 	kfree(err);
8006 }
8007 
get_tracing_log_err(struct trace_array * tr,int len)8008 static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr,
8009 						   int len)
8010 {
8011 	struct tracing_log_err *err;
8012 	char *cmd;
8013 
8014 	if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
8015 		err = alloc_tracing_log_err(len);
8016 		if (PTR_ERR(err) != -ENOMEM)
8017 			tr->n_err_log_entries++;
8018 
8019 		return err;
8020 	}
8021 	cmd = kzalloc(len, GFP_KERNEL);
8022 	if (!cmd)
8023 		return ERR_PTR(-ENOMEM);
8024 	err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
8025 	kfree(err->cmd);
8026 	err->cmd = cmd;
8027 	list_del(&err->list);
8028 
8029 	return err;
8030 }
8031 
8032 /**
8033  * err_pos - find the position of a string within a command for error careting
8034  * @cmd: The tracing command that caused the error
8035  * @str: The string to position the caret at within @cmd
8036  *
8037  * Finds the position of the first occurrence of @str within @cmd.  The
8038  * return value can be passed to tracing_log_err() for caret placement
8039  * within @cmd.
8040  *
8041  * Returns the index within @cmd of the first occurrence of @str or 0
8042  * if @str was not found.
8043  */
err_pos(char * cmd,const char * str)8044 unsigned int err_pos(char *cmd, const char *str)
8045 {
8046 	char *found;
8047 
8048 	if (WARN_ON(!strlen(cmd)))
8049 		return 0;
8050 
8051 	found = strstr(cmd, str);
8052 	if (found)
8053 		return found - cmd;
8054 
8055 	return 0;
8056 }
8057 
8058 /**
8059  * tracing_log_err - write an error to the tracing error log
8060  * @tr: The associated trace array for the error (NULL for top level array)
8061  * @loc: A string describing where the error occurred
8062  * @cmd: The tracing command that caused the error
8063  * @errs: The array of loc-specific static error strings
8064  * @type: The index into errs[], which produces the specific static err string
8065  * @pos: The position the caret should be placed in the cmd
8066  *
8067  * Writes an error into tracing/error_log of the form:
8068  *
8069  * <loc>: error: <text>
8070  *   Command: <cmd>
8071  *              ^
8072  *
8073  * tracing/error_log is a small log file containing the last
8074  * TRACING_LOG_ERRS_MAX errors (8).  Memory for errors isn't allocated
8075  * unless there has been a tracing error, and the error log can be
8076  * cleared and have its memory freed by writing the empty string in
8077  * truncation mode to it i.e. echo > tracing/error_log.
8078  *
8079  * NOTE: the @errs array along with the @type param are used to
8080  * produce a static error string - this string is not copied and saved
8081  * when the error is logged - only a pointer to it is saved.  See
8082  * existing callers for examples of how static strings are typically
8083  * defined for use with tracing_log_err().
8084  */
tracing_log_err(struct trace_array * tr,const char * loc,const char * cmd,const char ** errs,u8 type,u16 pos)8085 void tracing_log_err(struct trace_array *tr,
8086 		     const char *loc, const char *cmd,
8087 		     const char **errs, u8 type, u16 pos)
8088 {
8089 	struct tracing_log_err *err;
8090 	int len = 0;
8091 
8092 	if (!tr)
8093 		tr = &global_trace;
8094 
8095 	len += sizeof(CMD_PREFIX) + 2 * sizeof("\n") + strlen(cmd) + 1;
8096 
8097 	mutex_lock(&tracing_err_log_lock);
8098 	err = get_tracing_log_err(tr, len);
8099 	if (PTR_ERR(err) == -ENOMEM) {
8100 		mutex_unlock(&tracing_err_log_lock);
8101 		return;
8102 	}
8103 
8104 	snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
8105 	snprintf(err->cmd, len, "\n" CMD_PREFIX "%s\n", cmd);
8106 
8107 	err->info.errs = errs;
8108 	err->info.type = type;
8109 	err->info.pos = pos;
8110 	err->info.ts = local_clock();
8111 
8112 	list_add_tail(&err->list, &tr->err_log);
8113 	mutex_unlock(&tracing_err_log_lock);
8114 }
8115 
clear_tracing_err_log(struct trace_array * tr)8116 static void clear_tracing_err_log(struct trace_array *tr)
8117 {
8118 	struct tracing_log_err *err, *next;
8119 
8120 	mutex_lock(&tracing_err_log_lock);
8121 	list_for_each_entry_safe(err, next, &tr->err_log, list) {
8122 		list_del(&err->list);
8123 		free_tracing_log_err(err);
8124 	}
8125 
8126 	tr->n_err_log_entries = 0;
8127 	mutex_unlock(&tracing_err_log_lock);
8128 }
8129 
tracing_err_log_seq_start(struct seq_file * m,loff_t * pos)8130 static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
8131 {
8132 	struct trace_array *tr = m->private;
8133 
8134 	mutex_lock(&tracing_err_log_lock);
8135 
8136 	return seq_list_start(&tr->err_log, *pos);
8137 }
8138 
tracing_err_log_seq_next(struct seq_file * m,void * v,loff_t * pos)8139 static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
8140 {
8141 	struct trace_array *tr = m->private;
8142 
8143 	return seq_list_next(v, &tr->err_log, pos);
8144 }
8145 
tracing_err_log_seq_stop(struct seq_file * m,void * v)8146 static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
8147 {
8148 	mutex_unlock(&tracing_err_log_lock);
8149 }
8150 
tracing_err_log_show_pos(struct seq_file * m,u16 pos)8151 static void tracing_err_log_show_pos(struct seq_file *m, u16 pos)
8152 {
8153 	u16 i;
8154 
8155 	for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
8156 		seq_putc(m, ' ');
8157 	for (i = 0; i < pos; i++)
8158 		seq_putc(m, ' ');
8159 	seq_puts(m, "^\n");
8160 }
8161 
tracing_err_log_seq_show(struct seq_file * m,void * v)8162 static int tracing_err_log_seq_show(struct seq_file *m, void *v)
8163 {
8164 	struct tracing_log_err *err = v;
8165 
8166 	if (err) {
8167 		const char *err_text = err->info.errs[err->info.type];
8168 		u64 sec = err->info.ts;
8169 		u32 nsec;
8170 
8171 		nsec = do_div(sec, NSEC_PER_SEC);
8172 		seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
8173 			   err->loc, err_text);
8174 		seq_printf(m, "%s", err->cmd);
8175 		tracing_err_log_show_pos(m, err->info.pos);
8176 	}
8177 
8178 	return 0;
8179 }
8180 
8181 static const struct seq_operations tracing_err_log_seq_ops = {
8182 	.start  = tracing_err_log_seq_start,
8183 	.next   = tracing_err_log_seq_next,
8184 	.stop   = tracing_err_log_seq_stop,
8185 	.show   = tracing_err_log_seq_show
8186 };
8187 
tracing_err_log_open(struct inode * inode,struct file * file)8188 static int tracing_err_log_open(struct inode *inode, struct file *file)
8189 {
8190 	struct trace_array *tr = inode->i_private;
8191 	int ret = 0;
8192 
8193 	ret = tracing_check_open_get_tr(tr);
8194 	if (ret)
8195 		return ret;
8196 
8197 	/* If this file was opened for write, then erase contents */
8198 	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
8199 		clear_tracing_err_log(tr);
8200 
8201 	if (file->f_mode & FMODE_READ) {
8202 		ret = seq_open(file, &tracing_err_log_seq_ops);
8203 		if (!ret) {
8204 			struct seq_file *m = file->private_data;
8205 			m->private = tr;
8206 		} else {
8207 			trace_array_put(tr);
8208 		}
8209 	}
8210 	return ret;
8211 }
8212 
tracing_err_log_write(struct file * file,const char __user * buffer,size_t count,loff_t * ppos)8213 static ssize_t tracing_err_log_write(struct file *file,
8214 				     const char __user *buffer,
8215 				     size_t count, loff_t *ppos)
8216 {
8217 	return count;
8218 }
8219 
tracing_err_log_release(struct inode * inode,struct file * file)8220 static int tracing_err_log_release(struct inode *inode, struct file *file)
8221 {
8222 	struct trace_array *tr = inode->i_private;
8223 
8224 	trace_array_put(tr);
8225 
8226 	if (file->f_mode & FMODE_READ)
8227 		seq_release(inode, file);
8228 
8229 	return 0;
8230 }
8231 
8232 static const struct file_operations tracing_err_log_fops = {
8233 	.open           = tracing_err_log_open,
8234 	.write		= tracing_err_log_write,
8235 	.read           = seq_read,
8236 	.llseek         = tracing_lseek,
8237 	.release        = tracing_err_log_release,
8238 };
8239 
tracing_buffers_open(struct inode * inode,struct file * filp)8240 static int tracing_buffers_open(struct inode *inode, struct file *filp)
8241 {
8242 	struct trace_array *tr = inode->i_private;
8243 	struct ftrace_buffer_info *info;
8244 	int ret;
8245 
8246 	ret = tracing_check_open_get_tr(tr);
8247 	if (ret)
8248 		return ret;
8249 
8250 	info = kvzalloc(sizeof(*info), GFP_KERNEL);
8251 	if (!info) {
8252 		trace_array_put(tr);
8253 		return -ENOMEM;
8254 	}
8255 
8256 	mutex_lock(&trace_types_lock);
8257 
8258 	info->iter.tr		= tr;
8259 	info->iter.cpu_file	= tracing_get_cpu(inode);
8260 	info->iter.trace	= tr->current_trace;
8261 	info->iter.array_buffer = &tr->array_buffer;
8262 	info->spare		= NULL;
8263 	/* Force reading ring buffer for first read */
8264 	info->read		= (unsigned int)-1;
8265 
8266 	filp->private_data = info;
8267 
8268 	tr->trace_ref++;
8269 
8270 	mutex_unlock(&trace_types_lock);
8271 
8272 	ret = nonseekable_open(inode, filp);
8273 	if (ret < 0)
8274 		trace_array_put(tr);
8275 
8276 	return ret;
8277 }
8278 
8279 static __poll_t
tracing_buffers_poll(struct file * filp,poll_table * poll_table)8280 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
8281 {
8282 	struct ftrace_buffer_info *info = filp->private_data;
8283 	struct trace_iterator *iter = &info->iter;
8284 
8285 	return trace_poll(iter, filp, poll_table);
8286 }
8287 
8288 static ssize_t
tracing_buffers_read(struct file * filp,char __user * ubuf,size_t count,loff_t * ppos)8289 tracing_buffers_read(struct file *filp, char __user *ubuf,
8290 		     size_t count, loff_t *ppos)
8291 {
8292 	struct ftrace_buffer_info *info = filp->private_data;
8293 	struct trace_iterator *iter = &info->iter;
8294 	ssize_t ret = 0;
8295 	ssize_t size;
8296 
8297 	if (!count)
8298 		return 0;
8299 
8300 #ifdef CONFIG_TRACER_MAX_TRACE
8301 	if (iter->snapshot && iter->tr->current_trace->use_max_tr)
8302 		return -EBUSY;
8303 #endif
8304 
8305 	if (!info->spare) {
8306 		info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,
8307 							  iter->cpu_file);
8308 		if (IS_ERR(info->spare)) {
8309 			ret = PTR_ERR(info->spare);
8310 			info->spare = NULL;
8311 		} else {
8312 			info->spare_cpu = iter->cpu_file;
8313 		}
8314 	}
8315 	if (!info->spare)
8316 		return ret;
8317 
8318 	/* Do we have previous read data to read? */
8319 	if (info->read < PAGE_SIZE)
8320 		goto read;
8321 
8322  again:
8323 	trace_access_lock(iter->cpu_file);
8324 	ret = ring_buffer_read_page(iter->array_buffer->buffer,
8325 				    &info->spare,
8326 				    count,
8327 				    iter->cpu_file, 0);
8328 	trace_access_unlock(iter->cpu_file);
8329 
8330 	if (ret < 0) {
8331 		if (trace_empty(iter)) {
8332 			if ((filp->f_flags & O_NONBLOCK))
8333 				return -EAGAIN;
8334 
8335 			ret = wait_on_pipe(iter, 0);
8336 			if (ret)
8337 				return ret;
8338 
8339 			goto again;
8340 		}
8341 		return 0;
8342 	}
8343 
8344 	info->read = 0;
8345  read:
8346 	size = PAGE_SIZE - info->read;
8347 	if (size > count)
8348 		size = count;
8349 
8350 	ret = copy_to_user(ubuf, info->spare + info->read, size);
8351 	if (ret == size)
8352 		return -EFAULT;
8353 
8354 	size -= ret;
8355 
8356 	*ppos += size;
8357 	info->read += size;
8358 
8359 	return size;
8360 }
8361 
tracing_buffers_release(struct inode * inode,struct file * file)8362 static int tracing_buffers_release(struct inode *inode, struct file *file)
8363 {
8364 	struct ftrace_buffer_info *info = file->private_data;
8365 	struct trace_iterator *iter = &info->iter;
8366 
8367 	mutex_lock(&trace_types_lock);
8368 
8369 	iter->tr->trace_ref--;
8370 
8371 	__trace_array_put(iter->tr);
8372 
8373 	iter->wait_index++;
8374 	/* Make sure the waiters see the new wait_index */
8375 	smp_wmb();
8376 
8377 	ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
8378 
8379 	if (info->spare)
8380 		ring_buffer_free_read_page(iter->array_buffer->buffer,
8381 					   info->spare_cpu, info->spare);
8382 	kvfree(info);
8383 
8384 	mutex_unlock(&trace_types_lock);
8385 
8386 	return 0;
8387 }
8388 
8389 struct buffer_ref {
8390 	struct trace_buffer	*buffer;
8391 	void			*page;
8392 	int			cpu;
8393 	refcount_t		refcount;
8394 };
8395 
buffer_ref_release(struct buffer_ref * ref)8396 static void buffer_ref_release(struct buffer_ref *ref)
8397 {
8398 	if (!refcount_dec_and_test(&ref->refcount))
8399 		return;
8400 	ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
8401 	kfree(ref);
8402 }
8403 
buffer_pipe_buf_release(struct pipe_inode_info * pipe,struct pipe_buffer * buf)8404 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
8405 				    struct pipe_buffer *buf)
8406 {
8407 	struct buffer_ref *ref = (struct buffer_ref *)buf->private;
8408 
8409 	buffer_ref_release(ref);
8410 	buf->private = 0;
8411 }
8412 
buffer_pipe_buf_get(struct pipe_inode_info * pipe,struct pipe_buffer * buf)8413 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
8414 				struct pipe_buffer *buf)
8415 {
8416 	struct buffer_ref *ref = (struct buffer_ref *)buf->private;
8417 
8418 	if (refcount_read(&ref->refcount) > INT_MAX/2)
8419 		return false;
8420 
8421 	refcount_inc(&ref->refcount);
8422 	return true;
8423 }
8424 
8425 /* Pipe buffer operations for a buffer. */
8426 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
8427 	.release		= buffer_pipe_buf_release,
8428 	.get			= buffer_pipe_buf_get,
8429 };
8430 
8431 /*
8432  * Callback from splice_to_pipe(), if we need to release some pages
8433  * at the end of the spd in case we error'ed out in filling the pipe.
8434  */
buffer_spd_release(struct splice_pipe_desc * spd,unsigned int i)8435 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
8436 {
8437 	struct buffer_ref *ref =
8438 		(struct buffer_ref *)spd->partial[i].private;
8439 
8440 	buffer_ref_release(ref);
8441 	spd->partial[i].private = 0;
8442 }
8443 
8444 static ssize_t
tracing_buffers_splice_read(struct file * file,loff_t * ppos,struct pipe_inode_info * pipe,size_t len,unsigned int flags)8445 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
8446 			    struct pipe_inode_info *pipe, size_t len,
8447 			    unsigned int flags)
8448 {
8449 	struct ftrace_buffer_info *info = file->private_data;
8450 	struct trace_iterator *iter = &info->iter;
8451 	struct partial_page partial_def[PIPE_DEF_BUFFERS];
8452 	struct page *pages_def[PIPE_DEF_BUFFERS];
8453 	struct splice_pipe_desc spd = {
8454 		.pages		= pages_def,
8455 		.partial	= partial_def,
8456 		.nr_pages_max	= PIPE_DEF_BUFFERS,
8457 		.ops		= &buffer_pipe_buf_ops,
8458 		.spd_release	= buffer_spd_release,
8459 	};
8460 	struct buffer_ref *ref;
8461 	int entries, i;
8462 	ssize_t ret = 0;
8463 
8464 #ifdef CONFIG_TRACER_MAX_TRACE
8465 	if (iter->snapshot && iter->tr->current_trace->use_max_tr)
8466 		return -EBUSY;
8467 #endif
8468 
8469 	if (*ppos & (PAGE_SIZE - 1))
8470 		return -EINVAL;
8471 
8472 	if (len & (PAGE_SIZE - 1)) {
8473 		if (len < PAGE_SIZE)
8474 			return -EINVAL;
8475 		len &= PAGE_MASK;
8476 	}
8477 
8478 	if (splice_grow_spd(pipe, &spd))
8479 		return -ENOMEM;
8480 
8481  again:
8482 	trace_access_lock(iter->cpu_file);
8483 	entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
8484 
8485 	for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
8486 		struct page *page;
8487 		int r;
8488 
8489 		ref = kzalloc(sizeof(*ref), GFP_KERNEL);
8490 		if (!ref) {
8491 			ret = -ENOMEM;
8492 			break;
8493 		}
8494 
8495 		refcount_set(&ref->refcount, 1);
8496 		ref->buffer = iter->array_buffer->buffer;
8497 		ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
8498 		if (IS_ERR(ref->page)) {
8499 			ret = PTR_ERR(ref->page);
8500 			ref->page = NULL;
8501 			kfree(ref);
8502 			break;
8503 		}
8504 		ref->cpu = iter->cpu_file;
8505 
8506 		r = ring_buffer_read_page(ref->buffer, &ref->page,
8507 					  len, iter->cpu_file, 1);
8508 		if (r < 0) {
8509 			ring_buffer_free_read_page(ref->buffer, ref->cpu,
8510 						   ref->page);
8511 			kfree(ref);
8512 			break;
8513 		}
8514 
8515 		page = virt_to_page(ref->page);
8516 
8517 		spd.pages[i] = page;
8518 		spd.partial[i].len = PAGE_SIZE;
8519 		spd.partial[i].offset = 0;
8520 		spd.partial[i].private = (unsigned long)ref;
8521 		spd.nr_pages++;
8522 		*ppos += PAGE_SIZE;
8523 
8524 		entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
8525 	}
8526 
8527 	trace_access_unlock(iter->cpu_file);
8528 	spd.nr_pages = i;
8529 
8530 	/* did we read anything? */
8531 	if (!spd.nr_pages) {
8532 		long wait_index;
8533 
8534 		if (ret)
8535 			goto out;
8536 
8537 		ret = -EAGAIN;
8538 		if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
8539 			goto out;
8540 
8541 		wait_index = READ_ONCE(iter->wait_index);
8542 
8543 		ret = wait_on_pipe(iter, iter->snapshot ? 0 : iter->tr->buffer_percent);
8544 		if (ret)
8545 			goto out;
8546 
8547 		/* No need to wait after waking up when tracing is off */
8548 		if (!tracer_tracing_is_on(iter->tr))
8549 			goto out;
8550 
8551 		/* Make sure we see the new wait_index */
8552 		smp_rmb();
8553 		if (wait_index != iter->wait_index)
8554 			goto out;
8555 
8556 		goto again;
8557 	}
8558 
8559 	ret = splice_to_pipe(pipe, &spd);
8560 out:
8561 	splice_shrink_spd(&spd);
8562 
8563 	return ret;
8564 }
8565 
8566 /* An ioctl call with cmd 0 to the ring buffer file will wake up all waiters */
tracing_buffers_ioctl(struct file * file,unsigned int cmd,unsigned long arg)8567 static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
8568 {
8569 	struct ftrace_buffer_info *info = file->private_data;
8570 	struct trace_iterator *iter = &info->iter;
8571 
8572 	if (cmd)
8573 		return -ENOIOCTLCMD;
8574 
8575 	mutex_lock(&trace_types_lock);
8576 
8577 	iter->wait_index++;
8578 	/* Make sure the waiters see the new wait_index */
8579 	smp_wmb();
8580 
8581 	ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);
8582 
8583 	mutex_unlock(&trace_types_lock);
8584 	return 0;
8585 }
8586 
8587 static const struct file_operations tracing_buffers_fops = {
8588 	.open		= tracing_buffers_open,
8589 	.read		= tracing_buffers_read,
8590 	.poll		= tracing_buffers_poll,
8591 	.release	= tracing_buffers_release,
8592 	.splice_read	= tracing_buffers_splice_read,
8593 	.unlocked_ioctl = tracing_buffers_ioctl,
8594 	.llseek		= no_llseek,
8595 };
8596 
8597 static ssize_t
tracing_stats_read(struct file * filp,char __user * ubuf,size_t count,loff_t * ppos)8598 tracing_stats_read(struct file *filp, char __user *ubuf,
8599 		   size_t count, loff_t *ppos)
8600 {
8601 	struct inode *inode = file_inode(filp);
8602 	struct trace_array *tr = inode->i_private;
8603 	struct array_buffer *trace_buf = &tr->array_buffer;
8604 	int cpu = tracing_get_cpu(inode);
8605 	struct trace_seq *s;
8606 	unsigned long cnt;
8607 	unsigned long long t;
8608 	unsigned long usec_rem;
8609 
8610 	s = kmalloc(sizeof(*s), GFP_KERNEL);
8611 	if (!s)
8612 		return -ENOMEM;
8613 
8614 	trace_seq_init(s);
8615 
8616 	cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
8617 	trace_seq_printf(s, "entries: %ld\n", cnt);
8618 
8619 	cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
8620 	trace_seq_printf(s, "overrun: %ld\n", cnt);
8621 
8622 	cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
8623 	trace_seq_printf(s, "commit overrun: %ld\n", cnt);
8624 
8625 	cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
8626 	trace_seq_printf(s, "bytes: %ld\n", cnt);
8627 
8628 	if (trace_clocks[tr->clock_id].in_ns) {
8629 		/* local or global for trace_clock */
8630 		t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
8631 		usec_rem = do_div(t, USEC_PER_SEC);
8632 		trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
8633 								t, usec_rem);
8634 
8635 		t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer));
8636 		usec_rem = do_div(t, USEC_PER_SEC);
8637 		trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
8638 	} else {
8639 		/* counter or tsc mode for trace_clock */
8640 		trace_seq_printf(s, "oldest event ts: %llu\n",
8641 				ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
8642 
8643 		trace_seq_printf(s, "now ts: %llu\n",
8644 				ring_buffer_time_stamp(trace_buf->buffer));
8645 	}
8646 
8647 	cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
8648 	trace_seq_printf(s, "dropped events: %ld\n", cnt);
8649 
8650 	cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
8651 	trace_seq_printf(s, "read events: %ld\n", cnt);
8652 
8653 	count = simple_read_from_buffer(ubuf, count, ppos,
8654 					s->buffer, trace_seq_used(s));
8655 
8656 	kfree(s);
8657 
8658 	return count;
8659 }
8660 
8661 static const struct file_operations tracing_stats_fops = {
8662 	.open		= tracing_open_generic_tr,
8663 	.read		= tracing_stats_read,
8664 	.llseek		= generic_file_llseek,
8665 	.release	= tracing_release_generic_tr,
8666 };
8667 
8668 #ifdef CONFIG_DYNAMIC_FTRACE
8669 
8670 static ssize_t
tracing_read_dyn_info(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)8671 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
8672 		  size_t cnt, loff_t *ppos)
8673 {
8674 	ssize_t ret;
8675 	char *buf;
8676 	int r;
8677 
8678 	/* 256 should be plenty to hold the amount needed */
8679 	buf = kmalloc(256, GFP_KERNEL);
8680 	if (!buf)
8681 		return -ENOMEM;
8682 
8683 	r = scnprintf(buf, 256, "%ld pages:%ld groups: %ld\n",
8684 		      ftrace_update_tot_cnt,
8685 		      ftrace_number_of_pages,
8686 		      ftrace_number_of_groups);
8687 
8688 	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8689 	kfree(buf);
8690 	return ret;
8691 }
8692 
8693 static const struct file_operations tracing_dyn_info_fops = {
8694 	.open		= tracing_open_generic,
8695 	.read		= tracing_read_dyn_info,
8696 	.llseek		= generic_file_llseek,
8697 };
8698 #endif /* CONFIG_DYNAMIC_FTRACE */
8699 
8700 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
8701 static void
ftrace_snapshot(unsigned long ip,unsigned long parent_ip,struct trace_array * tr,struct ftrace_probe_ops * ops,void * data)8702 ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
8703 		struct trace_array *tr, struct ftrace_probe_ops *ops,
8704 		void *data)
8705 {
8706 	tracing_snapshot_instance(tr);
8707 }
8708 
8709 static void
ftrace_count_snapshot(unsigned long ip,unsigned long parent_ip,struct trace_array * tr,struct ftrace_probe_ops * ops,void * data)8710 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
8711 		      struct trace_array *tr, struct ftrace_probe_ops *ops,
8712 		      void *data)
8713 {
8714 	struct ftrace_func_mapper *mapper = data;
8715 	long *count = NULL;
8716 
8717 	if (mapper)
8718 		count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
8719 
8720 	if (count) {
8721 
8722 		if (*count <= 0)
8723 			return;
8724 
8725 		(*count)--;
8726 	}
8727 
8728 	tracing_snapshot_instance(tr);
8729 }
8730 
8731 static int
ftrace_snapshot_print(struct seq_file * m,unsigned long ip,struct ftrace_probe_ops * ops,void * data)8732 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
8733 		      struct ftrace_probe_ops *ops, void *data)
8734 {
8735 	struct ftrace_func_mapper *mapper = data;
8736 	long *count = NULL;
8737 
8738 	seq_printf(m, "%ps:", (void *)ip);
8739 
8740 	seq_puts(m, "snapshot");
8741 
8742 	if (mapper)
8743 		count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
8744 
8745 	if (count)
8746 		seq_printf(m, ":count=%ld\n", *count);
8747 	else
8748 		seq_puts(m, ":unlimited\n");
8749 
8750 	return 0;
8751 }
8752 
8753 static int
ftrace_snapshot_init(struct ftrace_probe_ops * ops,struct trace_array * tr,unsigned long ip,void * init_data,void ** data)8754 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
8755 		     unsigned long ip, void *init_data, void **data)
8756 {
8757 	struct ftrace_func_mapper *mapper = *data;
8758 
8759 	if (!mapper) {
8760 		mapper = allocate_ftrace_func_mapper();
8761 		if (!mapper)
8762 			return -ENOMEM;
8763 		*data = mapper;
8764 	}
8765 
8766 	return ftrace_func_mapper_add_ip(mapper, ip, init_data);
8767 }
8768 
8769 static void
ftrace_snapshot_free(struct ftrace_probe_ops * ops,struct trace_array * tr,unsigned long ip,void * data)8770 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
8771 		     unsigned long ip, void *data)
8772 {
8773 	struct ftrace_func_mapper *mapper = data;
8774 
8775 	if (!ip) {
8776 		if (!mapper)
8777 			return;
8778 		free_ftrace_func_mapper(mapper, NULL);
8779 		return;
8780 	}
8781 
8782 	ftrace_func_mapper_remove_ip(mapper, ip);
8783 }
8784 
8785 static struct ftrace_probe_ops snapshot_probe_ops = {
8786 	.func			= ftrace_snapshot,
8787 	.print			= ftrace_snapshot_print,
8788 };
8789 
8790 static struct ftrace_probe_ops snapshot_count_probe_ops = {
8791 	.func			= ftrace_count_snapshot,
8792 	.print			= ftrace_snapshot_print,
8793 	.init			= ftrace_snapshot_init,
8794 	.free			= ftrace_snapshot_free,
8795 };
8796 
8797 static int
ftrace_trace_snapshot_callback(struct trace_array * tr,struct ftrace_hash * hash,char * glob,char * cmd,char * param,int enable)8798 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
8799 			       char *glob, char *cmd, char *param, int enable)
8800 {
8801 	struct ftrace_probe_ops *ops;
8802 	void *count = (void *)-1;
8803 	char *number;
8804 	int ret;
8805 
8806 	if (!tr)
8807 		return -ENODEV;
8808 
8809 	/* hash funcs only work with set_ftrace_filter */
8810 	if (!enable)
8811 		return -EINVAL;
8812 
8813 	ops = param ? &snapshot_count_probe_ops :  &snapshot_probe_ops;
8814 
8815 	if (glob[0] == '!')
8816 		return unregister_ftrace_function_probe_func(glob+1, tr, ops);
8817 
8818 	if (!param)
8819 		goto out_reg;
8820 
8821 	number = strsep(&param, ":");
8822 
8823 	if (!strlen(number))
8824 		goto out_reg;
8825 
8826 	/*
8827 	 * We use the callback data field (which is a pointer)
8828 	 * as our counter.
8829 	 */
8830 	ret = kstrtoul(number, 0, (unsigned long *)&count);
8831 	if (ret)
8832 		return ret;
8833 
8834  out_reg:
8835 	ret = tracing_alloc_snapshot_instance(tr);
8836 	if (ret < 0)
8837 		goto out;
8838 
8839 	ret = register_ftrace_function_probe(glob, tr, ops, count);
8840 
8841  out:
8842 	return ret < 0 ? ret : 0;
8843 }
8844 
8845 static struct ftrace_func_command ftrace_snapshot_cmd = {
8846 	.name			= "snapshot",
8847 	.func			= ftrace_trace_snapshot_callback,
8848 };
8849 
register_snapshot_cmd(void)8850 static __init int register_snapshot_cmd(void)
8851 {
8852 	return register_ftrace_command(&ftrace_snapshot_cmd);
8853 }
8854 #else
register_snapshot_cmd(void)8855 static inline __init int register_snapshot_cmd(void) { return 0; }
8856 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
8857 
tracing_get_dentry(struct trace_array * tr)8858 static struct dentry *tracing_get_dentry(struct trace_array *tr)
8859 {
8860 	if (WARN_ON(!tr->dir))
8861 		return ERR_PTR(-ENODEV);
8862 
8863 	/* Top directory uses NULL as the parent */
8864 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
8865 		return NULL;
8866 
8867 	/* All sub buffers have a descriptor */
8868 	return tr->dir;
8869 }
8870 
tracing_dentry_percpu(struct trace_array * tr,int cpu)8871 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
8872 {
8873 	struct dentry *d_tracer;
8874 
8875 	if (tr->percpu_dir)
8876 		return tr->percpu_dir;
8877 
8878 	d_tracer = tracing_get_dentry(tr);
8879 	if (IS_ERR(d_tracer))
8880 		return NULL;
8881 
8882 	tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
8883 
8884 	MEM_FAIL(!tr->percpu_dir,
8885 		  "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
8886 
8887 	return tr->percpu_dir;
8888 }
8889 
8890 static struct dentry *
trace_create_cpu_file(const char * name,umode_t mode,struct dentry * parent,void * data,long cpu,const struct file_operations * fops)8891 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
8892 		      void *data, long cpu, const struct file_operations *fops)
8893 {
8894 	struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
8895 
8896 	if (ret) /* See tracing_get_cpu() */
8897 		d_inode(ret)->i_cdev = (void *)(cpu + 1);
8898 	return ret;
8899 }
8900 
8901 static void
tracing_init_tracefs_percpu(struct trace_array * tr,long cpu)8902 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
8903 {
8904 	struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
8905 	struct dentry *d_cpu;
8906 	char cpu_dir[30]; /* 30 characters should be more than enough */
8907 
8908 	if (!d_percpu)
8909 		return;
8910 
8911 	snprintf(cpu_dir, 30, "cpu%ld", cpu);
8912 	d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
8913 	if (!d_cpu) {
8914 		pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
8915 		return;
8916 	}
8917 
8918 	/* per cpu trace_pipe */
8919 	trace_create_cpu_file("trace_pipe", TRACE_MODE_READ, d_cpu,
8920 				tr, cpu, &tracing_pipe_fops);
8921 
8922 	/* per cpu trace */
8923 	trace_create_cpu_file("trace", TRACE_MODE_WRITE, d_cpu,
8924 				tr, cpu, &tracing_fops);
8925 
8926 	trace_create_cpu_file("trace_pipe_raw", TRACE_MODE_READ, d_cpu,
8927 				tr, cpu, &tracing_buffers_fops);
8928 
8929 	trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu,
8930 				tr, cpu, &tracing_stats_fops);
8931 
8932 	trace_create_cpu_file("buffer_size_kb", TRACE_MODE_READ, d_cpu,
8933 				tr, cpu, &tracing_entries_fops);
8934 
8935 #ifdef CONFIG_TRACER_SNAPSHOT
8936 	trace_create_cpu_file("snapshot", TRACE_MODE_WRITE, d_cpu,
8937 				tr, cpu, &snapshot_fops);
8938 
8939 	trace_create_cpu_file("snapshot_raw", TRACE_MODE_READ, d_cpu,
8940 				tr, cpu, &snapshot_raw_fops);
8941 #endif
8942 }
8943 
8944 #ifdef CONFIG_FTRACE_SELFTEST
8945 /* Let selftest have access to static functions in this file */
8946 #include "trace_selftest.c"
8947 #endif
8948 
8949 static ssize_t
trace_options_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)8950 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
8951 			loff_t *ppos)
8952 {
8953 	struct trace_option_dentry *topt = filp->private_data;
8954 	char *buf;
8955 
8956 	if (topt->flags->val & topt->opt->bit)
8957 		buf = "1\n";
8958 	else
8959 		buf = "0\n";
8960 
8961 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8962 }
8963 
8964 static ssize_t
trace_options_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)8965 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
8966 			 loff_t *ppos)
8967 {
8968 	struct trace_option_dentry *topt = filp->private_data;
8969 	unsigned long val;
8970 	int ret;
8971 
8972 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8973 	if (ret)
8974 		return ret;
8975 
8976 	if (val != 0 && val != 1)
8977 		return -EINVAL;
8978 
8979 	if (!!(topt->flags->val & topt->opt->bit) != val) {
8980 		mutex_lock(&trace_types_lock);
8981 		ret = __set_tracer_option(topt->tr, topt->flags,
8982 					  topt->opt, !val);
8983 		mutex_unlock(&trace_types_lock);
8984 		if (ret)
8985 			return ret;
8986 	}
8987 
8988 	*ppos += cnt;
8989 
8990 	return cnt;
8991 }
8992 
tracing_open_options(struct inode * inode,struct file * filp)8993 static int tracing_open_options(struct inode *inode, struct file *filp)
8994 {
8995 	struct trace_option_dentry *topt = inode->i_private;
8996 	int ret;
8997 
8998 	ret = tracing_check_open_get_tr(topt->tr);
8999 	if (ret)
9000 		return ret;
9001 
9002 	filp->private_data = inode->i_private;
9003 	return 0;
9004 }
9005 
tracing_release_options(struct inode * inode,struct file * file)9006 static int tracing_release_options(struct inode *inode, struct file *file)
9007 {
9008 	struct trace_option_dentry *topt = file->private_data;
9009 
9010 	trace_array_put(topt->tr);
9011 	return 0;
9012 }
9013 
9014 static const struct file_operations trace_options_fops = {
9015 	.open = tracing_open_options,
9016 	.read = trace_options_read,
9017 	.write = trace_options_write,
9018 	.llseek	= generic_file_llseek,
9019 	.release = tracing_release_options,
9020 };
9021 
9022 /*
9023  * In order to pass in both the trace_array descriptor as well as the index
9024  * to the flag that the trace option file represents, the trace_array
9025  * has a character array of trace_flags_index[], which holds the index
9026  * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
9027  * The address of this character array is passed to the flag option file
9028  * read/write callbacks.
9029  *
9030  * In order to extract both the index and the trace_array descriptor,
9031  * get_tr_index() uses the following algorithm.
9032  *
9033  *   idx = *ptr;
9034  *
9035  * As the pointer itself contains the address of the index (remember
9036  * index[1] == 1).
9037  *
9038  * Then to get the trace_array descriptor, by subtracting that index
9039  * from the ptr, we get to the start of the index itself.
9040  *
9041  *   ptr - idx == &index[0]
9042  *
9043  * Then a simple container_of() from that pointer gets us to the
9044  * trace_array descriptor.
9045  */
get_tr_index(void * data,struct trace_array ** ptr,unsigned int * pindex)9046 static void get_tr_index(void *data, struct trace_array **ptr,
9047 			 unsigned int *pindex)
9048 {
9049 	*pindex = *(unsigned char *)data;
9050 
9051 	*ptr = container_of(data - *pindex, struct trace_array,
9052 			    trace_flags_index);
9053 }
9054 
9055 static ssize_t
trace_options_core_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)9056 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
9057 			loff_t *ppos)
9058 {
9059 	void *tr_index = filp->private_data;
9060 	struct trace_array *tr;
9061 	unsigned int index;
9062 	char *buf;
9063 
9064 	get_tr_index(tr_index, &tr, &index);
9065 
9066 	if (tr->trace_flags & (1 << index))
9067 		buf = "1\n";
9068 	else
9069 		buf = "0\n";
9070 
9071 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
9072 }
9073 
9074 static ssize_t
trace_options_core_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)9075 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
9076 			 loff_t *ppos)
9077 {
9078 	void *tr_index = filp->private_data;
9079 	struct trace_array *tr;
9080 	unsigned int index;
9081 	unsigned long val;
9082 	int ret;
9083 
9084 	get_tr_index(tr_index, &tr, &index);
9085 
9086 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
9087 	if (ret)
9088 		return ret;
9089 
9090 	if (val != 0 && val != 1)
9091 		return -EINVAL;
9092 
9093 	mutex_lock(&event_mutex);
9094 	mutex_lock(&trace_types_lock);
9095 	ret = set_tracer_flag(tr, 1 << index, val);
9096 	mutex_unlock(&trace_types_lock);
9097 	mutex_unlock(&event_mutex);
9098 
9099 	if (ret < 0)
9100 		return ret;
9101 
9102 	*ppos += cnt;
9103 
9104 	return cnt;
9105 }
9106 
9107 static const struct file_operations trace_options_core_fops = {
9108 	.open = tracing_open_generic,
9109 	.read = trace_options_core_read,
9110 	.write = trace_options_core_write,
9111 	.llseek = generic_file_llseek,
9112 };
9113 
trace_create_file(const char * name,umode_t mode,struct dentry * parent,void * data,const struct file_operations * fops)9114 struct dentry *trace_create_file(const char *name,
9115 				 umode_t mode,
9116 				 struct dentry *parent,
9117 				 void *data,
9118 				 const struct file_operations *fops)
9119 {
9120 	struct dentry *ret;
9121 
9122 	ret = tracefs_create_file(name, mode, parent, data, fops);
9123 	if (!ret)
9124 		pr_warn("Could not create tracefs '%s' entry\n", name);
9125 
9126 	return ret;
9127 }
9128 
9129 
trace_options_init_dentry(struct trace_array * tr)9130 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
9131 {
9132 	struct dentry *d_tracer;
9133 
9134 	if (tr->options)
9135 		return tr->options;
9136 
9137 	d_tracer = tracing_get_dentry(tr);
9138 	if (IS_ERR(d_tracer))
9139 		return NULL;
9140 
9141 	tr->options = tracefs_create_dir("options", d_tracer);
9142 	if (!tr->options) {
9143 		pr_warn("Could not create tracefs directory 'options'\n");
9144 		return NULL;
9145 	}
9146 
9147 	return tr->options;
9148 }
9149 
9150 static void
create_trace_option_file(struct trace_array * tr,struct trace_option_dentry * topt,struct tracer_flags * flags,struct tracer_opt * opt)9151 create_trace_option_file(struct trace_array *tr,
9152 			 struct trace_option_dentry *topt,
9153 			 struct tracer_flags *flags,
9154 			 struct tracer_opt *opt)
9155 {
9156 	struct dentry *t_options;
9157 
9158 	t_options = trace_options_init_dentry(tr);
9159 	if (!t_options)
9160 		return;
9161 
9162 	topt->flags = flags;
9163 	topt->opt = opt;
9164 	topt->tr = tr;
9165 
9166 	topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE,
9167 					t_options, topt, &trace_options_fops);
9168 
9169 }
9170 
9171 static void
create_trace_option_files(struct trace_array * tr,struct tracer * tracer)9172 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
9173 {
9174 	struct trace_option_dentry *topts;
9175 	struct trace_options *tr_topts;
9176 	struct tracer_flags *flags;
9177 	struct tracer_opt *opts;
9178 	int cnt;
9179 	int i;
9180 
9181 	if (!tracer)
9182 		return;
9183 
9184 	flags = tracer->flags;
9185 
9186 	if (!flags || !flags->opts)
9187 		return;
9188 
9189 	/*
9190 	 * If this is an instance, only create flags for tracers
9191 	 * the instance may have.
9192 	 */
9193 	if (!trace_ok_for_array(tracer, tr))
9194 		return;
9195 
9196 	for (i = 0; i < tr->nr_topts; i++) {
9197 		/* Make sure there's no duplicate flags. */
9198 		if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
9199 			return;
9200 	}
9201 
9202 	opts = flags->opts;
9203 
9204 	for (cnt = 0; opts[cnt].name; cnt++)
9205 		;
9206 
9207 	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
9208 	if (!topts)
9209 		return;
9210 
9211 	tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
9212 			    GFP_KERNEL);
9213 	if (!tr_topts) {
9214 		kfree(topts);
9215 		return;
9216 	}
9217 
9218 	tr->topts = tr_topts;
9219 	tr->topts[tr->nr_topts].tracer = tracer;
9220 	tr->topts[tr->nr_topts].topts = topts;
9221 	tr->nr_topts++;
9222 
9223 	for (cnt = 0; opts[cnt].name; cnt++) {
9224 		create_trace_option_file(tr, &topts[cnt], flags,
9225 					 &opts[cnt]);
9226 		MEM_FAIL(topts[cnt].entry == NULL,
9227 			  "Failed to create trace option: %s",
9228 			  opts[cnt].name);
9229 	}
9230 }
9231 
9232 static struct dentry *
create_trace_option_core_file(struct trace_array * tr,const char * option,long index)9233 create_trace_option_core_file(struct trace_array *tr,
9234 			      const char *option, long index)
9235 {
9236 	struct dentry *t_options;
9237 
9238 	t_options = trace_options_init_dentry(tr);
9239 	if (!t_options)
9240 		return NULL;
9241 
9242 	return trace_create_file(option, TRACE_MODE_WRITE, t_options,
9243 				 (void *)&tr->trace_flags_index[index],
9244 				 &trace_options_core_fops);
9245 }
9246 
create_trace_options_dir(struct trace_array * tr)9247 static void create_trace_options_dir(struct trace_array *tr)
9248 {
9249 	struct dentry *t_options;
9250 	bool top_level = tr == &global_trace;
9251 	int i;
9252 
9253 	t_options = trace_options_init_dentry(tr);
9254 	if (!t_options)
9255 		return;
9256 
9257 	for (i = 0; trace_options[i]; i++) {
9258 		if (top_level ||
9259 		    !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
9260 			create_trace_option_core_file(tr, trace_options[i], i);
9261 	}
9262 }
9263 
9264 static ssize_t
rb_simple_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)9265 rb_simple_read(struct file *filp, char __user *ubuf,
9266 	       size_t cnt, loff_t *ppos)
9267 {
9268 	struct trace_array *tr = filp->private_data;
9269 	char buf[64];
9270 	int r;
9271 
9272 	r = tracer_tracing_is_on(tr);
9273 	r = sprintf(buf, "%d\n", r);
9274 
9275 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
9276 }
9277 
9278 static ssize_t
rb_simple_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)9279 rb_simple_write(struct file *filp, const char __user *ubuf,
9280 		size_t cnt, loff_t *ppos)
9281 {
9282 	struct trace_array *tr = filp->private_data;
9283 	struct trace_buffer *buffer = tr->array_buffer.buffer;
9284 	unsigned long val;
9285 	int ret;
9286 
9287 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
9288 	if (ret)
9289 		return ret;
9290 
9291 	if (buffer) {
9292 		mutex_lock(&trace_types_lock);
9293 		if (!!val == tracer_tracing_is_on(tr)) {
9294 			val = 0; /* do nothing */
9295 		} else if (val) {
9296 			tracer_tracing_on(tr);
9297 			if (tr->current_trace->start)
9298 				tr->current_trace->start(tr);
9299 		} else {
9300 			tracer_tracing_off(tr);
9301 			if (tr->current_trace->stop)
9302 				tr->current_trace->stop(tr);
9303 			/* Wake up any waiters */
9304 			ring_buffer_wake_waiters(buffer, RING_BUFFER_ALL_CPUS);
9305 		}
9306 		mutex_unlock(&trace_types_lock);
9307 	}
9308 
9309 	(*ppos)++;
9310 
9311 	return cnt;
9312 }
9313 
9314 static const struct file_operations rb_simple_fops = {
9315 	.open		= tracing_open_generic_tr,
9316 	.read		= rb_simple_read,
9317 	.write		= rb_simple_write,
9318 	.release	= tracing_release_generic_tr,
9319 	.llseek		= default_llseek,
9320 };
9321 
9322 static ssize_t
buffer_percent_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)9323 buffer_percent_read(struct file *filp, char __user *ubuf,
9324 		    size_t cnt, loff_t *ppos)
9325 {
9326 	struct trace_array *tr = filp->private_data;
9327 	char buf[64];
9328 	int r;
9329 
9330 	r = tr->buffer_percent;
9331 	r = sprintf(buf, "%d\n", r);
9332 
9333 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
9334 }
9335 
9336 static ssize_t
buffer_percent_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)9337 buffer_percent_write(struct file *filp, const char __user *ubuf,
9338 		     size_t cnt, loff_t *ppos)
9339 {
9340 	struct trace_array *tr = filp->private_data;
9341 	unsigned long val;
9342 	int ret;
9343 
9344 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
9345 	if (ret)
9346 		return ret;
9347 
9348 	if (val > 100)
9349 		return -EINVAL;
9350 
9351 	tr->buffer_percent = val;
9352 
9353 	(*ppos)++;
9354 
9355 	return cnt;
9356 }
9357 
9358 static const struct file_operations buffer_percent_fops = {
9359 	.open		= tracing_open_generic_tr,
9360 	.read		= buffer_percent_read,
9361 	.write		= buffer_percent_write,
9362 	.release	= tracing_release_generic_tr,
9363 	.llseek		= default_llseek,
9364 };
9365 
9366 static struct dentry *trace_instance_dir;
9367 
9368 static void
9369 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
9370 
9371 static int
allocate_trace_buffer(struct trace_array * tr,struct array_buffer * buf,int size)9372 allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size)
9373 {
9374 	enum ring_buffer_flags rb_flags;
9375 
9376 	rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
9377 
9378 	buf->tr = tr;
9379 
9380 	buf->buffer = ring_buffer_alloc(size, rb_flags);
9381 	if (!buf->buffer)
9382 		return -ENOMEM;
9383 
9384 	buf->data = alloc_percpu(struct trace_array_cpu);
9385 	if (!buf->data) {
9386 		ring_buffer_free(buf->buffer);
9387 		buf->buffer = NULL;
9388 		return -ENOMEM;
9389 	}
9390 
9391 	/* Allocate the first page for all buffers */
9392 	set_buffer_entries(&tr->array_buffer,
9393 			   ring_buffer_size(tr->array_buffer.buffer, 0));
9394 
9395 	return 0;
9396 }
9397 
free_trace_buffer(struct array_buffer * buf)9398 static void free_trace_buffer(struct array_buffer *buf)
9399 {
9400 	if (buf->buffer) {
9401 		ring_buffer_free(buf->buffer);
9402 		buf->buffer = NULL;
9403 		free_percpu(buf->data);
9404 		buf->data = NULL;
9405 	}
9406 }
9407 
allocate_trace_buffers(struct trace_array * tr,int size)9408 static int allocate_trace_buffers(struct trace_array *tr, int size)
9409 {
9410 	int ret;
9411 
9412 	ret = allocate_trace_buffer(tr, &tr->array_buffer, size);
9413 	if (ret)
9414 		return ret;
9415 
9416 #ifdef CONFIG_TRACER_MAX_TRACE
9417 	ret = allocate_trace_buffer(tr, &tr->max_buffer,
9418 				    allocate_snapshot ? size : 1);
9419 	if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) {
9420 		free_trace_buffer(&tr->array_buffer);
9421 		return -ENOMEM;
9422 	}
9423 	tr->allocated_snapshot = allocate_snapshot;
9424 
9425 	allocate_snapshot = false;
9426 #endif
9427 
9428 	return 0;
9429 }
9430 
free_trace_buffers(struct trace_array * tr)9431 static void free_trace_buffers(struct trace_array *tr)
9432 {
9433 	if (!tr)
9434 		return;
9435 
9436 	free_trace_buffer(&tr->array_buffer);
9437 
9438 #ifdef CONFIG_TRACER_MAX_TRACE
9439 	free_trace_buffer(&tr->max_buffer);
9440 #endif
9441 }
9442 
init_trace_flags_index(struct trace_array * tr)9443 static void init_trace_flags_index(struct trace_array *tr)
9444 {
9445 	int i;
9446 
9447 	/* Used by the trace options files */
9448 	for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
9449 		tr->trace_flags_index[i] = i;
9450 }
9451 
__update_tracer_options(struct trace_array * tr)9452 static void __update_tracer_options(struct trace_array *tr)
9453 {
9454 	struct tracer *t;
9455 
9456 	for (t = trace_types; t; t = t->next)
9457 		add_tracer_options(tr, t);
9458 }
9459 
update_tracer_options(struct trace_array * tr)9460 static void update_tracer_options(struct trace_array *tr)
9461 {
9462 	mutex_lock(&trace_types_lock);
9463 	tracer_options_updated = true;
9464 	__update_tracer_options(tr);
9465 	mutex_unlock(&trace_types_lock);
9466 }
9467 
9468 /* Must have trace_types_lock held */
trace_array_find(const char * instance)9469 struct trace_array *trace_array_find(const char *instance)
9470 {
9471 	struct trace_array *tr, *found = NULL;
9472 
9473 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9474 		if (tr->name && strcmp(tr->name, instance) == 0) {
9475 			found = tr;
9476 			break;
9477 		}
9478 	}
9479 
9480 	return found;
9481 }
9482 
trace_array_find_get(const char * instance)9483 struct trace_array *trace_array_find_get(const char *instance)
9484 {
9485 	struct trace_array *tr;
9486 
9487 	mutex_lock(&trace_types_lock);
9488 	tr = trace_array_find(instance);
9489 	if (tr)
9490 		tr->ref++;
9491 	mutex_unlock(&trace_types_lock);
9492 
9493 	return tr;
9494 }
9495 
trace_array_create_dir(struct trace_array * tr)9496 static int trace_array_create_dir(struct trace_array *tr)
9497 {
9498 	int ret;
9499 
9500 	tr->dir = tracefs_create_dir(tr->name, trace_instance_dir);
9501 	if (!tr->dir)
9502 		return -EINVAL;
9503 
9504 	ret = event_trace_add_tracer(tr->dir, tr);
9505 	if (ret) {
9506 		tracefs_remove(tr->dir);
9507 		return ret;
9508 	}
9509 
9510 	init_tracer_tracefs(tr, tr->dir);
9511 	__update_tracer_options(tr);
9512 
9513 	return ret;
9514 }
9515 
trace_array_create(const char * name)9516 static struct trace_array *trace_array_create(const char *name)
9517 {
9518 	struct trace_array *tr;
9519 	int ret;
9520 
9521 	ret = -ENOMEM;
9522 	tr = kzalloc(sizeof(*tr), GFP_KERNEL);
9523 	if (!tr)
9524 		return ERR_PTR(ret);
9525 
9526 	tr->name = kstrdup(name, GFP_KERNEL);
9527 	if (!tr->name)
9528 		goto out_free_tr;
9529 
9530 	if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
9531 		goto out_free_tr;
9532 
9533 	if (!zalloc_cpumask_var(&tr->pipe_cpumask, GFP_KERNEL))
9534 		goto out_free_tr;
9535 
9536 	tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
9537 
9538 	cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
9539 
9540 	raw_spin_lock_init(&tr->start_lock);
9541 
9542 	tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9543 
9544 	tr->current_trace = &nop_trace;
9545 
9546 	INIT_LIST_HEAD(&tr->systems);
9547 	INIT_LIST_HEAD(&tr->events);
9548 	INIT_LIST_HEAD(&tr->hist_vars);
9549 	INIT_LIST_HEAD(&tr->err_log);
9550 
9551 	if (allocate_trace_buffers(tr, trace_buf_size) < 0)
9552 		goto out_free_tr;
9553 
9554 	if (ftrace_allocate_ftrace_ops(tr) < 0)
9555 		goto out_free_tr;
9556 
9557 	ftrace_init_trace_array(tr);
9558 
9559 	init_trace_flags_index(tr);
9560 
9561 	if (trace_instance_dir) {
9562 		ret = trace_array_create_dir(tr);
9563 		if (ret)
9564 			goto out_free_tr;
9565 	} else
9566 		__trace_early_add_events(tr);
9567 
9568 	list_add(&tr->list, &ftrace_trace_arrays);
9569 
9570 	tr->ref++;
9571 
9572 	return tr;
9573 
9574  out_free_tr:
9575 	ftrace_free_ftrace_ops(tr);
9576 	free_trace_buffers(tr);
9577 	free_cpumask_var(tr->pipe_cpumask);
9578 	free_cpumask_var(tr->tracing_cpumask);
9579 	kfree(tr->name);
9580 	kfree(tr);
9581 
9582 	return ERR_PTR(ret);
9583 }
9584 
instance_mkdir(const char * name)9585 static int instance_mkdir(const char *name)
9586 {
9587 	struct trace_array *tr;
9588 	int ret;
9589 
9590 	mutex_lock(&event_mutex);
9591 	mutex_lock(&trace_types_lock);
9592 
9593 	ret = -EEXIST;
9594 	if (trace_array_find(name))
9595 		goto out_unlock;
9596 
9597 	tr = trace_array_create(name);
9598 
9599 	ret = PTR_ERR_OR_ZERO(tr);
9600 
9601 out_unlock:
9602 	mutex_unlock(&trace_types_lock);
9603 	mutex_unlock(&event_mutex);
9604 	return ret;
9605 }
9606 
9607 /**
9608  * trace_array_get_by_name - Create/Lookup a trace array, given its name.
9609  * @name: The name of the trace array to be looked up/created.
9610  *
9611  * Returns pointer to trace array with given name.
9612  * NULL, if it cannot be created.
9613  *
9614  * NOTE: This function increments the reference counter associated with the
9615  * trace array returned. This makes sure it cannot be freed while in use.
9616  * Use trace_array_put() once the trace array is no longer needed.
9617  * If the trace_array is to be freed, trace_array_destroy() needs to
9618  * be called after the trace_array_put(), or simply let user space delete
9619  * it from the tracefs instances directory. But until the
9620  * trace_array_put() is called, user space can not delete it.
9621  *
9622  */
trace_array_get_by_name(const char * name)9623 struct trace_array *trace_array_get_by_name(const char *name)
9624 {
9625 	struct trace_array *tr;
9626 
9627 	mutex_lock(&event_mutex);
9628 	mutex_lock(&trace_types_lock);
9629 
9630 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9631 		if (tr->name && strcmp(tr->name, name) == 0)
9632 			goto out_unlock;
9633 	}
9634 
9635 	tr = trace_array_create(name);
9636 
9637 	if (IS_ERR(tr))
9638 		tr = NULL;
9639 out_unlock:
9640 	if (tr)
9641 		tr->ref++;
9642 
9643 	mutex_unlock(&trace_types_lock);
9644 	mutex_unlock(&event_mutex);
9645 	return tr;
9646 }
9647 EXPORT_SYMBOL_GPL(trace_array_get_by_name);
9648 
__remove_instance(struct trace_array * tr)9649 static int __remove_instance(struct trace_array *tr)
9650 {
9651 	int i;
9652 
9653 	/* Reference counter for a newly created trace array = 1. */
9654 	if (tr->ref > 1 || (tr->current_trace && tr->trace_ref))
9655 		return -EBUSY;
9656 
9657 	list_del(&tr->list);
9658 
9659 	/* Disable all the flags that were enabled coming in */
9660 	for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
9661 		if ((1 << i) & ZEROED_TRACE_FLAGS)
9662 			set_tracer_flag(tr, 1 << i, 0);
9663 	}
9664 
9665 	tracing_set_nop(tr);
9666 	clear_ftrace_function_probes(tr);
9667 	event_trace_del_tracer(tr);
9668 	ftrace_clear_pids(tr);
9669 	ftrace_destroy_function_files(tr);
9670 	tracefs_remove(tr->dir);
9671 	free_percpu(tr->last_func_repeats);
9672 	free_trace_buffers(tr);
9673 	clear_tracing_err_log(tr);
9674 
9675 	for (i = 0; i < tr->nr_topts; i++) {
9676 		kfree(tr->topts[i].topts);
9677 	}
9678 	kfree(tr->topts);
9679 
9680 	free_cpumask_var(tr->pipe_cpumask);
9681 	free_cpumask_var(tr->tracing_cpumask);
9682 	kfree(tr->name);
9683 	kfree(tr);
9684 
9685 	return 0;
9686 }
9687 
trace_array_destroy(struct trace_array * this_tr)9688 int trace_array_destroy(struct trace_array *this_tr)
9689 {
9690 	struct trace_array *tr;
9691 	int ret;
9692 
9693 	if (!this_tr)
9694 		return -EINVAL;
9695 
9696 	mutex_lock(&event_mutex);
9697 	mutex_lock(&trace_types_lock);
9698 
9699 	ret = -ENODEV;
9700 
9701 	/* Making sure trace array exists before destroying it. */
9702 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9703 		if (tr == this_tr) {
9704 			ret = __remove_instance(tr);
9705 			break;
9706 		}
9707 	}
9708 
9709 	mutex_unlock(&trace_types_lock);
9710 	mutex_unlock(&event_mutex);
9711 
9712 	return ret;
9713 }
9714 EXPORT_SYMBOL_GPL(trace_array_destroy);
9715 
instance_rmdir(const char * name)9716 static int instance_rmdir(const char *name)
9717 {
9718 	struct trace_array *tr;
9719 	int ret;
9720 
9721 	mutex_lock(&event_mutex);
9722 	mutex_lock(&trace_types_lock);
9723 
9724 	ret = -ENODEV;
9725 	tr = trace_array_find(name);
9726 	if (tr)
9727 		ret = __remove_instance(tr);
9728 
9729 	mutex_unlock(&trace_types_lock);
9730 	mutex_unlock(&event_mutex);
9731 
9732 	return ret;
9733 }
9734 
create_trace_instances(struct dentry * d_tracer)9735 static __init void create_trace_instances(struct dentry *d_tracer)
9736 {
9737 	struct trace_array *tr;
9738 
9739 	trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
9740 							 instance_mkdir,
9741 							 instance_rmdir);
9742 	if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
9743 		return;
9744 
9745 	mutex_lock(&event_mutex);
9746 	mutex_lock(&trace_types_lock);
9747 
9748 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9749 		if (!tr->name)
9750 			continue;
9751 		if (MEM_FAIL(trace_array_create_dir(tr) < 0,
9752 			     "Failed to create instance directory\n"))
9753 			break;
9754 	}
9755 
9756 	mutex_unlock(&trace_types_lock);
9757 	mutex_unlock(&event_mutex);
9758 }
9759 
9760 static void
init_tracer_tracefs(struct trace_array * tr,struct dentry * d_tracer)9761 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
9762 {
9763 	int cpu;
9764 
9765 	trace_create_file("available_tracers", TRACE_MODE_READ, d_tracer,
9766 			tr, &show_traces_fops);
9767 
9768 	trace_create_file("current_tracer", TRACE_MODE_WRITE, d_tracer,
9769 			tr, &set_tracer_fops);
9770 
9771 	trace_create_file("tracing_cpumask", TRACE_MODE_WRITE, d_tracer,
9772 			  tr, &tracing_cpumask_fops);
9773 
9774 	trace_create_file("trace_options", TRACE_MODE_WRITE, d_tracer,
9775 			  tr, &tracing_iter_fops);
9776 
9777 	trace_create_file("trace", TRACE_MODE_WRITE, d_tracer,
9778 			  tr, &tracing_fops);
9779 
9780 	trace_create_file("trace_pipe", TRACE_MODE_READ, d_tracer,
9781 			  tr, &tracing_pipe_fops);
9782 
9783 	trace_create_file("buffer_size_kb", TRACE_MODE_WRITE, d_tracer,
9784 			  tr, &tracing_entries_fops);
9785 
9786 	trace_create_file("buffer_total_size_kb", TRACE_MODE_READ, d_tracer,
9787 			  tr, &tracing_total_entries_fops);
9788 
9789 	trace_create_file("free_buffer", 0200, d_tracer,
9790 			  tr, &tracing_free_buffer_fops);
9791 
9792 	trace_create_file("trace_marker", 0220, d_tracer,
9793 			  tr, &tracing_mark_fops);
9794 
9795 	tr->trace_marker_file = __find_event_file(tr, "ftrace", "print");
9796 
9797 	trace_create_file("trace_marker_raw", 0220, d_tracer,
9798 			  tr, &tracing_mark_raw_fops);
9799 
9800 	trace_create_file("trace_clock", TRACE_MODE_WRITE, d_tracer, tr,
9801 			  &trace_clock_fops);
9802 
9803 	trace_create_file("tracing_on", TRACE_MODE_WRITE, d_tracer,
9804 			  tr, &rb_simple_fops);
9805 
9806 	trace_create_file("timestamp_mode", TRACE_MODE_READ, d_tracer, tr,
9807 			  &trace_time_stamp_mode_fops);
9808 
9809 	tr->buffer_percent = 50;
9810 
9811 	trace_create_file("buffer_percent", TRACE_MODE_WRITE, d_tracer,
9812 			tr, &buffer_percent_fops);
9813 
9814 	create_trace_options_dir(tr);
9815 
9816 #ifdef CONFIG_TRACER_MAX_TRACE
9817 	trace_create_maxlat_file(tr, d_tracer);
9818 #endif
9819 
9820 	if (ftrace_create_function_files(tr, d_tracer))
9821 		MEM_FAIL(1, "Could not allocate function filter files");
9822 
9823 #ifdef CONFIG_TRACER_SNAPSHOT
9824 	trace_create_file("snapshot", TRACE_MODE_WRITE, d_tracer,
9825 			  tr, &snapshot_fops);
9826 #endif
9827 
9828 	trace_create_file("error_log", TRACE_MODE_WRITE, d_tracer,
9829 			  tr, &tracing_err_log_fops);
9830 
9831 	for_each_tracing_cpu(cpu)
9832 		tracing_init_tracefs_percpu(tr, cpu);
9833 
9834 	ftrace_init_tracefs(tr, d_tracer);
9835 }
9836 
trace_automount(struct dentry * mntpt,void * ingore)9837 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
9838 {
9839 	struct vfsmount *mnt;
9840 	struct file_system_type *type;
9841 
9842 	/*
9843 	 * To maintain backward compatibility for tools that mount
9844 	 * debugfs to get to the tracing facility, tracefs is automatically
9845 	 * mounted to the debugfs/tracing directory.
9846 	 */
9847 	type = get_fs_type("tracefs");
9848 	if (!type)
9849 		return NULL;
9850 	mnt = vfs_submount(mntpt, type, "tracefs", NULL);
9851 	put_filesystem(type);
9852 	if (IS_ERR(mnt))
9853 		return NULL;
9854 	mntget(mnt);
9855 
9856 	return mnt;
9857 }
9858 
9859 /**
9860  * tracing_init_dentry - initialize top level trace array
9861  *
9862  * This is called when creating files or directories in the tracing
9863  * directory. It is called via fs_initcall() by any of the boot up code
9864  * and expects to return the dentry of the top level tracing directory.
9865  */
tracing_init_dentry(void)9866 int tracing_init_dentry(void)
9867 {
9868 	struct trace_array *tr = &global_trace;
9869 
9870 	if (security_locked_down(LOCKDOWN_TRACEFS)) {
9871 		pr_warn("Tracing disabled due to lockdown\n");
9872 		return -EPERM;
9873 	}
9874 
9875 	/* The top level trace array uses  NULL as parent */
9876 	if (tr->dir)
9877 		return 0;
9878 
9879 	if (WARN_ON(!tracefs_initialized()))
9880 		return -ENODEV;
9881 
9882 	/*
9883 	 * As there may still be users that expect the tracing
9884 	 * files to exist in debugfs/tracing, we must automount
9885 	 * the tracefs file system there, so older tools still
9886 	 * work with the newer kernel.
9887 	 */
9888 	tr->dir = debugfs_create_automount("tracing", NULL,
9889 					   trace_automount, NULL);
9890 
9891 	return 0;
9892 }
9893 
9894 extern struct trace_eval_map *__start_ftrace_eval_maps[];
9895 extern struct trace_eval_map *__stop_ftrace_eval_maps[];
9896 
9897 static struct workqueue_struct *eval_map_wq __initdata;
9898 static struct work_struct eval_map_work __initdata;
9899 static struct work_struct tracerfs_init_work __initdata;
9900 
eval_map_work_func(struct work_struct * work)9901 static void __init eval_map_work_func(struct work_struct *work)
9902 {
9903 	int len;
9904 
9905 	len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
9906 	trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
9907 }
9908 
trace_eval_init(void)9909 static int __init trace_eval_init(void)
9910 {
9911 	INIT_WORK(&eval_map_work, eval_map_work_func);
9912 
9913 	eval_map_wq = alloc_workqueue("eval_map_wq", WQ_UNBOUND, 0);
9914 	if (!eval_map_wq) {
9915 		pr_err("Unable to allocate eval_map_wq\n");
9916 		/* Do work here */
9917 		eval_map_work_func(&eval_map_work);
9918 		return -ENOMEM;
9919 	}
9920 
9921 	queue_work(eval_map_wq, &eval_map_work);
9922 	return 0;
9923 }
9924 
9925 subsys_initcall(trace_eval_init);
9926 
trace_eval_sync(void)9927 static int __init trace_eval_sync(void)
9928 {
9929 	/* Make sure the eval map updates are finished */
9930 	if (eval_map_wq)
9931 		destroy_workqueue(eval_map_wq);
9932 	return 0;
9933 }
9934 
9935 late_initcall_sync(trace_eval_sync);
9936 
9937 
9938 #ifdef CONFIG_MODULES
trace_module_add_evals(struct module * mod)9939 static void trace_module_add_evals(struct module *mod)
9940 {
9941 	if (!mod->num_trace_evals)
9942 		return;
9943 
9944 	/*
9945 	 * Modules with bad taint do not have events created, do
9946 	 * not bother with enums either.
9947 	 */
9948 	if (trace_module_has_bad_taint(mod))
9949 		return;
9950 
9951 	trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
9952 }
9953 
9954 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
trace_module_remove_evals(struct module * mod)9955 static void trace_module_remove_evals(struct module *mod)
9956 {
9957 	union trace_eval_map_item *map;
9958 	union trace_eval_map_item **last = &trace_eval_maps;
9959 
9960 	if (!mod->num_trace_evals)
9961 		return;
9962 
9963 	mutex_lock(&trace_eval_mutex);
9964 
9965 	map = trace_eval_maps;
9966 
9967 	while (map) {
9968 		if (map->head.mod == mod)
9969 			break;
9970 		map = trace_eval_jmp_to_tail(map);
9971 		last = &map->tail.next;
9972 		map = map->tail.next;
9973 	}
9974 	if (!map)
9975 		goto out;
9976 
9977 	*last = trace_eval_jmp_to_tail(map)->tail.next;
9978 	kfree(map);
9979  out:
9980 	mutex_unlock(&trace_eval_mutex);
9981 }
9982 #else
trace_module_remove_evals(struct module * mod)9983 static inline void trace_module_remove_evals(struct module *mod) { }
9984 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9985 
trace_module_notify(struct notifier_block * self,unsigned long val,void * data)9986 static int trace_module_notify(struct notifier_block *self,
9987 			       unsigned long val, void *data)
9988 {
9989 	struct module *mod = data;
9990 
9991 	switch (val) {
9992 	case MODULE_STATE_COMING:
9993 		trace_module_add_evals(mod);
9994 		break;
9995 	case MODULE_STATE_GOING:
9996 		trace_module_remove_evals(mod);
9997 		break;
9998 	}
9999 
10000 	return NOTIFY_OK;
10001 }
10002 
10003 static struct notifier_block trace_module_nb = {
10004 	.notifier_call = trace_module_notify,
10005 	.priority = 0,
10006 };
10007 #endif /* CONFIG_MODULES */
10008 
tracer_init_tracefs_work_func(struct work_struct * work)10009 static __init void tracer_init_tracefs_work_func(struct work_struct *work)
10010 {
10011 
10012 	event_trace_init();
10013 
10014 	init_tracer_tracefs(&global_trace, NULL);
10015 	ftrace_init_tracefs_toplevel(&global_trace, NULL);
10016 
10017 	trace_create_file("tracing_thresh", TRACE_MODE_WRITE, NULL,
10018 			&global_trace, &tracing_thresh_fops);
10019 
10020 	trace_create_file("README", TRACE_MODE_READ, NULL,
10021 			NULL, &tracing_readme_fops);
10022 
10023 	trace_create_file("saved_cmdlines", TRACE_MODE_READ, NULL,
10024 			NULL, &tracing_saved_cmdlines_fops);
10025 
10026 	trace_create_file("saved_cmdlines_size", TRACE_MODE_WRITE, NULL,
10027 			  NULL, &tracing_saved_cmdlines_size_fops);
10028 
10029 	trace_create_file("saved_tgids", TRACE_MODE_READ, NULL,
10030 			NULL, &tracing_saved_tgids_fops);
10031 
10032 	trace_create_eval_file(NULL);
10033 
10034 #ifdef CONFIG_MODULES
10035 	register_module_notifier(&trace_module_nb);
10036 #endif
10037 
10038 #ifdef CONFIG_DYNAMIC_FTRACE
10039 	trace_create_file("dyn_ftrace_total_info", TRACE_MODE_READ, NULL,
10040 			NULL, &tracing_dyn_info_fops);
10041 #endif
10042 
10043 	create_trace_instances(NULL);
10044 
10045 	update_tracer_options(&global_trace);
10046 }
10047 
tracer_init_tracefs(void)10048 static __init int tracer_init_tracefs(void)
10049 {
10050 	int ret;
10051 
10052 	trace_access_lock_init();
10053 
10054 	ret = tracing_init_dentry();
10055 	if (ret)
10056 		return 0;
10057 
10058 	if (eval_map_wq) {
10059 		INIT_WORK(&tracerfs_init_work, tracer_init_tracefs_work_func);
10060 		queue_work(eval_map_wq, &tracerfs_init_work);
10061 	} else {
10062 		tracer_init_tracefs_work_func(NULL);
10063 	}
10064 
10065 	rv_init_interface();
10066 
10067 	return 0;
10068 }
10069 
10070 fs_initcall(tracer_init_tracefs);
10071 
10072 static int trace_die_panic_handler(struct notifier_block *self,
10073 				unsigned long ev, void *unused);
10074 
10075 static struct notifier_block trace_panic_notifier = {
10076 	.notifier_call = trace_die_panic_handler,
10077 	.priority = INT_MAX - 1,
10078 };
10079 
10080 static struct notifier_block trace_die_notifier = {
10081 	.notifier_call = trace_die_panic_handler,
10082 	.priority = INT_MAX - 1,
10083 };
10084 
10085 /*
10086  * The idea is to execute the following die/panic callback early, in order
10087  * to avoid showing irrelevant information in the trace (like other panic
10088  * notifier functions); we are the 2nd to run, after hung_task/rcu_stall
10089  * warnings get disabled (to prevent potential log flooding).
10090  */
trace_die_panic_handler(struct notifier_block * self,unsigned long ev,void * unused)10091 static int trace_die_panic_handler(struct notifier_block *self,
10092 				unsigned long ev, void *unused)
10093 {
10094 	if (!ftrace_dump_on_oops)
10095 		return NOTIFY_DONE;
10096 
10097 	/* The die notifier requires DIE_OOPS to trigger */
10098 	if (self == &trace_die_notifier && ev != DIE_OOPS)
10099 		return NOTIFY_DONE;
10100 
10101 	ftrace_dump(ftrace_dump_on_oops);
10102 
10103 	return NOTIFY_DONE;
10104 }
10105 
10106 /*
10107  * printk is set to max of 1024, we really don't need it that big.
10108  * Nothing should be printing 1000 characters anyway.
10109  */
10110 #define TRACE_MAX_PRINT		1000
10111 
10112 /*
10113  * Define here KERN_TRACE so that we have one place to modify
10114  * it if we decide to change what log level the ftrace dump
10115  * should be at.
10116  */
10117 #define KERN_TRACE		KERN_EMERG
10118 
10119 void
trace_printk_seq(struct trace_seq * s)10120 trace_printk_seq(struct trace_seq *s)
10121 {
10122 	/* Probably should print a warning here. */
10123 	if (s->seq.len >= TRACE_MAX_PRINT)
10124 		s->seq.len = TRACE_MAX_PRINT;
10125 
10126 	/*
10127 	 * More paranoid code. Although the buffer size is set to
10128 	 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
10129 	 * an extra layer of protection.
10130 	 */
10131 	if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
10132 		s->seq.len = s->seq.size - 1;
10133 
10134 	/* should be zero ended, but we are paranoid. */
10135 	s->buffer[s->seq.len] = 0;
10136 
10137 	printk(KERN_TRACE "%s", s->buffer);
10138 
10139 	trace_seq_init(s);
10140 }
10141 
trace_init_global_iter(struct trace_iterator * iter)10142 void trace_init_global_iter(struct trace_iterator *iter)
10143 {
10144 	iter->tr = &global_trace;
10145 	iter->trace = iter->tr->current_trace;
10146 	iter->cpu_file = RING_BUFFER_ALL_CPUS;
10147 	iter->array_buffer = &global_trace.array_buffer;
10148 
10149 	if (iter->trace && iter->trace->open)
10150 		iter->trace->open(iter);
10151 
10152 	/* Annotate start of buffers if we had overruns */
10153 	if (ring_buffer_overruns(iter->array_buffer->buffer))
10154 		iter->iter_flags |= TRACE_FILE_ANNOTATE;
10155 
10156 	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
10157 	if (trace_clocks[iter->tr->clock_id].in_ns)
10158 		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
10159 
10160 	/* Can not use kmalloc for iter.temp and iter.fmt */
10161 	iter->temp = static_temp_buf;
10162 	iter->temp_size = STATIC_TEMP_BUF_SIZE;
10163 	iter->fmt = static_fmt_buf;
10164 	iter->fmt_size = STATIC_FMT_BUF_SIZE;
10165 }
10166 
ftrace_dump(enum ftrace_dump_mode oops_dump_mode)10167 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
10168 {
10169 	/* use static because iter can be a bit big for the stack */
10170 	static struct trace_iterator iter;
10171 	static atomic_t dump_running;
10172 	struct trace_array *tr = &global_trace;
10173 	unsigned int old_userobj;
10174 	unsigned long flags;
10175 	int cnt = 0, cpu;
10176 
10177 	/* Only allow one dump user at a time. */
10178 	if (atomic_inc_return(&dump_running) != 1) {
10179 		atomic_dec(&dump_running);
10180 		return;
10181 	}
10182 
10183 	/*
10184 	 * Always turn off tracing when we dump.
10185 	 * We don't need to show trace output of what happens
10186 	 * between multiple crashes.
10187 	 *
10188 	 * If the user does a sysrq-z, then they can re-enable
10189 	 * tracing with echo 1 > tracing_on.
10190 	 */
10191 	tracing_off();
10192 
10193 	local_irq_save(flags);
10194 
10195 	/* Simulate the iterator */
10196 	trace_init_global_iter(&iter);
10197 
10198 	for_each_tracing_cpu(cpu) {
10199 		atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
10200 	}
10201 
10202 	old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
10203 
10204 	/* don't look at user memory in panic mode */
10205 	tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
10206 
10207 	switch (oops_dump_mode) {
10208 	case DUMP_ALL:
10209 		iter.cpu_file = RING_BUFFER_ALL_CPUS;
10210 		break;
10211 	case DUMP_ORIG:
10212 		iter.cpu_file = raw_smp_processor_id();
10213 		break;
10214 	case DUMP_NONE:
10215 		goto out_enable;
10216 	default:
10217 		printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
10218 		iter.cpu_file = RING_BUFFER_ALL_CPUS;
10219 	}
10220 
10221 	printk(KERN_TRACE "Dumping ftrace buffer:\n");
10222 
10223 	/* Did function tracer already get disabled? */
10224 	if (ftrace_is_dead()) {
10225 		printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
10226 		printk("#          MAY BE MISSING FUNCTION EVENTS\n");
10227 	}
10228 
10229 	/*
10230 	 * We need to stop all tracing on all CPUS to read
10231 	 * the next buffer. This is a bit expensive, but is
10232 	 * not done often. We fill all what we can read,
10233 	 * and then release the locks again.
10234 	 */
10235 
10236 	while (!trace_empty(&iter)) {
10237 
10238 		if (!cnt)
10239 			printk(KERN_TRACE "---------------------------------\n");
10240 
10241 		cnt++;
10242 
10243 		trace_iterator_reset(&iter);
10244 		iter.iter_flags |= TRACE_FILE_LAT_FMT;
10245 
10246 		if (trace_find_next_entry_inc(&iter) != NULL) {
10247 			int ret;
10248 
10249 			ret = print_trace_line(&iter);
10250 			if (ret != TRACE_TYPE_NO_CONSUME)
10251 				trace_consume(&iter);
10252 		}
10253 		touch_nmi_watchdog();
10254 
10255 		trace_printk_seq(&iter.seq);
10256 	}
10257 
10258 	if (!cnt)
10259 		printk(KERN_TRACE "   (ftrace buffer empty)\n");
10260 	else
10261 		printk(KERN_TRACE "---------------------------------\n");
10262 
10263  out_enable:
10264 	tr->trace_flags |= old_userobj;
10265 
10266 	for_each_tracing_cpu(cpu) {
10267 		atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
10268 	}
10269 	atomic_dec(&dump_running);
10270 	local_irq_restore(flags);
10271 }
10272 EXPORT_SYMBOL_GPL(ftrace_dump);
10273 
10274 #define WRITE_BUFSIZE  4096
10275 
trace_parse_run_command(struct file * file,const char __user * buffer,size_t count,loff_t * ppos,int (* createfn)(const char *))10276 ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
10277 				size_t count, loff_t *ppos,
10278 				int (*createfn)(const char *))
10279 {
10280 	char *kbuf, *buf, *tmp;
10281 	int ret = 0;
10282 	size_t done = 0;
10283 	size_t size;
10284 
10285 	kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
10286 	if (!kbuf)
10287 		return -ENOMEM;
10288 
10289 	while (done < count) {
10290 		size = count - done;
10291 
10292 		if (size >= WRITE_BUFSIZE)
10293 			size = WRITE_BUFSIZE - 1;
10294 
10295 		if (copy_from_user(kbuf, buffer + done, size)) {
10296 			ret = -EFAULT;
10297 			goto out;
10298 		}
10299 		kbuf[size] = '\0';
10300 		buf = kbuf;
10301 		do {
10302 			tmp = strchr(buf, '\n');
10303 			if (tmp) {
10304 				*tmp = '\0';
10305 				size = tmp - buf + 1;
10306 			} else {
10307 				size = strlen(buf);
10308 				if (done + size < count) {
10309 					if (buf != kbuf)
10310 						break;
10311 					/* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
10312 					pr_warn("Line length is too long: Should be less than %d\n",
10313 						WRITE_BUFSIZE - 2);
10314 					ret = -EINVAL;
10315 					goto out;
10316 				}
10317 			}
10318 			done += size;
10319 
10320 			/* Remove comments */
10321 			tmp = strchr(buf, '#');
10322 
10323 			if (tmp)
10324 				*tmp = '\0';
10325 
10326 			ret = createfn(buf);
10327 			if (ret)
10328 				goto out;
10329 			buf += size;
10330 
10331 		} while (done < count);
10332 	}
10333 	ret = done;
10334 
10335 out:
10336 	kfree(kbuf);
10337 
10338 	return ret;
10339 }
10340 
10341 #ifdef CONFIG_TRACER_MAX_TRACE
tr_needs_alloc_snapshot(const char * name)10342 __init static bool tr_needs_alloc_snapshot(const char *name)
10343 {
10344 	char *test;
10345 	int len = strlen(name);
10346 	bool ret;
10347 
10348 	if (!boot_snapshot_index)
10349 		return false;
10350 
10351 	if (strncmp(name, boot_snapshot_info, len) == 0 &&
10352 	    boot_snapshot_info[len] == '\t')
10353 		return true;
10354 
10355 	test = kmalloc(strlen(name) + 3, GFP_KERNEL);
10356 	if (!test)
10357 		return false;
10358 
10359 	sprintf(test, "\t%s\t", name);
10360 	ret = strstr(boot_snapshot_info, test) == NULL;
10361 	kfree(test);
10362 	return ret;
10363 }
10364 
do_allocate_snapshot(const char * name)10365 __init static void do_allocate_snapshot(const char *name)
10366 {
10367 	if (!tr_needs_alloc_snapshot(name))
10368 		return;
10369 
10370 	/*
10371 	 * When allocate_snapshot is set, the next call to
10372 	 * allocate_trace_buffers() (called by trace_array_get_by_name())
10373 	 * will allocate the snapshot buffer. That will alse clear
10374 	 * this flag.
10375 	 */
10376 	allocate_snapshot = true;
10377 }
10378 #else
do_allocate_snapshot(const char * name)10379 static inline void do_allocate_snapshot(const char *name) { }
10380 #endif
10381 
enable_instances(void)10382 __init static void enable_instances(void)
10383 {
10384 	struct trace_array *tr;
10385 	char *curr_str;
10386 	char *str;
10387 	char *tok;
10388 
10389 	/* A tab is always appended */
10390 	boot_instance_info[boot_instance_index - 1] = '\0';
10391 	str = boot_instance_info;
10392 
10393 	while ((curr_str = strsep(&str, "\t"))) {
10394 
10395 		tok = strsep(&curr_str, ",");
10396 
10397 		if (IS_ENABLED(CONFIG_TRACER_MAX_TRACE))
10398 			do_allocate_snapshot(tok);
10399 
10400 		tr = trace_array_get_by_name(tok);
10401 		if (!tr) {
10402 			pr_warn("Failed to create instance buffer %s\n", curr_str);
10403 			continue;
10404 		}
10405 		/* Allow user space to delete it */
10406 		trace_array_put(tr);
10407 
10408 		while ((tok = strsep(&curr_str, ","))) {
10409 			early_enable_events(tr, tok, true);
10410 		}
10411 	}
10412 }
10413 
tracer_alloc_buffers(void)10414 __init static int tracer_alloc_buffers(void)
10415 {
10416 	int ring_buf_size;
10417 	int ret = -ENOMEM;
10418 
10419 
10420 	if (security_locked_down(LOCKDOWN_TRACEFS)) {
10421 		pr_warn("Tracing disabled due to lockdown\n");
10422 		return -EPERM;
10423 	}
10424 
10425 	/*
10426 	 * Make sure we don't accidentally add more trace options
10427 	 * than we have bits for.
10428 	 */
10429 	BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
10430 
10431 	if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
10432 		goto out;
10433 
10434 	if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
10435 		goto out_free_buffer_mask;
10436 
10437 	/* Only allocate trace_printk buffers if a trace_printk exists */
10438 	if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
10439 		/* Must be called before global_trace.buffer is allocated */
10440 		trace_printk_init_buffers();
10441 
10442 	/* To save memory, keep the ring buffer size to its minimum */
10443 	if (ring_buffer_expanded)
10444 		ring_buf_size = trace_buf_size;
10445 	else
10446 		ring_buf_size = 1;
10447 
10448 	cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
10449 	cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
10450 
10451 	raw_spin_lock_init(&global_trace.start_lock);
10452 
10453 	/*
10454 	 * The prepare callbacks allocates some memory for the ring buffer. We
10455 	 * don't free the buffer if the CPU goes down. If we were to free
10456 	 * the buffer, then the user would lose any trace that was in the
10457 	 * buffer. The memory will be removed once the "instance" is removed.
10458 	 */
10459 	ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
10460 				      "trace/RB:prepare", trace_rb_cpu_prepare,
10461 				      NULL);
10462 	if (ret < 0)
10463 		goto out_free_cpumask;
10464 	/* Used for event triggers */
10465 	ret = -ENOMEM;
10466 	temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
10467 	if (!temp_buffer)
10468 		goto out_rm_hp_state;
10469 
10470 	if (trace_create_savedcmd() < 0)
10471 		goto out_free_temp_buffer;
10472 
10473 	if (!zalloc_cpumask_var(&global_trace.pipe_cpumask, GFP_KERNEL))
10474 		goto out_free_savedcmd;
10475 
10476 	/* TODO: make the number of buffers hot pluggable with CPUS */
10477 	if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
10478 		MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
10479 		goto out_free_pipe_cpumask;
10480 	}
10481 	if (global_trace.buffer_disabled)
10482 		tracing_off();
10483 
10484 	if (trace_boot_clock) {
10485 		ret = tracing_set_clock(&global_trace, trace_boot_clock);
10486 		if (ret < 0)
10487 			pr_warn("Trace clock %s not defined, going back to default\n",
10488 				trace_boot_clock);
10489 	}
10490 
10491 	/*
10492 	 * register_tracer() might reference current_trace, so it
10493 	 * needs to be set before we register anything. This is
10494 	 * just a bootstrap of current_trace anyway.
10495 	 */
10496 	global_trace.current_trace = &nop_trace;
10497 
10498 	global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
10499 
10500 	ftrace_init_global_array_ops(&global_trace);
10501 
10502 	init_trace_flags_index(&global_trace);
10503 
10504 	register_tracer(&nop_trace);
10505 
10506 	/* Function tracing may start here (via kernel command line) */
10507 	init_function_trace();
10508 
10509 	/* All seems OK, enable tracing */
10510 	tracing_disabled = 0;
10511 
10512 	atomic_notifier_chain_register(&panic_notifier_list,
10513 				       &trace_panic_notifier);
10514 
10515 	register_die_notifier(&trace_die_notifier);
10516 
10517 	global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
10518 
10519 	INIT_LIST_HEAD(&global_trace.systems);
10520 	INIT_LIST_HEAD(&global_trace.events);
10521 	INIT_LIST_HEAD(&global_trace.hist_vars);
10522 	INIT_LIST_HEAD(&global_trace.err_log);
10523 	list_add(&global_trace.list, &ftrace_trace_arrays);
10524 
10525 	apply_trace_boot_options();
10526 
10527 	register_snapshot_cmd();
10528 
10529 	test_can_verify();
10530 
10531 	return 0;
10532 
10533 out_free_pipe_cpumask:
10534 	free_cpumask_var(global_trace.pipe_cpumask);
10535 out_free_savedcmd:
10536 	free_saved_cmdlines_buffer(savedcmd);
10537 out_free_temp_buffer:
10538 	ring_buffer_free(temp_buffer);
10539 out_rm_hp_state:
10540 	cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
10541 out_free_cpumask:
10542 	free_cpumask_var(global_trace.tracing_cpumask);
10543 out_free_buffer_mask:
10544 	free_cpumask_var(tracing_buffer_mask);
10545 out:
10546 	return ret;
10547 }
10548 
ftrace_boot_snapshot(void)10549 void __init ftrace_boot_snapshot(void)
10550 {
10551 #ifdef CONFIG_TRACER_MAX_TRACE
10552 	struct trace_array *tr;
10553 
10554 	if (!snapshot_at_boot)
10555 		return;
10556 
10557 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
10558 		if (!tr->allocated_snapshot)
10559 			continue;
10560 
10561 		tracing_snapshot_instance(tr);
10562 		trace_array_puts(tr, "** Boot snapshot taken **\n");
10563 	}
10564 #endif
10565 }
10566 
early_trace_init(void)10567 void __init early_trace_init(void)
10568 {
10569 	if (tracepoint_printk) {
10570 		tracepoint_print_iter =
10571 			kzalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
10572 		if (MEM_FAIL(!tracepoint_print_iter,
10573 			     "Failed to allocate trace iterator\n"))
10574 			tracepoint_printk = 0;
10575 		else
10576 			static_key_enable(&tracepoint_printk_key.key);
10577 	}
10578 	tracer_alloc_buffers();
10579 
10580 	init_events();
10581 }
10582 
trace_init(void)10583 void __init trace_init(void)
10584 {
10585 	trace_event_init();
10586 
10587 	if (boot_instance_index)
10588 		enable_instances();
10589 }
10590 
clear_boot_tracer(void)10591 __init static void clear_boot_tracer(void)
10592 {
10593 	/*
10594 	 * The default tracer at boot buffer is an init section.
10595 	 * This function is called in lateinit. If we did not
10596 	 * find the boot tracer, then clear it out, to prevent
10597 	 * later registration from accessing the buffer that is
10598 	 * about to be freed.
10599 	 */
10600 	if (!default_bootup_tracer)
10601 		return;
10602 
10603 	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
10604 	       default_bootup_tracer);
10605 	default_bootup_tracer = NULL;
10606 }
10607 
10608 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
tracing_set_default_clock(void)10609 __init static void tracing_set_default_clock(void)
10610 {
10611 	/* sched_clock_stable() is determined in late_initcall */
10612 	if (!trace_boot_clock && !sched_clock_stable()) {
10613 		if (security_locked_down(LOCKDOWN_TRACEFS)) {
10614 			pr_warn("Can not set tracing clock due to lockdown\n");
10615 			return;
10616 		}
10617 
10618 		printk(KERN_WARNING
10619 		       "Unstable clock detected, switching default tracing clock to \"global\"\n"
10620 		       "If you want to keep using the local clock, then add:\n"
10621 		       "  \"trace_clock=local\"\n"
10622 		       "on the kernel command line\n");
10623 		tracing_set_clock(&global_trace, "global");
10624 	}
10625 }
10626 #else
tracing_set_default_clock(void)10627 static inline void tracing_set_default_clock(void) { }
10628 #endif
10629 
late_trace_init(void)10630 __init static int late_trace_init(void)
10631 {
10632 	if (tracepoint_printk && tracepoint_printk_stop_on_boot) {
10633 		static_key_disable(&tracepoint_printk_key.key);
10634 		tracepoint_printk = 0;
10635 	}
10636 
10637 	tracing_set_default_clock();
10638 	clear_boot_tracer();
10639 	return 0;
10640 }
10641 
10642 late_initcall_sync(late_trace_init);
10643