1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Performance events core code:
4  *
5  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
6  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
8  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9  */
10 
11 #include <linux/fs.h>
12 #include <linux/mm.h>
13 #include <linux/cpu.h>
14 #include <linux/smp.h>
15 #include <linux/idr.h>
16 #include <linux/file.h>
17 #include <linux/poll.h>
18 #include <linux/slab.h>
19 #include <linux/hash.h>
20 #include <linux/tick.h>
21 #include <linux/sysfs.h>
22 #include <linux/dcache.h>
23 #include <linux/percpu.h>
24 #include <linux/ptrace.h>
25 #include <linux/reboot.h>
26 #include <linux/vmstat.h>
27 #include <linux/device.h>
28 #include <linux/export.h>
29 #include <linux/vmalloc.h>
30 #include <linux/hardirq.h>
31 #include <linux/hugetlb.h>
32 #include <linux/rculist.h>
33 #include <linux/uaccess.h>
34 #include <linux/syscalls.h>
35 #include <linux/anon_inodes.h>
36 #include <linux/kernel_stat.h>
37 #include <linux/cgroup.h>
38 #include <linux/perf_event.h>
39 #include <linux/trace_events.h>
40 #include <linux/hw_breakpoint.h>
41 #include <linux/mm_types.h>
42 #include <linux/module.h>
43 #include <linux/mman.h>
44 #include <linux/compat.h>
45 #include <linux/bpf.h>
46 #include <linux/filter.h>
47 #include <linux/namei.h>
48 #include <linux/parser.h>
49 #include <linux/sched/clock.h>
50 #include <linux/sched/mm.h>
51 #include <linux/proc_ns.h>
52 #include <linux/mount.h>
53 #include <linux/min_heap.h>
54 #include <linux/highmem.h>
55 #include <linux/pgtable.h>
56 #include <linux/buildid.h>
57 #include <linux/task_work.h>
58 
59 #include "internal.h"
60 
61 #include <asm/irq_regs.h>
62 
63 typedef int (*remote_function_f)(void *);
64 
65 struct remote_function_call {
66 	struct task_struct	*p;
67 	remote_function_f	func;
68 	void			*info;
69 	int			ret;
70 };
71 
remote_function(void * data)72 static void remote_function(void *data)
73 {
74 	struct remote_function_call *tfc = data;
75 	struct task_struct *p = tfc->p;
76 
77 	if (p) {
78 		/* -EAGAIN */
79 		if (task_cpu(p) != smp_processor_id())
80 			return;
81 
82 		/*
83 		 * Now that we're on right CPU with IRQs disabled, we can test
84 		 * if we hit the right task without races.
85 		 */
86 
87 		tfc->ret = -ESRCH; /* No such (running) process */
88 		if (p != current)
89 			return;
90 	}
91 
92 	tfc->ret = tfc->func(tfc->info);
93 }
94 
95 /**
96  * task_function_call - call a function on the cpu on which a task runs
97  * @p:		the task to evaluate
98  * @func:	the function to be called
99  * @info:	the function call argument
100  *
101  * Calls the function @func when the task is currently running. This might
102  * be on the current CPU, which just calls the function directly.  This will
103  * retry due to any failures in smp_call_function_single(), such as if the
104  * task_cpu() goes offline concurrently.
105  *
106  * returns @func return value or -ESRCH or -ENXIO when the process isn't running
107  */
108 static int
task_function_call(struct task_struct * p,remote_function_f func,void * info)109 task_function_call(struct task_struct *p, remote_function_f func, void *info)
110 {
111 	struct remote_function_call data = {
112 		.p	= p,
113 		.func	= func,
114 		.info	= info,
115 		.ret	= -EAGAIN,
116 	};
117 	int ret;
118 
119 	for (;;) {
120 		ret = smp_call_function_single(task_cpu(p), remote_function,
121 					       &data, 1);
122 		if (!ret)
123 			ret = data.ret;
124 
125 		if (ret != -EAGAIN)
126 			break;
127 
128 		cond_resched();
129 	}
130 
131 	return ret;
132 }
133 
134 /**
135  * cpu_function_call - call a function on the cpu
136  * @cpu:	target cpu to queue this function
137  * @func:	the function to be called
138  * @info:	the function call argument
139  *
140  * Calls the function @func on the remote cpu.
141  *
142  * returns: @func return value or -ENXIO when the cpu is offline
143  */
cpu_function_call(int cpu,remote_function_f func,void * info)144 static int cpu_function_call(int cpu, remote_function_f func, void *info)
145 {
146 	struct remote_function_call data = {
147 		.p	= NULL,
148 		.func	= func,
149 		.info	= info,
150 		.ret	= -ENXIO, /* No such CPU */
151 	};
152 
153 	smp_call_function_single(cpu, remote_function, &data, 1);
154 
155 	return data.ret;
156 }
157 
perf_ctx_lock(struct perf_cpu_context * cpuctx,struct perf_event_context * ctx)158 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
159 			  struct perf_event_context *ctx)
160 {
161 	raw_spin_lock(&cpuctx->ctx.lock);
162 	if (ctx)
163 		raw_spin_lock(&ctx->lock);
164 }
165 
perf_ctx_unlock(struct perf_cpu_context * cpuctx,struct perf_event_context * ctx)166 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
167 			    struct perf_event_context *ctx)
168 {
169 	if (ctx)
170 		raw_spin_unlock(&ctx->lock);
171 	raw_spin_unlock(&cpuctx->ctx.lock);
172 }
173 
174 #define TASK_TOMBSTONE ((void *)-1L)
175 
is_kernel_event(struct perf_event * event)176 static bool is_kernel_event(struct perf_event *event)
177 {
178 	return READ_ONCE(event->owner) == TASK_TOMBSTONE;
179 }
180 
181 static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
182 
perf_cpu_task_ctx(void)183 struct perf_event_context *perf_cpu_task_ctx(void)
184 {
185 	lockdep_assert_irqs_disabled();
186 	return this_cpu_ptr(&perf_cpu_context)->task_ctx;
187 }
188 
189 /*
190  * On task ctx scheduling...
191  *
192  * When !ctx->nr_events a task context will not be scheduled. This means
193  * we can disable the scheduler hooks (for performance) without leaving
194  * pending task ctx state.
195  *
196  * This however results in two special cases:
197  *
198  *  - removing the last event from a task ctx; this is relatively straight
199  *    forward and is done in __perf_remove_from_context.
200  *
201  *  - adding the first event to a task ctx; this is tricky because we cannot
202  *    rely on ctx->is_active and therefore cannot use event_function_call().
203  *    See perf_install_in_context().
204  *
205  * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
206  */
207 
208 typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *,
209 			struct perf_event_context *, void *);
210 
211 struct event_function_struct {
212 	struct perf_event *event;
213 	event_f func;
214 	void *data;
215 };
216 
event_function(void * info)217 static int event_function(void *info)
218 {
219 	struct event_function_struct *efs = info;
220 	struct perf_event *event = efs->event;
221 	struct perf_event_context *ctx = event->ctx;
222 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
223 	struct perf_event_context *task_ctx = cpuctx->task_ctx;
224 	int ret = 0;
225 
226 	lockdep_assert_irqs_disabled();
227 
228 	perf_ctx_lock(cpuctx, task_ctx);
229 	/*
230 	 * Since we do the IPI call without holding ctx->lock things can have
231 	 * changed, double check we hit the task we set out to hit.
232 	 */
233 	if (ctx->task) {
234 		if (ctx->task != current) {
235 			ret = -ESRCH;
236 			goto unlock;
237 		}
238 
239 		/*
240 		 * We only use event_function_call() on established contexts,
241 		 * and event_function() is only ever called when active (or
242 		 * rather, we'll have bailed in task_function_call() or the
243 		 * above ctx->task != current test), therefore we must have
244 		 * ctx->is_active here.
245 		 */
246 		WARN_ON_ONCE(!ctx->is_active);
247 		/*
248 		 * And since we have ctx->is_active, cpuctx->task_ctx must
249 		 * match.
250 		 */
251 		WARN_ON_ONCE(task_ctx != ctx);
252 	} else {
253 		WARN_ON_ONCE(&cpuctx->ctx != ctx);
254 	}
255 
256 	efs->func(event, cpuctx, ctx, efs->data);
257 unlock:
258 	perf_ctx_unlock(cpuctx, task_ctx);
259 
260 	return ret;
261 }
262 
event_function_call(struct perf_event * event,event_f func,void * data)263 static void event_function_call(struct perf_event *event, event_f func, void *data)
264 {
265 	struct perf_event_context *ctx = event->ctx;
266 	struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */
267 	struct event_function_struct efs = {
268 		.event = event,
269 		.func = func,
270 		.data = data,
271 	};
272 
273 	if (!event->parent) {
274 		/*
275 		 * If this is a !child event, we must hold ctx::mutex to
276 		 * stabilize the event->ctx relation. See
277 		 * perf_event_ctx_lock().
278 		 */
279 		lockdep_assert_held(&ctx->mutex);
280 	}
281 
282 	if (!task) {
283 		cpu_function_call(event->cpu, event_function, &efs);
284 		return;
285 	}
286 
287 	if (task == TASK_TOMBSTONE)
288 		return;
289 
290 again:
291 	if (!task_function_call(task, event_function, &efs))
292 		return;
293 
294 	raw_spin_lock_irq(&ctx->lock);
295 	/*
296 	 * Reload the task pointer, it might have been changed by
297 	 * a concurrent perf_event_context_sched_out().
298 	 */
299 	task = ctx->task;
300 	if (task == TASK_TOMBSTONE) {
301 		raw_spin_unlock_irq(&ctx->lock);
302 		return;
303 	}
304 	if (ctx->is_active) {
305 		raw_spin_unlock_irq(&ctx->lock);
306 		goto again;
307 	}
308 	func(event, NULL, ctx, data);
309 	raw_spin_unlock_irq(&ctx->lock);
310 }
311 
312 /*
313  * Similar to event_function_call() + event_function(), but hard assumes IRQs
314  * are already disabled and we're on the right CPU.
315  */
event_function_local(struct perf_event * event,event_f func,void * data)316 static void event_function_local(struct perf_event *event, event_f func, void *data)
317 {
318 	struct perf_event_context *ctx = event->ctx;
319 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
320 	struct task_struct *task = READ_ONCE(ctx->task);
321 	struct perf_event_context *task_ctx = NULL;
322 
323 	lockdep_assert_irqs_disabled();
324 
325 	if (task) {
326 		if (task == TASK_TOMBSTONE)
327 			return;
328 
329 		task_ctx = ctx;
330 	}
331 
332 	perf_ctx_lock(cpuctx, task_ctx);
333 
334 	task = ctx->task;
335 	if (task == TASK_TOMBSTONE)
336 		goto unlock;
337 
338 	if (task) {
339 		/*
340 		 * We must be either inactive or active and the right task,
341 		 * otherwise we're screwed, since we cannot IPI to somewhere
342 		 * else.
343 		 */
344 		if (ctx->is_active) {
345 			if (WARN_ON_ONCE(task != current))
346 				goto unlock;
347 
348 			if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))
349 				goto unlock;
350 		}
351 	} else {
352 		WARN_ON_ONCE(&cpuctx->ctx != ctx);
353 	}
354 
355 	func(event, cpuctx, ctx, data);
356 unlock:
357 	perf_ctx_unlock(cpuctx, task_ctx);
358 }
359 
360 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
361 		       PERF_FLAG_FD_OUTPUT  |\
362 		       PERF_FLAG_PID_CGROUP |\
363 		       PERF_FLAG_FD_CLOEXEC)
364 
365 /*
366  * branch priv levels that need permission checks
367  */
368 #define PERF_SAMPLE_BRANCH_PERM_PLM \
369 	(PERF_SAMPLE_BRANCH_KERNEL |\
370 	 PERF_SAMPLE_BRANCH_HV)
371 
372 enum event_type_t {
373 	EVENT_FLEXIBLE = 0x1,
374 	EVENT_PINNED = 0x2,
375 	EVENT_TIME = 0x4,
376 	/* see ctx_resched() for details */
377 	EVENT_CPU = 0x8,
378 	EVENT_CGROUP = 0x10,
379 	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
380 };
381 
382 /*
383  * perf_sched_events : >0 events exist
384  */
385 
386 static void perf_sched_delayed(struct work_struct *work);
387 DEFINE_STATIC_KEY_FALSE(perf_sched_events);
388 static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed);
389 static DEFINE_MUTEX(perf_sched_mutex);
390 static atomic_t perf_sched_count;
391 
392 static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
393 
394 static atomic_t nr_mmap_events __read_mostly;
395 static atomic_t nr_comm_events __read_mostly;
396 static atomic_t nr_namespaces_events __read_mostly;
397 static atomic_t nr_task_events __read_mostly;
398 static atomic_t nr_freq_events __read_mostly;
399 static atomic_t nr_switch_events __read_mostly;
400 static atomic_t nr_ksymbol_events __read_mostly;
401 static atomic_t nr_bpf_events __read_mostly;
402 static atomic_t nr_cgroup_events __read_mostly;
403 static atomic_t nr_text_poke_events __read_mostly;
404 static atomic_t nr_build_id_events __read_mostly;
405 
406 static LIST_HEAD(pmus);
407 static DEFINE_MUTEX(pmus_lock);
408 static struct srcu_struct pmus_srcu;
409 static cpumask_var_t perf_online_mask;
410 static struct kmem_cache *perf_event_cache;
411 
412 /*
413  * perf event paranoia level:
414  *  -1 - not paranoid at all
415  *   0 - disallow raw tracepoint access for unpriv
416  *   1 - disallow cpu events for unpriv
417  *   2 - disallow kernel profiling for unpriv
418  */
419 int sysctl_perf_event_paranoid __read_mostly = 2;
420 
421 /* Minimum for 512 kiB + 1 user control page */
422 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
423 
424 /*
425  * max perf event sample rate
426  */
427 #define DEFAULT_MAX_SAMPLE_RATE		100000
428 #define DEFAULT_SAMPLE_PERIOD_NS	(NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
429 #define DEFAULT_CPU_TIME_MAX_PERCENT	25
430 
431 int sysctl_perf_event_sample_rate __read_mostly	= DEFAULT_MAX_SAMPLE_RATE;
432 
433 static int max_samples_per_tick __read_mostly	= DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
434 static int perf_sample_period_ns __read_mostly	= DEFAULT_SAMPLE_PERIOD_NS;
435 
436 static int perf_sample_allowed_ns __read_mostly =
437 	DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
438 
update_perf_cpu_limits(void)439 static void update_perf_cpu_limits(void)
440 {
441 	u64 tmp = perf_sample_period_ns;
442 
443 	tmp *= sysctl_perf_cpu_time_max_percent;
444 	tmp = div_u64(tmp, 100);
445 	if (!tmp)
446 		tmp = 1;
447 
448 	WRITE_ONCE(perf_sample_allowed_ns, tmp);
449 }
450 
451 static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc);
452 
perf_proc_update_handler(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)453 int perf_proc_update_handler(struct ctl_table *table, int write,
454 		void *buffer, size_t *lenp, loff_t *ppos)
455 {
456 	int ret;
457 	int perf_cpu = sysctl_perf_cpu_time_max_percent;
458 	/*
459 	 * If throttling is disabled don't allow the write:
460 	 */
461 	if (write && (perf_cpu == 100 || perf_cpu == 0))
462 		return -EINVAL;
463 
464 	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
465 	if (ret || !write)
466 		return ret;
467 
468 	max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
469 	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
470 	update_perf_cpu_limits();
471 
472 	return 0;
473 }
474 
475 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
476 
perf_cpu_time_max_percent_handler(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)477 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
478 		void *buffer, size_t *lenp, loff_t *ppos)
479 {
480 	int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
481 
482 	if (ret || !write)
483 		return ret;
484 
485 	if (sysctl_perf_cpu_time_max_percent == 100 ||
486 	    sysctl_perf_cpu_time_max_percent == 0) {
487 		printk(KERN_WARNING
488 		       "perf: Dynamic interrupt throttling disabled, can hang your system!\n");
489 		WRITE_ONCE(perf_sample_allowed_ns, 0);
490 	} else {
491 		update_perf_cpu_limits();
492 	}
493 
494 	return 0;
495 }
496 
497 /*
498  * perf samples are done in some very critical code paths (NMIs).
499  * If they take too much CPU time, the system can lock up and not
500  * get any real work done.  This will drop the sample rate when
501  * we detect that events are taking too long.
502  */
503 #define NR_ACCUMULATED_SAMPLES 128
504 static DEFINE_PER_CPU(u64, running_sample_length);
505 
506 static u64 __report_avg;
507 static u64 __report_allowed;
508 
perf_duration_warn(struct irq_work * w)509 static void perf_duration_warn(struct irq_work *w)
510 {
511 	printk_ratelimited(KERN_INFO
512 		"perf: interrupt took too long (%lld > %lld), lowering "
513 		"kernel.perf_event_max_sample_rate to %d\n",
514 		__report_avg, __report_allowed,
515 		sysctl_perf_event_sample_rate);
516 }
517 
518 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
519 
perf_sample_event_took(u64 sample_len_ns)520 void perf_sample_event_took(u64 sample_len_ns)
521 {
522 	u64 max_len = READ_ONCE(perf_sample_allowed_ns);
523 	u64 running_len;
524 	u64 avg_len;
525 	u32 max;
526 
527 	if (max_len == 0)
528 		return;
529 
530 	/* Decay the counter by 1 average sample. */
531 	running_len = __this_cpu_read(running_sample_length);
532 	running_len -= running_len/NR_ACCUMULATED_SAMPLES;
533 	running_len += sample_len_ns;
534 	__this_cpu_write(running_sample_length, running_len);
535 
536 	/*
537 	 * Note: this will be biased artifically low until we have
538 	 * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us
539 	 * from having to maintain a count.
540 	 */
541 	avg_len = running_len/NR_ACCUMULATED_SAMPLES;
542 	if (avg_len <= max_len)
543 		return;
544 
545 	__report_avg = avg_len;
546 	__report_allowed = max_len;
547 
548 	/*
549 	 * Compute a throttle threshold 25% below the current duration.
550 	 */
551 	avg_len += avg_len / 4;
552 	max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent;
553 	if (avg_len < max)
554 		max /= (u32)avg_len;
555 	else
556 		max = 1;
557 
558 	WRITE_ONCE(perf_sample_allowed_ns, avg_len);
559 	WRITE_ONCE(max_samples_per_tick, max);
560 
561 	sysctl_perf_event_sample_rate = max * HZ;
562 	perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
563 
564 	if (!irq_work_queue(&perf_duration_work)) {
565 		early_printk("perf: interrupt took too long (%lld > %lld), lowering "
566 			     "kernel.perf_event_max_sample_rate to %d\n",
567 			     __report_avg, __report_allowed,
568 			     sysctl_perf_event_sample_rate);
569 	}
570 }
571 
572 static atomic64_t perf_event_id;
573 
574 static void update_context_time(struct perf_event_context *ctx);
575 static u64 perf_event_time(struct perf_event *event);
576 
perf_event_print_debug(void)577 void __weak perf_event_print_debug(void)	{ }
578 
perf_clock(void)579 static inline u64 perf_clock(void)
580 {
581 	return local_clock();
582 }
583 
perf_event_clock(struct perf_event * event)584 static inline u64 perf_event_clock(struct perf_event *event)
585 {
586 	return event->clock();
587 }
588 
589 /*
590  * State based event timekeeping...
591  *
592  * The basic idea is to use event->state to determine which (if any) time
593  * fields to increment with the current delta. This means we only need to
594  * update timestamps when we change state or when they are explicitly requested
595  * (read).
596  *
597  * Event groups make things a little more complicated, but not terribly so. The
598  * rules for a group are that if the group leader is OFF the entire group is
599  * OFF, irrespecive of what the group member states are. This results in
600  * __perf_effective_state().
601  *
602  * A futher ramification is that when a group leader flips between OFF and
603  * !OFF, we need to update all group member times.
604  *
605  *
606  * NOTE: perf_event_time() is based on the (cgroup) context time, and thus we
607  * need to make sure the relevant context time is updated before we try and
608  * update our timestamps.
609  */
610 
611 static __always_inline enum perf_event_state
__perf_effective_state(struct perf_event * event)612 __perf_effective_state(struct perf_event *event)
613 {
614 	struct perf_event *leader = event->group_leader;
615 
616 	if (leader->state <= PERF_EVENT_STATE_OFF)
617 		return leader->state;
618 
619 	return event->state;
620 }
621 
622 static __always_inline void
__perf_update_times(struct perf_event * event,u64 now,u64 * enabled,u64 * running)623 __perf_update_times(struct perf_event *event, u64 now, u64 *enabled, u64 *running)
624 {
625 	enum perf_event_state state = __perf_effective_state(event);
626 	u64 delta = now - event->tstamp;
627 
628 	*enabled = event->total_time_enabled;
629 	if (state >= PERF_EVENT_STATE_INACTIVE)
630 		*enabled += delta;
631 
632 	*running = event->total_time_running;
633 	if (state >= PERF_EVENT_STATE_ACTIVE)
634 		*running += delta;
635 }
636 
perf_event_update_time(struct perf_event * event)637 static void perf_event_update_time(struct perf_event *event)
638 {
639 	u64 now = perf_event_time(event);
640 
641 	__perf_update_times(event, now, &event->total_time_enabled,
642 					&event->total_time_running);
643 	event->tstamp = now;
644 }
645 
perf_event_update_sibling_time(struct perf_event * leader)646 static void perf_event_update_sibling_time(struct perf_event *leader)
647 {
648 	struct perf_event *sibling;
649 
650 	for_each_sibling_event(sibling, leader)
651 		perf_event_update_time(sibling);
652 }
653 
654 static void
perf_event_set_state(struct perf_event * event,enum perf_event_state state)655 perf_event_set_state(struct perf_event *event, enum perf_event_state state)
656 {
657 	if (event->state == state)
658 		return;
659 
660 	perf_event_update_time(event);
661 	/*
662 	 * If a group leader gets enabled/disabled all its siblings
663 	 * are affected too.
664 	 */
665 	if ((event->state < 0) ^ (state < 0))
666 		perf_event_update_sibling_time(event);
667 
668 	WRITE_ONCE(event->state, state);
669 }
670 
671 /*
672  * UP store-release, load-acquire
673  */
674 
675 #define __store_release(ptr, val)					\
676 do {									\
677 	barrier();							\
678 	WRITE_ONCE(*(ptr), (val));					\
679 } while (0)
680 
681 #define __load_acquire(ptr)						\
682 ({									\
683 	__unqual_scalar_typeof(*(ptr)) ___p = READ_ONCE(*(ptr));	\
684 	barrier();							\
685 	___p;								\
686 })
687 
perf_ctx_disable(struct perf_event_context * ctx,bool cgroup)688 static void perf_ctx_disable(struct perf_event_context *ctx, bool cgroup)
689 {
690 	struct perf_event_pmu_context *pmu_ctx;
691 
692 	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
693 		if (cgroup && !pmu_ctx->nr_cgroups)
694 			continue;
695 		perf_pmu_disable(pmu_ctx->pmu);
696 	}
697 }
698 
perf_ctx_enable(struct perf_event_context * ctx,bool cgroup)699 static void perf_ctx_enable(struct perf_event_context *ctx, bool cgroup)
700 {
701 	struct perf_event_pmu_context *pmu_ctx;
702 
703 	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
704 		if (cgroup && !pmu_ctx->nr_cgroups)
705 			continue;
706 		perf_pmu_enable(pmu_ctx->pmu);
707 	}
708 }
709 
710 static void ctx_sched_out(struct perf_event_context *ctx, enum event_type_t event_type);
711 static void ctx_sched_in(struct perf_event_context *ctx, enum event_type_t event_type);
712 
713 #ifdef CONFIG_CGROUP_PERF
714 
715 static inline bool
perf_cgroup_match(struct perf_event * event)716 perf_cgroup_match(struct perf_event *event)
717 {
718 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
719 
720 	/* @event doesn't care about cgroup */
721 	if (!event->cgrp)
722 		return true;
723 
724 	/* wants specific cgroup scope but @cpuctx isn't associated with any */
725 	if (!cpuctx->cgrp)
726 		return false;
727 
728 	/*
729 	 * Cgroup scoping is recursive.  An event enabled for a cgroup is
730 	 * also enabled for all its descendant cgroups.  If @cpuctx's
731 	 * cgroup is a descendant of @event's (the test covers identity
732 	 * case), it's a match.
733 	 */
734 	return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
735 				    event->cgrp->css.cgroup);
736 }
737 
perf_detach_cgroup(struct perf_event * event)738 static inline void perf_detach_cgroup(struct perf_event *event)
739 {
740 	css_put(&event->cgrp->css);
741 	event->cgrp = NULL;
742 }
743 
is_cgroup_event(struct perf_event * event)744 static inline int is_cgroup_event(struct perf_event *event)
745 {
746 	return event->cgrp != NULL;
747 }
748 
perf_cgroup_event_time(struct perf_event * event)749 static inline u64 perf_cgroup_event_time(struct perf_event *event)
750 {
751 	struct perf_cgroup_info *t;
752 
753 	t = per_cpu_ptr(event->cgrp->info, event->cpu);
754 	return t->time;
755 }
756 
perf_cgroup_event_time_now(struct perf_event * event,u64 now)757 static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
758 {
759 	struct perf_cgroup_info *t;
760 
761 	t = per_cpu_ptr(event->cgrp->info, event->cpu);
762 	if (!__load_acquire(&t->active))
763 		return t->time;
764 	now += READ_ONCE(t->timeoffset);
765 	return now;
766 }
767 
__update_cgrp_time(struct perf_cgroup_info * info,u64 now,bool adv)768 static inline void __update_cgrp_time(struct perf_cgroup_info *info, u64 now, bool adv)
769 {
770 	if (adv)
771 		info->time += now - info->timestamp;
772 	info->timestamp = now;
773 	/*
774 	 * see update_context_time()
775 	 */
776 	WRITE_ONCE(info->timeoffset, info->time - info->timestamp);
777 }
778 
update_cgrp_time_from_cpuctx(struct perf_cpu_context * cpuctx,bool final)779 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final)
780 {
781 	struct perf_cgroup *cgrp = cpuctx->cgrp;
782 	struct cgroup_subsys_state *css;
783 	struct perf_cgroup_info *info;
784 
785 	if (cgrp) {
786 		u64 now = perf_clock();
787 
788 		for (css = &cgrp->css; css; css = css->parent) {
789 			cgrp = container_of(css, struct perf_cgroup, css);
790 			info = this_cpu_ptr(cgrp->info);
791 
792 			__update_cgrp_time(info, now, true);
793 			if (final)
794 				__store_release(&info->active, 0);
795 		}
796 	}
797 }
798 
update_cgrp_time_from_event(struct perf_event * event)799 static inline void update_cgrp_time_from_event(struct perf_event *event)
800 {
801 	struct perf_cgroup_info *info;
802 
803 	/*
804 	 * ensure we access cgroup data only when needed and
805 	 * when we know the cgroup is pinned (css_get)
806 	 */
807 	if (!is_cgroup_event(event))
808 		return;
809 
810 	info = this_cpu_ptr(event->cgrp->info);
811 	/*
812 	 * Do not update time when cgroup is not active
813 	 */
814 	if (info->active)
815 		__update_cgrp_time(info, perf_clock(), true);
816 }
817 
818 static inline void
perf_cgroup_set_timestamp(struct perf_cpu_context * cpuctx)819 perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx)
820 {
821 	struct perf_event_context *ctx = &cpuctx->ctx;
822 	struct perf_cgroup *cgrp = cpuctx->cgrp;
823 	struct perf_cgroup_info *info;
824 	struct cgroup_subsys_state *css;
825 
826 	/*
827 	 * ctx->lock held by caller
828 	 * ensure we do not access cgroup data
829 	 * unless we have the cgroup pinned (css_get)
830 	 */
831 	if (!cgrp)
832 		return;
833 
834 	WARN_ON_ONCE(!ctx->nr_cgroups);
835 
836 	for (css = &cgrp->css; css; css = css->parent) {
837 		cgrp = container_of(css, struct perf_cgroup, css);
838 		info = this_cpu_ptr(cgrp->info);
839 		__update_cgrp_time(info, ctx->timestamp, false);
840 		__store_release(&info->active, 1);
841 	}
842 }
843 
844 /*
845  * reschedule events based on the cgroup constraint of task.
846  */
perf_cgroup_switch(struct task_struct * task)847 static void perf_cgroup_switch(struct task_struct *task)
848 {
849 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
850 	struct perf_cgroup *cgrp;
851 
852 	/*
853 	 * cpuctx->cgrp is set when the first cgroup event enabled,
854 	 * and is cleared when the last cgroup event disabled.
855 	 */
856 	if (READ_ONCE(cpuctx->cgrp) == NULL)
857 		return;
858 
859 	WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);
860 
861 	cgrp = perf_cgroup_from_task(task, NULL);
862 	if (READ_ONCE(cpuctx->cgrp) == cgrp)
863 		return;
864 
865 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
866 	perf_ctx_disable(&cpuctx->ctx, true);
867 
868 	ctx_sched_out(&cpuctx->ctx, EVENT_ALL|EVENT_CGROUP);
869 	/*
870 	 * must not be done before ctxswout due
871 	 * to update_cgrp_time_from_cpuctx() in
872 	 * ctx_sched_out()
873 	 */
874 	cpuctx->cgrp = cgrp;
875 	/*
876 	 * set cgrp before ctxsw in to allow
877 	 * perf_cgroup_set_timestamp() in ctx_sched_in()
878 	 * to not have to pass task around
879 	 */
880 	ctx_sched_in(&cpuctx->ctx, EVENT_ALL|EVENT_CGROUP);
881 
882 	perf_ctx_enable(&cpuctx->ctx, true);
883 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
884 }
885 
perf_cgroup_ensure_storage(struct perf_event * event,struct cgroup_subsys_state * css)886 static int perf_cgroup_ensure_storage(struct perf_event *event,
887 				struct cgroup_subsys_state *css)
888 {
889 	struct perf_cpu_context *cpuctx;
890 	struct perf_event **storage;
891 	int cpu, heap_size, ret = 0;
892 
893 	/*
894 	 * Allow storage to have sufficent space for an iterator for each
895 	 * possibly nested cgroup plus an iterator for events with no cgroup.
896 	 */
897 	for (heap_size = 1; css; css = css->parent)
898 		heap_size++;
899 
900 	for_each_possible_cpu(cpu) {
901 		cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
902 		if (heap_size <= cpuctx->heap_size)
903 			continue;
904 
905 		storage = kmalloc_node(heap_size * sizeof(struct perf_event *),
906 				       GFP_KERNEL, cpu_to_node(cpu));
907 		if (!storage) {
908 			ret = -ENOMEM;
909 			break;
910 		}
911 
912 		raw_spin_lock_irq(&cpuctx->ctx.lock);
913 		if (cpuctx->heap_size < heap_size) {
914 			swap(cpuctx->heap, storage);
915 			if (storage == cpuctx->heap_default)
916 				storage = NULL;
917 			cpuctx->heap_size = heap_size;
918 		}
919 		raw_spin_unlock_irq(&cpuctx->ctx.lock);
920 
921 		kfree(storage);
922 	}
923 
924 	return ret;
925 }
926 
perf_cgroup_connect(int fd,struct perf_event * event,struct perf_event_attr * attr,struct perf_event * group_leader)927 static inline int perf_cgroup_connect(int fd, struct perf_event *event,
928 				      struct perf_event_attr *attr,
929 				      struct perf_event *group_leader)
930 {
931 	struct perf_cgroup *cgrp;
932 	struct cgroup_subsys_state *css;
933 	struct fd f = fdget(fd);
934 	int ret = 0;
935 
936 	if (!f.file)
937 		return -EBADF;
938 
939 	css = css_tryget_online_from_dir(f.file->f_path.dentry,
940 					 &perf_event_cgrp_subsys);
941 	if (IS_ERR(css)) {
942 		ret = PTR_ERR(css);
943 		goto out;
944 	}
945 
946 	ret = perf_cgroup_ensure_storage(event, css);
947 	if (ret)
948 		goto out;
949 
950 	cgrp = container_of(css, struct perf_cgroup, css);
951 	event->cgrp = cgrp;
952 
953 	/*
954 	 * all events in a group must monitor
955 	 * the same cgroup because a task belongs
956 	 * to only one perf cgroup at a time
957 	 */
958 	if (group_leader && group_leader->cgrp != cgrp) {
959 		perf_detach_cgroup(event);
960 		ret = -EINVAL;
961 	}
962 out:
963 	fdput(f);
964 	return ret;
965 }
966 
967 static inline void
perf_cgroup_event_enable(struct perf_event * event,struct perf_event_context * ctx)968 perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
969 {
970 	struct perf_cpu_context *cpuctx;
971 
972 	if (!is_cgroup_event(event))
973 		return;
974 
975 	event->pmu_ctx->nr_cgroups++;
976 
977 	/*
978 	 * Because cgroup events are always per-cpu events,
979 	 * @ctx == &cpuctx->ctx.
980 	 */
981 	cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
982 
983 	if (ctx->nr_cgroups++)
984 		return;
985 
986 	cpuctx->cgrp = perf_cgroup_from_task(current, ctx);
987 }
988 
989 static inline void
perf_cgroup_event_disable(struct perf_event * event,struct perf_event_context * ctx)990 perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
991 {
992 	struct perf_cpu_context *cpuctx;
993 
994 	if (!is_cgroup_event(event))
995 		return;
996 
997 	event->pmu_ctx->nr_cgroups--;
998 
999 	/*
1000 	 * Because cgroup events are always per-cpu events,
1001 	 * @ctx == &cpuctx->ctx.
1002 	 */
1003 	cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
1004 
1005 	if (--ctx->nr_cgroups)
1006 		return;
1007 
1008 	cpuctx->cgrp = NULL;
1009 }
1010 
1011 #else /* !CONFIG_CGROUP_PERF */
1012 
1013 static inline bool
perf_cgroup_match(struct perf_event * event)1014 perf_cgroup_match(struct perf_event *event)
1015 {
1016 	return true;
1017 }
1018 
perf_detach_cgroup(struct perf_event * event)1019 static inline void perf_detach_cgroup(struct perf_event *event)
1020 {}
1021 
is_cgroup_event(struct perf_event * event)1022 static inline int is_cgroup_event(struct perf_event *event)
1023 {
1024 	return 0;
1025 }
1026 
update_cgrp_time_from_event(struct perf_event * event)1027 static inline void update_cgrp_time_from_event(struct perf_event *event)
1028 {
1029 }
1030 
update_cgrp_time_from_cpuctx(struct perf_cpu_context * cpuctx,bool final)1031 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx,
1032 						bool final)
1033 {
1034 }
1035 
perf_cgroup_connect(pid_t pid,struct perf_event * event,struct perf_event_attr * attr,struct perf_event * group_leader)1036 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
1037 				      struct perf_event_attr *attr,
1038 				      struct perf_event *group_leader)
1039 {
1040 	return -EINVAL;
1041 }
1042 
1043 static inline void
perf_cgroup_set_timestamp(struct perf_cpu_context * cpuctx)1044 perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx)
1045 {
1046 }
1047 
perf_cgroup_event_time(struct perf_event * event)1048 static inline u64 perf_cgroup_event_time(struct perf_event *event)
1049 {
1050 	return 0;
1051 }
1052 
perf_cgroup_event_time_now(struct perf_event * event,u64 now)1053 static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)
1054 {
1055 	return 0;
1056 }
1057 
1058 static inline void
perf_cgroup_event_enable(struct perf_event * event,struct perf_event_context * ctx)1059 perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
1060 {
1061 }
1062 
1063 static inline void
perf_cgroup_event_disable(struct perf_event * event,struct perf_event_context * ctx)1064 perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
1065 {
1066 }
1067 
perf_cgroup_switch(struct task_struct * task)1068 static void perf_cgroup_switch(struct task_struct *task)
1069 {
1070 }
1071 #endif
1072 
1073 /*
1074  * set default to be dependent on timer tick just
1075  * like original code
1076  */
1077 #define PERF_CPU_HRTIMER (1000 / HZ)
1078 /*
1079  * function must be called with interrupts disabled
1080  */
perf_mux_hrtimer_handler(struct hrtimer * hr)1081 static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)
1082 {
1083 	struct perf_cpu_pmu_context *cpc;
1084 	bool rotations;
1085 
1086 	lockdep_assert_irqs_disabled();
1087 
1088 	cpc = container_of(hr, struct perf_cpu_pmu_context, hrtimer);
1089 	rotations = perf_rotate_context(cpc);
1090 
1091 	raw_spin_lock(&cpc->hrtimer_lock);
1092 	if (rotations)
1093 		hrtimer_forward_now(hr, cpc->hrtimer_interval);
1094 	else
1095 		cpc->hrtimer_active = 0;
1096 	raw_spin_unlock(&cpc->hrtimer_lock);
1097 
1098 	return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART;
1099 }
1100 
__perf_mux_hrtimer_init(struct perf_cpu_pmu_context * cpc,int cpu)1101 static void __perf_mux_hrtimer_init(struct perf_cpu_pmu_context *cpc, int cpu)
1102 {
1103 	struct hrtimer *timer = &cpc->hrtimer;
1104 	struct pmu *pmu = cpc->epc.pmu;
1105 	u64 interval;
1106 
1107 	/*
1108 	 * check default is sane, if not set then force to
1109 	 * default interval (1/tick)
1110 	 */
1111 	interval = pmu->hrtimer_interval_ms;
1112 	if (interval < 1)
1113 		interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
1114 
1115 	cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);
1116 
1117 	raw_spin_lock_init(&cpc->hrtimer_lock);
1118 	hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1119 	timer->function = perf_mux_hrtimer_handler;
1120 }
1121 
perf_mux_hrtimer_restart(struct perf_cpu_pmu_context * cpc)1122 static int perf_mux_hrtimer_restart(struct perf_cpu_pmu_context *cpc)
1123 {
1124 	struct hrtimer *timer = &cpc->hrtimer;
1125 	unsigned long flags;
1126 
1127 	raw_spin_lock_irqsave(&cpc->hrtimer_lock, flags);
1128 	if (!cpc->hrtimer_active) {
1129 		cpc->hrtimer_active = 1;
1130 		hrtimer_forward_now(timer, cpc->hrtimer_interval);
1131 		hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD);
1132 	}
1133 	raw_spin_unlock_irqrestore(&cpc->hrtimer_lock, flags);
1134 
1135 	return 0;
1136 }
1137 
perf_mux_hrtimer_restart_ipi(void * arg)1138 static int perf_mux_hrtimer_restart_ipi(void *arg)
1139 {
1140 	return perf_mux_hrtimer_restart(arg);
1141 }
1142 
perf_pmu_disable(struct pmu * pmu)1143 void perf_pmu_disable(struct pmu *pmu)
1144 {
1145 	int *count = this_cpu_ptr(pmu->pmu_disable_count);
1146 	if (!(*count)++)
1147 		pmu->pmu_disable(pmu);
1148 }
1149 
perf_pmu_enable(struct pmu * pmu)1150 void perf_pmu_enable(struct pmu *pmu)
1151 {
1152 	int *count = this_cpu_ptr(pmu->pmu_disable_count);
1153 	if (!--(*count))
1154 		pmu->pmu_enable(pmu);
1155 }
1156 
perf_assert_pmu_disabled(struct pmu * pmu)1157 static void perf_assert_pmu_disabled(struct pmu *pmu)
1158 {
1159 	WARN_ON_ONCE(*this_cpu_ptr(pmu->pmu_disable_count) == 0);
1160 }
1161 
get_ctx(struct perf_event_context * ctx)1162 static void get_ctx(struct perf_event_context *ctx)
1163 {
1164 	refcount_inc(&ctx->refcount);
1165 }
1166 
alloc_task_ctx_data(struct pmu * pmu)1167 static void *alloc_task_ctx_data(struct pmu *pmu)
1168 {
1169 	if (pmu->task_ctx_cache)
1170 		return kmem_cache_zalloc(pmu->task_ctx_cache, GFP_KERNEL);
1171 
1172 	return NULL;
1173 }
1174 
free_task_ctx_data(struct pmu * pmu,void * task_ctx_data)1175 static void free_task_ctx_data(struct pmu *pmu, void *task_ctx_data)
1176 {
1177 	if (pmu->task_ctx_cache && task_ctx_data)
1178 		kmem_cache_free(pmu->task_ctx_cache, task_ctx_data);
1179 }
1180 
free_ctx(struct rcu_head * head)1181 static void free_ctx(struct rcu_head *head)
1182 {
1183 	struct perf_event_context *ctx;
1184 
1185 	ctx = container_of(head, struct perf_event_context, rcu_head);
1186 	kfree(ctx);
1187 }
1188 
put_ctx(struct perf_event_context * ctx)1189 static void put_ctx(struct perf_event_context *ctx)
1190 {
1191 	if (refcount_dec_and_test(&ctx->refcount)) {
1192 		if (ctx->parent_ctx)
1193 			put_ctx(ctx->parent_ctx);
1194 		if (ctx->task && ctx->task != TASK_TOMBSTONE)
1195 			put_task_struct(ctx->task);
1196 		call_rcu(&ctx->rcu_head, free_ctx);
1197 	}
1198 }
1199 
1200 /*
1201  * Because of perf_event::ctx migration in sys_perf_event_open::move_group and
1202  * perf_pmu_migrate_context() we need some magic.
1203  *
1204  * Those places that change perf_event::ctx will hold both
1205  * perf_event_ctx::mutex of the 'old' and 'new' ctx value.
1206  *
1207  * Lock ordering is by mutex address. There are two other sites where
1208  * perf_event_context::mutex nests and those are:
1209  *
1210  *  - perf_event_exit_task_context()	[ child , 0 ]
1211  *      perf_event_exit_event()
1212  *        put_event()			[ parent, 1 ]
1213  *
1214  *  - perf_event_init_context()		[ parent, 0 ]
1215  *      inherit_task_group()
1216  *        inherit_group()
1217  *          inherit_event()
1218  *            perf_event_alloc()
1219  *              perf_init_event()
1220  *                perf_try_init_event()	[ child , 1 ]
1221  *
1222  * While it appears there is an obvious deadlock here -- the parent and child
1223  * nesting levels are inverted between the two. This is in fact safe because
1224  * life-time rules separate them. That is an exiting task cannot fork, and a
1225  * spawning task cannot (yet) exit.
1226  *
1227  * But remember that these are parent<->child context relations, and
1228  * migration does not affect children, therefore these two orderings should not
1229  * interact.
1230  *
1231  * The change in perf_event::ctx does not affect children (as claimed above)
1232  * because the sys_perf_event_open() case will install a new event and break
1233  * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
1234  * concerned with cpuctx and that doesn't have children.
1235  *
1236  * The places that change perf_event::ctx will issue:
1237  *
1238  *   perf_remove_from_context();
1239  *   synchronize_rcu();
1240  *   perf_install_in_context();
1241  *
1242  * to affect the change. The remove_from_context() + synchronize_rcu() should
1243  * quiesce the event, after which we can install it in the new location. This
1244  * means that only external vectors (perf_fops, prctl) can perturb the event
1245  * while in transit. Therefore all such accessors should also acquire
1246  * perf_event_context::mutex to serialize against this.
1247  *
1248  * However; because event->ctx can change while we're waiting to acquire
1249  * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
1250  * function.
1251  *
1252  * Lock order:
1253  *    exec_update_lock
1254  *	task_struct::perf_event_mutex
1255  *	  perf_event_context::mutex
1256  *	    perf_event::child_mutex;
1257  *	      perf_event_context::lock
1258  *	    perf_event::mmap_mutex
1259  *	    mmap_lock
1260  *	      perf_addr_filters_head::lock
1261  *
1262  *    cpu_hotplug_lock
1263  *      pmus_lock
1264  *	  cpuctx->mutex / perf_event_context::mutex
1265  */
1266 static struct perf_event_context *
perf_event_ctx_lock_nested(struct perf_event * event,int nesting)1267 perf_event_ctx_lock_nested(struct perf_event *event, int nesting)
1268 {
1269 	struct perf_event_context *ctx;
1270 
1271 again:
1272 	rcu_read_lock();
1273 	ctx = READ_ONCE(event->ctx);
1274 	if (!refcount_inc_not_zero(&ctx->refcount)) {
1275 		rcu_read_unlock();
1276 		goto again;
1277 	}
1278 	rcu_read_unlock();
1279 
1280 	mutex_lock_nested(&ctx->mutex, nesting);
1281 	if (event->ctx != ctx) {
1282 		mutex_unlock(&ctx->mutex);
1283 		put_ctx(ctx);
1284 		goto again;
1285 	}
1286 
1287 	return ctx;
1288 }
1289 
1290 static inline struct perf_event_context *
perf_event_ctx_lock(struct perf_event * event)1291 perf_event_ctx_lock(struct perf_event *event)
1292 {
1293 	return perf_event_ctx_lock_nested(event, 0);
1294 }
1295 
perf_event_ctx_unlock(struct perf_event * event,struct perf_event_context * ctx)1296 static void perf_event_ctx_unlock(struct perf_event *event,
1297 				  struct perf_event_context *ctx)
1298 {
1299 	mutex_unlock(&ctx->mutex);
1300 	put_ctx(ctx);
1301 }
1302 
1303 /*
1304  * This must be done under the ctx->lock, such as to serialize against
1305  * context_equiv(), therefore we cannot call put_ctx() since that might end up
1306  * calling scheduler related locks and ctx->lock nests inside those.
1307  */
1308 static __must_check struct perf_event_context *
unclone_ctx(struct perf_event_context * ctx)1309 unclone_ctx(struct perf_event_context *ctx)
1310 {
1311 	struct perf_event_context *parent_ctx = ctx->parent_ctx;
1312 
1313 	lockdep_assert_held(&ctx->lock);
1314 
1315 	if (parent_ctx)
1316 		ctx->parent_ctx = NULL;
1317 	ctx->generation++;
1318 
1319 	return parent_ctx;
1320 }
1321 
perf_event_pid_type(struct perf_event * event,struct task_struct * p,enum pid_type type)1322 static u32 perf_event_pid_type(struct perf_event *event, struct task_struct *p,
1323 				enum pid_type type)
1324 {
1325 	u32 nr;
1326 	/*
1327 	 * only top level events have the pid namespace they were created in
1328 	 */
1329 	if (event->parent)
1330 		event = event->parent;
1331 
1332 	nr = __task_pid_nr_ns(p, type, event->ns);
1333 	/* avoid -1 if it is idle thread or runs in another ns */
1334 	if (!nr && !pid_alive(p))
1335 		nr = -1;
1336 	return nr;
1337 }
1338 
perf_event_pid(struct perf_event * event,struct task_struct * p)1339 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
1340 {
1341 	return perf_event_pid_type(event, p, PIDTYPE_TGID);
1342 }
1343 
perf_event_tid(struct perf_event * event,struct task_struct * p)1344 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
1345 {
1346 	return perf_event_pid_type(event, p, PIDTYPE_PID);
1347 }
1348 
1349 /*
1350  * If we inherit events we want to return the parent event id
1351  * to userspace.
1352  */
primary_event_id(struct perf_event * event)1353 static u64 primary_event_id(struct perf_event *event)
1354 {
1355 	u64 id = event->id;
1356 
1357 	if (event->parent)
1358 		id = event->parent->id;
1359 
1360 	return id;
1361 }
1362 
1363 /*
1364  * Get the perf_event_context for a task and lock it.
1365  *
1366  * This has to cope with the fact that until it is locked,
1367  * the context could get moved to another task.
1368  */
1369 static struct perf_event_context *
perf_lock_task_context(struct task_struct * task,unsigned long * flags)1370 perf_lock_task_context(struct task_struct *task, unsigned long *flags)
1371 {
1372 	struct perf_event_context *ctx;
1373 
1374 retry:
1375 	/*
1376 	 * One of the few rules of preemptible RCU is that one cannot do
1377 	 * rcu_read_unlock() while holding a scheduler (or nested) lock when
1378 	 * part of the read side critical section was irqs-enabled -- see
1379 	 * rcu_read_unlock_special().
1380 	 *
1381 	 * Since ctx->lock nests under rq->lock we must ensure the entire read
1382 	 * side critical section has interrupts disabled.
1383 	 */
1384 	local_irq_save(*flags);
1385 	rcu_read_lock();
1386 	ctx = rcu_dereference(task->perf_event_ctxp);
1387 	if (ctx) {
1388 		/*
1389 		 * If this context is a clone of another, it might
1390 		 * get swapped for another underneath us by
1391 		 * perf_event_task_sched_out, though the
1392 		 * rcu_read_lock() protects us from any context
1393 		 * getting freed.  Lock the context and check if it
1394 		 * got swapped before we could get the lock, and retry
1395 		 * if so.  If we locked the right context, then it
1396 		 * can't get swapped on us any more.
1397 		 */
1398 		raw_spin_lock(&ctx->lock);
1399 		if (ctx != rcu_dereference(task->perf_event_ctxp)) {
1400 			raw_spin_unlock(&ctx->lock);
1401 			rcu_read_unlock();
1402 			local_irq_restore(*flags);
1403 			goto retry;
1404 		}
1405 
1406 		if (ctx->task == TASK_TOMBSTONE ||
1407 		    !refcount_inc_not_zero(&ctx->refcount)) {
1408 			raw_spin_unlock(&ctx->lock);
1409 			ctx = NULL;
1410 		} else {
1411 			WARN_ON_ONCE(ctx->task != task);
1412 		}
1413 	}
1414 	rcu_read_unlock();
1415 	if (!ctx)
1416 		local_irq_restore(*flags);
1417 	return ctx;
1418 }
1419 
1420 /*
1421  * Get the context for a task and increment its pin_count so it
1422  * can't get swapped to another task.  This also increments its
1423  * reference count so that the context can't get freed.
1424  */
1425 static struct perf_event_context *
perf_pin_task_context(struct task_struct * task)1426 perf_pin_task_context(struct task_struct *task)
1427 {
1428 	struct perf_event_context *ctx;
1429 	unsigned long flags;
1430 
1431 	ctx = perf_lock_task_context(task, &flags);
1432 	if (ctx) {
1433 		++ctx->pin_count;
1434 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
1435 	}
1436 	return ctx;
1437 }
1438 
perf_unpin_context(struct perf_event_context * ctx)1439 static void perf_unpin_context(struct perf_event_context *ctx)
1440 {
1441 	unsigned long flags;
1442 
1443 	raw_spin_lock_irqsave(&ctx->lock, flags);
1444 	--ctx->pin_count;
1445 	raw_spin_unlock_irqrestore(&ctx->lock, flags);
1446 }
1447 
1448 /*
1449  * Update the record of the current time in a context.
1450  */
__update_context_time(struct perf_event_context * ctx,bool adv)1451 static void __update_context_time(struct perf_event_context *ctx, bool adv)
1452 {
1453 	u64 now = perf_clock();
1454 
1455 	lockdep_assert_held(&ctx->lock);
1456 
1457 	if (adv)
1458 		ctx->time += now - ctx->timestamp;
1459 	ctx->timestamp = now;
1460 
1461 	/*
1462 	 * The above: time' = time + (now - timestamp), can be re-arranged
1463 	 * into: time` = now + (time - timestamp), which gives a single value
1464 	 * offset to compute future time without locks on.
1465 	 *
1466 	 * See perf_event_time_now(), which can be used from NMI context where
1467 	 * it's (obviously) not possible to acquire ctx->lock in order to read
1468 	 * both the above values in a consistent manner.
1469 	 */
1470 	WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp);
1471 }
1472 
update_context_time(struct perf_event_context * ctx)1473 static void update_context_time(struct perf_event_context *ctx)
1474 {
1475 	__update_context_time(ctx, true);
1476 }
1477 
perf_event_time(struct perf_event * event)1478 static u64 perf_event_time(struct perf_event *event)
1479 {
1480 	struct perf_event_context *ctx = event->ctx;
1481 
1482 	if (unlikely(!ctx))
1483 		return 0;
1484 
1485 	if (is_cgroup_event(event))
1486 		return perf_cgroup_event_time(event);
1487 
1488 	return ctx->time;
1489 }
1490 
perf_event_time_now(struct perf_event * event,u64 now)1491 static u64 perf_event_time_now(struct perf_event *event, u64 now)
1492 {
1493 	struct perf_event_context *ctx = event->ctx;
1494 
1495 	if (unlikely(!ctx))
1496 		return 0;
1497 
1498 	if (is_cgroup_event(event))
1499 		return perf_cgroup_event_time_now(event, now);
1500 
1501 	if (!(__load_acquire(&ctx->is_active) & EVENT_TIME))
1502 		return ctx->time;
1503 
1504 	now += READ_ONCE(ctx->timeoffset);
1505 	return now;
1506 }
1507 
get_event_type(struct perf_event * event)1508 static enum event_type_t get_event_type(struct perf_event *event)
1509 {
1510 	struct perf_event_context *ctx = event->ctx;
1511 	enum event_type_t event_type;
1512 
1513 	lockdep_assert_held(&ctx->lock);
1514 
1515 	/*
1516 	 * It's 'group type', really, because if our group leader is
1517 	 * pinned, so are we.
1518 	 */
1519 	if (event->group_leader != event)
1520 		event = event->group_leader;
1521 
1522 	event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE;
1523 	if (!ctx->task)
1524 		event_type |= EVENT_CPU;
1525 
1526 	return event_type;
1527 }
1528 
1529 /*
1530  * Helper function to initialize event group nodes.
1531  */
init_event_group(struct perf_event * event)1532 static void init_event_group(struct perf_event *event)
1533 {
1534 	RB_CLEAR_NODE(&event->group_node);
1535 	event->group_index = 0;
1536 }
1537 
1538 /*
1539  * Extract pinned or flexible groups from the context
1540  * based on event attrs bits.
1541  */
1542 static struct perf_event_groups *
get_event_groups(struct perf_event * event,struct perf_event_context * ctx)1543 get_event_groups(struct perf_event *event, struct perf_event_context *ctx)
1544 {
1545 	if (event->attr.pinned)
1546 		return &ctx->pinned_groups;
1547 	else
1548 		return &ctx->flexible_groups;
1549 }
1550 
1551 /*
1552  * Helper function to initializes perf_event_group trees.
1553  */
perf_event_groups_init(struct perf_event_groups * groups)1554 static void perf_event_groups_init(struct perf_event_groups *groups)
1555 {
1556 	groups->tree = RB_ROOT;
1557 	groups->index = 0;
1558 }
1559 
event_cgroup(const struct perf_event * event)1560 static inline struct cgroup *event_cgroup(const struct perf_event *event)
1561 {
1562 	struct cgroup *cgroup = NULL;
1563 
1564 #ifdef CONFIG_CGROUP_PERF
1565 	if (event->cgrp)
1566 		cgroup = event->cgrp->css.cgroup;
1567 #endif
1568 
1569 	return cgroup;
1570 }
1571 
1572 /*
1573  * Compare function for event groups;
1574  *
1575  * Implements complex key that first sorts by CPU and then by virtual index
1576  * which provides ordering when rotating groups for the same CPU.
1577  */
1578 static __always_inline int
perf_event_groups_cmp(const int left_cpu,const struct pmu * left_pmu,const struct cgroup * left_cgroup,const u64 left_group_index,const struct perf_event * right)1579 perf_event_groups_cmp(const int left_cpu, const struct pmu *left_pmu,
1580 		      const struct cgroup *left_cgroup, const u64 left_group_index,
1581 		      const struct perf_event *right)
1582 {
1583 	if (left_cpu < right->cpu)
1584 		return -1;
1585 	if (left_cpu > right->cpu)
1586 		return 1;
1587 
1588 	if (left_pmu) {
1589 		if (left_pmu < right->pmu_ctx->pmu)
1590 			return -1;
1591 		if (left_pmu > right->pmu_ctx->pmu)
1592 			return 1;
1593 	}
1594 
1595 #ifdef CONFIG_CGROUP_PERF
1596 	{
1597 		const struct cgroup *right_cgroup = event_cgroup(right);
1598 
1599 		if (left_cgroup != right_cgroup) {
1600 			if (!left_cgroup) {
1601 				/*
1602 				 * Left has no cgroup but right does, no
1603 				 * cgroups come first.
1604 				 */
1605 				return -1;
1606 			}
1607 			if (!right_cgroup) {
1608 				/*
1609 				 * Right has no cgroup but left does, no
1610 				 * cgroups come first.
1611 				 */
1612 				return 1;
1613 			}
1614 			/* Two dissimilar cgroups, order by id. */
1615 			if (cgroup_id(left_cgroup) < cgroup_id(right_cgroup))
1616 				return -1;
1617 
1618 			return 1;
1619 		}
1620 	}
1621 #endif
1622 
1623 	if (left_group_index < right->group_index)
1624 		return -1;
1625 	if (left_group_index > right->group_index)
1626 		return 1;
1627 
1628 	return 0;
1629 }
1630 
1631 #define __node_2_pe(node) \
1632 	rb_entry((node), struct perf_event, group_node)
1633 
__group_less(struct rb_node * a,const struct rb_node * b)1634 static inline bool __group_less(struct rb_node *a, const struct rb_node *b)
1635 {
1636 	struct perf_event *e = __node_2_pe(a);
1637 	return perf_event_groups_cmp(e->cpu, e->pmu_ctx->pmu, event_cgroup(e),
1638 				     e->group_index, __node_2_pe(b)) < 0;
1639 }
1640 
1641 struct __group_key {
1642 	int cpu;
1643 	struct pmu *pmu;
1644 	struct cgroup *cgroup;
1645 };
1646 
__group_cmp(const void * key,const struct rb_node * node)1647 static inline int __group_cmp(const void *key, const struct rb_node *node)
1648 {
1649 	const struct __group_key *a = key;
1650 	const struct perf_event *b = __node_2_pe(node);
1651 
1652 	/* partial/subtree match: @cpu, @pmu, @cgroup; ignore: @group_index */
1653 	return perf_event_groups_cmp(a->cpu, a->pmu, a->cgroup, b->group_index, b);
1654 }
1655 
1656 static inline int
__group_cmp_ignore_cgroup(const void * key,const struct rb_node * node)1657 __group_cmp_ignore_cgroup(const void *key, const struct rb_node *node)
1658 {
1659 	const struct __group_key *a = key;
1660 	const struct perf_event *b = __node_2_pe(node);
1661 
1662 	/* partial/subtree match: @cpu, @pmu, ignore: @cgroup, @group_index */
1663 	return perf_event_groups_cmp(a->cpu, a->pmu, event_cgroup(b),
1664 				     b->group_index, b);
1665 }
1666 
1667 /*
1668  * Insert @event into @groups' tree; using
1669  *   {@event->cpu, @event->pmu_ctx->pmu, event_cgroup(@event), ++@groups->index}
1670  * as key. This places it last inside the {cpu,pmu,cgroup} subtree.
1671  */
1672 static void
perf_event_groups_insert(struct perf_event_groups * groups,struct perf_event * event)1673 perf_event_groups_insert(struct perf_event_groups *groups,
1674 			 struct perf_event *event)
1675 {
1676 	event->group_index = ++groups->index;
1677 
1678 	rb_add(&event->group_node, &groups->tree, __group_less);
1679 }
1680 
1681 /*
1682  * Helper function to insert event into the pinned or flexible groups.
1683  */
1684 static void
add_event_to_groups(struct perf_event * event,struct perf_event_context * ctx)1685 add_event_to_groups(struct perf_event *event, struct perf_event_context *ctx)
1686 {
1687 	struct perf_event_groups *groups;
1688 
1689 	groups = get_event_groups(event, ctx);
1690 	perf_event_groups_insert(groups, event);
1691 }
1692 
1693 /*
1694  * Delete a group from a tree.
1695  */
1696 static void
perf_event_groups_delete(struct perf_event_groups * groups,struct perf_event * event)1697 perf_event_groups_delete(struct perf_event_groups *groups,
1698 			 struct perf_event *event)
1699 {
1700 	WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) ||
1701 		     RB_EMPTY_ROOT(&groups->tree));
1702 
1703 	rb_erase(&event->group_node, &groups->tree);
1704 	init_event_group(event);
1705 }
1706 
1707 /*
1708  * Helper function to delete event from its groups.
1709  */
1710 static void
del_event_from_groups(struct perf_event * event,struct perf_event_context * ctx)1711 del_event_from_groups(struct perf_event *event, struct perf_event_context *ctx)
1712 {
1713 	struct perf_event_groups *groups;
1714 
1715 	groups = get_event_groups(event, ctx);
1716 	perf_event_groups_delete(groups, event);
1717 }
1718 
1719 /*
1720  * Get the leftmost event in the {cpu,pmu,cgroup} subtree.
1721  */
1722 static struct perf_event *
perf_event_groups_first(struct perf_event_groups * groups,int cpu,struct pmu * pmu,struct cgroup * cgrp)1723 perf_event_groups_first(struct perf_event_groups *groups, int cpu,
1724 			struct pmu *pmu, struct cgroup *cgrp)
1725 {
1726 	struct __group_key key = {
1727 		.cpu = cpu,
1728 		.pmu = pmu,
1729 		.cgroup = cgrp,
1730 	};
1731 	struct rb_node *node;
1732 
1733 	node = rb_find_first(&key, &groups->tree, __group_cmp);
1734 	if (node)
1735 		return __node_2_pe(node);
1736 
1737 	return NULL;
1738 }
1739 
1740 static struct perf_event *
perf_event_groups_next(struct perf_event * event,struct pmu * pmu)1741 perf_event_groups_next(struct perf_event *event, struct pmu *pmu)
1742 {
1743 	struct __group_key key = {
1744 		.cpu = event->cpu,
1745 		.pmu = pmu,
1746 		.cgroup = event_cgroup(event),
1747 	};
1748 	struct rb_node *next;
1749 
1750 	next = rb_next_match(&key, &event->group_node, __group_cmp);
1751 	if (next)
1752 		return __node_2_pe(next);
1753 
1754 	return NULL;
1755 }
1756 
1757 #define perf_event_groups_for_cpu_pmu(event, groups, cpu, pmu)		\
1758 	for (event = perf_event_groups_first(groups, cpu, pmu, NULL);	\
1759 	     event; event = perf_event_groups_next(event, pmu))
1760 
1761 /*
1762  * Iterate through the whole groups tree.
1763  */
1764 #define perf_event_groups_for_each(event, groups)			\
1765 	for (event = rb_entry_safe(rb_first(&((groups)->tree)),		\
1766 				typeof(*event), group_node); event;	\
1767 		event = rb_entry_safe(rb_next(&event->group_node),	\
1768 				typeof(*event), group_node))
1769 
1770 /*
1771  * Add an event from the lists for its context.
1772  * Must be called with ctx->mutex and ctx->lock held.
1773  */
1774 static void
list_add_event(struct perf_event * event,struct perf_event_context * ctx)1775 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
1776 {
1777 	lockdep_assert_held(&ctx->lock);
1778 
1779 	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
1780 	event->attach_state |= PERF_ATTACH_CONTEXT;
1781 
1782 	event->tstamp = perf_event_time(event);
1783 
1784 	/*
1785 	 * If we're a stand alone event or group leader, we go to the context
1786 	 * list, group events are kept attached to the group so that
1787 	 * perf_group_detach can, at all times, locate all siblings.
1788 	 */
1789 	if (event->group_leader == event) {
1790 		event->group_caps = event->event_caps;
1791 		add_event_to_groups(event, ctx);
1792 	}
1793 
1794 	list_add_rcu(&event->event_entry, &ctx->event_list);
1795 	ctx->nr_events++;
1796 	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)
1797 		ctx->nr_user++;
1798 	if (event->attr.inherit_stat)
1799 		ctx->nr_stat++;
1800 
1801 	if (event->state > PERF_EVENT_STATE_OFF)
1802 		perf_cgroup_event_enable(event, ctx);
1803 
1804 	ctx->generation++;
1805 	event->pmu_ctx->nr_events++;
1806 }
1807 
1808 /*
1809  * Initialize event state based on the perf_event_attr::disabled.
1810  */
perf_event__state_init(struct perf_event * event)1811 static inline void perf_event__state_init(struct perf_event *event)
1812 {
1813 	event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
1814 					      PERF_EVENT_STATE_INACTIVE;
1815 }
1816 
__perf_event_read_size(u64 read_format,int nr_siblings)1817 static int __perf_event_read_size(u64 read_format, int nr_siblings)
1818 {
1819 	int entry = sizeof(u64); /* value */
1820 	int size = 0;
1821 	int nr = 1;
1822 
1823 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1824 		size += sizeof(u64);
1825 
1826 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1827 		size += sizeof(u64);
1828 
1829 	if (read_format & PERF_FORMAT_ID)
1830 		entry += sizeof(u64);
1831 
1832 	if (read_format & PERF_FORMAT_LOST)
1833 		entry += sizeof(u64);
1834 
1835 	if (read_format & PERF_FORMAT_GROUP) {
1836 		nr += nr_siblings;
1837 		size += sizeof(u64);
1838 	}
1839 
1840 	/*
1841 	 * Since perf_event_validate_size() limits this to 16k and inhibits
1842 	 * adding more siblings, this will never overflow.
1843 	 */
1844 	return size + nr * entry;
1845 }
1846 
__perf_event_header_size(struct perf_event * event,u64 sample_type)1847 static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
1848 {
1849 	struct perf_sample_data *data;
1850 	u16 size = 0;
1851 
1852 	if (sample_type & PERF_SAMPLE_IP)
1853 		size += sizeof(data->ip);
1854 
1855 	if (sample_type & PERF_SAMPLE_ADDR)
1856 		size += sizeof(data->addr);
1857 
1858 	if (sample_type & PERF_SAMPLE_PERIOD)
1859 		size += sizeof(data->period);
1860 
1861 	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE)
1862 		size += sizeof(data->weight.full);
1863 
1864 	if (sample_type & PERF_SAMPLE_READ)
1865 		size += event->read_size;
1866 
1867 	if (sample_type & PERF_SAMPLE_DATA_SRC)
1868 		size += sizeof(data->data_src.val);
1869 
1870 	if (sample_type & PERF_SAMPLE_TRANSACTION)
1871 		size += sizeof(data->txn);
1872 
1873 	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
1874 		size += sizeof(data->phys_addr);
1875 
1876 	if (sample_type & PERF_SAMPLE_CGROUP)
1877 		size += sizeof(data->cgroup);
1878 
1879 	if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
1880 		size += sizeof(data->data_page_size);
1881 
1882 	if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
1883 		size += sizeof(data->code_page_size);
1884 
1885 	event->header_size = size;
1886 }
1887 
1888 /*
1889  * Called at perf_event creation and when events are attached/detached from a
1890  * group.
1891  */
perf_event__header_size(struct perf_event * event)1892 static void perf_event__header_size(struct perf_event *event)
1893 {
1894 	event->read_size =
1895 		__perf_event_read_size(event->attr.read_format,
1896 				       event->group_leader->nr_siblings);
1897 	__perf_event_header_size(event, event->attr.sample_type);
1898 }
1899 
perf_event__id_header_size(struct perf_event * event)1900 static void perf_event__id_header_size(struct perf_event *event)
1901 {
1902 	struct perf_sample_data *data;
1903 	u64 sample_type = event->attr.sample_type;
1904 	u16 size = 0;
1905 
1906 	if (sample_type & PERF_SAMPLE_TID)
1907 		size += sizeof(data->tid_entry);
1908 
1909 	if (sample_type & PERF_SAMPLE_TIME)
1910 		size += sizeof(data->time);
1911 
1912 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
1913 		size += sizeof(data->id);
1914 
1915 	if (sample_type & PERF_SAMPLE_ID)
1916 		size += sizeof(data->id);
1917 
1918 	if (sample_type & PERF_SAMPLE_STREAM_ID)
1919 		size += sizeof(data->stream_id);
1920 
1921 	if (sample_type & PERF_SAMPLE_CPU)
1922 		size += sizeof(data->cpu_entry);
1923 
1924 	event->id_header_size = size;
1925 }
1926 
1927 /*
1928  * Check that adding an event to the group does not result in anybody
1929  * overflowing the 64k event limit imposed by the output buffer.
1930  *
1931  * Specifically, check that the read_size for the event does not exceed 16k,
1932  * read_size being the one term that grows with groups size. Since read_size
1933  * depends on per-event read_format, also (re)check the existing events.
1934  *
1935  * This leaves 48k for the constant size fields and things like callchains,
1936  * branch stacks and register sets.
1937  */
perf_event_validate_size(struct perf_event * event)1938 static bool perf_event_validate_size(struct perf_event *event)
1939 {
1940 	struct perf_event *sibling, *group_leader = event->group_leader;
1941 
1942 	if (__perf_event_read_size(event->attr.read_format,
1943 				   group_leader->nr_siblings + 1) > 16*1024)
1944 		return false;
1945 
1946 	if (__perf_event_read_size(group_leader->attr.read_format,
1947 				   group_leader->nr_siblings + 1) > 16*1024)
1948 		return false;
1949 
1950 	/*
1951 	 * When creating a new group leader, group_leader->ctx is initialized
1952 	 * after the size has been validated, but we cannot safely use
1953 	 * for_each_sibling_event() until group_leader->ctx is set. A new group
1954 	 * leader cannot have any siblings yet, so we can safely skip checking
1955 	 * the non-existent siblings.
1956 	 */
1957 	if (event == group_leader)
1958 		return true;
1959 
1960 	for_each_sibling_event(sibling, group_leader) {
1961 		if (__perf_event_read_size(sibling->attr.read_format,
1962 					   group_leader->nr_siblings + 1) > 16*1024)
1963 			return false;
1964 	}
1965 
1966 	return true;
1967 }
1968 
perf_group_attach(struct perf_event * event)1969 static void perf_group_attach(struct perf_event *event)
1970 {
1971 	struct perf_event *group_leader = event->group_leader, *pos;
1972 
1973 	lockdep_assert_held(&event->ctx->lock);
1974 
1975 	/*
1976 	 * We can have double attach due to group movement (move_group) in
1977 	 * perf_event_open().
1978 	 */
1979 	if (event->attach_state & PERF_ATTACH_GROUP)
1980 		return;
1981 
1982 	event->attach_state |= PERF_ATTACH_GROUP;
1983 
1984 	if (group_leader == event)
1985 		return;
1986 
1987 	WARN_ON_ONCE(group_leader->ctx != event->ctx);
1988 
1989 	group_leader->group_caps &= event->event_caps;
1990 
1991 	list_add_tail(&event->sibling_list, &group_leader->sibling_list);
1992 	group_leader->nr_siblings++;
1993 	group_leader->group_generation++;
1994 
1995 	perf_event__header_size(group_leader);
1996 
1997 	for_each_sibling_event(pos, group_leader)
1998 		perf_event__header_size(pos);
1999 }
2000 
2001 /*
2002  * Remove an event from the lists for its context.
2003  * Must be called with ctx->mutex and ctx->lock held.
2004  */
2005 static void
list_del_event(struct perf_event * event,struct perf_event_context * ctx)2006 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
2007 {
2008 	WARN_ON_ONCE(event->ctx != ctx);
2009 	lockdep_assert_held(&ctx->lock);
2010 
2011 	/*
2012 	 * We can have double detach due to exit/hot-unplug + close.
2013 	 */
2014 	if (!(event->attach_state & PERF_ATTACH_CONTEXT))
2015 		return;
2016 
2017 	event->attach_state &= ~PERF_ATTACH_CONTEXT;
2018 
2019 	ctx->nr_events--;
2020 	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)
2021 		ctx->nr_user--;
2022 	if (event->attr.inherit_stat)
2023 		ctx->nr_stat--;
2024 
2025 	list_del_rcu(&event->event_entry);
2026 
2027 	if (event->group_leader == event)
2028 		del_event_from_groups(event, ctx);
2029 
2030 	/*
2031 	 * If event was in error state, then keep it
2032 	 * that way, otherwise bogus counts will be
2033 	 * returned on read(). The only way to get out
2034 	 * of error state is by explicit re-enabling
2035 	 * of the event
2036 	 */
2037 	if (event->state > PERF_EVENT_STATE_OFF) {
2038 		perf_cgroup_event_disable(event, ctx);
2039 		perf_event_set_state(event, PERF_EVENT_STATE_OFF);
2040 	}
2041 
2042 	ctx->generation++;
2043 	event->pmu_ctx->nr_events--;
2044 }
2045 
2046 static int
perf_aux_output_match(struct perf_event * event,struct perf_event * aux_event)2047 perf_aux_output_match(struct perf_event *event, struct perf_event *aux_event)
2048 {
2049 	if (!has_aux(aux_event))
2050 		return 0;
2051 
2052 	if (!event->pmu->aux_output_match)
2053 		return 0;
2054 
2055 	return event->pmu->aux_output_match(aux_event);
2056 }
2057 
2058 static void put_event(struct perf_event *event);
2059 static void event_sched_out(struct perf_event *event,
2060 			    struct perf_event_context *ctx);
2061 
perf_put_aux_event(struct perf_event * event)2062 static void perf_put_aux_event(struct perf_event *event)
2063 {
2064 	struct perf_event_context *ctx = event->ctx;
2065 	struct perf_event *iter;
2066 
2067 	/*
2068 	 * If event uses aux_event tear down the link
2069 	 */
2070 	if (event->aux_event) {
2071 		iter = event->aux_event;
2072 		event->aux_event = NULL;
2073 		put_event(iter);
2074 		return;
2075 	}
2076 
2077 	/*
2078 	 * If the event is an aux_event, tear down all links to
2079 	 * it from other events.
2080 	 */
2081 	for_each_sibling_event(iter, event->group_leader) {
2082 		if (iter->aux_event != event)
2083 			continue;
2084 
2085 		iter->aux_event = NULL;
2086 		put_event(event);
2087 
2088 		/*
2089 		 * If it's ACTIVE, schedule it out and put it into ERROR
2090 		 * state so that we don't try to schedule it again. Note
2091 		 * that perf_event_enable() will clear the ERROR status.
2092 		 */
2093 		event_sched_out(iter, ctx);
2094 		perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
2095 	}
2096 }
2097 
perf_need_aux_event(struct perf_event * event)2098 static bool perf_need_aux_event(struct perf_event *event)
2099 {
2100 	return !!event->attr.aux_output || !!event->attr.aux_sample_size;
2101 }
2102 
perf_get_aux_event(struct perf_event * event,struct perf_event * group_leader)2103 static int perf_get_aux_event(struct perf_event *event,
2104 			      struct perf_event *group_leader)
2105 {
2106 	/*
2107 	 * Our group leader must be an aux event if we want to be
2108 	 * an aux_output. This way, the aux event will precede its
2109 	 * aux_output events in the group, and therefore will always
2110 	 * schedule first.
2111 	 */
2112 	if (!group_leader)
2113 		return 0;
2114 
2115 	/*
2116 	 * aux_output and aux_sample_size are mutually exclusive.
2117 	 */
2118 	if (event->attr.aux_output && event->attr.aux_sample_size)
2119 		return 0;
2120 
2121 	if (event->attr.aux_output &&
2122 	    !perf_aux_output_match(event, group_leader))
2123 		return 0;
2124 
2125 	if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux)
2126 		return 0;
2127 
2128 	if (!atomic_long_inc_not_zero(&group_leader->refcount))
2129 		return 0;
2130 
2131 	/*
2132 	 * Link aux_outputs to their aux event; this is undone in
2133 	 * perf_group_detach() by perf_put_aux_event(). When the
2134 	 * group in torn down, the aux_output events loose their
2135 	 * link to the aux_event and can't schedule any more.
2136 	 */
2137 	event->aux_event = group_leader;
2138 
2139 	return 1;
2140 }
2141 
get_event_list(struct perf_event * event)2142 static inline struct list_head *get_event_list(struct perf_event *event)
2143 {
2144 	return event->attr.pinned ? &event->pmu_ctx->pinned_active :
2145 				    &event->pmu_ctx->flexible_active;
2146 }
2147 
2148 /*
2149  * Events that have PERF_EV_CAP_SIBLING require being part of a group and
2150  * cannot exist on their own, schedule them out and move them into the ERROR
2151  * state. Also see _perf_event_enable(), it will not be able to recover
2152  * this ERROR state.
2153  */
perf_remove_sibling_event(struct perf_event * event)2154 static inline void perf_remove_sibling_event(struct perf_event *event)
2155 {
2156 	event_sched_out(event, event->ctx);
2157 	perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
2158 }
2159 
perf_group_detach(struct perf_event * event)2160 static void perf_group_detach(struct perf_event *event)
2161 {
2162 	struct perf_event *leader = event->group_leader;
2163 	struct perf_event *sibling, *tmp;
2164 	struct perf_event_context *ctx = event->ctx;
2165 
2166 	lockdep_assert_held(&ctx->lock);
2167 
2168 	/*
2169 	 * We can have double detach due to exit/hot-unplug + close.
2170 	 */
2171 	if (!(event->attach_state & PERF_ATTACH_GROUP))
2172 		return;
2173 
2174 	event->attach_state &= ~PERF_ATTACH_GROUP;
2175 
2176 	perf_put_aux_event(event);
2177 
2178 	/*
2179 	 * If this is a sibling, remove it from its group.
2180 	 */
2181 	if (leader != event) {
2182 		list_del_init(&event->sibling_list);
2183 		event->group_leader->nr_siblings--;
2184 		event->group_leader->group_generation++;
2185 		goto out;
2186 	}
2187 
2188 	/*
2189 	 * If this was a group event with sibling events then
2190 	 * upgrade the siblings to singleton events by adding them
2191 	 * to whatever list we are on.
2192 	 */
2193 	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {
2194 
2195 		if (sibling->event_caps & PERF_EV_CAP_SIBLING)
2196 			perf_remove_sibling_event(sibling);
2197 
2198 		sibling->group_leader = sibling;
2199 		list_del_init(&sibling->sibling_list);
2200 
2201 		/* Inherit group flags from the previous leader */
2202 		sibling->group_caps = event->group_caps;
2203 
2204 		if (sibling->attach_state & PERF_ATTACH_CONTEXT) {
2205 			add_event_to_groups(sibling, event->ctx);
2206 
2207 			if (sibling->state == PERF_EVENT_STATE_ACTIVE)
2208 				list_add_tail(&sibling->active_list, get_event_list(sibling));
2209 		}
2210 
2211 		WARN_ON_ONCE(sibling->ctx != event->ctx);
2212 	}
2213 
2214 out:
2215 	for_each_sibling_event(tmp, leader)
2216 		perf_event__header_size(tmp);
2217 
2218 	perf_event__header_size(leader);
2219 }
2220 
2221 static void sync_child_event(struct perf_event *child_event);
2222 
perf_child_detach(struct perf_event * event)2223 static void perf_child_detach(struct perf_event *event)
2224 {
2225 	struct perf_event *parent_event = event->parent;
2226 
2227 	if (!(event->attach_state & PERF_ATTACH_CHILD))
2228 		return;
2229 
2230 	event->attach_state &= ~PERF_ATTACH_CHILD;
2231 
2232 	if (WARN_ON_ONCE(!parent_event))
2233 		return;
2234 
2235 	lockdep_assert_held(&parent_event->child_mutex);
2236 
2237 	sync_child_event(event);
2238 	list_del_init(&event->child_list);
2239 }
2240 
is_orphaned_event(struct perf_event * event)2241 static bool is_orphaned_event(struct perf_event *event)
2242 {
2243 	return event->state == PERF_EVENT_STATE_DEAD;
2244 }
2245 
2246 static inline int
event_filter_match(struct perf_event * event)2247 event_filter_match(struct perf_event *event)
2248 {
2249 	return (event->cpu == -1 || event->cpu == smp_processor_id()) &&
2250 	       perf_cgroup_match(event);
2251 }
2252 
2253 static void
event_sched_out(struct perf_event * event,struct perf_event_context * ctx)2254 event_sched_out(struct perf_event *event, struct perf_event_context *ctx)
2255 {
2256 	struct perf_event_pmu_context *epc = event->pmu_ctx;
2257 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);
2258 	enum perf_event_state state = PERF_EVENT_STATE_INACTIVE;
2259 
2260 	// XXX cpc serialization, probably per-cpu IRQ disabled
2261 
2262 	WARN_ON_ONCE(event->ctx != ctx);
2263 	lockdep_assert_held(&ctx->lock);
2264 
2265 	if (event->state != PERF_EVENT_STATE_ACTIVE)
2266 		return;
2267 
2268 	/*
2269 	 * Asymmetry; we only schedule events _IN_ through ctx_sched_in(), but
2270 	 * we can schedule events _OUT_ individually through things like
2271 	 * __perf_remove_from_context().
2272 	 */
2273 	list_del_init(&event->active_list);
2274 
2275 	perf_pmu_disable(event->pmu);
2276 
2277 	event->pmu->del(event, 0);
2278 	event->oncpu = -1;
2279 
2280 	if (event->pending_disable) {
2281 		event->pending_disable = 0;
2282 		perf_cgroup_event_disable(event, ctx);
2283 		state = PERF_EVENT_STATE_OFF;
2284 	}
2285 
2286 	if (event->pending_sigtrap) {
2287 		bool dec = true;
2288 
2289 		event->pending_sigtrap = 0;
2290 		if (state != PERF_EVENT_STATE_OFF &&
2291 		    !event->pending_work) {
2292 			event->pending_work = 1;
2293 			dec = false;
2294 			WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));
2295 			task_work_add(current, &event->pending_task, TWA_RESUME);
2296 		}
2297 		if (dec)
2298 			local_dec(&event->ctx->nr_pending);
2299 	}
2300 
2301 	perf_event_set_state(event, state);
2302 
2303 	if (!is_software_event(event))
2304 		cpc->active_oncpu--;
2305 	if (event->attr.freq && event->attr.sample_freq)
2306 		ctx->nr_freq--;
2307 	if (event->attr.exclusive || !cpc->active_oncpu)
2308 		cpc->exclusive = 0;
2309 
2310 	perf_pmu_enable(event->pmu);
2311 }
2312 
2313 static void
group_sched_out(struct perf_event * group_event,struct perf_event_context * ctx)2314 group_sched_out(struct perf_event *group_event, struct perf_event_context *ctx)
2315 {
2316 	struct perf_event *event;
2317 
2318 	if (group_event->state != PERF_EVENT_STATE_ACTIVE)
2319 		return;
2320 
2321 	perf_assert_pmu_disabled(group_event->pmu_ctx->pmu);
2322 
2323 	event_sched_out(group_event, ctx);
2324 
2325 	/*
2326 	 * Schedule out siblings (if any):
2327 	 */
2328 	for_each_sibling_event(event, group_event)
2329 		event_sched_out(event, ctx);
2330 }
2331 
2332 #define DETACH_GROUP	0x01UL
2333 #define DETACH_CHILD	0x02UL
2334 #define DETACH_DEAD	0x04UL
2335 
2336 /*
2337  * Cross CPU call to remove a performance event
2338  *
2339  * We disable the event on the hardware level first. After that we
2340  * remove it from the context list.
2341  */
2342 static void
__perf_remove_from_context(struct perf_event * event,struct perf_cpu_context * cpuctx,struct perf_event_context * ctx,void * info)2343 __perf_remove_from_context(struct perf_event *event,
2344 			   struct perf_cpu_context *cpuctx,
2345 			   struct perf_event_context *ctx,
2346 			   void *info)
2347 {
2348 	struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx;
2349 	unsigned long flags = (unsigned long)info;
2350 
2351 	if (ctx->is_active & EVENT_TIME) {
2352 		update_context_time(ctx);
2353 		update_cgrp_time_from_cpuctx(cpuctx, false);
2354 	}
2355 
2356 	/*
2357 	 * Ensure event_sched_out() switches to OFF, at the very least
2358 	 * this avoids raising perf_pending_task() at this time.
2359 	 */
2360 	if (flags & DETACH_DEAD)
2361 		event->pending_disable = 1;
2362 	event_sched_out(event, ctx);
2363 	if (flags & DETACH_GROUP)
2364 		perf_group_detach(event);
2365 	if (flags & DETACH_CHILD)
2366 		perf_child_detach(event);
2367 	list_del_event(event, ctx);
2368 	if (flags & DETACH_DEAD)
2369 		event->state = PERF_EVENT_STATE_DEAD;
2370 
2371 	if (!pmu_ctx->nr_events) {
2372 		pmu_ctx->rotate_necessary = 0;
2373 
2374 		if (ctx->task && ctx->is_active) {
2375 			struct perf_cpu_pmu_context *cpc;
2376 
2377 			cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);
2378 			WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
2379 			cpc->task_epc = NULL;
2380 		}
2381 	}
2382 
2383 	if (!ctx->nr_events && ctx->is_active) {
2384 		if (ctx == &cpuctx->ctx)
2385 			update_cgrp_time_from_cpuctx(cpuctx, true);
2386 
2387 		ctx->is_active = 0;
2388 		if (ctx->task) {
2389 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);
2390 			cpuctx->task_ctx = NULL;
2391 		}
2392 	}
2393 }
2394 
2395 /*
2396  * Remove the event from a task's (or a CPU's) list of events.
2397  *
2398  * If event->ctx is a cloned context, callers must make sure that
2399  * every task struct that event->ctx->task could possibly point to
2400  * remains valid.  This is OK when called from perf_release since
2401  * that only calls us on the top-level context, which can't be a clone.
2402  * When called from perf_event_exit_task, it's OK because the
2403  * context has been detached from its task.
2404  */
perf_remove_from_context(struct perf_event * event,unsigned long flags)2405 static void perf_remove_from_context(struct perf_event *event, unsigned long flags)
2406 {
2407 	struct perf_event_context *ctx = event->ctx;
2408 
2409 	lockdep_assert_held(&ctx->mutex);
2410 
2411 	/*
2412 	 * Because of perf_event_exit_task(), perf_remove_from_context() ought
2413 	 * to work in the face of TASK_TOMBSTONE, unlike every other
2414 	 * event_function_call() user.
2415 	 */
2416 	raw_spin_lock_irq(&ctx->lock);
2417 	if (!ctx->is_active) {
2418 		__perf_remove_from_context(event, this_cpu_ptr(&perf_cpu_context),
2419 					   ctx, (void *)flags);
2420 		raw_spin_unlock_irq(&ctx->lock);
2421 		return;
2422 	}
2423 	raw_spin_unlock_irq(&ctx->lock);
2424 
2425 	event_function_call(event, __perf_remove_from_context, (void *)flags);
2426 }
2427 
2428 /*
2429  * Cross CPU call to disable a performance event
2430  */
__perf_event_disable(struct perf_event * event,struct perf_cpu_context * cpuctx,struct perf_event_context * ctx,void * info)2431 static void __perf_event_disable(struct perf_event *event,
2432 				 struct perf_cpu_context *cpuctx,
2433 				 struct perf_event_context *ctx,
2434 				 void *info)
2435 {
2436 	if (event->state < PERF_EVENT_STATE_INACTIVE)
2437 		return;
2438 
2439 	if (ctx->is_active & EVENT_TIME) {
2440 		update_context_time(ctx);
2441 		update_cgrp_time_from_event(event);
2442 	}
2443 
2444 	perf_pmu_disable(event->pmu_ctx->pmu);
2445 
2446 	if (event == event->group_leader)
2447 		group_sched_out(event, ctx);
2448 	else
2449 		event_sched_out(event, ctx);
2450 
2451 	perf_event_set_state(event, PERF_EVENT_STATE_OFF);
2452 	perf_cgroup_event_disable(event, ctx);
2453 
2454 	perf_pmu_enable(event->pmu_ctx->pmu);
2455 }
2456 
2457 /*
2458  * Disable an event.
2459  *
2460  * If event->ctx is a cloned context, callers must make sure that
2461  * every task struct that event->ctx->task could possibly point to
2462  * remains valid.  This condition is satisfied when called through
2463  * perf_event_for_each_child or perf_event_for_each because they
2464  * hold the top-level event's child_mutex, so any descendant that
2465  * goes to exit will block in perf_event_exit_event().
2466  *
2467  * When called from perf_pending_irq it's OK because event->ctx
2468  * is the current context on this CPU and preemption is disabled,
2469  * hence we can't get into perf_event_task_sched_out for this context.
2470  */
_perf_event_disable(struct perf_event * event)2471 static void _perf_event_disable(struct perf_event *event)
2472 {
2473 	struct perf_event_context *ctx = event->ctx;
2474 
2475 	raw_spin_lock_irq(&ctx->lock);
2476 	if (event->state <= PERF_EVENT_STATE_OFF) {
2477 		raw_spin_unlock_irq(&ctx->lock);
2478 		return;
2479 	}
2480 	raw_spin_unlock_irq(&ctx->lock);
2481 
2482 	event_function_call(event, __perf_event_disable, NULL);
2483 }
2484 
perf_event_disable_local(struct perf_event * event)2485 void perf_event_disable_local(struct perf_event *event)
2486 {
2487 	event_function_local(event, __perf_event_disable, NULL);
2488 }
2489 
2490 /*
2491  * Strictly speaking kernel users cannot create groups and therefore this
2492  * interface does not need the perf_event_ctx_lock() magic.
2493  */
perf_event_disable(struct perf_event * event)2494 void perf_event_disable(struct perf_event *event)
2495 {
2496 	struct perf_event_context *ctx;
2497 
2498 	ctx = perf_event_ctx_lock(event);
2499 	_perf_event_disable(event);
2500 	perf_event_ctx_unlock(event, ctx);
2501 }
2502 EXPORT_SYMBOL_GPL(perf_event_disable);
2503 
perf_event_disable_inatomic(struct perf_event * event)2504 void perf_event_disable_inatomic(struct perf_event *event)
2505 {
2506 	event->pending_disable = 1;
2507 	irq_work_queue(&event->pending_irq);
2508 }
2509 
2510 #define MAX_INTERRUPTS (~0ULL)
2511 
2512 static void perf_log_throttle(struct perf_event *event, int enable);
2513 static void perf_log_itrace_start(struct perf_event *event);
2514 
2515 static int
event_sched_in(struct perf_event * event,struct perf_event_context * ctx)2516 event_sched_in(struct perf_event *event, struct perf_event_context *ctx)
2517 {
2518 	struct perf_event_pmu_context *epc = event->pmu_ctx;
2519 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);
2520 	int ret = 0;
2521 
2522 	WARN_ON_ONCE(event->ctx != ctx);
2523 
2524 	lockdep_assert_held(&ctx->lock);
2525 
2526 	if (event->state <= PERF_EVENT_STATE_OFF)
2527 		return 0;
2528 
2529 	WRITE_ONCE(event->oncpu, smp_processor_id());
2530 	/*
2531 	 * Order event::oncpu write to happen before the ACTIVE state is
2532 	 * visible. This allows perf_event_{stop,read}() to observe the correct
2533 	 * ->oncpu if it sees ACTIVE.
2534 	 */
2535 	smp_wmb();
2536 	perf_event_set_state(event, PERF_EVENT_STATE_ACTIVE);
2537 
2538 	/*
2539 	 * Unthrottle events, since we scheduled we might have missed several
2540 	 * ticks already, also for a heavily scheduling task there is little
2541 	 * guarantee it'll get a tick in a timely manner.
2542 	 */
2543 	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
2544 		perf_log_throttle(event, 1);
2545 		event->hw.interrupts = 0;
2546 	}
2547 
2548 	perf_pmu_disable(event->pmu);
2549 
2550 	perf_log_itrace_start(event);
2551 
2552 	if (event->pmu->add(event, PERF_EF_START)) {
2553 		perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
2554 		event->oncpu = -1;
2555 		ret = -EAGAIN;
2556 		goto out;
2557 	}
2558 
2559 	if (!is_software_event(event))
2560 		cpc->active_oncpu++;
2561 	if (event->attr.freq && event->attr.sample_freq)
2562 		ctx->nr_freq++;
2563 
2564 	if (event->attr.exclusive)
2565 		cpc->exclusive = 1;
2566 
2567 out:
2568 	perf_pmu_enable(event->pmu);
2569 
2570 	return ret;
2571 }
2572 
2573 static int
group_sched_in(struct perf_event * group_event,struct perf_event_context * ctx)2574 group_sched_in(struct perf_event *group_event, struct perf_event_context *ctx)
2575 {
2576 	struct perf_event *event, *partial_group = NULL;
2577 	struct pmu *pmu = group_event->pmu_ctx->pmu;
2578 
2579 	if (group_event->state == PERF_EVENT_STATE_OFF)
2580 		return 0;
2581 
2582 	pmu->start_txn(pmu, PERF_PMU_TXN_ADD);
2583 
2584 	if (event_sched_in(group_event, ctx))
2585 		goto error;
2586 
2587 	/*
2588 	 * Schedule in siblings as one group (if any):
2589 	 */
2590 	for_each_sibling_event(event, group_event) {
2591 		if (event_sched_in(event, ctx)) {
2592 			partial_group = event;
2593 			goto group_error;
2594 		}
2595 	}
2596 
2597 	if (!pmu->commit_txn(pmu))
2598 		return 0;
2599 
2600 group_error:
2601 	/*
2602 	 * Groups can be scheduled in as one unit only, so undo any
2603 	 * partial group before returning:
2604 	 * The events up to the failed event are scheduled out normally.
2605 	 */
2606 	for_each_sibling_event(event, group_event) {
2607 		if (event == partial_group)
2608 			break;
2609 
2610 		event_sched_out(event, ctx);
2611 	}
2612 	event_sched_out(group_event, ctx);
2613 
2614 error:
2615 	pmu->cancel_txn(pmu);
2616 	return -EAGAIN;
2617 }
2618 
2619 /*
2620  * Work out whether we can put this event group on the CPU now.
2621  */
group_can_go_on(struct perf_event * event,int can_add_hw)2622 static int group_can_go_on(struct perf_event *event, int can_add_hw)
2623 {
2624 	struct perf_event_pmu_context *epc = event->pmu_ctx;
2625 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(epc->pmu->cpu_pmu_context);
2626 
2627 	/*
2628 	 * Groups consisting entirely of software events can always go on.
2629 	 */
2630 	if (event->group_caps & PERF_EV_CAP_SOFTWARE)
2631 		return 1;
2632 	/*
2633 	 * If an exclusive group is already on, no other hardware
2634 	 * events can go on.
2635 	 */
2636 	if (cpc->exclusive)
2637 		return 0;
2638 	/*
2639 	 * If this group is exclusive and there are already
2640 	 * events on the CPU, it can't go on.
2641 	 */
2642 	if (event->attr.exclusive && !list_empty(get_event_list(event)))
2643 		return 0;
2644 	/*
2645 	 * Otherwise, try to add it if all previous groups were able
2646 	 * to go on.
2647 	 */
2648 	return can_add_hw;
2649 }
2650 
add_event_to_ctx(struct perf_event * event,struct perf_event_context * ctx)2651 static void add_event_to_ctx(struct perf_event *event,
2652 			       struct perf_event_context *ctx)
2653 {
2654 	list_add_event(event, ctx);
2655 	perf_group_attach(event);
2656 }
2657 
task_ctx_sched_out(struct perf_event_context * ctx,enum event_type_t event_type)2658 static void task_ctx_sched_out(struct perf_event_context *ctx,
2659 				enum event_type_t event_type)
2660 {
2661 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
2662 
2663 	if (!cpuctx->task_ctx)
2664 		return;
2665 
2666 	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2667 		return;
2668 
2669 	ctx_sched_out(ctx, event_type);
2670 }
2671 
perf_event_sched_in(struct perf_cpu_context * cpuctx,struct perf_event_context * ctx)2672 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
2673 				struct perf_event_context *ctx)
2674 {
2675 	ctx_sched_in(&cpuctx->ctx, EVENT_PINNED);
2676 	if (ctx)
2677 		 ctx_sched_in(ctx, EVENT_PINNED);
2678 	ctx_sched_in(&cpuctx->ctx, EVENT_FLEXIBLE);
2679 	if (ctx)
2680 		 ctx_sched_in(ctx, EVENT_FLEXIBLE);
2681 }
2682 
2683 /*
2684  * We want to maintain the following priority of scheduling:
2685  *  - CPU pinned (EVENT_CPU | EVENT_PINNED)
2686  *  - task pinned (EVENT_PINNED)
2687  *  - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE)
2688  *  - task flexible (EVENT_FLEXIBLE).
2689  *
2690  * In order to avoid unscheduling and scheduling back in everything every
2691  * time an event is added, only do it for the groups of equal priority and
2692  * below.
2693  *
2694  * This can be called after a batch operation on task events, in which case
2695  * event_type is a bit mask of the types of events involved. For CPU events,
2696  * event_type is only either EVENT_PINNED or EVENT_FLEXIBLE.
2697  */
2698 /*
2699  * XXX: ctx_resched() reschedule entire perf_event_context while adding new
2700  * event to the context or enabling existing event in the context. We can
2701  * probably optimize it by rescheduling only affected pmu_ctx.
2702  */
ctx_resched(struct perf_cpu_context * cpuctx,struct perf_event_context * task_ctx,enum event_type_t event_type)2703 static void ctx_resched(struct perf_cpu_context *cpuctx,
2704 			struct perf_event_context *task_ctx,
2705 			enum event_type_t event_type)
2706 {
2707 	bool cpu_event = !!(event_type & EVENT_CPU);
2708 
2709 	/*
2710 	 * If pinned groups are involved, flexible groups also need to be
2711 	 * scheduled out.
2712 	 */
2713 	if (event_type & EVENT_PINNED)
2714 		event_type |= EVENT_FLEXIBLE;
2715 
2716 	event_type &= EVENT_ALL;
2717 
2718 	perf_ctx_disable(&cpuctx->ctx, false);
2719 	if (task_ctx) {
2720 		perf_ctx_disable(task_ctx, false);
2721 		task_ctx_sched_out(task_ctx, event_type);
2722 	}
2723 
2724 	/*
2725 	 * Decide which cpu ctx groups to schedule out based on the types
2726 	 * of events that caused rescheduling:
2727 	 *  - EVENT_CPU: schedule out corresponding groups;
2728 	 *  - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups;
2729 	 *  - otherwise, do nothing more.
2730 	 */
2731 	if (cpu_event)
2732 		ctx_sched_out(&cpuctx->ctx, event_type);
2733 	else if (event_type & EVENT_PINNED)
2734 		ctx_sched_out(&cpuctx->ctx, EVENT_FLEXIBLE);
2735 
2736 	perf_event_sched_in(cpuctx, task_ctx);
2737 
2738 	perf_ctx_enable(&cpuctx->ctx, false);
2739 	if (task_ctx)
2740 		perf_ctx_enable(task_ctx, false);
2741 }
2742 
perf_pmu_resched(struct pmu * pmu)2743 void perf_pmu_resched(struct pmu *pmu)
2744 {
2745 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
2746 	struct perf_event_context *task_ctx = cpuctx->task_ctx;
2747 
2748 	perf_ctx_lock(cpuctx, task_ctx);
2749 	ctx_resched(cpuctx, task_ctx, EVENT_ALL|EVENT_CPU);
2750 	perf_ctx_unlock(cpuctx, task_ctx);
2751 }
2752 
2753 /*
2754  * Cross CPU call to install and enable a performance event
2755  *
2756  * Very similar to remote_function() + event_function() but cannot assume that
2757  * things like ctx->is_active and cpuctx->task_ctx are set.
2758  */
__perf_install_in_context(void * info)2759 static int  __perf_install_in_context(void *info)
2760 {
2761 	struct perf_event *event = info;
2762 	struct perf_event_context *ctx = event->ctx;
2763 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
2764 	struct perf_event_context *task_ctx = cpuctx->task_ctx;
2765 	bool reprogram = true;
2766 	int ret = 0;
2767 
2768 	raw_spin_lock(&cpuctx->ctx.lock);
2769 	if (ctx->task) {
2770 		raw_spin_lock(&ctx->lock);
2771 		task_ctx = ctx;
2772 
2773 		reprogram = (ctx->task == current);
2774 
2775 		/*
2776 		 * If the task is running, it must be running on this CPU,
2777 		 * otherwise we cannot reprogram things.
2778 		 *
2779 		 * If its not running, we don't care, ctx->lock will
2780 		 * serialize against it becoming runnable.
2781 		 */
2782 		if (task_curr(ctx->task) && !reprogram) {
2783 			ret = -ESRCH;
2784 			goto unlock;
2785 		}
2786 
2787 		WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx);
2788 	} else if (task_ctx) {
2789 		raw_spin_lock(&task_ctx->lock);
2790 	}
2791 
2792 #ifdef CONFIG_CGROUP_PERF
2793 	if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {
2794 		/*
2795 		 * If the current cgroup doesn't match the event's
2796 		 * cgroup, we should not try to schedule it.
2797 		 */
2798 		struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx);
2799 		reprogram = cgroup_is_descendant(cgrp->css.cgroup,
2800 					event->cgrp->css.cgroup);
2801 	}
2802 #endif
2803 
2804 	if (reprogram) {
2805 		ctx_sched_out(ctx, EVENT_TIME);
2806 		add_event_to_ctx(event, ctx);
2807 		ctx_resched(cpuctx, task_ctx, get_event_type(event));
2808 	} else {
2809 		add_event_to_ctx(event, ctx);
2810 	}
2811 
2812 unlock:
2813 	perf_ctx_unlock(cpuctx, task_ctx);
2814 
2815 	return ret;
2816 }
2817 
2818 static bool exclusive_event_installable(struct perf_event *event,
2819 					struct perf_event_context *ctx);
2820 
2821 /*
2822  * Attach a performance event to a context.
2823  *
2824  * Very similar to event_function_call, see comment there.
2825  */
2826 static void
perf_install_in_context(struct perf_event_context * ctx,struct perf_event * event,int cpu)2827 perf_install_in_context(struct perf_event_context *ctx,
2828 			struct perf_event *event,
2829 			int cpu)
2830 {
2831 	struct task_struct *task = READ_ONCE(ctx->task);
2832 
2833 	lockdep_assert_held(&ctx->mutex);
2834 
2835 	WARN_ON_ONCE(!exclusive_event_installable(event, ctx));
2836 
2837 	if (event->cpu != -1)
2838 		WARN_ON_ONCE(event->cpu != cpu);
2839 
2840 	/*
2841 	 * Ensures that if we can observe event->ctx, both the event and ctx
2842 	 * will be 'complete'. See perf_iterate_sb_cpu().
2843 	 */
2844 	smp_store_release(&event->ctx, ctx);
2845 
2846 	/*
2847 	 * perf_event_attr::disabled events will not run and can be initialized
2848 	 * without IPI. Except when this is the first event for the context, in
2849 	 * that case we need the magic of the IPI to set ctx->is_active.
2850 	 *
2851 	 * The IOC_ENABLE that is sure to follow the creation of a disabled
2852 	 * event will issue the IPI and reprogram the hardware.
2853 	 */
2854 	if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF &&
2855 	    ctx->nr_events && !is_cgroup_event(event)) {
2856 		raw_spin_lock_irq(&ctx->lock);
2857 		if (ctx->task == TASK_TOMBSTONE) {
2858 			raw_spin_unlock_irq(&ctx->lock);
2859 			return;
2860 		}
2861 		add_event_to_ctx(event, ctx);
2862 		raw_spin_unlock_irq(&ctx->lock);
2863 		return;
2864 	}
2865 
2866 	if (!task) {
2867 		cpu_function_call(cpu, __perf_install_in_context, event);
2868 		return;
2869 	}
2870 
2871 	/*
2872 	 * Should not happen, we validate the ctx is still alive before calling.
2873 	 */
2874 	if (WARN_ON_ONCE(task == TASK_TOMBSTONE))
2875 		return;
2876 
2877 	/*
2878 	 * Installing events is tricky because we cannot rely on ctx->is_active
2879 	 * to be set in case this is the nr_events 0 -> 1 transition.
2880 	 *
2881 	 * Instead we use task_curr(), which tells us if the task is running.
2882 	 * However, since we use task_curr() outside of rq::lock, we can race
2883 	 * against the actual state. This means the result can be wrong.
2884 	 *
2885 	 * If we get a false positive, we retry, this is harmless.
2886 	 *
2887 	 * If we get a false negative, things are complicated. If we are after
2888 	 * perf_event_context_sched_in() ctx::lock will serialize us, and the
2889 	 * value must be correct. If we're before, it doesn't matter since
2890 	 * perf_event_context_sched_in() will program the counter.
2891 	 *
2892 	 * However, this hinges on the remote context switch having observed
2893 	 * our task->perf_event_ctxp[] store, such that it will in fact take
2894 	 * ctx::lock in perf_event_context_sched_in().
2895 	 *
2896 	 * We do this by task_function_call(), if the IPI fails to hit the task
2897 	 * we know any future context switch of task must see the
2898 	 * perf_event_ctpx[] store.
2899 	 */
2900 
2901 	/*
2902 	 * This smp_mb() orders the task->perf_event_ctxp[] store with the
2903 	 * task_cpu() load, such that if the IPI then does not find the task
2904 	 * running, a future context switch of that task must observe the
2905 	 * store.
2906 	 */
2907 	smp_mb();
2908 again:
2909 	if (!task_function_call(task, __perf_install_in_context, event))
2910 		return;
2911 
2912 	raw_spin_lock_irq(&ctx->lock);
2913 	task = ctx->task;
2914 	if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) {
2915 		/*
2916 		 * Cannot happen because we already checked above (which also
2917 		 * cannot happen), and we hold ctx->mutex, which serializes us
2918 		 * against perf_event_exit_task_context().
2919 		 */
2920 		raw_spin_unlock_irq(&ctx->lock);
2921 		return;
2922 	}
2923 	/*
2924 	 * If the task is not running, ctx->lock will avoid it becoming so,
2925 	 * thus we can safely install the event.
2926 	 */
2927 	if (task_curr(task)) {
2928 		raw_spin_unlock_irq(&ctx->lock);
2929 		goto again;
2930 	}
2931 	add_event_to_ctx(event, ctx);
2932 	raw_spin_unlock_irq(&ctx->lock);
2933 }
2934 
2935 /*
2936  * Cross CPU call to enable a performance event
2937  */
__perf_event_enable(struct perf_event * event,struct perf_cpu_context * cpuctx,struct perf_event_context * ctx,void * info)2938 static void __perf_event_enable(struct perf_event *event,
2939 				struct perf_cpu_context *cpuctx,
2940 				struct perf_event_context *ctx,
2941 				void *info)
2942 {
2943 	struct perf_event *leader = event->group_leader;
2944 	struct perf_event_context *task_ctx;
2945 
2946 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||
2947 	    event->state <= PERF_EVENT_STATE_ERROR)
2948 		return;
2949 
2950 	if (ctx->is_active)
2951 		ctx_sched_out(ctx, EVENT_TIME);
2952 
2953 	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
2954 	perf_cgroup_event_enable(event, ctx);
2955 
2956 	if (!ctx->is_active)
2957 		return;
2958 
2959 	if (!event_filter_match(event)) {
2960 		ctx_sched_in(ctx, EVENT_TIME);
2961 		return;
2962 	}
2963 
2964 	/*
2965 	 * If the event is in a group and isn't the group leader,
2966 	 * then don't put it on unless the group is on.
2967 	 */
2968 	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) {
2969 		ctx_sched_in(ctx, EVENT_TIME);
2970 		return;
2971 	}
2972 
2973 	task_ctx = cpuctx->task_ctx;
2974 	if (ctx->task)
2975 		WARN_ON_ONCE(task_ctx != ctx);
2976 
2977 	ctx_resched(cpuctx, task_ctx, get_event_type(event));
2978 }
2979 
2980 /*
2981  * Enable an event.
2982  *
2983  * If event->ctx is a cloned context, callers must make sure that
2984  * every task struct that event->ctx->task could possibly point to
2985  * remains valid.  This condition is satisfied when called through
2986  * perf_event_for_each_child or perf_event_for_each as described
2987  * for perf_event_disable.
2988  */
_perf_event_enable(struct perf_event * event)2989 static void _perf_event_enable(struct perf_event *event)
2990 {
2991 	struct perf_event_context *ctx = event->ctx;
2992 
2993 	raw_spin_lock_irq(&ctx->lock);
2994 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||
2995 	    event->state <  PERF_EVENT_STATE_ERROR) {
2996 out:
2997 		raw_spin_unlock_irq(&ctx->lock);
2998 		return;
2999 	}
3000 
3001 	/*
3002 	 * If the event is in error state, clear that first.
3003 	 *
3004 	 * That way, if we see the event in error state below, we know that it
3005 	 * has gone back into error state, as distinct from the task having
3006 	 * been scheduled away before the cross-call arrived.
3007 	 */
3008 	if (event->state == PERF_EVENT_STATE_ERROR) {
3009 		/*
3010 		 * Detached SIBLING events cannot leave ERROR state.
3011 		 */
3012 		if (event->event_caps & PERF_EV_CAP_SIBLING &&
3013 		    event->group_leader == event)
3014 			goto out;
3015 
3016 		event->state = PERF_EVENT_STATE_OFF;
3017 	}
3018 	raw_spin_unlock_irq(&ctx->lock);
3019 
3020 	event_function_call(event, __perf_event_enable, NULL);
3021 }
3022 
3023 /*
3024  * See perf_event_disable();
3025  */
perf_event_enable(struct perf_event * event)3026 void perf_event_enable(struct perf_event *event)
3027 {
3028 	struct perf_event_context *ctx;
3029 
3030 	ctx = perf_event_ctx_lock(event);
3031 	_perf_event_enable(event);
3032 	perf_event_ctx_unlock(event, ctx);
3033 }
3034 EXPORT_SYMBOL_GPL(perf_event_enable);
3035 
3036 struct stop_event_data {
3037 	struct perf_event	*event;
3038 	unsigned int		restart;
3039 };
3040 
__perf_event_stop(void * info)3041 static int __perf_event_stop(void *info)
3042 {
3043 	struct stop_event_data *sd = info;
3044 	struct perf_event *event = sd->event;
3045 
3046 	/* if it's already INACTIVE, do nothing */
3047 	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
3048 		return 0;
3049 
3050 	/* matches smp_wmb() in event_sched_in() */
3051 	smp_rmb();
3052 
3053 	/*
3054 	 * There is a window with interrupts enabled before we get here,
3055 	 * so we need to check again lest we try to stop another CPU's event.
3056 	 */
3057 	if (READ_ONCE(event->oncpu) != smp_processor_id())
3058 		return -EAGAIN;
3059 
3060 	event->pmu->stop(event, PERF_EF_UPDATE);
3061 
3062 	/*
3063 	 * May race with the actual stop (through perf_pmu_output_stop()),
3064 	 * but it is only used for events with AUX ring buffer, and such
3065 	 * events will refuse to restart because of rb::aux_mmap_count==0,
3066 	 * see comments in perf_aux_output_begin().
3067 	 *
3068 	 * Since this is happening on an event-local CPU, no trace is lost
3069 	 * while restarting.
3070 	 */
3071 	if (sd->restart)
3072 		event->pmu->start(event, 0);
3073 
3074 	return 0;
3075 }
3076 
perf_event_stop(struct perf_event * event,int restart)3077 static int perf_event_stop(struct perf_event *event, int restart)
3078 {
3079 	struct stop_event_data sd = {
3080 		.event		= event,
3081 		.restart	= restart,
3082 	};
3083 	int ret = 0;
3084 
3085 	do {
3086 		if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
3087 			return 0;
3088 
3089 		/* matches smp_wmb() in event_sched_in() */
3090 		smp_rmb();
3091 
3092 		/*
3093 		 * We only want to restart ACTIVE events, so if the event goes
3094 		 * inactive here (event->oncpu==-1), there's nothing more to do;
3095 		 * fall through with ret==-ENXIO.
3096 		 */
3097 		ret = cpu_function_call(READ_ONCE(event->oncpu),
3098 					__perf_event_stop, &sd);
3099 	} while (ret == -EAGAIN);
3100 
3101 	return ret;
3102 }
3103 
3104 /*
3105  * In order to contain the amount of racy and tricky in the address filter
3106  * configuration management, it is a two part process:
3107  *
3108  * (p1) when userspace mappings change as a result of (1) or (2) or (3) below,
3109  *      we update the addresses of corresponding vmas in
3110  *	event::addr_filter_ranges array and bump the event::addr_filters_gen;
3111  * (p2) when an event is scheduled in (pmu::add), it calls
3112  *      perf_event_addr_filters_sync() which calls pmu::addr_filters_sync()
3113  *      if the generation has changed since the previous call.
3114  *
3115  * If (p1) happens while the event is active, we restart it to force (p2).
3116  *
3117  * (1) perf_addr_filters_apply(): adjusting filters' offsets based on
3118  *     pre-existing mappings, called once when new filters arrive via SET_FILTER
3119  *     ioctl;
3120  * (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly
3121  *     registered mapping, called for every new mmap(), with mm::mmap_lock down
3122  *     for reading;
3123  * (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process
3124  *     of exec.
3125  */
perf_event_addr_filters_sync(struct perf_event * event)3126 void perf_event_addr_filters_sync(struct perf_event *event)
3127 {
3128 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
3129 
3130 	if (!has_addr_filter(event))
3131 		return;
3132 
3133 	raw_spin_lock(&ifh->lock);
3134 	if (event->addr_filters_gen != event->hw.addr_filters_gen) {
3135 		event->pmu->addr_filters_sync(event);
3136 		event->hw.addr_filters_gen = event->addr_filters_gen;
3137 	}
3138 	raw_spin_unlock(&ifh->lock);
3139 }
3140 EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync);
3141 
_perf_event_refresh(struct perf_event * event,int refresh)3142 static int _perf_event_refresh(struct perf_event *event, int refresh)
3143 {
3144 	/*
3145 	 * not supported on inherited events
3146 	 */
3147 	if (event->attr.inherit || !is_sampling_event(event))
3148 		return -EINVAL;
3149 
3150 	atomic_add(refresh, &event->event_limit);
3151 	_perf_event_enable(event);
3152 
3153 	return 0;
3154 }
3155 
3156 /*
3157  * See perf_event_disable()
3158  */
perf_event_refresh(struct perf_event * event,int refresh)3159 int perf_event_refresh(struct perf_event *event, int refresh)
3160 {
3161 	struct perf_event_context *ctx;
3162 	int ret;
3163 
3164 	ctx = perf_event_ctx_lock(event);
3165 	ret = _perf_event_refresh(event, refresh);
3166 	perf_event_ctx_unlock(event, ctx);
3167 
3168 	return ret;
3169 }
3170 EXPORT_SYMBOL_GPL(perf_event_refresh);
3171 
perf_event_modify_breakpoint(struct perf_event * bp,struct perf_event_attr * attr)3172 static int perf_event_modify_breakpoint(struct perf_event *bp,
3173 					 struct perf_event_attr *attr)
3174 {
3175 	int err;
3176 
3177 	_perf_event_disable(bp);
3178 
3179 	err = modify_user_hw_breakpoint_check(bp, attr, true);
3180 
3181 	if (!bp->attr.disabled)
3182 		_perf_event_enable(bp);
3183 
3184 	return err;
3185 }
3186 
3187 /*
3188  * Copy event-type-independent attributes that may be modified.
3189  */
perf_event_modify_copy_attr(struct perf_event_attr * to,const struct perf_event_attr * from)3190 static void perf_event_modify_copy_attr(struct perf_event_attr *to,
3191 					const struct perf_event_attr *from)
3192 {
3193 	to->sig_data = from->sig_data;
3194 }
3195 
perf_event_modify_attr(struct perf_event * event,struct perf_event_attr * attr)3196 static int perf_event_modify_attr(struct perf_event *event,
3197 				  struct perf_event_attr *attr)
3198 {
3199 	int (*func)(struct perf_event *, struct perf_event_attr *);
3200 	struct perf_event *child;
3201 	int err;
3202 
3203 	if (event->attr.type != attr->type)
3204 		return -EINVAL;
3205 
3206 	switch (event->attr.type) {
3207 	case PERF_TYPE_BREAKPOINT:
3208 		func = perf_event_modify_breakpoint;
3209 		break;
3210 	default:
3211 		/* Place holder for future additions. */
3212 		return -EOPNOTSUPP;
3213 	}
3214 
3215 	WARN_ON_ONCE(event->ctx->parent_ctx);
3216 
3217 	mutex_lock(&event->child_mutex);
3218 	/*
3219 	 * Event-type-independent attributes must be copied before event-type
3220 	 * modification, which will validate that final attributes match the
3221 	 * source attributes after all relevant attributes have been copied.
3222 	 */
3223 	perf_event_modify_copy_attr(&event->attr, attr);
3224 	err = func(event, attr);
3225 	if (err)
3226 		goto out;
3227 	list_for_each_entry(child, &event->child_list, child_list) {
3228 		perf_event_modify_copy_attr(&child->attr, attr);
3229 		err = func(child, attr);
3230 		if (err)
3231 			goto out;
3232 	}
3233 out:
3234 	mutex_unlock(&event->child_mutex);
3235 	return err;
3236 }
3237 
__pmu_ctx_sched_out(struct perf_event_pmu_context * pmu_ctx,enum event_type_t event_type)3238 static void __pmu_ctx_sched_out(struct perf_event_pmu_context *pmu_ctx,
3239 				enum event_type_t event_type)
3240 {
3241 	struct perf_event_context *ctx = pmu_ctx->ctx;
3242 	struct perf_event *event, *tmp;
3243 	struct pmu *pmu = pmu_ctx->pmu;
3244 
3245 	if (ctx->task && !ctx->is_active) {
3246 		struct perf_cpu_pmu_context *cpc;
3247 
3248 		cpc = this_cpu_ptr(pmu->cpu_pmu_context);
3249 		WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
3250 		cpc->task_epc = NULL;
3251 	}
3252 
3253 	if (!event_type)
3254 		return;
3255 
3256 	perf_pmu_disable(pmu);
3257 	if (event_type & EVENT_PINNED) {
3258 		list_for_each_entry_safe(event, tmp,
3259 					 &pmu_ctx->pinned_active,
3260 					 active_list)
3261 			group_sched_out(event, ctx);
3262 	}
3263 
3264 	if (event_type & EVENT_FLEXIBLE) {
3265 		list_for_each_entry_safe(event, tmp,
3266 					 &pmu_ctx->flexible_active,
3267 					 active_list)
3268 			group_sched_out(event, ctx);
3269 		/*
3270 		 * Since we cleared EVENT_FLEXIBLE, also clear
3271 		 * rotate_necessary, is will be reset by
3272 		 * ctx_flexible_sched_in() when needed.
3273 		 */
3274 		pmu_ctx->rotate_necessary = 0;
3275 	}
3276 	perf_pmu_enable(pmu);
3277 }
3278 
3279 static void
ctx_sched_out(struct perf_event_context * ctx,enum event_type_t event_type)3280 ctx_sched_out(struct perf_event_context *ctx, enum event_type_t event_type)
3281 {
3282 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3283 	struct perf_event_pmu_context *pmu_ctx;
3284 	int is_active = ctx->is_active;
3285 	bool cgroup = event_type & EVENT_CGROUP;
3286 
3287 	event_type &= ~EVENT_CGROUP;
3288 
3289 	lockdep_assert_held(&ctx->lock);
3290 
3291 	if (likely(!ctx->nr_events)) {
3292 		/*
3293 		 * See __perf_remove_from_context().
3294 		 */
3295 		WARN_ON_ONCE(ctx->is_active);
3296 		if (ctx->task)
3297 			WARN_ON_ONCE(cpuctx->task_ctx);
3298 		return;
3299 	}
3300 
3301 	/*
3302 	 * Always update time if it was set; not only when it changes.
3303 	 * Otherwise we can 'forget' to update time for any but the last
3304 	 * context we sched out. For example:
3305 	 *
3306 	 *   ctx_sched_out(.event_type = EVENT_FLEXIBLE)
3307 	 *   ctx_sched_out(.event_type = EVENT_PINNED)
3308 	 *
3309 	 * would only update time for the pinned events.
3310 	 */
3311 	if (is_active & EVENT_TIME) {
3312 		/* update (and stop) ctx time */
3313 		update_context_time(ctx);
3314 		update_cgrp_time_from_cpuctx(cpuctx, ctx == &cpuctx->ctx);
3315 		/*
3316 		 * CPU-release for the below ->is_active store,
3317 		 * see __load_acquire() in perf_event_time_now()
3318 		 */
3319 		barrier();
3320 	}
3321 
3322 	ctx->is_active &= ~event_type;
3323 	if (!(ctx->is_active & EVENT_ALL))
3324 		ctx->is_active = 0;
3325 
3326 	if (ctx->task) {
3327 		WARN_ON_ONCE(cpuctx->task_ctx != ctx);
3328 		if (!ctx->is_active)
3329 			cpuctx->task_ctx = NULL;
3330 	}
3331 
3332 	is_active ^= ctx->is_active; /* changed bits */
3333 
3334 	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
3335 		if (cgroup && !pmu_ctx->nr_cgroups)
3336 			continue;
3337 		__pmu_ctx_sched_out(pmu_ctx, is_active);
3338 	}
3339 }
3340 
3341 /*
3342  * Test whether two contexts are equivalent, i.e. whether they have both been
3343  * cloned from the same version of the same context.
3344  *
3345  * Equivalence is measured using a generation number in the context that is
3346  * incremented on each modification to it; see unclone_ctx(), list_add_event()
3347  * and list_del_event().
3348  */
context_equiv(struct perf_event_context * ctx1,struct perf_event_context * ctx2)3349 static int context_equiv(struct perf_event_context *ctx1,
3350 			 struct perf_event_context *ctx2)
3351 {
3352 	lockdep_assert_held(&ctx1->lock);
3353 	lockdep_assert_held(&ctx2->lock);
3354 
3355 	/* Pinning disables the swap optimization */
3356 	if (ctx1->pin_count || ctx2->pin_count)
3357 		return 0;
3358 
3359 	/* If ctx1 is the parent of ctx2 */
3360 	if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
3361 		return 1;
3362 
3363 	/* If ctx2 is the parent of ctx1 */
3364 	if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
3365 		return 1;
3366 
3367 	/*
3368 	 * If ctx1 and ctx2 have the same parent; we flatten the parent
3369 	 * hierarchy, see perf_event_init_context().
3370 	 */
3371 	if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
3372 			ctx1->parent_gen == ctx2->parent_gen)
3373 		return 1;
3374 
3375 	/* Unmatched */
3376 	return 0;
3377 }
3378 
__perf_event_sync_stat(struct perf_event * event,struct perf_event * next_event)3379 static void __perf_event_sync_stat(struct perf_event *event,
3380 				     struct perf_event *next_event)
3381 {
3382 	u64 value;
3383 
3384 	if (!event->attr.inherit_stat)
3385 		return;
3386 
3387 	/*
3388 	 * Update the event value, we cannot use perf_event_read()
3389 	 * because we're in the middle of a context switch and have IRQs
3390 	 * disabled, which upsets smp_call_function_single(), however
3391 	 * we know the event must be on the current CPU, therefore we
3392 	 * don't need to use it.
3393 	 */
3394 	if (event->state == PERF_EVENT_STATE_ACTIVE)
3395 		event->pmu->read(event);
3396 
3397 	perf_event_update_time(event);
3398 
3399 	/*
3400 	 * In order to keep per-task stats reliable we need to flip the event
3401 	 * values when we flip the contexts.
3402 	 */
3403 	value = local64_read(&next_event->count);
3404 	value = local64_xchg(&event->count, value);
3405 	local64_set(&next_event->count, value);
3406 
3407 	swap(event->total_time_enabled, next_event->total_time_enabled);
3408 	swap(event->total_time_running, next_event->total_time_running);
3409 
3410 	/*
3411 	 * Since we swizzled the values, update the user visible data too.
3412 	 */
3413 	perf_event_update_userpage(event);
3414 	perf_event_update_userpage(next_event);
3415 }
3416 
perf_event_sync_stat(struct perf_event_context * ctx,struct perf_event_context * next_ctx)3417 static void perf_event_sync_stat(struct perf_event_context *ctx,
3418 				   struct perf_event_context *next_ctx)
3419 {
3420 	struct perf_event *event, *next_event;
3421 
3422 	if (!ctx->nr_stat)
3423 		return;
3424 
3425 	update_context_time(ctx);
3426 
3427 	event = list_first_entry(&ctx->event_list,
3428 				   struct perf_event, event_entry);
3429 
3430 	next_event = list_first_entry(&next_ctx->event_list,
3431 					struct perf_event, event_entry);
3432 
3433 	while (&event->event_entry != &ctx->event_list &&
3434 	       &next_event->event_entry != &next_ctx->event_list) {
3435 
3436 		__perf_event_sync_stat(event, next_event);
3437 
3438 		event = list_next_entry(event, event_entry);
3439 		next_event = list_next_entry(next_event, event_entry);
3440 	}
3441 }
3442 
3443 #define double_list_for_each_entry(pos1, pos2, head1, head2, member)	\
3444 	for (pos1 = list_first_entry(head1, typeof(*pos1), member),	\
3445 	     pos2 = list_first_entry(head2, typeof(*pos2), member);	\
3446 	     !list_entry_is_head(pos1, head1, member) &&		\
3447 	     !list_entry_is_head(pos2, head2, member);			\
3448 	     pos1 = list_next_entry(pos1, member),			\
3449 	     pos2 = list_next_entry(pos2, member))
3450 
perf_event_swap_task_ctx_data(struct perf_event_context * prev_ctx,struct perf_event_context * next_ctx)3451 static void perf_event_swap_task_ctx_data(struct perf_event_context *prev_ctx,
3452 					  struct perf_event_context *next_ctx)
3453 {
3454 	struct perf_event_pmu_context *prev_epc, *next_epc;
3455 
3456 	if (!prev_ctx->nr_task_data)
3457 		return;
3458 
3459 	double_list_for_each_entry(prev_epc, next_epc,
3460 				   &prev_ctx->pmu_ctx_list, &next_ctx->pmu_ctx_list,
3461 				   pmu_ctx_entry) {
3462 
3463 		if (WARN_ON_ONCE(prev_epc->pmu != next_epc->pmu))
3464 			continue;
3465 
3466 		/*
3467 		 * PMU specific parts of task perf context can require
3468 		 * additional synchronization. As an example of such
3469 		 * synchronization see implementation details of Intel
3470 		 * LBR call stack data profiling;
3471 		 */
3472 		if (prev_epc->pmu->swap_task_ctx)
3473 			prev_epc->pmu->swap_task_ctx(prev_epc, next_epc);
3474 		else
3475 			swap(prev_epc->task_ctx_data, next_epc->task_ctx_data);
3476 	}
3477 }
3478 
perf_ctx_sched_task_cb(struct perf_event_context * ctx,bool sched_in)3479 static void perf_ctx_sched_task_cb(struct perf_event_context *ctx, bool sched_in)
3480 {
3481 	struct perf_event_pmu_context *pmu_ctx;
3482 	struct perf_cpu_pmu_context *cpc;
3483 
3484 	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
3485 		cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);
3486 
3487 		if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task)
3488 			pmu_ctx->pmu->sched_task(pmu_ctx, sched_in);
3489 	}
3490 }
3491 
3492 static void
perf_event_context_sched_out(struct task_struct * task,struct task_struct * next)3493 perf_event_context_sched_out(struct task_struct *task, struct task_struct *next)
3494 {
3495 	struct perf_event_context *ctx = task->perf_event_ctxp;
3496 	struct perf_event_context *next_ctx;
3497 	struct perf_event_context *parent, *next_parent;
3498 	int do_switch = 1;
3499 
3500 	if (likely(!ctx))
3501 		return;
3502 
3503 	rcu_read_lock();
3504 	next_ctx = rcu_dereference(next->perf_event_ctxp);
3505 	if (!next_ctx)
3506 		goto unlock;
3507 
3508 	parent = rcu_dereference(ctx->parent_ctx);
3509 	next_parent = rcu_dereference(next_ctx->parent_ctx);
3510 
3511 	/* If neither context have a parent context; they cannot be clones. */
3512 	if (!parent && !next_parent)
3513 		goto unlock;
3514 
3515 	if (next_parent == ctx || next_ctx == parent || next_parent == parent) {
3516 		/*
3517 		 * Looks like the two contexts are clones, so we might be
3518 		 * able to optimize the context switch.  We lock both
3519 		 * contexts and check that they are clones under the
3520 		 * lock (including re-checking that neither has been
3521 		 * uncloned in the meantime).  It doesn't matter which
3522 		 * order we take the locks because no other cpu could
3523 		 * be trying to lock both of these tasks.
3524 		 */
3525 		raw_spin_lock(&ctx->lock);
3526 		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
3527 		if (context_equiv(ctx, next_ctx)) {
3528 
3529 			perf_ctx_disable(ctx, false);
3530 
3531 			/* PMIs are disabled; ctx->nr_pending is stable. */
3532 			if (local_read(&ctx->nr_pending) ||
3533 			    local_read(&next_ctx->nr_pending)) {
3534 				/*
3535 				 * Must not swap out ctx when there's pending
3536 				 * events that rely on the ctx->task relation.
3537 				 */
3538 				raw_spin_unlock(&next_ctx->lock);
3539 				rcu_read_unlock();
3540 				goto inside_switch;
3541 			}
3542 
3543 			WRITE_ONCE(ctx->task, next);
3544 			WRITE_ONCE(next_ctx->task, task);
3545 
3546 			perf_ctx_sched_task_cb(ctx, false);
3547 			perf_event_swap_task_ctx_data(ctx, next_ctx);
3548 
3549 			perf_ctx_enable(ctx, false);
3550 
3551 			/*
3552 			 * RCU_INIT_POINTER here is safe because we've not
3553 			 * modified the ctx and the above modification of
3554 			 * ctx->task and ctx->task_ctx_data are immaterial
3555 			 * since those values are always verified under
3556 			 * ctx->lock which we're now holding.
3557 			 */
3558 			RCU_INIT_POINTER(task->perf_event_ctxp, next_ctx);
3559 			RCU_INIT_POINTER(next->perf_event_ctxp, ctx);
3560 
3561 			do_switch = 0;
3562 
3563 			perf_event_sync_stat(ctx, next_ctx);
3564 		}
3565 		raw_spin_unlock(&next_ctx->lock);
3566 		raw_spin_unlock(&ctx->lock);
3567 	}
3568 unlock:
3569 	rcu_read_unlock();
3570 
3571 	if (do_switch) {
3572 		raw_spin_lock(&ctx->lock);
3573 		perf_ctx_disable(ctx, false);
3574 
3575 inside_switch:
3576 		perf_ctx_sched_task_cb(ctx, false);
3577 		task_ctx_sched_out(ctx, EVENT_ALL);
3578 
3579 		perf_ctx_enable(ctx, false);
3580 		raw_spin_unlock(&ctx->lock);
3581 	}
3582 }
3583 
3584 static DEFINE_PER_CPU(struct list_head, sched_cb_list);
3585 static DEFINE_PER_CPU(int, perf_sched_cb_usages);
3586 
perf_sched_cb_dec(struct pmu * pmu)3587 void perf_sched_cb_dec(struct pmu *pmu)
3588 {
3589 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(pmu->cpu_pmu_context);
3590 
3591 	this_cpu_dec(perf_sched_cb_usages);
3592 	barrier();
3593 
3594 	if (!--cpc->sched_cb_usage)
3595 		list_del(&cpc->sched_cb_entry);
3596 }
3597 
3598 
perf_sched_cb_inc(struct pmu * pmu)3599 void perf_sched_cb_inc(struct pmu *pmu)
3600 {
3601 	struct perf_cpu_pmu_context *cpc = this_cpu_ptr(pmu->cpu_pmu_context);
3602 
3603 	if (!cpc->sched_cb_usage++)
3604 		list_add(&cpc->sched_cb_entry, this_cpu_ptr(&sched_cb_list));
3605 
3606 	barrier();
3607 	this_cpu_inc(perf_sched_cb_usages);
3608 }
3609 
3610 /*
3611  * This function provides the context switch callback to the lower code
3612  * layer. It is invoked ONLY when the context switch callback is enabled.
3613  *
3614  * This callback is relevant even to per-cpu events; for example multi event
3615  * PEBS requires this to provide PID/TID information. This requires we flush
3616  * all queued PEBS records before we context switch to a new task.
3617  */
__perf_pmu_sched_task(struct perf_cpu_pmu_context * cpc,bool sched_in)3618 static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc, bool sched_in)
3619 {
3620 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3621 	struct pmu *pmu;
3622 
3623 	pmu = cpc->epc.pmu;
3624 
3625 	/* software PMUs will not have sched_task */
3626 	if (WARN_ON_ONCE(!pmu->sched_task))
3627 		return;
3628 
3629 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
3630 	perf_pmu_disable(pmu);
3631 
3632 	pmu->sched_task(cpc->task_epc, sched_in);
3633 
3634 	perf_pmu_enable(pmu);
3635 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
3636 }
3637 
perf_pmu_sched_task(struct task_struct * prev,struct task_struct * next,bool sched_in)3638 static void perf_pmu_sched_task(struct task_struct *prev,
3639 				struct task_struct *next,
3640 				bool sched_in)
3641 {
3642 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3643 	struct perf_cpu_pmu_context *cpc;
3644 
3645 	/* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */
3646 	if (prev == next || cpuctx->task_ctx)
3647 		return;
3648 
3649 	list_for_each_entry(cpc, this_cpu_ptr(&sched_cb_list), sched_cb_entry)
3650 		__perf_pmu_sched_task(cpc, sched_in);
3651 }
3652 
3653 static void perf_event_switch(struct task_struct *task,
3654 			      struct task_struct *next_prev, bool sched_in);
3655 
3656 /*
3657  * Called from scheduler to remove the events of the current task,
3658  * with interrupts disabled.
3659  *
3660  * We stop each event and update the event value in event->count.
3661  *
3662  * This does not protect us against NMI, but disable()
3663  * sets the disabled bit in the control field of event _before_
3664  * accessing the event control register. If a NMI hits, then it will
3665  * not restart the event.
3666  */
__perf_event_task_sched_out(struct task_struct * task,struct task_struct * next)3667 void __perf_event_task_sched_out(struct task_struct *task,
3668 				 struct task_struct *next)
3669 {
3670 	if (__this_cpu_read(perf_sched_cb_usages))
3671 		perf_pmu_sched_task(task, next, false);
3672 
3673 	if (atomic_read(&nr_switch_events))
3674 		perf_event_switch(task, next, false);
3675 
3676 	perf_event_context_sched_out(task, next);
3677 
3678 	/*
3679 	 * if cgroup events exist on this CPU, then we need
3680 	 * to check if we have to switch out PMU state.
3681 	 * cgroup event are system-wide mode only
3682 	 */
3683 	perf_cgroup_switch(next);
3684 }
3685 
perf_less_group_idx(const void * l,const void * r)3686 static bool perf_less_group_idx(const void *l, const void *r)
3687 {
3688 	const struct perf_event *le = *(const struct perf_event **)l;
3689 	const struct perf_event *re = *(const struct perf_event **)r;
3690 
3691 	return le->group_index < re->group_index;
3692 }
3693 
swap_ptr(void * l,void * r)3694 static void swap_ptr(void *l, void *r)
3695 {
3696 	void **lp = l, **rp = r;
3697 
3698 	swap(*lp, *rp);
3699 }
3700 
3701 static const struct min_heap_callbacks perf_min_heap = {
3702 	.elem_size = sizeof(struct perf_event *),
3703 	.less = perf_less_group_idx,
3704 	.swp = swap_ptr,
3705 };
3706 
__heap_add(struct min_heap * heap,struct perf_event * event)3707 static void __heap_add(struct min_heap *heap, struct perf_event *event)
3708 {
3709 	struct perf_event **itrs = heap->data;
3710 
3711 	if (event) {
3712 		itrs[heap->nr] = event;
3713 		heap->nr++;
3714 	}
3715 }
3716 
__link_epc(struct perf_event_pmu_context * pmu_ctx)3717 static void __link_epc(struct perf_event_pmu_context *pmu_ctx)
3718 {
3719 	struct perf_cpu_pmu_context *cpc;
3720 
3721 	if (!pmu_ctx->ctx->task)
3722 		return;
3723 
3724 	cpc = this_cpu_ptr(pmu_ctx->pmu->cpu_pmu_context);
3725 	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);
3726 	cpc->task_epc = pmu_ctx;
3727 }
3728 
visit_groups_merge(struct perf_event_context * ctx,struct perf_event_groups * groups,int cpu,struct pmu * pmu,int (* func)(struct perf_event *,void *),void * data)3729 static noinline int visit_groups_merge(struct perf_event_context *ctx,
3730 				struct perf_event_groups *groups, int cpu,
3731 				struct pmu *pmu,
3732 				int (*func)(struct perf_event *, void *),
3733 				void *data)
3734 {
3735 #ifdef CONFIG_CGROUP_PERF
3736 	struct cgroup_subsys_state *css = NULL;
3737 #endif
3738 	struct perf_cpu_context *cpuctx = NULL;
3739 	/* Space for per CPU and/or any CPU event iterators. */
3740 	struct perf_event *itrs[2];
3741 	struct min_heap event_heap;
3742 	struct perf_event **evt;
3743 	int ret;
3744 
3745 	if (pmu->filter && pmu->filter(pmu, cpu))
3746 		return 0;
3747 
3748 	if (!ctx->task) {
3749 		cpuctx = this_cpu_ptr(&perf_cpu_context);
3750 		event_heap = (struct min_heap){
3751 			.data = cpuctx->heap,
3752 			.nr = 0,
3753 			.size = cpuctx->heap_size,
3754 		};
3755 
3756 		lockdep_assert_held(&cpuctx->ctx.lock);
3757 
3758 #ifdef CONFIG_CGROUP_PERF
3759 		if (cpuctx->cgrp)
3760 			css = &cpuctx->cgrp->css;
3761 #endif
3762 	} else {
3763 		event_heap = (struct min_heap){
3764 			.data = itrs,
3765 			.nr = 0,
3766 			.size = ARRAY_SIZE(itrs),
3767 		};
3768 		/* Events not within a CPU context may be on any CPU. */
3769 		__heap_add(&event_heap, perf_event_groups_first(groups, -1, pmu, NULL));
3770 	}
3771 	evt = event_heap.data;
3772 
3773 	__heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, NULL));
3774 
3775 #ifdef CONFIG_CGROUP_PERF
3776 	for (; css; css = css->parent)
3777 		__heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, css->cgroup));
3778 #endif
3779 
3780 	if (event_heap.nr) {
3781 		__link_epc((*evt)->pmu_ctx);
3782 		perf_assert_pmu_disabled((*evt)->pmu_ctx->pmu);
3783 	}
3784 
3785 	min_heapify_all(&event_heap, &perf_min_heap);
3786 
3787 	while (event_heap.nr) {
3788 		ret = func(*evt, data);
3789 		if (ret)
3790 			return ret;
3791 
3792 		*evt = perf_event_groups_next(*evt, pmu);
3793 		if (*evt)
3794 			min_heapify(&event_heap, 0, &perf_min_heap);
3795 		else
3796 			min_heap_pop(&event_heap, &perf_min_heap);
3797 	}
3798 
3799 	return 0;
3800 }
3801 
3802 /*
3803  * Because the userpage is strictly per-event (there is no concept of context,
3804  * so there cannot be a context indirection), every userpage must be updated
3805  * when context time starts :-(
3806  *
3807  * IOW, we must not miss EVENT_TIME edges.
3808  */
event_update_userpage(struct perf_event * event)3809 static inline bool event_update_userpage(struct perf_event *event)
3810 {
3811 	if (likely(!atomic_read(&event->mmap_count)))
3812 		return false;
3813 
3814 	perf_event_update_time(event);
3815 	perf_event_update_userpage(event);
3816 
3817 	return true;
3818 }
3819 
group_update_userpage(struct perf_event * group_event)3820 static inline void group_update_userpage(struct perf_event *group_event)
3821 {
3822 	struct perf_event *event;
3823 
3824 	if (!event_update_userpage(group_event))
3825 		return;
3826 
3827 	for_each_sibling_event(event, group_event)
3828 		event_update_userpage(event);
3829 }
3830 
merge_sched_in(struct perf_event * event,void * data)3831 static int merge_sched_in(struct perf_event *event, void *data)
3832 {
3833 	struct perf_event_context *ctx = event->ctx;
3834 	int *can_add_hw = data;
3835 
3836 	if (event->state <= PERF_EVENT_STATE_OFF)
3837 		return 0;
3838 
3839 	if (!event_filter_match(event))
3840 		return 0;
3841 
3842 	if (group_can_go_on(event, *can_add_hw)) {
3843 		if (!group_sched_in(event, ctx))
3844 			list_add_tail(&event->active_list, get_event_list(event));
3845 	}
3846 
3847 	if (event->state == PERF_EVENT_STATE_INACTIVE) {
3848 		*can_add_hw = 0;
3849 		if (event->attr.pinned) {
3850 			perf_cgroup_event_disable(event, ctx);
3851 			perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
3852 		} else {
3853 			struct perf_cpu_pmu_context *cpc;
3854 
3855 			event->pmu_ctx->rotate_necessary = 1;
3856 			cpc = this_cpu_ptr(event->pmu_ctx->pmu->cpu_pmu_context);
3857 			perf_mux_hrtimer_restart(cpc);
3858 			group_update_userpage(event);
3859 		}
3860 	}
3861 
3862 	return 0;
3863 }
3864 
pmu_groups_sched_in(struct perf_event_context * ctx,struct perf_event_groups * groups,struct pmu * pmu)3865 static void pmu_groups_sched_in(struct perf_event_context *ctx,
3866 				struct perf_event_groups *groups,
3867 				struct pmu *pmu)
3868 {
3869 	int can_add_hw = 1;
3870 	visit_groups_merge(ctx, groups, smp_processor_id(), pmu,
3871 			   merge_sched_in, &can_add_hw);
3872 }
3873 
ctx_groups_sched_in(struct perf_event_context * ctx,struct perf_event_groups * groups,bool cgroup)3874 static void ctx_groups_sched_in(struct perf_event_context *ctx,
3875 				struct perf_event_groups *groups,
3876 				bool cgroup)
3877 {
3878 	struct perf_event_pmu_context *pmu_ctx;
3879 
3880 	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
3881 		if (cgroup && !pmu_ctx->nr_cgroups)
3882 			continue;
3883 		pmu_groups_sched_in(ctx, groups, pmu_ctx->pmu);
3884 	}
3885 }
3886 
__pmu_ctx_sched_in(struct perf_event_context * ctx,struct pmu * pmu)3887 static void __pmu_ctx_sched_in(struct perf_event_context *ctx,
3888 			       struct pmu *pmu)
3889 {
3890 	pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu);
3891 }
3892 
3893 static void
ctx_sched_in(struct perf_event_context * ctx,enum event_type_t event_type)3894 ctx_sched_in(struct perf_event_context *ctx, enum event_type_t event_type)
3895 {
3896 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3897 	int is_active = ctx->is_active;
3898 	bool cgroup = event_type & EVENT_CGROUP;
3899 
3900 	event_type &= ~EVENT_CGROUP;
3901 
3902 	lockdep_assert_held(&ctx->lock);
3903 
3904 	if (likely(!ctx->nr_events))
3905 		return;
3906 
3907 	if (!(is_active & EVENT_TIME)) {
3908 		/* start ctx time */
3909 		__update_context_time(ctx, false);
3910 		perf_cgroup_set_timestamp(cpuctx);
3911 		/*
3912 		 * CPU-release for the below ->is_active store,
3913 		 * see __load_acquire() in perf_event_time_now()
3914 		 */
3915 		barrier();
3916 	}
3917 
3918 	ctx->is_active |= (event_type | EVENT_TIME);
3919 	if (ctx->task) {
3920 		if (!is_active)
3921 			cpuctx->task_ctx = ctx;
3922 		else
3923 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);
3924 	}
3925 
3926 	is_active ^= ctx->is_active; /* changed bits */
3927 
3928 	/*
3929 	 * First go through the list and put on any pinned groups
3930 	 * in order to give them the best chance of going on.
3931 	 */
3932 	if (is_active & EVENT_PINNED)
3933 		ctx_groups_sched_in(ctx, &ctx->pinned_groups, cgroup);
3934 
3935 	/* Then walk through the lower prio flexible groups */
3936 	if (is_active & EVENT_FLEXIBLE)
3937 		ctx_groups_sched_in(ctx, &ctx->flexible_groups, cgroup);
3938 }
3939 
perf_event_context_sched_in(struct task_struct * task)3940 static void perf_event_context_sched_in(struct task_struct *task)
3941 {
3942 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
3943 	struct perf_event_context *ctx;
3944 
3945 	rcu_read_lock();
3946 	ctx = rcu_dereference(task->perf_event_ctxp);
3947 	if (!ctx)
3948 		goto rcu_unlock;
3949 
3950 	if (cpuctx->task_ctx == ctx) {
3951 		perf_ctx_lock(cpuctx, ctx);
3952 		perf_ctx_disable(ctx, false);
3953 
3954 		perf_ctx_sched_task_cb(ctx, true);
3955 
3956 		perf_ctx_enable(ctx, false);
3957 		perf_ctx_unlock(cpuctx, ctx);
3958 		goto rcu_unlock;
3959 	}
3960 
3961 	perf_ctx_lock(cpuctx, ctx);
3962 	/*
3963 	 * We must check ctx->nr_events while holding ctx->lock, such
3964 	 * that we serialize against perf_install_in_context().
3965 	 */
3966 	if (!ctx->nr_events)
3967 		goto unlock;
3968 
3969 	perf_ctx_disable(ctx, false);
3970 	/*
3971 	 * We want to keep the following priority order:
3972 	 * cpu pinned (that don't need to move), task pinned,
3973 	 * cpu flexible, task flexible.
3974 	 *
3975 	 * However, if task's ctx is not carrying any pinned
3976 	 * events, no need to flip the cpuctx's events around.
3977 	 */
3978 	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) {
3979 		perf_ctx_disable(&cpuctx->ctx, false);
3980 		ctx_sched_out(&cpuctx->ctx, EVENT_FLEXIBLE);
3981 	}
3982 
3983 	perf_event_sched_in(cpuctx, ctx);
3984 
3985 	perf_ctx_sched_task_cb(cpuctx->task_ctx, true);
3986 
3987 	if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))
3988 		perf_ctx_enable(&cpuctx->ctx, false);
3989 
3990 	perf_ctx_enable(ctx, false);
3991 
3992 unlock:
3993 	perf_ctx_unlock(cpuctx, ctx);
3994 rcu_unlock:
3995 	rcu_read_unlock();
3996 }
3997 
3998 /*
3999  * Called from scheduler to add the events of the current task
4000  * with interrupts disabled.
4001  *
4002  * We restore the event value and then enable it.
4003  *
4004  * This does not protect us against NMI, but enable()
4005  * sets the enabled bit in the control field of event _before_
4006  * accessing the event control register. If a NMI hits, then it will
4007  * keep the event running.
4008  */
__perf_event_task_sched_in(struct task_struct * prev,struct task_struct * task)4009 void __perf_event_task_sched_in(struct task_struct *prev,
4010 				struct task_struct *task)
4011 {
4012 	perf_event_context_sched_in(task);
4013 
4014 	if (atomic_read(&nr_switch_events))
4015 		perf_event_switch(task, prev, true);
4016 
4017 	if (__this_cpu_read(perf_sched_cb_usages))
4018 		perf_pmu_sched_task(prev, task, true);
4019 }
4020 
perf_calculate_period(struct perf_event * event,u64 nsec,u64 count)4021 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
4022 {
4023 	u64 frequency = event->attr.sample_freq;
4024 	u64 sec = NSEC_PER_SEC;
4025 	u64 divisor, dividend;
4026 
4027 	int count_fls, nsec_fls, frequency_fls, sec_fls;
4028 
4029 	count_fls = fls64(count);
4030 	nsec_fls = fls64(nsec);
4031 	frequency_fls = fls64(frequency);
4032 	sec_fls = 30;
4033 
4034 	/*
4035 	 * We got @count in @nsec, with a target of sample_freq HZ
4036 	 * the target period becomes:
4037 	 *
4038 	 *             @count * 10^9
4039 	 * period = -------------------
4040 	 *          @nsec * sample_freq
4041 	 *
4042 	 */
4043 
4044 	/*
4045 	 * Reduce accuracy by one bit such that @a and @b converge
4046 	 * to a similar magnitude.
4047 	 */
4048 #define REDUCE_FLS(a, b)		\
4049 do {					\
4050 	if (a##_fls > b##_fls) {	\
4051 		a >>= 1;		\
4052 		a##_fls--;		\
4053 	} else {			\
4054 		b >>= 1;		\
4055 		b##_fls--;		\
4056 	}				\
4057 } while (0)
4058 
4059 	/*
4060 	 * Reduce accuracy until either term fits in a u64, then proceed with
4061 	 * the other, so that finally we can do a u64/u64 division.
4062 	 */
4063 	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
4064 		REDUCE_FLS(nsec, frequency);
4065 		REDUCE_FLS(sec, count);
4066 	}
4067 
4068 	if (count_fls + sec_fls > 64) {
4069 		divisor = nsec * frequency;
4070 
4071 		while (count_fls + sec_fls > 64) {
4072 			REDUCE_FLS(count, sec);
4073 			divisor >>= 1;
4074 		}
4075 
4076 		dividend = count * sec;
4077 	} else {
4078 		dividend = count * sec;
4079 
4080 		while (nsec_fls + frequency_fls > 64) {
4081 			REDUCE_FLS(nsec, frequency);
4082 			dividend >>= 1;
4083 		}
4084 
4085 		divisor = nsec * frequency;
4086 	}
4087 
4088 	if (!divisor)
4089 		return dividend;
4090 
4091 	return div64_u64(dividend, divisor);
4092 }
4093 
4094 static DEFINE_PER_CPU(int, perf_throttled_count);
4095 static DEFINE_PER_CPU(u64, perf_throttled_seq);
4096 
perf_adjust_period(struct perf_event * event,u64 nsec,u64 count,bool disable)4097 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
4098 {
4099 	struct hw_perf_event *hwc = &event->hw;
4100 	s64 period, sample_period;
4101 	s64 delta;
4102 
4103 	period = perf_calculate_period(event, nsec, count);
4104 
4105 	delta = (s64)(period - hwc->sample_period);
4106 	delta = (delta + 7) / 8; /* low pass filter */
4107 
4108 	sample_period = hwc->sample_period + delta;
4109 
4110 	if (!sample_period)
4111 		sample_period = 1;
4112 
4113 	hwc->sample_period = sample_period;
4114 
4115 	if (local64_read(&hwc->period_left) > 8*sample_period) {
4116 		if (disable)
4117 			event->pmu->stop(event, PERF_EF_UPDATE);
4118 
4119 		local64_set(&hwc->period_left, 0);
4120 
4121 		if (disable)
4122 			event->pmu->start(event, PERF_EF_RELOAD);
4123 	}
4124 }
4125 
4126 /*
4127  * combine freq adjustment with unthrottling to avoid two passes over the
4128  * events. At the same time, make sure, having freq events does not change
4129  * the rate of unthrottling as that would introduce bias.
4130  */
4131 static void
perf_adjust_freq_unthr_context(struct perf_event_context * ctx,bool unthrottle)4132 perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
4133 {
4134 	struct perf_event *event;
4135 	struct hw_perf_event *hwc;
4136 	u64 now, period = TICK_NSEC;
4137 	s64 delta;
4138 
4139 	/*
4140 	 * only need to iterate over all events iff:
4141 	 * - context have events in frequency mode (needs freq adjust)
4142 	 * - there are events to unthrottle on this cpu
4143 	 */
4144 	if (!(ctx->nr_freq || unthrottle))
4145 		return;
4146 
4147 	raw_spin_lock(&ctx->lock);
4148 
4149 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4150 		if (event->state != PERF_EVENT_STATE_ACTIVE)
4151 			continue;
4152 
4153 		// XXX use visit thingy to avoid the -1,cpu match
4154 		if (!event_filter_match(event))
4155 			continue;
4156 
4157 		perf_pmu_disable(event->pmu);
4158 
4159 		hwc = &event->hw;
4160 
4161 		if (hwc->interrupts == MAX_INTERRUPTS) {
4162 			hwc->interrupts = 0;
4163 			perf_log_throttle(event, 1);
4164 			event->pmu->start(event, 0);
4165 		}
4166 
4167 		if (!event->attr.freq || !event->attr.sample_freq)
4168 			goto next;
4169 
4170 		/*
4171 		 * stop the event and update event->count
4172 		 */
4173 		event->pmu->stop(event, PERF_EF_UPDATE);
4174 
4175 		now = local64_read(&event->count);
4176 		delta = now - hwc->freq_count_stamp;
4177 		hwc->freq_count_stamp = now;
4178 
4179 		/*
4180 		 * restart the event
4181 		 * reload only if value has changed
4182 		 * we have stopped the event so tell that
4183 		 * to perf_adjust_period() to avoid stopping it
4184 		 * twice.
4185 		 */
4186 		if (delta > 0)
4187 			perf_adjust_period(event, period, delta, false);
4188 
4189 		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
4190 	next:
4191 		perf_pmu_enable(event->pmu);
4192 	}
4193 
4194 	raw_spin_unlock(&ctx->lock);
4195 }
4196 
4197 /*
4198  * Move @event to the tail of the @ctx's elegible events.
4199  */
rotate_ctx(struct perf_event_context * ctx,struct perf_event * event)4200 static void rotate_ctx(struct perf_event_context *ctx, struct perf_event *event)
4201 {
4202 	/*
4203 	 * Rotate the first entry last of non-pinned groups. Rotation might be
4204 	 * disabled by the inheritance code.
4205 	 */
4206 	if (ctx->rotate_disable)
4207 		return;
4208 
4209 	perf_event_groups_delete(&ctx->flexible_groups, event);
4210 	perf_event_groups_insert(&ctx->flexible_groups, event);
4211 }
4212 
4213 /* pick an event from the flexible_groups to rotate */
4214 static inline struct perf_event *
ctx_event_to_rotate(struct perf_event_pmu_context * pmu_ctx)4215 ctx_event_to_rotate(struct perf_event_pmu_context *pmu_ctx)
4216 {
4217 	struct perf_event *event;
4218 	struct rb_node *node;
4219 	struct rb_root *tree;
4220 	struct __group_key key = {
4221 		.pmu = pmu_ctx->pmu,
4222 	};
4223 
4224 	/* pick the first active flexible event */
4225 	event = list_first_entry_or_null(&pmu_ctx->flexible_active,
4226 					 struct perf_event, active_list);
4227 	if (event)
4228 		goto out;
4229 
4230 	/* if no active flexible event, pick the first event */
4231 	tree = &pmu_ctx->ctx->flexible_groups.tree;
4232 
4233 	if (!pmu_ctx->ctx->task) {
4234 		key.cpu = smp_processor_id();
4235 
4236 		node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup);
4237 		if (node)
4238 			event = __node_2_pe(node);
4239 		goto out;
4240 	}
4241 
4242 	key.cpu = -1;
4243 	node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup);
4244 	if (node) {
4245 		event = __node_2_pe(node);
4246 		goto out;
4247 	}
4248 
4249 	key.cpu = smp_processor_id();
4250 	node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup);
4251 	if (node)
4252 		event = __node_2_pe(node);
4253 
4254 out:
4255 	/*
4256 	 * Unconditionally clear rotate_necessary; if ctx_flexible_sched_in()
4257 	 * finds there are unschedulable events, it will set it again.
4258 	 */
4259 	pmu_ctx->rotate_necessary = 0;
4260 
4261 	return event;
4262 }
4263 
perf_rotate_context(struct perf_cpu_pmu_context * cpc)4264 static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc)
4265 {
4266 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4267 	struct perf_event_pmu_context *cpu_epc, *task_epc = NULL;
4268 	struct perf_event *cpu_event = NULL, *task_event = NULL;
4269 	int cpu_rotate, task_rotate;
4270 	struct pmu *pmu;
4271 
4272 	/*
4273 	 * Since we run this from IRQ context, nobody can install new
4274 	 * events, thus the event count values are stable.
4275 	 */
4276 
4277 	cpu_epc = &cpc->epc;
4278 	pmu = cpu_epc->pmu;
4279 	task_epc = cpc->task_epc;
4280 
4281 	cpu_rotate = cpu_epc->rotate_necessary;
4282 	task_rotate = task_epc ? task_epc->rotate_necessary : 0;
4283 
4284 	if (!(cpu_rotate || task_rotate))
4285 		return false;
4286 
4287 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
4288 	perf_pmu_disable(pmu);
4289 
4290 	if (task_rotate)
4291 		task_event = ctx_event_to_rotate(task_epc);
4292 	if (cpu_rotate)
4293 		cpu_event = ctx_event_to_rotate(cpu_epc);
4294 
4295 	/*
4296 	 * As per the order given at ctx_resched() first 'pop' task flexible
4297 	 * and then, if needed CPU flexible.
4298 	 */
4299 	if (task_event || (task_epc && cpu_event)) {
4300 		update_context_time(task_epc->ctx);
4301 		__pmu_ctx_sched_out(task_epc, EVENT_FLEXIBLE);
4302 	}
4303 
4304 	if (cpu_event) {
4305 		update_context_time(&cpuctx->ctx);
4306 		__pmu_ctx_sched_out(cpu_epc, EVENT_FLEXIBLE);
4307 		rotate_ctx(&cpuctx->ctx, cpu_event);
4308 		__pmu_ctx_sched_in(&cpuctx->ctx, pmu);
4309 	}
4310 
4311 	if (task_event)
4312 		rotate_ctx(task_epc->ctx, task_event);
4313 
4314 	if (task_event || (task_epc && cpu_event))
4315 		__pmu_ctx_sched_in(task_epc->ctx, pmu);
4316 
4317 	perf_pmu_enable(pmu);
4318 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
4319 
4320 	return true;
4321 }
4322 
perf_event_task_tick(void)4323 void perf_event_task_tick(void)
4324 {
4325 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4326 	struct perf_event_context *ctx;
4327 	int throttled;
4328 
4329 	lockdep_assert_irqs_disabled();
4330 
4331 	__this_cpu_inc(perf_throttled_seq);
4332 	throttled = __this_cpu_xchg(perf_throttled_count, 0);
4333 	tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
4334 
4335 	perf_adjust_freq_unthr_context(&cpuctx->ctx, !!throttled);
4336 
4337 	rcu_read_lock();
4338 	ctx = rcu_dereference(current->perf_event_ctxp);
4339 	if (ctx)
4340 		perf_adjust_freq_unthr_context(ctx, !!throttled);
4341 	rcu_read_unlock();
4342 }
4343 
event_enable_on_exec(struct perf_event * event,struct perf_event_context * ctx)4344 static int event_enable_on_exec(struct perf_event *event,
4345 				struct perf_event_context *ctx)
4346 {
4347 	if (!event->attr.enable_on_exec)
4348 		return 0;
4349 
4350 	event->attr.enable_on_exec = 0;
4351 	if (event->state >= PERF_EVENT_STATE_INACTIVE)
4352 		return 0;
4353 
4354 	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
4355 
4356 	return 1;
4357 }
4358 
4359 /*
4360  * Enable all of a task's events that have been marked enable-on-exec.
4361  * This expects task == current.
4362  */
perf_event_enable_on_exec(struct perf_event_context * ctx)4363 static void perf_event_enable_on_exec(struct perf_event_context *ctx)
4364 {
4365 	struct perf_event_context *clone_ctx = NULL;
4366 	enum event_type_t event_type = 0;
4367 	struct perf_cpu_context *cpuctx;
4368 	struct perf_event *event;
4369 	unsigned long flags;
4370 	int enabled = 0;
4371 
4372 	local_irq_save(flags);
4373 	if (WARN_ON_ONCE(current->perf_event_ctxp != ctx))
4374 		goto out;
4375 
4376 	if (!ctx->nr_events)
4377 		goto out;
4378 
4379 	cpuctx = this_cpu_ptr(&perf_cpu_context);
4380 	perf_ctx_lock(cpuctx, ctx);
4381 	ctx_sched_out(ctx, EVENT_TIME);
4382 
4383 	list_for_each_entry(event, &ctx->event_list, event_entry) {
4384 		enabled |= event_enable_on_exec(event, ctx);
4385 		event_type |= get_event_type(event);
4386 	}
4387 
4388 	/*
4389 	 * Unclone and reschedule this context if we enabled any event.
4390 	 */
4391 	if (enabled) {
4392 		clone_ctx = unclone_ctx(ctx);
4393 		ctx_resched(cpuctx, ctx, event_type);
4394 	} else {
4395 		ctx_sched_in(ctx, EVENT_TIME);
4396 	}
4397 	perf_ctx_unlock(cpuctx, ctx);
4398 
4399 out:
4400 	local_irq_restore(flags);
4401 
4402 	if (clone_ctx)
4403 		put_ctx(clone_ctx);
4404 }
4405 
4406 static void perf_remove_from_owner(struct perf_event *event);
4407 static void perf_event_exit_event(struct perf_event *event,
4408 				  struct perf_event_context *ctx);
4409 
4410 /*
4411  * Removes all events from the current task that have been marked
4412  * remove-on-exec, and feeds their values back to parent events.
4413  */
perf_event_remove_on_exec(struct perf_event_context * ctx)4414 static void perf_event_remove_on_exec(struct perf_event_context *ctx)
4415 {
4416 	struct perf_event_context *clone_ctx = NULL;
4417 	struct perf_event *event, *next;
4418 	unsigned long flags;
4419 	bool modified = false;
4420 
4421 	mutex_lock(&ctx->mutex);
4422 
4423 	if (WARN_ON_ONCE(ctx->task != current))
4424 		goto unlock;
4425 
4426 	list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) {
4427 		if (!event->attr.remove_on_exec)
4428 			continue;
4429 
4430 		if (!is_kernel_event(event))
4431 			perf_remove_from_owner(event);
4432 
4433 		modified = true;
4434 
4435 		perf_event_exit_event(event, ctx);
4436 	}
4437 
4438 	raw_spin_lock_irqsave(&ctx->lock, flags);
4439 	if (modified)
4440 		clone_ctx = unclone_ctx(ctx);
4441 	raw_spin_unlock_irqrestore(&ctx->lock, flags);
4442 
4443 unlock:
4444 	mutex_unlock(&ctx->mutex);
4445 
4446 	if (clone_ctx)
4447 		put_ctx(clone_ctx);
4448 }
4449 
4450 struct perf_read_data {
4451 	struct perf_event *event;
4452 	bool group;
4453 	int ret;
4454 };
4455 
__perf_event_read_cpu(struct perf_event * event,int event_cpu)4456 static int __perf_event_read_cpu(struct perf_event *event, int event_cpu)
4457 {
4458 	u16 local_pkg, event_pkg;
4459 
4460 	if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {
4461 		int local_cpu = smp_processor_id();
4462 
4463 		event_pkg = topology_physical_package_id(event_cpu);
4464 		local_pkg = topology_physical_package_id(local_cpu);
4465 
4466 		if (event_pkg == local_pkg)
4467 			return local_cpu;
4468 	}
4469 
4470 	return event_cpu;
4471 }
4472 
4473 /*
4474  * Cross CPU call to read the hardware event
4475  */
__perf_event_read(void * info)4476 static void __perf_event_read(void *info)
4477 {
4478 	struct perf_read_data *data = info;
4479 	struct perf_event *sub, *event = data->event;
4480 	struct perf_event_context *ctx = event->ctx;
4481 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
4482 	struct pmu *pmu = event->pmu;
4483 
4484 	/*
4485 	 * If this is a task context, we need to check whether it is
4486 	 * the current task context of this cpu.  If not it has been
4487 	 * scheduled out before the smp call arrived.  In that case
4488 	 * event->count would have been updated to a recent sample
4489 	 * when the event was scheduled out.
4490 	 */
4491 	if (ctx->task && cpuctx->task_ctx != ctx)
4492 		return;
4493 
4494 	raw_spin_lock(&ctx->lock);
4495 	if (ctx->is_active & EVENT_TIME) {
4496 		update_context_time(ctx);
4497 		update_cgrp_time_from_event(event);
4498 	}
4499 
4500 	perf_event_update_time(event);
4501 	if (data->group)
4502 		perf_event_update_sibling_time(event);
4503 
4504 	if (event->state != PERF_EVENT_STATE_ACTIVE)
4505 		goto unlock;
4506 
4507 	if (!data->group) {
4508 		pmu->read(event);
4509 		data->ret = 0;
4510 		goto unlock;
4511 	}
4512 
4513 	pmu->start_txn(pmu, PERF_PMU_TXN_READ);
4514 
4515 	pmu->read(event);
4516 
4517 	for_each_sibling_event(sub, event) {
4518 		if (sub->state == PERF_EVENT_STATE_ACTIVE) {
4519 			/*
4520 			 * Use sibling's PMU rather than @event's since
4521 			 * sibling could be on different (eg: software) PMU.
4522 			 */
4523 			sub->pmu->read(sub);
4524 		}
4525 	}
4526 
4527 	data->ret = pmu->commit_txn(pmu);
4528 
4529 unlock:
4530 	raw_spin_unlock(&ctx->lock);
4531 }
4532 
perf_event_count(struct perf_event * event)4533 static inline u64 perf_event_count(struct perf_event *event)
4534 {
4535 	return local64_read(&event->count) + atomic64_read(&event->child_count);
4536 }
4537 
calc_timer_values(struct perf_event * event,u64 * now,u64 * enabled,u64 * running)4538 static void calc_timer_values(struct perf_event *event,
4539 				u64 *now,
4540 				u64 *enabled,
4541 				u64 *running)
4542 {
4543 	u64 ctx_time;
4544 
4545 	*now = perf_clock();
4546 	ctx_time = perf_event_time_now(event, *now);
4547 	__perf_update_times(event, ctx_time, enabled, running);
4548 }
4549 
4550 /*
4551  * NMI-safe method to read a local event, that is an event that
4552  * is:
4553  *   - either for the current task, or for this CPU
4554  *   - does not have inherit set, for inherited task events
4555  *     will not be local and we cannot read them atomically
4556  *   - must not have a pmu::count method
4557  */
perf_event_read_local(struct perf_event * event,u64 * value,u64 * enabled,u64 * running)4558 int perf_event_read_local(struct perf_event *event, u64 *value,
4559 			  u64 *enabled, u64 *running)
4560 {
4561 	unsigned long flags;
4562 	int ret = 0;
4563 
4564 	/*
4565 	 * Disabling interrupts avoids all counter scheduling (context
4566 	 * switches, timer based rotation and IPIs).
4567 	 */
4568 	local_irq_save(flags);
4569 
4570 	/*
4571 	 * It must not be an event with inherit set, we cannot read
4572 	 * all child counters from atomic context.
4573 	 */
4574 	if (event->attr.inherit) {
4575 		ret = -EOPNOTSUPP;
4576 		goto out;
4577 	}
4578 
4579 	/* If this is a per-task event, it must be for current */
4580 	if ((event->attach_state & PERF_ATTACH_TASK) &&
4581 	    event->hw.target != current) {
4582 		ret = -EINVAL;
4583 		goto out;
4584 	}
4585 
4586 	/* If this is a per-CPU event, it must be for this CPU */
4587 	if (!(event->attach_state & PERF_ATTACH_TASK) &&
4588 	    event->cpu != smp_processor_id()) {
4589 		ret = -EINVAL;
4590 		goto out;
4591 	}
4592 
4593 	/* If this is a pinned event it must be running on this CPU */
4594 	if (event->attr.pinned && event->oncpu != smp_processor_id()) {
4595 		ret = -EBUSY;
4596 		goto out;
4597 	}
4598 
4599 	/*
4600 	 * If the event is currently on this CPU, its either a per-task event,
4601 	 * or local to this CPU. Furthermore it means its ACTIVE (otherwise
4602 	 * oncpu == -1).
4603 	 */
4604 	if (event->oncpu == smp_processor_id())
4605 		event->pmu->read(event);
4606 
4607 	*value = local64_read(&event->count);
4608 	if (enabled || running) {
4609 		u64 __enabled, __running, __now;
4610 
4611 		calc_timer_values(event, &__now, &__enabled, &__running);
4612 		if (enabled)
4613 			*enabled = __enabled;
4614 		if (running)
4615 			*running = __running;
4616 	}
4617 out:
4618 	local_irq_restore(flags);
4619 
4620 	return ret;
4621 }
4622 
perf_event_read(struct perf_event * event,bool group)4623 static int perf_event_read(struct perf_event *event, bool group)
4624 {
4625 	enum perf_event_state state = READ_ONCE(event->state);
4626 	int event_cpu, ret = 0;
4627 
4628 	/*
4629 	 * If event is enabled and currently active on a CPU, update the
4630 	 * value in the event structure:
4631 	 */
4632 again:
4633 	if (state == PERF_EVENT_STATE_ACTIVE) {
4634 		struct perf_read_data data;
4635 
4636 		/*
4637 		 * Orders the ->state and ->oncpu loads such that if we see
4638 		 * ACTIVE we must also see the right ->oncpu.
4639 		 *
4640 		 * Matches the smp_wmb() from event_sched_in().
4641 		 */
4642 		smp_rmb();
4643 
4644 		event_cpu = READ_ONCE(event->oncpu);
4645 		if ((unsigned)event_cpu >= nr_cpu_ids)
4646 			return 0;
4647 
4648 		data = (struct perf_read_data){
4649 			.event = event,
4650 			.group = group,
4651 			.ret = 0,
4652 		};
4653 
4654 		preempt_disable();
4655 		event_cpu = __perf_event_read_cpu(event, event_cpu);
4656 
4657 		/*
4658 		 * Purposely ignore the smp_call_function_single() return
4659 		 * value.
4660 		 *
4661 		 * If event_cpu isn't a valid CPU it means the event got
4662 		 * scheduled out and that will have updated the event count.
4663 		 *
4664 		 * Therefore, either way, we'll have an up-to-date event count
4665 		 * after this.
4666 		 */
4667 		(void)smp_call_function_single(event_cpu, __perf_event_read, &data, 1);
4668 		preempt_enable();
4669 		ret = data.ret;
4670 
4671 	} else if (state == PERF_EVENT_STATE_INACTIVE) {
4672 		struct perf_event_context *ctx = event->ctx;
4673 		unsigned long flags;
4674 
4675 		raw_spin_lock_irqsave(&ctx->lock, flags);
4676 		state = event->state;
4677 		if (state != PERF_EVENT_STATE_INACTIVE) {
4678 			raw_spin_unlock_irqrestore(&ctx->lock, flags);
4679 			goto again;
4680 		}
4681 
4682 		/*
4683 		 * May read while context is not active (e.g., thread is
4684 		 * blocked), in that case we cannot update context time
4685 		 */
4686 		if (ctx->is_active & EVENT_TIME) {
4687 			update_context_time(ctx);
4688 			update_cgrp_time_from_event(event);
4689 		}
4690 
4691 		perf_event_update_time(event);
4692 		if (group)
4693 			perf_event_update_sibling_time(event);
4694 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
4695 	}
4696 
4697 	return ret;
4698 }
4699 
4700 /*
4701  * Initialize the perf_event context in a task_struct:
4702  */
__perf_event_init_context(struct perf_event_context * ctx)4703 static void __perf_event_init_context(struct perf_event_context *ctx)
4704 {
4705 	raw_spin_lock_init(&ctx->lock);
4706 	mutex_init(&ctx->mutex);
4707 	INIT_LIST_HEAD(&ctx->pmu_ctx_list);
4708 	perf_event_groups_init(&ctx->pinned_groups);
4709 	perf_event_groups_init(&ctx->flexible_groups);
4710 	INIT_LIST_HEAD(&ctx->event_list);
4711 	refcount_set(&ctx->refcount, 1);
4712 }
4713 
4714 static void
__perf_init_event_pmu_context(struct perf_event_pmu_context * epc,struct pmu * pmu)4715 __perf_init_event_pmu_context(struct perf_event_pmu_context *epc, struct pmu *pmu)
4716 {
4717 	epc->pmu = pmu;
4718 	INIT_LIST_HEAD(&epc->pmu_ctx_entry);
4719 	INIT_LIST_HEAD(&epc->pinned_active);
4720 	INIT_LIST_HEAD(&epc->flexible_active);
4721 	atomic_set(&epc->refcount, 1);
4722 }
4723 
4724 static struct perf_event_context *
alloc_perf_context(struct task_struct * task)4725 alloc_perf_context(struct task_struct *task)
4726 {
4727 	struct perf_event_context *ctx;
4728 
4729 	ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
4730 	if (!ctx)
4731 		return NULL;
4732 
4733 	__perf_event_init_context(ctx);
4734 	if (task)
4735 		ctx->task = get_task_struct(task);
4736 
4737 	return ctx;
4738 }
4739 
4740 static struct task_struct *
find_lively_task_by_vpid(pid_t vpid)4741 find_lively_task_by_vpid(pid_t vpid)
4742 {
4743 	struct task_struct *task;
4744 
4745 	rcu_read_lock();
4746 	if (!vpid)
4747 		task = current;
4748 	else
4749 		task = find_task_by_vpid(vpid);
4750 	if (task)
4751 		get_task_struct(task);
4752 	rcu_read_unlock();
4753 
4754 	if (!task)
4755 		return ERR_PTR(-ESRCH);
4756 
4757 	return task;
4758 }
4759 
4760 /*
4761  * Returns a matching context with refcount and pincount.
4762  */
4763 static struct perf_event_context *
find_get_context(struct task_struct * task,struct perf_event * event)4764 find_get_context(struct task_struct *task, struct perf_event *event)
4765 {
4766 	struct perf_event_context *ctx, *clone_ctx = NULL;
4767 	struct perf_cpu_context *cpuctx;
4768 	unsigned long flags;
4769 	int err;
4770 
4771 	if (!task) {
4772 		/* Must be root to operate on a CPU event: */
4773 		err = perf_allow_cpu(&event->attr);
4774 		if (err)
4775 			return ERR_PTR(err);
4776 
4777 		cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);
4778 		ctx = &cpuctx->ctx;
4779 		get_ctx(ctx);
4780 		raw_spin_lock_irqsave(&ctx->lock, flags);
4781 		++ctx->pin_count;
4782 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
4783 
4784 		return ctx;
4785 	}
4786 
4787 	err = -EINVAL;
4788 retry:
4789 	ctx = perf_lock_task_context(task, &flags);
4790 	if (ctx) {
4791 		clone_ctx = unclone_ctx(ctx);
4792 		++ctx->pin_count;
4793 
4794 		raw_spin_unlock_irqrestore(&ctx->lock, flags);
4795 
4796 		if (clone_ctx)
4797 			put_ctx(clone_ctx);
4798 	} else {
4799 		ctx = alloc_perf_context(task);
4800 		err = -ENOMEM;
4801 		if (!ctx)
4802 			goto errout;
4803 
4804 		err = 0;
4805 		mutex_lock(&task->perf_event_mutex);
4806 		/*
4807 		 * If it has already passed perf_event_exit_task().
4808 		 * we must see PF_EXITING, it takes this mutex too.
4809 		 */
4810 		if (task->flags & PF_EXITING)
4811 			err = -ESRCH;
4812 		else if (task->perf_event_ctxp)
4813 			err = -EAGAIN;
4814 		else {
4815 			get_ctx(ctx);
4816 			++ctx->pin_count;
4817 			rcu_assign_pointer(task->perf_event_ctxp, ctx);
4818 		}
4819 		mutex_unlock(&task->perf_event_mutex);
4820 
4821 		if (unlikely(err)) {
4822 			put_ctx(ctx);
4823 
4824 			if (err == -EAGAIN)
4825 				goto retry;
4826 			goto errout;
4827 		}
4828 	}
4829 
4830 	return ctx;
4831 
4832 errout:
4833 	return ERR_PTR(err);
4834 }
4835 
4836 static struct perf_event_pmu_context *
find_get_pmu_context(struct pmu * pmu,struct perf_event_context * ctx,struct perf_event * event)4837 find_get_pmu_context(struct pmu *pmu, struct perf_event_context *ctx,
4838 		     struct perf_event *event)
4839 {
4840 	struct perf_event_pmu_context *new = NULL, *epc;
4841 	void *task_ctx_data = NULL;
4842 
4843 	if (!ctx->task) {
4844 		/*
4845 		 * perf_pmu_migrate_context() / __perf_pmu_install_event()
4846 		 * relies on the fact that find_get_pmu_context() cannot fail
4847 		 * for CPU contexts.
4848 		 */
4849 		struct perf_cpu_pmu_context *cpc;
4850 
4851 		cpc = per_cpu_ptr(pmu->cpu_pmu_context, event->cpu);
4852 		epc = &cpc->epc;
4853 		raw_spin_lock_irq(&ctx->lock);
4854 		if (!epc->ctx) {
4855 			atomic_set(&epc->refcount, 1);
4856 			epc->embedded = 1;
4857 			list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);
4858 			epc->ctx = ctx;
4859 		} else {
4860 			WARN_ON_ONCE(epc->ctx != ctx);
4861 			atomic_inc(&epc->refcount);
4862 		}
4863 		raw_spin_unlock_irq(&ctx->lock);
4864 		return epc;
4865 	}
4866 
4867 	new = kzalloc(sizeof(*epc), GFP_KERNEL);
4868 	if (!new)
4869 		return ERR_PTR(-ENOMEM);
4870 
4871 	if (event->attach_state & PERF_ATTACH_TASK_DATA) {
4872 		task_ctx_data = alloc_task_ctx_data(pmu);
4873 		if (!task_ctx_data) {
4874 			kfree(new);
4875 			return ERR_PTR(-ENOMEM);
4876 		}
4877 	}
4878 
4879 	__perf_init_event_pmu_context(new, pmu);
4880 
4881 	/*
4882 	 * XXX
4883 	 *
4884 	 * lockdep_assert_held(&ctx->mutex);
4885 	 *
4886 	 * can't because perf_event_init_task() doesn't actually hold the
4887 	 * child_ctx->mutex.
4888 	 */
4889 
4890 	raw_spin_lock_irq(&ctx->lock);
4891 	list_for_each_entry(epc, &ctx->pmu_ctx_list, pmu_ctx_entry) {
4892 		if (epc->pmu == pmu) {
4893 			WARN_ON_ONCE(epc->ctx != ctx);
4894 			atomic_inc(&epc->refcount);
4895 			goto found_epc;
4896 		}
4897 	}
4898 
4899 	epc = new;
4900 	new = NULL;
4901 
4902 	list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list);
4903 	epc->ctx = ctx;
4904 
4905 found_epc:
4906 	if (task_ctx_data && !epc->task_ctx_data) {
4907 		epc->task_ctx_data = task_ctx_data;
4908 		task_ctx_data = NULL;
4909 		ctx->nr_task_data++;
4910 	}
4911 	raw_spin_unlock_irq(&ctx->lock);
4912 
4913 	free_task_ctx_data(pmu, task_ctx_data);
4914 	kfree(new);
4915 
4916 	return epc;
4917 }
4918 
get_pmu_ctx(struct perf_event_pmu_context * epc)4919 static void get_pmu_ctx(struct perf_event_pmu_context *epc)
4920 {
4921 	WARN_ON_ONCE(!atomic_inc_not_zero(&epc->refcount));
4922 }
4923 
free_epc_rcu(struct rcu_head * head)4924 static void free_epc_rcu(struct rcu_head *head)
4925 {
4926 	struct perf_event_pmu_context *epc = container_of(head, typeof(*epc), rcu_head);
4927 
4928 	kfree(epc->task_ctx_data);
4929 	kfree(epc);
4930 }
4931 
put_pmu_ctx(struct perf_event_pmu_context * epc)4932 static void put_pmu_ctx(struct perf_event_pmu_context *epc)
4933 {
4934 	struct perf_event_context *ctx = epc->ctx;
4935 	unsigned long flags;
4936 
4937 	/*
4938 	 * XXX
4939 	 *
4940 	 * lockdep_assert_held(&ctx->mutex);
4941 	 *
4942 	 * can't because of the call-site in _free_event()/put_event()
4943 	 * which isn't always called under ctx->mutex.
4944 	 */
4945 	if (!atomic_dec_and_raw_lock_irqsave(&epc->refcount, &ctx->lock, flags))
4946 		return;
4947 
4948 	WARN_ON_ONCE(list_empty(&epc->pmu_ctx_entry));
4949 
4950 	list_del_init(&epc->pmu_ctx_entry);
4951 	epc->ctx = NULL;
4952 
4953 	WARN_ON_ONCE(!list_empty(&epc->pinned_active));
4954 	WARN_ON_ONCE(!list_empty(&epc->flexible_active));
4955 
4956 	raw_spin_unlock_irqrestore(&ctx->lock, flags);
4957 
4958 	if (epc->embedded)
4959 		return;
4960 
4961 	call_rcu(&epc->rcu_head, free_epc_rcu);
4962 }
4963 
4964 static void perf_event_free_filter(struct perf_event *event);
4965 
free_event_rcu(struct rcu_head * head)4966 static void free_event_rcu(struct rcu_head *head)
4967 {
4968 	struct perf_event *event = container_of(head, typeof(*event), rcu_head);
4969 
4970 	if (event->ns)
4971 		put_pid_ns(event->ns);
4972 	perf_event_free_filter(event);
4973 	kmem_cache_free(perf_event_cache, event);
4974 }
4975 
4976 static void ring_buffer_attach(struct perf_event *event,
4977 			       struct perf_buffer *rb);
4978 
detach_sb_event(struct perf_event * event)4979 static void detach_sb_event(struct perf_event *event)
4980 {
4981 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
4982 
4983 	raw_spin_lock(&pel->lock);
4984 	list_del_rcu(&event->sb_list);
4985 	raw_spin_unlock(&pel->lock);
4986 }
4987 
is_sb_event(struct perf_event * event)4988 static bool is_sb_event(struct perf_event *event)
4989 {
4990 	struct perf_event_attr *attr = &event->attr;
4991 
4992 	if (event->parent)
4993 		return false;
4994 
4995 	if (event->attach_state & PERF_ATTACH_TASK)
4996 		return false;
4997 
4998 	if (attr->mmap || attr->mmap_data || attr->mmap2 ||
4999 	    attr->comm || attr->comm_exec ||
5000 	    attr->task || attr->ksymbol ||
5001 	    attr->context_switch || attr->text_poke ||
5002 	    attr->bpf_event)
5003 		return true;
5004 	return false;
5005 }
5006 
unaccount_pmu_sb_event(struct perf_event * event)5007 static void unaccount_pmu_sb_event(struct perf_event *event)
5008 {
5009 	if (is_sb_event(event))
5010 		detach_sb_event(event);
5011 }
5012 
5013 #ifdef CONFIG_NO_HZ_FULL
5014 static DEFINE_SPINLOCK(nr_freq_lock);
5015 #endif
5016 
unaccount_freq_event_nohz(void)5017 static void unaccount_freq_event_nohz(void)
5018 {
5019 #ifdef CONFIG_NO_HZ_FULL
5020 	spin_lock(&nr_freq_lock);
5021 	if (atomic_dec_and_test(&nr_freq_events))
5022 		tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS);
5023 	spin_unlock(&nr_freq_lock);
5024 #endif
5025 }
5026 
unaccount_freq_event(void)5027 static void unaccount_freq_event(void)
5028 {
5029 	if (tick_nohz_full_enabled())
5030 		unaccount_freq_event_nohz();
5031 	else
5032 		atomic_dec(&nr_freq_events);
5033 }
5034 
unaccount_event(struct perf_event * event)5035 static void unaccount_event(struct perf_event *event)
5036 {
5037 	bool dec = false;
5038 
5039 	if (event->parent)
5040 		return;
5041 
5042 	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
5043 		dec = true;
5044 	if (event->attr.mmap || event->attr.mmap_data)
5045 		atomic_dec(&nr_mmap_events);
5046 	if (event->attr.build_id)
5047 		atomic_dec(&nr_build_id_events);
5048 	if (event->attr.comm)
5049 		atomic_dec(&nr_comm_events);
5050 	if (event->attr.namespaces)
5051 		atomic_dec(&nr_namespaces_events);
5052 	if (event->attr.cgroup)
5053 		atomic_dec(&nr_cgroup_events);
5054 	if (event->attr.task)
5055 		atomic_dec(&nr_task_events);
5056 	if (event->attr.freq)
5057 		unaccount_freq_event();
5058 	if (event->attr.context_switch) {
5059 		dec = true;
5060 		atomic_dec(&nr_switch_events);
5061 	}
5062 	if (is_cgroup_event(event))
5063 		dec = true;
5064 	if (has_branch_stack(event))
5065 		dec = true;
5066 	if (event->attr.ksymbol)
5067 		atomic_dec(&nr_ksymbol_events);
5068 	if (event->attr.bpf_event)
5069 		atomic_dec(&nr_bpf_events);
5070 	if (event->attr.text_poke)
5071 		atomic_dec(&nr_text_poke_events);
5072 
5073 	if (dec) {
5074 		if (!atomic_add_unless(&perf_sched_count, -1, 1))
5075 			schedule_delayed_work(&perf_sched_work, HZ);
5076 	}
5077 
5078 	unaccount_pmu_sb_event(event);
5079 }
5080 
perf_sched_delayed(struct work_struct * work)5081 static void perf_sched_delayed(struct work_struct *work)
5082 {
5083 	mutex_lock(&perf_sched_mutex);
5084 	if (atomic_dec_and_test(&perf_sched_count))
5085 		static_branch_disable(&perf_sched_events);
5086 	mutex_unlock(&perf_sched_mutex);
5087 }
5088 
5089 /*
5090  * The following implement mutual exclusion of events on "exclusive" pmus
5091  * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled
5092  * at a time, so we disallow creating events that might conflict, namely:
5093  *
5094  *  1) cpu-wide events in the presence of per-task events,
5095  *  2) per-task events in the presence of cpu-wide events,
5096  *  3) two matching events on the same perf_event_context.
5097  *
5098  * The former two cases are handled in the allocation path (perf_event_alloc(),
5099  * _free_event()), the latter -- before the first perf_install_in_context().
5100  */
exclusive_event_init(struct perf_event * event)5101 static int exclusive_event_init(struct perf_event *event)
5102 {
5103 	struct pmu *pmu = event->pmu;
5104 
5105 	if (!is_exclusive_pmu(pmu))
5106 		return 0;
5107 
5108 	/*
5109 	 * Prevent co-existence of per-task and cpu-wide events on the
5110 	 * same exclusive pmu.
5111 	 *
5112 	 * Negative pmu::exclusive_cnt means there are cpu-wide
5113 	 * events on this "exclusive" pmu, positive means there are
5114 	 * per-task events.
5115 	 *
5116 	 * Since this is called in perf_event_alloc() path, event::ctx
5117 	 * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK
5118 	 * to mean "per-task event", because unlike other attach states it
5119 	 * never gets cleared.
5120 	 */
5121 	if (event->attach_state & PERF_ATTACH_TASK) {
5122 		if (!atomic_inc_unless_negative(&pmu->exclusive_cnt))
5123 			return -EBUSY;
5124 	} else {
5125 		if (!atomic_dec_unless_positive(&pmu->exclusive_cnt))
5126 			return -EBUSY;
5127 	}
5128 
5129 	return 0;
5130 }
5131 
exclusive_event_destroy(struct perf_event * event)5132 static void exclusive_event_destroy(struct perf_event *event)
5133 {
5134 	struct pmu *pmu = event->pmu;
5135 
5136 	if (!is_exclusive_pmu(pmu))
5137 		return;
5138 
5139 	/* see comment in exclusive_event_init() */
5140 	if (event->attach_state & PERF_ATTACH_TASK)
5141 		atomic_dec(&pmu->exclusive_cnt);
5142 	else
5143 		atomic_inc(&pmu->exclusive_cnt);
5144 }
5145 
exclusive_event_match(struct perf_event * e1,struct perf_event * e2)5146 static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2)
5147 {
5148 	if ((e1->pmu == e2->pmu) &&
5149 	    (e1->cpu == e2->cpu ||
5150 	     e1->cpu == -1 ||
5151 	     e2->cpu == -1))
5152 		return true;
5153 	return false;
5154 }
5155 
exclusive_event_installable(struct perf_event * event,struct perf_event_context * ctx)5156 static bool exclusive_event_installable(struct perf_event *event,
5157 					struct perf_event_context *ctx)
5158 {
5159 	struct perf_event *iter_event;
5160 	struct pmu *pmu = event->pmu;
5161 
5162 	lockdep_assert_held(&ctx->mutex);
5163 
5164 	if (!is_exclusive_pmu(pmu))
5165 		return true;
5166 
5167 	list_for_each_entry(iter_event, &ctx->event_list, event_entry) {
5168 		if (exclusive_event_match(iter_event, event))
5169 			return false;
5170 	}
5171 
5172 	return true;
5173 }
5174 
5175 static void perf_addr_filters_splice(struct perf_event *event,
5176 				       struct list_head *head);
5177 
_free_event(struct perf_event * event)5178 static void _free_event(struct perf_event *event)
5179 {
5180 	irq_work_sync(&event->pending_irq);
5181 
5182 	unaccount_event(event);
5183 
5184 	security_perf_event_free(event);
5185 
5186 	if (event->rb) {
5187 		/*
5188 		 * Can happen when we close an event with re-directed output.
5189 		 *
5190 		 * Since we have a 0 refcount, perf_mmap_close() will skip
5191 		 * over us; possibly making our ring_buffer_put() the last.
5192 		 */
5193 		mutex_lock(&event->mmap_mutex);
5194 		ring_buffer_attach(event, NULL);
5195 		mutex_unlock(&event->mmap_mutex);
5196 	}
5197 
5198 	if (is_cgroup_event(event))
5199 		perf_detach_cgroup(event);
5200 
5201 	if (!event->parent) {
5202 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
5203 			put_callchain_buffers();
5204 	}
5205 
5206 	perf_event_free_bpf_prog(event);
5207 	perf_addr_filters_splice(event, NULL);
5208 	kfree(event->addr_filter_ranges);
5209 
5210 	if (event->destroy)
5211 		event->destroy(event);
5212 
5213 	/*
5214 	 * Must be after ->destroy(), due to uprobe_perf_close() using
5215 	 * hw.target.
5216 	 */
5217 	if (event->hw.target)
5218 		put_task_struct(event->hw.target);
5219 
5220 	if (event->pmu_ctx)
5221 		put_pmu_ctx(event->pmu_ctx);
5222 
5223 	/*
5224 	 * perf_event_free_task() relies on put_ctx() being 'last', in particular
5225 	 * all task references must be cleaned up.
5226 	 */
5227 	if (event->ctx)
5228 		put_ctx(event->ctx);
5229 
5230 	exclusive_event_destroy(event);
5231 	module_put(event->pmu->module);
5232 
5233 	call_rcu(&event->rcu_head, free_event_rcu);
5234 }
5235 
5236 /*
5237  * Used to free events which have a known refcount of 1, such as in error paths
5238  * where the event isn't exposed yet and inherited events.
5239  */
free_event(struct perf_event * event)5240 static void free_event(struct perf_event *event)
5241 {
5242 	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
5243 				"unexpected event refcount: %ld; ptr=%p\n",
5244 				atomic_long_read(&event->refcount), event)) {
5245 		/* leak to avoid use-after-free */
5246 		return;
5247 	}
5248 
5249 	_free_event(event);
5250 }
5251 
5252 /*
5253  * Remove user event from the owner task.
5254  */
perf_remove_from_owner(struct perf_event * event)5255 static void perf_remove_from_owner(struct perf_event *event)
5256 {
5257 	struct task_struct *owner;
5258 
5259 	rcu_read_lock();
5260 	/*
5261 	 * Matches the smp_store_release() in perf_event_exit_task(). If we
5262 	 * observe !owner it means the list deletion is complete and we can
5263 	 * indeed free this event, otherwise we need to serialize on
5264 	 * owner->perf_event_mutex.
5265 	 */
5266 	owner = READ_ONCE(event->owner);
5267 	if (owner) {
5268 		/*
5269 		 * Since delayed_put_task_struct() also drops the last
5270 		 * task reference we can safely take a new reference
5271 		 * while holding the rcu_read_lock().
5272 		 */
5273 		get_task_struct(owner);
5274 	}
5275 	rcu_read_unlock();
5276 
5277 	if (owner) {
5278 		/*
5279 		 * If we're here through perf_event_exit_task() we're already
5280 		 * holding ctx->mutex which would be an inversion wrt. the
5281 		 * normal lock order.
5282 		 *
5283 		 * However we can safely take this lock because its the child
5284 		 * ctx->mutex.
5285 		 */
5286 		mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING);
5287 
5288 		/*
5289 		 * We have to re-check the event->owner field, if it is cleared
5290 		 * we raced with perf_event_exit_task(), acquiring the mutex
5291 		 * ensured they're done, and we can proceed with freeing the
5292 		 * event.
5293 		 */
5294 		if (event->owner) {
5295 			list_del_init(&event->owner_entry);
5296 			smp_store_release(&event->owner, NULL);
5297 		}
5298 		mutex_unlock(&owner->perf_event_mutex);
5299 		put_task_struct(owner);
5300 	}
5301 }
5302 
put_event(struct perf_event * event)5303 static void put_event(struct perf_event *event)
5304 {
5305 	if (!atomic_long_dec_and_test(&event->refcount))
5306 		return;
5307 
5308 	_free_event(event);
5309 }
5310 
5311 /*
5312  * Kill an event dead; while event:refcount will preserve the event
5313  * object, it will not preserve its functionality. Once the last 'user'
5314  * gives up the object, we'll destroy the thing.
5315  */
perf_event_release_kernel(struct perf_event * event)5316 int perf_event_release_kernel(struct perf_event *event)
5317 {
5318 	struct perf_event_context *ctx = event->ctx;
5319 	struct perf_event *child, *tmp;
5320 	LIST_HEAD(free_list);
5321 
5322 	/*
5323 	 * If we got here through err_alloc: free_event(event); we will not
5324 	 * have attached to a context yet.
5325 	 */
5326 	if (!ctx) {
5327 		WARN_ON_ONCE(event->attach_state &
5328 				(PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP));
5329 		goto no_ctx;
5330 	}
5331 
5332 	if (!is_kernel_event(event))
5333 		perf_remove_from_owner(event);
5334 
5335 	ctx = perf_event_ctx_lock(event);
5336 	WARN_ON_ONCE(ctx->parent_ctx);
5337 
5338 	/*
5339 	 * Mark this event as STATE_DEAD, there is no external reference to it
5340 	 * anymore.
5341 	 *
5342 	 * Anybody acquiring event->child_mutex after the below loop _must_
5343 	 * also see this, most importantly inherit_event() which will avoid
5344 	 * placing more children on the list.
5345 	 *
5346 	 * Thus this guarantees that we will in fact observe and kill _ALL_
5347 	 * child events.
5348 	 */
5349 	perf_remove_from_context(event, DETACH_GROUP|DETACH_DEAD);
5350 
5351 	perf_event_ctx_unlock(event, ctx);
5352 
5353 again:
5354 	mutex_lock(&event->child_mutex);
5355 	list_for_each_entry(child, &event->child_list, child_list) {
5356 
5357 		/*
5358 		 * Cannot change, child events are not migrated, see the
5359 		 * comment with perf_event_ctx_lock_nested().
5360 		 */
5361 		ctx = READ_ONCE(child->ctx);
5362 		/*
5363 		 * Since child_mutex nests inside ctx::mutex, we must jump
5364 		 * through hoops. We start by grabbing a reference on the ctx.
5365 		 *
5366 		 * Since the event cannot get freed while we hold the
5367 		 * child_mutex, the context must also exist and have a !0
5368 		 * reference count.
5369 		 */
5370 		get_ctx(ctx);
5371 
5372 		/*
5373 		 * Now that we have a ctx ref, we can drop child_mutex, and
5374 		 * acquire ctx::mutex without fear of it going away. Then we
5375 		 * can re-acquire child_mutex.
5376 		 */
5377 		mutex_unlock(&event->child_mutex);
5378 		mutex_lock(&ctx->mutex);
5379 		mutex_lock(&event->child_mutex);
5380 
5381 		/*
5382 		 * Now that we hold ctx::mutex and child_mutex, revalidate our
5383 		 * state, if child is still the first entry, it didn't get freed
5384 		 * and we can continue doing so.
5385 		 */
5386 		tmp = list_first_entry_or_null(&event->child_list,
5387 					       struct perf_event, child_list);
5388 		if (tmp == child) {
5389 			perf_remove_from_context(child, DETACH_GROUP);
5390 			list_move(&child->child_list, &free_list);
5391 			/*
5392 			 * This matches the refcount bump in inherit_event();
5393 			 * this can't be the last reference.
5394 			 */
5395 			put_event(event);
5396 		}
5397 
5398 		mutex_unlock(&event->child_mutex);
5399 		mutex_unlock(&ctx->mutex);
5400 		put_ctx(ctx);
5401 		goto again;
5402 	}
5403 	mutex_unlock(&event->child_mutex);
5404 
5405 	list_for_each_entry_safe(child, tmp, &free_list, child_list) {
5406 		void *var = &child->ctx->refcount;
5407 
5408 		list_del(&child->child_list);
5409 		free_event(child);
5410 
5411 		/*
5412 		 * Wake any perf_event_free_task() waiting for this event to be
5413 		 * freed.
5414 		 */
5415 		smp_mb(); /* pairs with wait_var_event() */
5416 		wake_up_var(var);
5417 	}
5418 
5419 no_ctx:
5420 	put_event(event); /* Must be the 'last' reference */
5421 	return 0;
5422 }
5423 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
5424 
5425 /*
5426  * Called when the last reference to the file is gone.
5427  */
perf_release(struct inode * inode,struct file * file)5428 static int perf_release(struct inode *inode, struct file *file)
5429 {
5430 	perf_event_release_kernel(file->private_data);
5431 	return 0;
5432 }
5433 
__perf_event_read_value(struct perf_event * event,u64 * enabled,u64 * running)5434 static u64 __perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
5435 {
5436 	struct perf_event *child;
5437 	u64 total = 0;
5438 
5439 	*enabled = 0;
5440 	*running = 0;
5441 
5442 	mutex_lock(&event->child_mutex);
5443 
5444 	(void)perf_event_read(event, false);
5445 	total += perf_event_count(event);
5446 
5447 	*enabled += event->total_time_enabled +
5448 			atomic64_read(&event->child_total_time_enabled);
5449 	*running += event->total_time_running +
5450 			atomic64_read(&event->child_total_time_running);
5451 
5452 	list_for_each_entry(child, &event->child_list, child_list) {
5453 		(void)perf_event_read(child, false);
5454 		total += perf_event_count(child);
5455 		*enabled += child->total_time_enabled;
5456 		*running += child->total_time_running;
5457 	}
5458 	mutex_unlock(&event->child_mutex);
5459 
5460 	return total;
5461 }
5462 
perf_event_read_value(struct perf_event * event,u64 * enabled,u64 * running)5463 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
5464 {
5465 	struct perf_event_context *ctx;
5466 	u64 count;
5467 
5468 	ctx = perf_event_ctx_lock(event);
5469 	count = __perf_event_read_value(event, enabled, running);
5470 	perf_event_ctx_unlock(event, ctx);
5471 
5472 	return count;
5473 }
5474 EXPORT_SYMBOL_GPL(perf_event_read_value);
5475 
__perf_read_group_add(struct perf_event * leader,u64 read_format,u64 * values)5476 static int __perf_read_group_add(struct perf_event *leader,
5477 					u64 read_format, u64 *values)
5478 {
5479 	struct perf_event_context *ctx = leader->ctx;
5480 	struct perf_event *sub, *parent;
5481 	unsigned long flags;
5482 	int n = 1; /* skip @nr */
5483 	int ret;
5484 
5485 	ret = perf_event_read(leader, true);
5486 	if (ret)
5487 		return ret;
5488 
5489 	raw_spin_lock_irqsave(&ctx->lock, flags);
5490 	/*
5491 	 * Verify the grouping between the parent and child (inherited)
5492 	 * events is still in tact.
5493 	 *
5494 	 * Specifically:
5495 	 *  - leader->ctx->lock pins leader->sibling_list
5496 	 *  - parent->child_mutex pins parent->child_list
5497 	 *  - parent->ctx->mutex pins parent->sibling_list
5498 	 *
5499 	 * Because parent->ctx != leader->ctx (and child_list nests inside
5500 	 * ctx->mutex), group destruction is not atomic between children, also
5501 	 * see perf_event_release_kernel(). Additionally, parent can grow the
5502 	 * group.
5503 	 *
5504 	 * Therefore it is possible to have parent and child groups in a
5505 	 * different configuration and summing over such a beast makes no sense
5506 	 * what so ever.
5507 	 *
5508 	 * Reject this.
5509 	 */
5510 	parent = leader->parent;
5511 	if (parent &&
5512 	    (parent->group_generation != leader->group_generation ||
5513 	     parent->nr_siblings != leader->nr_siblings)) {
5514 		ret = -ECHILD;
5515 		goto unlock;
5516 	}
5517 
5518 	/*
5519 	 * Since we co-schedule groups, {enabled,running} times of siblings
5520 	 * will be identical to those of the leader, so we only publish one
5521 	 * set.
5522 	 */
5523 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
5524 		values[n++] += leader->total_time_enabled +
5525 			atomic64_read(&leader->child_total_time_enabled);
5526 	}
5527 
5528 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
5529 		values[n++] += leader->total_time_running +
5530 			atomic64_read(&leader->child_total_time_running);
5531 	}
5532 
5533 	/*
5534 	 * Write {count,id} tuples for every sibling.
5535 	 */
5536 	values[n++] += perf_event_count(leader);
5537 	if (read_format & PERF_FORMAT_ID)
5538 		values[n++] = primary_event_id(leader);
5539 	if (read_format & PERF_FORMAT_LOST)
5540 		values[n++] = atomic64_read(&leader->lost_samples);
5541 
5542 	for_each_sibling_event(sub, leader) {
5543 		values[n++] += perf_event_count(sub);
5544 		if (read_format & PERF_FORMAT_ID)
5545 			values[n++] = primary_event_id(sub);
5546 		if (read_format & PERF_FORMAT_LOST)
5547 			values[n++] = atomic64_read(&sub->lost_samples);
5548 	}
5549 
5550 unlock:
5551 	raw_spin_unlock_irqrestore(&ctx->lock, flags);
5552 	return ret;
5553 }
5554 
perf_read_group(struct perf_event * event,u64 read_format,char __user * buf)5555 static int perf_read_group(struct perf_event *event,
5556 				   u64 read_format, char __user *buf)
5557 {
5558 	struct perf_event *leader = event->group_leader, *child;
5559 	struct perf_event_context *ctx = leader->ctx;
5560 	int ret;
5561 	u64 *values;
5562 
5563 	lockdep_assert_held(&ctx->mutex);
5564 
5565 	values = kzalloc(event->read_size, GFP_KERNEL);
5566 	if (!values)
5567 		return -ENOMEM;
5568 
5569 	values[0] = 1 + leader->nr_siblings;
5570 
5571 	mutex_lock(&leader->child_mutex);
5572 
5573 	ret = __perf_read_group_add(leader, read_format, values);
5574 	if (ret)
5575 		goto unlock;
5576 
5577 	list_for_each_entry(child, &leader->child_list, child_list) {
5578 		ret = __perf_read_group_add(child, read_format, values);
5579 		if (ret)
5580 			goto unlock;
5581 	}
5582 
5583 	mutex_unlock(&leader->child_mutex);
5584 
5585 	ret = event->read_size;
5586 	if (copy_to_user(buf, values, event->read_size))
5587 		ret = -EFAULT;
5588 	goto out;
5589 
5590 unlock:
5591 	mutex_unlock(&leader->child_mutex);
5592 out:
5593 	kfree(values);
5594 	return ret;
5595 }
5596 
perf_read_one(struct perf_event * event,u64 read_format,char __user * buf)5597 static int perf_read_one(struct perf_event *event,
5598 				 u64 read_format, char __user *buf)
5599 {
5600 	u64 enabled, running;
5601 	u64 values[5];
5602 	int n = 0;
5603 
5604 	values[n++] = __perf_event_read_value(event, &enabled, &running);
5605 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
5606 		values[n++] = enabled;
5607 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
5608 		values[n++] = running;
5609 	if (read_format & PERF_FORMAT_ID)
5610 		values[n++] = primary_event_id(event);
5611 	if (read_format & PERF_FORMAT_LOST)
5612 		values[n++] = atomic64_read(&event->lost_samples);
5613 
5614 	if (copy_to_user(buf, values, n * sizeof(u64)))
5615 		return -EFAULT;
5616 
5617 	return n * sizeof(u64);
5618 }
5619 
is_event_hup(struct perf_event * event)5620 static bool is_event_hup(struct perf_event *event)
5621 {
5622 	bool no_children;
5623 
5624 	if (event->state > PERF_EVENT_STATE_EXIT)
5625 		return false;
5626 
5627 	mutex_lock(&event->child_mutex);
5628 	no_children = list_empty(&event->child_list);
5629 	mutex_unlock(&event->child_mutex);
5630 	return no_children;
5631 }
5632 
5633 /*
5634  * Read the performance event - simple non blocking version for now
5635  */
5636 static ssize_t
__perf_read(struct perf_event * event,char __user * buf,size_t count)5637 __perf_read(struct perf_event *event, char __user *buf, size_t count)
5638 {
5639 	u64 read_format = event->attr.read_format;
5640 	int ret;
5641 
5642 	/*
5643 	 * Return end-of-file for a read on an event that is in
5644 	 * error state (i.e. because it was pinned but it couldn't be
5645 	 * scheduled on to the CPU at some point).
5646 	 */
5647 	if (event->state == PERF_EVENT_STATE_ERROR)
5648 		return 0;
5649 
5650 	if (count < event->read_size)
5651 		return -ENOSPC;
5652 
5653 	WARN_ON_ONCE(event->ctx->parent_ctx);
5654 	if (read_format & PERF_FORMAT_GROUP)
5655 		ret = perf_read_group(event, read_format, buf);
5656 	else
5657 		ret = perf_read_one(event, read_format, buf);
5658 
5659 	return ret;
5660 }
5661 
5662 static ssize_t
perf_read(struct file * file,char __user * buf,size_t count,loff_t * ppos)5663 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
5664 {
5665 	struct perf_event *event = file->private_data;
5666 	struct perf_event_context *ctx;
5667 	int ret;
5668 
5669 	ret = security_perf_event_read(event);
5670 	if (ret)
5671 		return ret;
5672 
5673 	ctx = perf_event_ctx_lock(event);
5674 	ret = __perf_read(event, buf, count);
5675 	perf_event_ctx_unlock(event, ctx);
5676 
5677 	return ret;
5678 }
5679 
perf_poll(struct file * file,poll_table * wait)5680 static __poll_t perf_poll(struct file *file, poll_table *wait)
5681 {
5682 	struct perf_event *event = file->private_data;
5683 	struct perf_buffer *rb;
5684 	__poll_t events = EPOLLHUP;
5685 
5686 	poll_wait(file, &event->waitq, wait);
5687 
5688 	if (is_event_hup(event))
5689 		return events;
5690 
5691 	/*
5692 	 * Pin the event->rb by taking event->mmap_mutex; otherwise
5693 	 * perf_event_set_output() can swizzle our rb and make us miss wakeups.
5694 	 */
5695 	mutex_lock(&event->mmap_mutex);
5696 	rb = event->rb;
5697 	if (rb)
5698 		events = atomic_xchg(&rb->poll, 0);
5699 	mutex_unlock(&event->mmap_mutex);
5700 	return events;
5701 }
5702 
_perf_event_reset(struct perf_event * event)5703 static void _perf_event_reset(struct perf_event *event)
5704 {
5705 	(void)perf_event_read(event, false);
5706 	local64_set(&event->count, 0);
5707 	perf_event_update_userpage(event);
5708 }
5709 
5710 /* Assume it's not an event with inherit set. */
perf_event_pause(struct perf_event * event,bool reset)5711 u64 perf_event_pause(struct perf_event *event, bool reset)
5712 {
5713 	struct perf_event_context *ctx;
5714 	u64 count;
5715 
5716 	ctx = perf_event_ctx_lock(event);
5717 	WARN_ON_ONCE(event->attr.inherit);
5718 	_perf_event_disable(event);
5719 	count = local64_read(&event->count);
5720 	if (reset)
5721 		local64_set(&event->count, 0);
5722 	perf_event_ctx_unlock(event, ctx);
5723 
5724 	return count;
5725 }
5726 EXPORT_SYMBOL_GPL(perf_event_pause);
5727 
5728 /*
5729  * Holding the top-level event's child_mutex means that any
5730  * descendant process that has inherited this event will block
5731  * in perf_event_exit_event() if it goes to exit, thus satisfying the
5732  * task existence requirements of perf_event_enable/disable.
5733  */
perf_event_for_each_child(struct perf_event * event,void (* func)(struct perf_event *))5734 static void perf_event_for_each_child(struct perf_event *event,
5735 					void (*func)(struct perf_event *))
5736 {
5737 	struct perf_event *child;
5738 
5739 	WARN_ON_ONCE(event->ctx->parent_ctx);
5740 
5741 	mutex_lock(&event->child_mutex);
5742 	func(event);
5743 	list_for_each_entry(child, &event->child_list, child_list)
5744 		func(child);
5745 	mutex_unlock(&event->child_mutex);
5746 }
5747 
perf_event_for_each(struct perf_event * event,void (* func)(struct perf_event *))5748 static void perf_event_for_each(struct perf_event *event,
5749 				  void (*func)(struct perf_event *))
5750 {
5751 	struct perf_event_context *ctx = event->ctx;
5752 	struct perf_event *sibling;
5753 
5754 	lockdep_assert_held(&ctx->mutex);
5755 
5756 	event = event->group_leader;
5757 
5758 	perf_event_for_each_child(event, func);
5759 	for_each_sibling_event(sibling, event)
5760 		perf_event_for_each_child(sibling, func);
5761 }
5762 
__perf_event_period(struct perf_event * event,struct perf_cpu_context * cpuctx,struct perf_event_context * ctx,void * info)5763 static void __perf_event_period(struct perf_event *event,
5764 				struct perf_cpu_context *cpuctx,
5765 				struct perf_event_context *ctx,
5766 				void *info)
5767 {
5768 	u64 value = *((u64 *)info);
5769 	bool active;
5770 
5771 	if (event->attr.freq) {
5772 		event->attr.sample_freq = value;
5773 	} else {
5774 		event->attr.sample_period = value;
5775 		event->hw.sample_period = value;
5776 	}
5777 
5778 	active = (event->state == PERF_EVENT_STATE_ACTIVE);
5779 	if (active) {
5780 		perf_pmu_disable(event->pmu);
5781 		/*
5782 		 * We could be throttled; unthrottle now to avoid the tick
5783 		 * trying to unthrottle while we already re-started the event.
5784 		 */
5785 		if (event->hw.interrupts == MAX_INTERRUPTS) {
5786 			event->hw.interrupts = 0;
5787 			perf_log_throttle(event, 1);
5788 		}
5789 		event->pmu->stop(event, PERF_EF_UPDATE);
5790 	}
5791 
5792 	local64_set(&event->hw.period_left, 0);
5793 
5794 	if (active) {
5795 		event->pmu->start(event, PERF_EF_RELOAD);
5796 		perf_pmu_enable(event->pmu);
5797 	}
5798 }
5799 
perf_event_check_period(struct perf_event * event,u64 value)5800 static int perf_event_check_period(struct perf_event *event, u64 value)
5801 {
5802 	return event->pmu->check_period(event, value);
5803 }
5804 
_perf_event_period(struct perf_event * event,u64 value)5805 static int _perf_event_period(struct perf_event *event, u64 value)
5806 {
5807 	if (!is_sampling_event(event))
5808 		return -EINVAL;
5809 
5810 	if (!value)
5811 		return -EINVAL;
5812 
5813 	if (event->attr.freq && value > sysctl_perf_event_sample_rate)
5814 		return -EINVAL;
5815 
5816 	if (perf_event_check_period(event, value))
5817 		return -EINVAL;
5818 
5819 	if (!event->attr.freq && (value & (1ULL << 63)))
5820 		return -EINVAL;
5821 
5822 	event_function_call(event, __perf_event_period, &value);
5823 
5824 	return 0;
5825 }
5826 
perf_event_period(struct perf_event * event,u64 value)5827 int perf_event_period(struct perf_event *event, u64 value)
5828 {
5829 	struct perf_event_context *ctx;
5830 	int ret;
5831 
5832 	ctx = perf_event_ctx_lock(event);
5833 	ret = _perf_event_period(event, value);
5834 	perf_event_ctx_unlock(event, ctx);
5835 
5836 	return ret;
5837 }
5838 EXPORT_SYMBOL_GPL(perf_event_period);
5839 
5840 static const struct file_operations perf_fops;
5841 
perf_fget_light(int fd,struct fd * p)5842 static inline int perf_fget_light(int fd, struct fd *p)
5843 {
5844 	struct fd f = fdget(fd);
5845 	if (!f.file)
5846 		return -EBADF;
5847 
5848 	if (f.file->f_op != &perf_fops) {
5849 		fdput(f);
5850 		return -EBADF;
5851 	}
5852 	*p = f;
5853 	return 0;
5854 }
5855 
5856 static int perf_event_set_output(struct perf_event *event,
5857 				 struct perf_event *output_event);
5858 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
5859 static int perf_copy_attr(struct perf_event_attr __user *uattr,
5860 			  struct perf_event_attr *attr);
5861 
_perf_ioctl(struct perf_event * event,unsigned int cmd,unsigned long arg)5862 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
5863 {
5864 	void (*func)(struct perf_event *);
5865 	u32 flags = arg;
5866 
5867 	switch (cmd) {
5868 	case PERF_EVENT_IOC_ENABLE:
5869 		func = _perf_event_enable;
5870 		break;
5871 	case PERF_EVENT_IOC_DISABLE:
5872 		func = _perf_event_disable;
5873 		break;
5874 	case PERF_EVENT_IOC_RESET:
5875 		func = _perf_event_reset;
5876 		break;
5877 
5878 	case PERF_EVENT_IOC_REFRESH:
5879 		return _perf_event_refresh(event, arg);
5880 
5881 	case PERF_EVENT_IOC_PERIOD:
5882 	{
5883 		u64 value;
5884 
5885 		if (copy_from_user(&value, (u64 __user *)arg, sizeof(value)))
5886 			return -EFAULT;
5887 
5888 		return _perf_event_period(event, value);
5889 	}
5890 	case PERF_EVENT_IOC_ID:
5891 	{
5892 		u64 id = primary_event_id(event);
5893 
5894 		if (copy_to_user((void __user *)arg, &id, sizeof(id)))
5895 			return -EFAULT;
5896 		return 0;
5897 	}
5898 
5899 	case PERF_EVENT_IOC_SET_OUTPUT:
5900 	{
5901 		int ret;
5902 		if (arg != -1) {
5903 			struct perf_event *output_event;
5904 			struct fd output;
5905 			ret = perf_fget_light(arg, &output);
5906 			if (ret)
5907 				return ret;
5908 			output_event = output.file->private_data;
5909 			ret = perf_event_set_output(event, output_event);
5910 			fdput(output);
5911 		} else {
5912 			ret = perf_event_set_output(event, NULL);
5913 		}
5914 		return ret;
5915 	}
5916 
5917 	case PERF_EVENT_IOC_SET_FILTER:
5918 		return perf_event_set_filter(event, (void __user *)arg);
5919 
5920 	case PERF_EVENT_IOC_SET_BPF:
5921 	{
5922 		struct bpf_prog *prog;
5923 		int err;
5924 
5925 		prog = bpf_prog_get(arg);
5926 		if (IS_ERR(prog))
5927 			return PTR_ERR(prog);
5928 
5929 		err = perf_event_set_bpf_prog(event, prog, 0);
5930 		if (err) {
5931 			bpf_prog_put(prog);
5932 			return err;
5933 		}
5934 
5935 		return 0;
5936 	}
5937 
5938 	case PERF_EVENT_IOC_PAUSE_OUTPUT: {
5939 		struct perf_buffer *rb;
5940 
5941 		rcu_read_lock();
5942 		rb = rcu_dereference(event->rb);
5943 		if (!rb || !rb->nr_pages) {
5944 			rcu_read_unlock();
5945 			return -EINVAL;
5946 		}
5947 		rb_toggle_paused(rb, !!arg);
5948 		rcu_read_unlock();
5949 		return 0;
5950 	}
5951 
5952 	case PERF_EVENT_IOC_QUERY_BPF:
5953 		return perf_event_query_prog_array(event, (void __user *)arg);
5954 
5955 	case PERF_EVENT_IOC_MODIFY_ATTRIBUTES: {
5956 		struct perf_event_attr new_attr;
5957 		int err = perf_copy_attr((struct perf_event_attr __user *)arg,
5958 					 &new_attr);
5959 
5960 		if (err)
5961 			return err;
5962 
5963 		return perf_event_modify_attr(event,  &new_attr);
5964 	}
5965 	default:
5966 		return -ENOTTY;
5967 	}
5968 
5969 	if (flags & PERF_IOC_FLAG_GROUP)
5970 		perf_event_for_each(event, func);
5971 	else
5972 		perf_event_for_each_child(event, func);
5973 
5974 	return 0;
5975 }
5976 
perf_ioctl(struct file * file,unsigned int cmd,unsigned long arg)5977 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
5978 {
5979 	struct perf_event *event = file->private_data;
5980 	struct perf_event_context *ctx;
5981 	long ret;
5982 
5983 	/* Treat ioctl like writes as it is likely a mutating operation. */
5984 	ret = security_perf_event_write(event);
5985 	if (ret)
5986 		return ret;
5987 
5988 	ctx = perf_event_ctx_lock(event);
5989 	ret = _perf_ioctl(event, cmd, arg);
5990 	perf_event_ctx_unlock(event, ctx);
5991 
5992 	return ret;
5993 }
5994 
5995 #ifdef CONFIG_COMPAT
perf_compat_ioctl(struct file * file,unsigned int cmd,unsigned long arg)5996 static long perf_compat_ioctl(struct file *file, unsigned int cmd,
5997 				unsigned long arg)
5998 {
5999 	switch (_IOC_NR(cmd)) {
6000 	case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
6001 	case _IOC_NR(PERF_EVENT_IOC_ID):
6002 	case _IOC_NR(PERF_EVENT_IOC_QUERY_BPF):
6003 	case _IOC_NR(PERF_EVENT_IOC_MODIFY_ATTRIBUTES):
6004 		/* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
6005 		if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
6006 			cmd &= ~IOCSIZE_MASK;
6007 			cmd |= sizeof(void *) << IOCSIZE_SHIFT;
6008 		}
6009 		break;
6010 	}
6011 	return perf_ioctl(file, cmd, arg);
6012 }
6013 #else
6014 # define perf_compat_ioctl NULL
6015 #endif
6016 
perf_event_task_enable(void)6017 int perf_event_task_enable(void)
6018 {
6019 	struct perf_event_context *ctx;
6020 	struct perf_event *event;
6021 
6022 	mutex_lock(&current->perf_event_mutex);
6023 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
6024 		ctx = perf_event_ctx_lock(event);
6025 		perf_event_for_each_child(event, _perf_event_enable);
6026 		perf_event_ctx_unlock(event, ctx);
6027 	}
6028 	mutex_unlock(&current->perf_event_mutex);
6029 
6030 	return 0;
6031 }
6032 
perf_event_task_disable(void)6033 int perf_event_task_disable(void)
6034 {
6035 	struct perf_event_context *ctx;
6036 	struct perf_event *event;
6037 
6038 	mutex_lock(&current->perf_event_mutex);
6039 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {
6040 		ctx = perf_event_ctx_lock(event);
6041 		perf_event_for_each_child(event, _perf_event_disable);
6042 		perf_event_ctx_unlock(event, ctx);
6043 	}
6044 	mutex_unlock(&current->perf_event_mutex);
6045 
6046 	return 0;
6047 }
6048 
perf_event_index(struct perf_event * event)6049 static int perf_event_index(struct perf_event *event)
6050 {
6051 	if (event->hw.state & PERF_HES_STOPPED)
6052 		return 0;
6053 
6054 	if (event->state != PERF_EVENT_STATE_ACTIVE)
6055 		return 0;
6056 
6057 	return event->pmu->event_idx(event);
6058 }
6059 
perf_event_init_userpage(struct perf_event * event)6060 static void perf_event_init_userpage(struct perf_event *event)
6061 {
6062 	struct perf_event_mmap_page *userpg;
6063 	struct perf_buffer *rb;
6064 
6065 	rcu_read_lock();
6066 	rb = rcu_dereference(event->rb);
6067 	if (!rb)
6068 		goto unlock;
6069 
6070 	userpg = rb->user_page;
6071 
6072 	/* Allow new userspace to detect that bit 0 is deprecated */
6073 	userpg->cap_bit0_is_deprecated = 1;
6074 	userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
6075 	userpg->data_offset = PAGE_SIZE;
6076 	userpg->data_size = perf_data_size(rb);
6077 
6078 unlock:
6079 	rcu_read_unlock();
6080 }
6081 
arch_perf_update_userpage(struct perf_event * event,struct perf_event_mmap_page * userpg,u64 now)6082 void __weak arch_perf_update_userpage(
6083 	struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now)
6084 {
6085 }
6086 
6087 /*
6088  * Callers need to ensure there can be no nesting of this function, otherwise
6089  * the seqlock logic goes bad. We can not serialize this because the arch
6090  * code calls this from NMI context.
6091  */
perf_event_update_userpage(struct perf_event * event)6092 void perf_event_update_userpage(struct perf_event *event)
6093 {
6094 	struct perf_event_mmap_page *userpg;
6095 	struct perf_buffer *rb;
6096 	u64 enabled, running, now;
6097 
6098 	rcu_read_lock();
6099 	rb = rcu_dereference(event->rb);
6100 	if (!rb)
6101 		goto unlock;
6102 
6103 	/*
6104 	 * compute total_time_enabled, total_time_running
6105 	 * based on snapshot values taken when the event
6106 	 * was last scheduled in.
6107 	 *
6108 	 * we cannot simply called update_context_time()
6109 	 * because of locking issue as we can be called in
6110 	 * NMI context
6111 	 */
6112 	calc_timer_values(event, &now, &enabled, &running);
6113 
6114 	userpg = rb->user_page;
6115 	/*
6116 	 * Disable preemption to guarantee consistent time stamps are stored to
6117 	 * the user page.
6118 	 */
6119 	preempt_disable();
6120 	++userpg->lock;
6121 	barrier();
6122 	userpg->index = perf_event_index(event);
6123 	userpg->offset = perf_event_count(event);
6124 	if (userpg->index)
6125 		userpg->offset -= local64_read(&event->hw.prev_count);
6126 
6127 	userpg->time_enabled = enabled +
6128 			atomic64_read(&event->child_total_time_enabled);
6129 
6130 	userpg->time_running = running +
6131 			atomic64_read(&event->child_total_time_running);
6132 
6133 	arch_perf_update_userpage(event, userpg, now);
6134 
6135 	barrier();
6136 	++userpg->lock;
6137 	preempt_enable();
6138 unlock:
6139 	rcu_read_unlock();
6140 }
6141 EXPORT_SYMBOL_GPL(perf_event_update_userpage);
6142 
perf_mmap_fault(struct vm_fault * vmf)6143 static vm_fault_t perf_mmap_fault(struct vm_fault *vmf)
6144 {
6145 	struct perf_event *event = vmf->vma->vm_file->private_data;
6146 	struct perf_buffer *rb;
6147 	vm_fault_t ret = VM_FAULT_SIGBUS;
6148 
6149 	if (vmf->flags & FAULT_FLAG_MKWRITE) {
6150 		if (vmf->pgoff == 0)
6151 			ret = 0;
6152 		return ret;
6153 	}
6154 
6155 	rcu_read_lock();
6156 	rb = rcu_dereference(event->rb);
6157 	if (!rb)
6158 		goto unlock;
6159 
6160 	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
6161 		goto unlock;
6162 
6163 	vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
6164 	if (!vmf->page)
6165 		goto unlock;
6166 
6167 	get_page(vmf->page);
6168 	vmf->page->mapping = vmf->vma->vm_file->f_mapping;
6169 	vmf->page->index   = vmf->pgoff;
6170 
6171 	ret = 0;
6172 unlock:
6173 	rcu_read_unlock();
6174 
6175 	return ret;
6176 }
6177 
ring_buffer_attach(struct perf_event * event,struct perf_buffer * rb)6178 static void ring_buffer_attach(struct perf_event *event,
6179 			       struct perf_buffer *rb)
6180 {
6181 	struct perf_buffer *old_rb = NULL;
6182 	unsigned long flags;
6183 
6184 	WARN_ON_ONCE(event->parent);
6185 
6186 	if (event->rb) {
6187 		/*
6188 		 * Should be impossible, we set this when removing
6189 		 * event->rb_entry and wait/clear when adding event->rb_entry.
6190 		 */
6191 		WARN_ON_ONCE(event->rcu_pending);
6192 
6193 		old_rb = event->rb;
6194 		spin_lock_irqsave(&old_rb->event_lock, flags);
6195 		list_del_rcu(&event->rb_entry);
6196 		spin_unlock_irqrestore(&old_rb->event_lock, flags);
6197 
6198 		event->rcu_batches = get_state_synchronize_rcu();
6199 		event->rcu_pending = 1;
6200 	}
6201 
6202 	if (rb) {
6203 		if (event->rcu_pending) {
6204 			cond_synchronize_rcu(event->rcu_batches);
6205 			event->rcu_pending = 0;
6206 		}
6207 
6208 		spin_lock_irqsave(&rb->event_lock, flags);
6209 		list_add_rcu(&event->rb_entry, &rb->event_list);
6210 		spin_unlock_irqrestore(&rb->event_lock, flags);
6211 	}
6212 
6213 	/*
6214 	 * Avoid racing with perf_mmap_close(AUX): stop the event
6215 	 * before swizzling the event::rb pointer; if it's getting
6216 	 * unmapped, its aux_mmap_count will be 0 and it won't
6217 	 * restart. See the comment in __perf_pmu_output_stop().
6218 	 *
6219 	 * Data will inevitably be lost when set_output is done in
6220 	 * mid-air, but then again, whoever does it like this is
6221 	 * not in for the data anyway.
6222 	 */
6223 	if (has_aux(event))
6224 		perf_event_stop(event, 0);
6225 
6226 	rcu_assign_pointer(event->rb, rb);
6227 
6228 	if (old_rb) {
6229 		ring_buffer_put(old_rb);
6230 		/*
6231 		 * Since we detached before setting the new rb, so that we
6232 		 * could attach the new rb, we could have missed a wakeup.
6233 		 * Provide it now.
6234 		 */
6235 		wake_up_all(&event->waitq);
6236 	}
6237 }
6238 
ring_buffer_wakeup(struct perf_event * event)6239 static void ring_buffer_wakeup(struct perf_event *event)
6240 {
6241 	struct perf_buffer *rb;
6242 
6243 	if (event->parent)
6244 		event = event->parent;
6245 
6246 	rcu_read_lock();
6247 	rb = rcu_dereference(event->rb);
6248 	if (rb) {
6249 		list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
6250 			wake_up_all(&event->waitq);
6251 	}
6252 	rcu_read_unlock();
6253 }
6254 
ring_buffer_get(struct perf_event * event)6255 struct perf_buffer *ring_buffer_get(struct perf_event *event)
6256 {
6257 	struct perf_buffer *rb;
6258 
6259 	if (event->parent)
6260 		event = event->parent;
6261 
6262 	rcu_read_lock();
6263 	rb = rcu_dereference(event->rb);
6264 	if (rb) {
6265 		if (!refcount_inc_not_zero(&rb->refcount))
6266 			rb = NULL;
6267 	}
6268 	rcu_read_unlock();
6269 
6270 	return rb;
6271 }
6272 
ring_buffer_put(struct perf_buffer * rb)6273 void ring_buffer_put(struct perf_buffer *rb)
6274 {
6275 	if (!refcount_dec_and_test(&rb->refcount))
6276 		return;
6277 
6278 	WARN_ON_ONCE(!list_empty(&rb->event_list));
6279 
6280 	call_rcu(&rb->rcu_head, rb_free_rcu);
6281 }
6282 
perf_mmap_open(struct vm_area_struct * vma)6283 static void perf_mmap_open(struct vm_area_struct *vma)
6284 {
6285 	struct perf_event *event = vma->vm_file->private_data;
6286 
6287 	atomic_inc(&event->mmap_count);
6288 	atomic_inc(&event->rb->mmap_count);
6289 
6290 	if (vma->vm_pgoff)
6291 		atomic_inc(&event->rb->aux_mmap_count);
6292 
6293 	if (event->pmu->event_mapped)
6294 		event->pmu->event_mapped(event, vma->vm_mm);
6295 }
6296 
6297 static void perf_pmu_output_stop(struct perf_event *event);
6298 
6299 /*
6300  * A buffer can be mmap()ed multiple times; either directly through the same
6301  * event, or through other events by use of perf_event_set_output().
6302  *
6303  * In order to undo the VM accounting done by perf_mmap() we need to destroy
6304  * the buffer here, where we still have a VM context. This means we need
6305  * to detach all events redirecting to us.
6306  */
perf_mmap_close(struct vm_area_struct * vma)6307 static void perf_mmap_close(struct vm_area_struct *vma)
6308 {
6309 	struct perf_event *event = vma->vm_file->private_data;
6310 	struct perf_buffer *rb = ring_buffer_get(event);
6311 	struct user_struct *mmap_user = rb->mmap_user;
6312 	int mmap_locked = rb->mmap_locked;
6313 	unsigned long size = perf_data_size(rb);
6314 	bool detach_rest = false;
6315 
6316 	if (event->pmu->event_unmapped)
6317 		event->pmu->event_unmapped(event, vma->vm_mm);
6318 
6319 	/*
6320 	 * rb->aux_mmap_count will always drop before rb->mmap_count and
6321 	 * event->mmap_count, so it is ok to use event->mmap_mutex to
6322 	 * serialize with perf_mmap here.
6323 	 */
6324 	if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&
6325 	    atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) {
6326 		/*
6327 		 * Stop all AUX events that are writing to this buffer,
6328 		 * so that we can free its AUX pages and corresponding PMU
6329 		 * data. Note that after rb::aux_mmap_count dropped to zero,
6330 		 * they won't start any more (see perf_aux_output_begin()).
6331 		 */
6332 		perf_pmu_output_stop(event);
6333 
6334 		/* now it's safe to free the pages */
6335 		atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->locked_vm);
6336 		atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm);
6337 
6338 		/* this has to be the last one */
6339 		rb_free_aux(rb);
6340 		WARN_ON_ONCE(refcount_read(&rb->aux_refcount));
6341 
6342 		mutex_unlock(&event->mmap_mutex);
6343 	}
6344 
6345 	if (atomic_dec_and_test(&rb->mmap_count))
6346 		detach_rest = true;
6347 
6348 	if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
6349 		goto out_put;
6350 
6351 	ring_buffer_attach(event, NULL);
6352 	mutex_unlock(&event->mmap_mutex);
6353 
6354 	/* If there's still other mmap()s of this buffer, we're done. */
6355 	if (!detach_rest)
6356 		goto out_put;
6357 
6358 	/*
6359 	 * No other mmap()s, detach from all other events that might redirect
6360 	 * into the now unreachable buffer. Somewhat complicated by the
6361 	 * fact that rb::event_lock otherwise nests inside mmap_mutex.
6362 	 */
6363 again:
6364 	rcu_read_lock();
6365 	list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
6366 		if (!atomic_long_inc_not_zero(&event->refcount)) {
6367 			/*
6368 			 * This event is en-route to free_event() which will
6369 			 * detach it and remove it from the list.
6370 			 */
6371 			continue;
6372 		}
6373 		rcu_read_unlock();
6374 
6375 		mutex_lock(&event->mmap_mutex);
6376 		/*
6377 		 * Check we didn't race with perf_event_set_output() which can
6378 		 * swizzle the rb from under us while we were waiting to
6379 		 * acquire mmap_mutex.
6380 		 *
6381 		 * If we find a different rb; ignore this event, a next
6382 		 * iteration will no longer find it on the list. We have to
6383 		 * still restart the iteration to make sure we're not now
6384 		 * iterating the wrong list.
6385 		 */
6386 		if (event->rb == rb)
6387 			ring_buffer_attach(event, NULL);
6388 
6389 		mutex_unlock(&event->mmap_mutex);
6390 		put_event(event);
6391 
6392 		/*
6393 		 * Restart the iteration; either we're on the wrong list or
6394 		 * destroyed its integrity by doing a deletion.
6395 		 */
6396 		goto again;
6397 	}
6398 	rcu_read_unlock();
6399 
6400 	/*
6401 	 * It could be there's still a few 0-ref events on the list; they'll
6402 	 * get cleaned up by free_event() -- they'll also still have their
6403 	 * ref on the rb and will free it whenever they are done with it.
6404 	 *
6405 	 * Aside from that, this buffer is 'fully' detached and unmapped,
6406 	 * undo the VM accounting.
6407 	 */
6408 
6409 	atomic_long_sub((size >> PAGE_SHIFT) + 1 - mmap_locked,
6410 			&mmap_user->locked_vm);
6411 	atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm);
6412 	free_uid(mmap_user);
6413 
6414 out_put:
6415 	ring_buffer_put(rb); /* could be last */
6416 }
6417 
6418 static const struct vm_operations_struct perf_mmap_vmops = {
6419 	.open		= perf_mmap_open,
6420 	.close		= perf_mmap_close, /* non mergeable */
6421 	.fault		= perf_mmap_fault,
6422 	.page_mkwrite	= perf_mmap_fault,
6423 };
6424 
perf_mmap(struct file * file,struct vm_area_struct * vma)6425 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
6426 {
6427 	struct perf_event *event = file->private_data;
6428 	unsigned long user_locked, user_lock_limit;
6429 	struct user_struct *user = current_user();
6430 	struct perf_buffer *rb = NULL;
6431 	unsigned long locked, lock_limit;
6432 	unsigned long vma_size;
6433 	unsigned long nr_pages;
6434 	long user_extra = 0, extra = 0;
6435 	int ret = 0, flags = 0;
6436 
6437 	/*
6438 	 * Don't allow mmap() of inherited per-task counters. This would
6439 	 * create a performance issue due to all children writing to the
6440 	 * same rb.
6441 	 */
6442 	if (event->cpu == -1 && event->attr.inherit)
6443 		return -EINVAL;
6444 
6445 	if (!(vma->vm_flags & VM_SHARED))
6446 		return -EINVAL;
6447 
6448 	ret = security_perf_event_read(event);
6449 	if (ret)
6450 		return ret;
6451 
6452 	vma_size = vma->vm_end - vma->vm_start;
6453 
6454 	if (vma->vm_pgoff == 0) {
6455 		nr_pages = (vma_size / PAGE_SIZE) - 1;
6456 	} else {
6457 		/*
6458 		 * AUX area mapping: if rb->aux_nr_pages != 0, it's already
6459 		 * mapped, all subsequent mappings should have the same size
6460 		 * and offset. Must be above the normal perf buffer.
6461 		 */
6462 		u64 aux_offset, aux_size;
6463 
6464 		if (!event->rb)
6465 			return -EINVAL;
6466 
6467 		nr_pages = vma_size / PAGE_SIZE;
6468 
6469 		mutex_lock(&event->mmap_mutex);
6470 		ret = -EINVAL;
6471 
6472 		rb = event->rb;
6473 		if (!rb)
6474 			goto aux_unlock;
6475 
6476 		aux_offset = READ_ONCE(rb->user_page->aux_offset);
6477 		aux_size = READ_ONCE(rb->user_page->aux_size);
6478 
6479 		if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
6480 			goto aux_unlock;
6481 
6482 		if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)
6483 			goto aux_unlock;
6484 
6485 		/* already mapped with a different offset */
6486 		if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)
6487 			goto aux_unlock;
6488 
6489 		if (aux_size != vma_size || aux_size != nr_pages * PAGE_SIZE)
6490 			goto aux_unlock;
6491 
6492 		/* already mapped with a different size */
6493 		if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)
6494 			goto aux_unlock;
6495 
6496 		if (!is_power_of_2(nr_pages))
6497 			goto aux_unlock;
6498 
6499 		if (!atomic_inc_not_zero(&rb->mmap_count))
6500 			goto aux_unlock;
6501 
6502 		if (rb_has_aux(rb)) {
6503 			atomic_inc(&rb->aux_mmap_count);
6504 			ret = 0;
6505 			goto unlock;
6506 		}
6507 
6508 		atomic_set(&rb->aux_mmap_count, 1);
6509 		user_extra = nr_pages;
6510 
6511 		goto accounting;
6512 	}
6513 
6514 	/*
6515 	 * If we have rb pages ensure they're a power-of-two number, so we
6516 	 * can do bitmasks instead of modulo.
6517 	 */
6518 	if (nr_pages != 0 && !is_power_of_2(nr_pages))
6519 		return -EINVAL;
6520 
6521 	if (vma_size != PAGE_SIZE * (1 + nr_pages))
6522 		return -EINVAL;
6523 
6524 	WARN_ON_ONCE(event->ctx->parent_ctx);
6525 again:
6526 	mutex_lock(&event->mmap_mutex);
6527 	if (event->rb) {
6528 		if (data_page_nr(event->rb) != nr_pages) {
6529 			ret = -EINVAL;
6530 			goto unlock;
6531 		}
6532 
6533 		if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
6534 			/*
6535 			 * Raced against perf_mmap_close(); remove the
6536 			 * event and try again.
6537 			 */
6538 			ring_buffer_attach(event, NULL);
6539 			mutex_unlock(&event->mmap_mutex);
6540 			goto again;
6541 		}
6542 
6543 		goto unlock;
6544 	}
6545 
6546 	user_extra = nr_pages + 1;
6547 
6548 accounting:
6549 	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
6550 
6551 	/*
6552 	 * Increase the limit linearly with more CPUs:
6553 	 */
6554 	user_lock_limit *= num_online_cpus();
6555 
6556 	user_locked = atomic_long_read(&user->locked_vm);
6557 
6558 	/*
6559 	 * sysctl_perf_event_mlock may have changed, so that
6560 	 *     user->locked_vm > user_lock_limit
6561 	 */
6562 	if (user_locked > user_lock_limit)
6563 		user_locked = user_lock_limit;
6564 	user_locked += user_extra;
6565 
6566 	if (user_locked > user_lock_limit) {
6567 		/*
6568 		 * charge locked_vm until it hits user_lock_limit;
6569 		 * charge the rest from pinned_vm
6570 		 */
6571 		extra = user_locked - user_lock_limit;
6572 		user_extra -= extra;
6573 	}
6574 
6575 	lock_limit = rlimit(RLIMIT_MEMLOCK);
6576 	lock_limit >>= PAGE_SHIFT;
6577 	locked = atomic64_read(&vma->vm_mm->pinned_vm) + extra;
6578 
6579 	if ((locked > lock_limit) && perf_is_paranoid() &&
6580 		!capable(CAP_IPC_LOCK)) {
6581 		ret = -EPERM;
6582 		goto unlock;
6583 	}
6584 
6585 	WARN_ON(!rb && event->rb);
6586 
6587 	if (vma->vm_flags & VM_WRITE)
6588 		flags |= RING_BUFFER_WRITABLE;
6589 
6590 	if (!rb) {
6591 		rb = rb_alloc(nr_pages,
6592 			      event->attr.watermark ? event->attr.wakeup_watermark : 0,
6593 			      event->cpu, flags);
6594 
6595 		if (!rb) {
6596 			ret = -ENOMEM;
6597 			goto unlock;
6598 		}
6599 
6600 		atomic_set(&rb->mmap_count, 1);
6601 		rb->mmap_user = get_current_user();
6602 		rb->mmap_locked = extra;
6603 
6604 		ring_buffer_attach(event, rb);
6605 
6606 		perf_event_update_time(event);
6607 		perf_event_init_userpage(event);
6608 		perf_event_update_userpage(event);
6609 	} else {
6610 		ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,
6611 				   event->attr.aux_watermark, flags);
6612 		if (!ret)
6613 			rb->aux_mmap_locked = extra;
6614 	}
6615 
6616 unlock:
6617 	if (!ret) {
6618 		atomic_long_add(user_extra, &user->locked_vm);
6619 		atomic64_add(extra, &vma->vm_mm->pinned_vm);
6620 
6621 		atomic_inc(&event->mmap_count);
6622 	} else if (rb) {
6623 		atomic_dec(&rb->mmap_count);
6624 	}
6625 aux_unlock:
6626 	mutex_unlock(&event->mmap_mutex);
6627 
6628 	/*
6629 	 * Since pinned accounting is per vm we cannot allow fork() to copy our
6630 	 * vma.
6631 	 */
6632 	vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP);
6633 	vma->vm_ops = &perf_mmap_vmops;
6634 
6635 	if (event->pmu->event_mapped)
6636 		event->pmu->event_mapped(event, vma->vm_mm);
6637 
6638 	return ret;
6639 }
6640 
perf_fasync(int fd,struct file * filp,int on)6641 static int perf_fasync(int fd, struct file *filp, int on)
6642 {
6643 	struct inode *inode = file_inode(filp);
6644 	struct perf_event *event = filp->private_data;
6645 	int retval;
6646 
6647 	inode_lock(inode);
6648 	retval = fasync_helper(fd, filp, on, &event->fasync);
6649 	inode_unlock(inode);
6650 
6651 	if (retval < 0)
6652 		return retval;
6653 
6654 	return 0;
6655 }
6656 
6657 static const struct file_operations perf_fops = {
6658 	.llseek			= no_llseek,
6659 	.release		= perf_release,
6660 	.read			= perf_read,
6661 	.poll			= perf_poll,
6662 	.unlocked_ioctl		= perf_ioctl,
6663 	.compat_ioctl		= perf_compat_ioctl,
6664 	.mmap			= perf_mmap,
6665 	.fasync			= perf_fasync,
6666 };
6667 
6668 /*
6669  * Perf event wakeup
6670  *
6671  * If there's data, ensure we set the poll() state and publish everything
6672  * to user-space before waking everybody up.
6673  */
6674 
perf_event_fasync(struct perf_event * event)6675 static inline struct fasync_struct **perf_event_fasync(struct perf_event *event)
6676 {
6677 	/* only the parent has fasync state */
6678 	if (event->parent)
6679 		event = event->parent;
6680 	return &event->fasync;
6681 }
6682 
perf_event_wakeup(struct perf_event * event)6683 void perf_event_wakeup(struct perf_event *event)
6684 {
6685 	ring_buffer_wakeup(event);
6686 
6687 	if (event->pending_kill) {
6688 		kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);
6689 		event->pending_kill = 0;
6690 	}
6691 }
6692 
perf_sigtrap(struct perf_event * event)6693 static void perf_sigtrap(struct perf_event *event)
6694 {
6695 	/*
6696 	 * We'd expect this to only occur if the irq_work is delayed and either
6697 	 * ctx->task or current has changed in the meantime. This can be the
6698 	 * case on architectures that do not implement arch_irq_work_raise().
6699 	 */
6700 	if (WARN_ON_ONCE(event->ctx->task != current))
6701 		return;
6702 
6703 	/*
6704 	 * Both perf_pending_task() and perf_pending_irq() can race with the
6705 	 * task exiting.
6706 	 */
6707 	if (current->flags & PF_EXITING)
6708 		return;
6709 
6710 	send_sig_perf((void __user *)event->pending_addr,
6711 		      event->orig_type, event->attr.sig_data);
6712 }
6713 
6714 /*
6715  * Deliver the pending work in-event-context or follow the context.
6716  */
__perf_pending_irq(struct perf_event * event)6717 static void __perf_pending_irq(struct perf_event *event)
6718 {
6719 	int cpu = READ_ONCE(event->oncpu);
6720 
6721 	/*
6722 	 * If the event isn't running; we done. event_sched_out() will have
6723 	 * taken care of things.
6724 	 */
6725 	if (cpu < 0)
6726 		return;
6727 
6728 	/*
6729 	 * Yay, we hit home and are in the context of the event.
6730 	 */
6731 	if (cpu == smp_processor_id()) {
6732 		if (event->pending_sigtrap) {
6733 			event->pending_sigtrap = 0;
6734 			perf_sigtrap(event);
6735 			local_dec(&event->ctx->nr_pending);
6736 		}
6737 		if (event->pending_disable) {
6738 			event->pending_disable = 0;
6739 			perf_event_disable_local(event);
6740 		}
6741 		return;
6742 	}
6743 
6744 	/*
6745 	 *  CPU-A			CPU-B
6746 	 *
6747 	 *  perf_event_disable_inatomic()
6748 	 *    @pending_disable = CPU-A;
6749 	 *    irq_work_queue();
6750 	 *
6751 	 *  sched-out
6752 	 *    @pending_disable = -1;
6753 	 *
6754 	 *				sched-in
6755 	 *				perf_event_disable_inatomic()
6756 	 *				  @pending_disable = CPU-B;
6757 	 *				  irq_work_queue(); // FAILS
6758 	 *
6759 	 *  irq_work_run()
6760 	 *    perf_pending_irq()
6761 	 *
6762 	 * But the event runs on CPU-B and wants disabling there.
6763 	 */
6764 	irq_work_queue_on(&event->pending_irq, cpu);
6765 }
6766 
perf_pending_irq(struct irq_work * entry)6767 static void perf_pending_irq(struct irq_work *entry)
6768 {
6769 	struct perf_event *event = container_of(entry, struct perf_event, pending_irq);
6770 	int rctx;
6771 
6772 	/*
6773 	 * If we 'fail' here, that's OK, it means recursion is already disabled
6774 	 * and we won't recurse 'further'.
6775 	 */
6776 	rctx = perf_swevent_get_recursion_context();
6777 
6778 	/*
6779 	 * The wakeup isn't bound to the context of the event -- it can happen
6780 	 * irrespective of where the event is.
6781 	 */
6782 	if (event->pending_wakeup) {
6783 		event->pending_wakeup = 0;
6784 		perf_event_wakeup(event);
6785 	}
6786 
6787 	__perf_pending_irq(event);
6788 
6789 	if (rctx >= 0)
6790 		perf_swevent_put_recursion_context(rctx);
6791 }
6792 
perf_pending_task(struct callback_head * head)6793 static void perf_pending_task(struct callback_head *head)
6794 {
6795 	struct perf_event *event = container_of(head, struct perf_event, pending_task);
6796 	int rctx;
6797 
6798 	/*
6799 	 * If we 'fail' here, that's OK, it means recursion is already disabled
6800 	 * and we won't recurse 'further'.
6801 	 */
6802 	preempt_disable_notrace();
6803 	rctx = perf_swevent_get_recursion_context();
6804 
6805 	if (event->pending_work) {
6806 		event->pending_work = 0;
6807 		perf_sigtrap(event);
6808 		local_dec(&event->ctx->nr_pending);
6809 	}
6810 
6811 	if (rctx >= 0)
6812 		perf_swevent_put_recursion_context(rctx);
6813 	preempt_enable_notrace();
6814 
6815 	put_event(event);
6816 }
6817 
6818 #ifdef CONFIG_GUEST_PERF_EVENTS
6819 struct perf_guest_info_callbacks __rcu *perf_guest_cbs;
6820 
6821 DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
6822 DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
6823 DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
6824 
perf_register_guest_info_callbacks(struct perf_guest_info_callbacks * cbs)6825 void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
6826 {
6827 	if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs)))
6828 		return;
6829 
6830 	rcu_assign_pointer(perf_guest_cbs, cbs);
6831 	static_call_update(__perf_guest_state, cbs->state);
6832 	static_call_update(__perf_guest_get_ip, cbs->get_ip);
6833 
6834 	/* Implementing ->handle_intel_pt_intr is optional. */
6835 	if (cbs->handle_intel_pt_intr)
6836 		static_call_update(__perf_guest_handle_intel_pt_intr,
6837 				   cbs->handle_intel_pt_intr);
6838 }
6839 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
6840 
perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks * cbs)6841 void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
6842 {
6843 	if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs) != cbs))
6844 		return;
6845 
6846 	rcu_assign_pointer(perf_guest_cbs, NULL);
6847 	static_call_update(__perf_guest_state, (void *)&__static_call_return0);
6848 	static_call_update(__perf_guest_get_ip, (void *)&__static_call_return0);
6849 	static_call_update(__perf_guest_handle_intel_pt_intr,
6850 			   (void *)&__static_call_return0);
6851 	synchronize_rcu();
6852 }
6853 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
6854 #endif
6855 
6856 static void
perf_output_sample_regs(struct perf_output_handle * handle,struct pt_regs * regs,u64 mask)6857 perf_output_sample_regs(struct perf_output_handle *handle,
6858 			struct pt_regs *regs, u64 mask)
6859 {
6860 	int bit;
6861 	DECLARE_BITMAP(_mask, 64);
6862 
6863 	bitmap_from_u64(_mask, mask);
6864 	for_each_set_bit(bit, _mask, sizeof(mask) * BITS_PER_BYTE) {
6865 		u64 val;
6866 
6867 		val = perf_reg_value(regs, bit);
6868 		perf_output_put(handle, val);
6869 	}
6870 }
6871 
perf_sample_regs_user(struct perf_regs * regs_user,struct pt_regs * regs)6872 static void perf_sample_regs_user(struct perf_regs *regs_user,
6873 				  struct pt_regs *regs)
6874 {
6875 	if (user_mode(regs)) {
6876 		regs_user->abi = perf_reg_abi(current);
6877 		regs_user->regs = regs;
6878 	} else if (!(current->flags & PF_KTHREAD)) {
6879 		perf_get_regs_user(regs_user, regs);
6880 	} else {
6881 		regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
6882 		regs_user->regs = NULL;
6883 	}
6884 }
6885 
perf_sample_regs_intr(struct perf_regs * regs_intr,struct pt_regs * regs)6886 static void perf_sample_regs_intr(struct perf_regs *regs_intr,
6887 				  struct pt_regs *regs)
6888 {
6889 	regs_intr->regs = regs;
6890 	regs_intr->abi  = perf_reg_abi(current);
6891 }
6892 
6893 
6894 /*
6895  * Get remaining task size from user stack pointer.
6896  *
6897  * It'd be better to take stack vma map and limit this more
6898  * precisely, but there's no way to get it safely under interrupt,
6899  * so using TASK_SIZE as limit.
6900  */
perf_ustack_task_size(struct pt_regs * regs)6901 static u64 perf_ustack_task_size(struct pt_regs *regs)
6902 {
6903 	unsigned long addr = perf_user_stack_pointer(regs);
6904 
6905 	if (!addr || addr >= TASK_SIZE)
6906 		return 0;
6907 
6908 	return TASK_SIZE - addr;
6909 }
6910 
6911 static u16
perf_sample_ustack_size(u16 stack_size,u16 header_size,struct pt_regs * regs)6912 perf_sample_ustack_size(u16 stack_size, u16 header_size,
6913 			struct pt_regs *regs)
6914 {
6915 	u64 task_size;
6916 
6917 	/* No regs, no stack pointer, no dump. */
6918 	if (!regs)
6919 		return 0;
6920 
6921 	/*
6922 	 * Check if we fit in with the requested stack size into the:
6923 	 * - TASK_SIZE
6924 	 *   If we don't, we limit the size to the TASK_SIZE.
6925 	 *
6926 	 * - remaining sample size
6927 	 *   If we don't, we customize the stack size to
6928 	 *   fit in to the remaining sample size.
6929 	 */
6930 
6931 	task_size  = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
6932 	stack_size = min(stack_size, (u16) task_size);
6933 
6934 	/* Current header size plus static size and dynamic size. */
6935 	header_size += 2 * sizeof(u64);
6936 
6937 	/* Do we fit in with the current stack dump size? */
6938 	if ((u16) (header_size + stack_size) < header_size) {
6939 		/*
6940 		 * If we overflow the maximum size for the sample,
6941 		 * we customize the stack dump size to fit in.
6942 		 */
6943 		stack_size = USHRT_MAX - header_size - sizeof(u64);
6944 		stack_size = round_up(stack_size, sizeof(u64));
6945 	}
6946 
6947 	return stack_size;
6948 }
6949 
6950 static void
perf_output_sample_ustack(struct perf_output_handle * handle,u64 dump_size,struct pt_regs * regs)6951 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
6952 			  struct pt_regs *regs)
6953 {
6954 	/* Case of a kernel thread, nothing to dump */
6955 	if (!regs) {
6956 		u64 size = 0;
6957 		perf_output_put(handle, size);
6958 	} else {
6959 		unsigned long sp;
6960 		unsigned int rem;
6961 		u64 dyn_size;
6962 
6963 		/*
6964 		 * We dump:
6965 		 * static size
6966 		 *   - the size requested by user or the best one we can fit
6967 		 *     in to the sample max size
6968 		 * data
6969 		 *   - user stack dump data
6970 		 * dynamic size
6971 		 *   - the actual dumped size
6972 		 */
6973 
6974 		/* Static size. */
6975 		perf_output_put(handle, dump_size);
6976 
6977 		/* Data. */
6978 		sp = perf_user_stack_pointer(regs);
6979 		rem = __output_copy_user(handle, (void *) sp, dump_size);
6980 		dyn_size = dump_size - rem;
6981 
6982 		perf_output_skip(handle, rem);
6983 
6984 		/* Dynamic size. */
6985 		perf_output_put(handle, dyn_size);
6986 	}
6987 }
6988 
perf_prepare_sample_aux(struct perf_event * event,struct perf_sample_data * data,size_t size)6989 static unsigned long perf_prepare_sample_aux(struct perf_event *event,
6990 					  struct perf_sample_data *data,
6991 					  size_t size)
6992 {
6993 	struct perf_event *sampler = event->aux_event;
6994 	struct perf_buffer *rb;
6995 
6996 	data->aux_size = 0;
6997 
6998 	if (!sampler)
6999 		goto out;
7000 
7001 	if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE))
7002 		goto out;
7003 
7004 	if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id()))
7005 		goto out;
7006 
7007 	rb = ring_buffer_get(sampler);
7008 	if (!rb)
7009 		goto out;
7010 
7011 	/*
7012 	 * If this is an NMI hit inside sampling code, don't take
7013 	 * the sample. See also perf_aux_sample_output().
7014 	 */
7015 	if (READ_ONCE(rb->aux_in_sampling)) {
7016 		data->aux_size = 0;
7017 	} else {
7018 		size = min_t(size_t, size, perf_aux_size(rb));
7019 		data->aux_size = ALIGN(size, sizeof(u64));
7020 	}
7021 	ring_buffer_put(rb);
7022 
7023 out:
7024 	return data->aux_size;
7025 }
7026 
perf_pmu_snapshot_aux(struct perf_buffer * rb,struct perf_event * event,struct perf_output_handle * handle,unsigned long size)7027 static long perf_pmu_snapshot_aux(struct perf_buffer *rb,
7028                                  struct perf_event *event,
7029                                  struct perf_output_handle *handle,
7030                                  unsigned long size)
7031 {
7032 	unsigned long flags;
7033 	long ret;
7034 
7035 	/*
7036 	 * Normal ->start()/->stop() callbacks run in IRQ mode in scheduler
7037 	 * paths. If we start calling them in NMI context, they may race with
7038 	 * the IRQ ones, that is, for example, re-starting an event that's just
7039 	 * been stopped, which is why we're using a separate callback that
7040 	 * doesn't change the event state.
7041 	 *
7042 	 * IRQs need to be disabled to prevent IPIs from racing with us.
7043 	 */
7044 	local_irq_save(flags);
7045 	/*
7046 	 * Guard against NMI hits inside the critical section;
7047 	 * see also perf_prepare_sample_aux().
7048 	 */
7049 	WRITE_ONCE(rb->aux_in_sampling, 1);
7050 	barrier();
7051 
7052 	ret = event->pmu->snapshot_aux(event, handle, size);
7053 
7054 	barrier();
7055 	WRITE_ONCE(rb->aux_in_sampling, 0);
7056 	local_irq_restore(flags);
7057 
7058 	return ret;
7059 }
7060 
perf_aux_sample_output(struct perf_event * event,struct perf_output_handle * handle,struct perf_sample_data * data)7061 static void perf_aux_sample_output(struct perf_event *event,
7062 				   struct perf_output_handle *handle,
7063 				   struct perf_sample_data *data)
7064 {
7065 	struct perf_event *sampler = event->aux_event;
7066 	struct perf_buffer *rb;
7067 	unsigned long pad;
7068 	long size;
7069 
7070 	if (WARN_ON_ONCE(!sampler || !data->aux_size))
7071 		return;
7072 
7073 	rb = ring_buffer_get(sampler);
7074 	if (!rb)
7075 		return;
7076 
7077 	size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size);
7078 
7079 	/*
7080 	 * An error here means that perf_output_copy() failed (returned a
7081 	 * non-zero surplus that it didn't copy), which in its current
7082 	 * enlightened implementation is not possible. If that changes, we'd
7083 	 * like to know.
7084 	 */
7085 	if (WARN_ON_ONCE(size < 0))
7086 		goto out_put;
7087 
7088 	/*
7089 	 * The pad comes from ALIGN()ing data->aux_size up to u64 in
7090 	 * perf_prepare_sample_aux(), so should not be more than that.
7091 	 */
7092 	pad = data->aux_size - size;
7093 	if (WARN_ON_ONCE(pad >= sizeof(u64)))
7094 		pad = 8;
7095 
7096 	if (pad) {
7097 		u64 zero = 0;
7098 		perf_output_copy(handle, &zero, pad);
7099 	}
7100 
7101 out_put:
7102 	ring_buffer_put(rb);
7103 }
7104 
7105 /*
7106  * A set of common sample data types saved even for non-sample records
7107  * when event->attr.sample_id_all is set.
7108  */
7109 #define PERF_SAMPLE_ID_ALL  (PERF_SAMPLE_TID | PERF_SAMPLE_TIME |	\
7110 			     PERF_SAMPLE_ID | PERF_SAMPLE_STREAM_ID |	\
7111 			     PERF_SAMPLE_CPU | PERF_SAMPLE_IDENTIFIER)
7112 
__perf_event_header__init_id(struct perf_sample_data * data,struct perf_event * event,u64 sample_type)7113 static void __perf_event_header__init_id(struct perf_sample_data *data,
7114 					 struct perf_event *event,
7115 					 u64 sample_type)
7116 {
7117 	data->type = event->attr.sample_type;
7118 	data->sample_flags |= data->type & PERF_SAMPLE_ID_ALL;
7119 
7120 	if (sample_type & PERF_SAMPLE_TID) {
7121 		/* namespace issues */
7122 		data->tid_entry.pid = perf_event_pid(event, current);
7123 		data->tid_entry.tid = perf_event_tid(event, current);
7124 	}
7125 
7126 	if (sample_type & PERF_SAMPLE_TIME)
7127 		data->time = perf_event_clock(event);
7128 
7129 	if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
7130 		data->id = primary_event_id(event);
7131 
7132 	if (sample_type & PERF_SAMPLE_STREAM_ID)
7133 		data->stream_id = event->id;
7134 
7135 	if (sample_type & PERF_SAMPLE_CPU) {
7136 		data->cpu_entry.cpu	 = raw_smp_processor_id();
7137 		data->cpu_entry.reserved = 0;
7138 	}
7139 }
7140 
perf_event_header__init_id(struct perf_event_header * header,struct perf_sample_data * data,struct perf_event * event)7141 void perf_event_header__init_id(struct perf_event_header *header,
7142 				struct perf_sample_data *data,
7143 				struct perf_event *event)
7144 {
7145 	if (event->attr.sample_id_all) {
7146 		header->size += event->id_header_size;
7147 		__perf_event_header__init_id(data, event, event->attr.sample_type);
7148 	}
7149 }
7150 
__perf_event__output_id_sample(struct perf_output_handle * handle,struct perf_sample_data * data)7151 static void __perf_event__output_id_sample(struct perf_output_handle *handle,
7152 					   struct perf_sample_data *data)
7153 {
7154 	u64 sample_type = data->type;
7155 
7156 	if (sample_type & PERF_SAMPLE_TID)
7157 		perf_output_put(handle, data->tid_entry);
7158 
7159 	if (sample_type & PERF_SAMPLE_TIME)
7160 		perf_output_put(handle, data->time);
7161 
7162 	if (sample_type & PERF_SAMPLE_ID)
7163 		perf_output_put(handle, data->id);
7164 
7165 	if (sample_type & PERF_SAMPLE_STREAM_ID)
7166 		perf_output_put(handle, data->stream_id);
7167 
7168 	if (sample_type & PERF_SAMPLE_CPU)
7169 		perf_output_put(handle, data->cpu_entry);
7170 
7171 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
7172 		perf_output_put(handle, data->id);
7173 }
7174 
perf_event__output_id_sample(struct perf_event * event,struct perf_output_handle * handle,struct perf_sample_data * sample)7175 void perf_event__output_id_sample(struct perf_event *event,
7176 				  struct perf_output_handle *handle,
7177 				  struct perf_sample_data *sample)
7178 {
7179 	if (event->attr.sample_id_all)
7180 		__perf_event__output_id_sample(handle, sample);
7181 }
7182 
perf_output_read_one(struct perf_output_handle * handle,struct perf_event * event,u64 enabled,u64 running)7183 static void perf_output_read_one(struct perf_output_handle *handle,
7184 				 struct perf_event *event,
7185 				 u64 enabled, u64 running)
7186 {
7187 	u64 read_format = event->attr.read_format;
7188 	u64 values[5];
7189 	int n = 0;
7190 
7191 	values[n++] = perf_event_count(event);
7192 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
7193 		values[n++] = enabled +
7194 			atomic64_read(&event->child_total_time_enabled);
7195 	}
7196 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
7197 		values[n++] = running +
7198 			atomic64_read(&event->child_total_time_running);
7199 	}
7200 	if (read_format & PERF_FORMAT_ID)
7201 		values[n++] = primary_event_id(event);
7202 	if (read_format & PERF_FORMAT_LOST)
7203 		values[n++] = atomic64_read(&event->lost_samples);
7204 
7205 	__output_copy(handle, values, n * sizeof(u64));
7206 }
7207 
perf_output_read_group(struct perf_output_handle * handle,struct perf_event * event,u64 enabled,u64 running)7208 static void perf_output_read_group(struct perf_output_handle *handle,
7209 			    struct perf_event *event,
7210 			    u64 enabled, u64 running)
7211 {
7212 	struct perf_event *leader = event->group_leader, *sub;
7213 	u64 read_format = event->attr.read_format;
7214 	unsigned long flags;
7215 	u64 values[6];
7216 	int n = 0;
7217 
7218 	/*
7219 	 * Disabling interrupts avoids all counter scheduling
7220 	 * (context switches, timer based rotation and IPIs).
7221 	 */
7222 	local_irq_save(flags);
7223 
7224 	values[n++] = 1 + leader->nr_siblings;
7225 
7226 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
7227 		values[n++] = enabled;
7228 
7229 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
7230 		values[n++] = running;
7231 
7232 	if ((leader != event) &&
7233 	    (leader->state == PERF_EVENT_STATE_ACTIVE))
7234 		leader->pmu->read(leader);
7235 
7236 	values[n++] = perf_event_count(leader);
7237 	if (read_format & PERF_FORMAT_ID)
7238 		values[n++] = primary_event_id(leader);
7239 	if (read_format & PERF_FORMAT_LOST)
7240 		values[n++] = atomic64_read(&leader->lost_samples);
7241 
7242 	__output_copy(handle, values, n * sizeof(u64));
7243 
7244 	for_each_sibling_event(sub, leader) {
7245 		n = 0;
7246 
7247 		if ((sub != event) &&
7248 		    (sub->state == PERF_EVENT_STATE_ACTIVE))
7249 			sub->pmu->read(sub);
7250 
7251 		values[n++] = perf_event_count(sub);
7252 		if (read_format & PERF_FORMAT_ID)
7253 			values[n++] = primary_event_id(sub);
7254 		if (read_format & PERF_FORMAT_LOST)
7255 			values[n++] = atomic64_read(&sub->lost_samples);
7256 
7257 		__output_copy(handle, values, n * sizeof(u64));
7258 	}
7259 
7260 	local_irq_restore(flags);
7261 }
7262 
7263 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
7264 				 PERF_FORMAT_TOTAL_TIME_RUNNING)
7265 
7266 /*
7267  * XXX PERF_SAMPLE_READ vs inherited events seems difficult.
7268  *
7269  * The problem is that its both hard and excessively expensive to iterate the
7270  * child list, not to mention that its impossible to IPI the children running
7271  * on another CPU, from interrupt/NMI context.
7272  */
perf_output_read(struct perf_output_handle * handle,struct perf_event * event)7273 static void perf_output_read(struct perf_output_handle *handle,
7274 			     struct perf_event *event)
7275 {
7276 	u64 enabled = 0, running = 0, now;
7277 	u64 read_format = event->attr.read_format;
7278 
7279 	/*
7280 	 * compute total_time_enabled, total_time_running
7281 	 * based on snapshot values taken when the event
7282 	 * was last scheduled in.
7283 	 *
7284 	 * we cannot simply called update_context_time()
7285 	 * because of locking issue as we are called in
7286 	 * NMI context
7287 	 */
7288 	if (read_format & PERF_FORMAT_TOTAL_TIMES)
7289 		calc_timer_values(event, &now, &enabled, &running);
7290 
7291 	if (event->attr.read_format & PERF_FORMAT_GROUP)
7292 		perf_output_read_group(handle, event, enabled, running);
7293 	else
7294 		perf_output_read_one(handle, event, enabled, running);
7295 }
7296 
perf_output_sample(struct perf_output_handle * handle,struct perf_event_header * header,struct perf_sample_data * data,struct perf_event * event)7297 void perf_output_sample(struct perf_output_handle *handle,
7298 			struct perf_event_header *header,
7299 			struct perf_sample_data *data,
7300 			struct perf_event *event)
7301 {
7302 	u64 sample_type = data->type;
7303 
7304 	perf_output_put(handle, *header);
7305 
7306 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
7307 		perf_output_put(handle, data->id);
7308 
7309 	if (sample_type & PERF_SAMPLE_IP)
7310 		perf_output_put(handle, data->ip);
7311 
7312 	if (sample_type & PERF_SAMPLE_TID)
7313 		perf_output_put(handle, data->tid_entry);
7314 
7315 	if (sample_type & PERF_SAMPLE_TIME)
7316 		perf_output_put(handle, data->time);
7317 
7318 	if (sample_type & PERF_SAMPLE_ADDR)
7319 		perf_output_put(handle, data->addr);
7320 
7321 	if (sample_type & PERF_SAMPLE_ID)
7322 		perf_output_put(handle, data->id);
7323 
7324 	if (sample_type & PERF_SAMPLE_STREAM_ID)
7325 		perf_output_put(handle, data->stream_id);
7326 
7327 	if (sample_type & PERF_SAMPLE_CPU)
7328 		perf_output_put(handle, data->cpu_entry);
7329 
7330 	if (sample_type & PERF_SAMPLE_PERIOD)
7331 		perf_output_put(handle, data->period);
7332 
7333 	if (sample_type & PERF_SAMPLE_READ)
7334 		perf_output_read(handle, event);
7335 
7336 	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
7337 		int size = 1;
7338 
7339 		size += data->callchain->nr;
7340 		size *= sizeof(u64);
7341 		__output_copy(handle, data->callchain, size);
7342 	}
7343 
7344 	if (sample_type & PERF_SAMPLE_RAW) {
7345 		struct perf_raw_record *raw = data->raw;
7346 
7347 		if (raw) {
7348 			struct perf_raw_frag *frag = &raw->frag;
7349 
7350 			perf_output_put(handle, raw->size);
7351 			do {
7352 				if (frag->copy) {
7353 					__output_custom(handle, frag->copy,
7354 							frag->data, frag->size);
7355 				} else {
7356 					__output_copy(handle, frag->data,
7357 						      frag->size);
7358 				}
7359 				if (perf_raw_frag_last(frag))
7360 					break;
7361 				frag = frag->next;
7362 			} while (1);
7363 			if (frag->pad)
7364 				__output_skip(handle, NULL, frag->pad);
7365 		} else {
7366 			struct {
7367 				u32	size;
7368 				u32	data;
7369 			} raw = {
7370 				.size = sizeof(u32),
7371 				.data = 0,
7372 			};
7373 			perf_output_put(handle, raw);
7374 		}
7375 	}
7376 
7377 	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
7378 		if (data->br_stack) {
7379 			size_t size;
7380 
7381 			size = data->br_stack->nr
7382 			     * sizeof(struct perf_branch_entry);
7383 
7384 			perf_output_put(handle, data->br_stack->nr);
7385 			if (branch_sample_hw_index(event))
7386 				perf_output_put(handle, data->br_stack->hw_idx);
7387 			perf_output_copy(handle, data->br_stack->entries, size);
7388 		} else {
7389 			/*
7390 			 * we always store at least the value of nr
7391 			 */
7392 			u64 nr = 0;
7393 			perf_output_put(handle, nr);
7394 		}
7395 	}
7396 
7397 	if (sample_type & PERF_SAMPLE_REGS_USER) {
7398 		u64 abi = data->regs_user.abi;
7399 
7400 		/*
7401 		 * If there are no regs to dump, notice it through
7402 		 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
7403 		 */
7404 		perf_output_put(handle, abi);
7405 
7406 		if (abi) {
7407 			u64 mask = event->attr.sample_regs_user;
7408 			perf_output_sample_regs(handle,
7409 						data->regs_user.regs,
7410 						mask);
7411 		}
7412 	}
7413 
7414 	if (sample_type & PERF_SAMPLE_STACK_USER) {
7415 		perf_output_sample_ustack(handle,
7416 					  data->stack_user_size,
7417 					  data->regs_user.regs);
7418 	}
7419 
7420 	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE)
7421 		perf_output_put(handle, data->weight.full);
7422 
7423 	if (sample_type & PERF_SAMPLE_DATA_SRC)
7424 		perf_output_put(handle, data->data_src.val);
7425 
7426 	if (sample_type & PERF_SAMPLE_TRANSACTION)
7427 		perf_output_put(handle, data->txn);
7428 
7429 	if (sample_type & PERF_SAMPLE_REGS_INTR) {
7430 		u64 abi = data->regs_intr.abi;
7431 		/*
7432 		 * If there are no regs to dump, notice it through
7433 		 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
7434 		 */
7435 		perf_output_put(handle, abi);
7436 
7437 		if (abi) {
7438 			u64 mask = event->attr.sample_regs_intr;
7439 
7440 			perf_output_sample_regs(handle,
7441 						data->regs_intr.regs,
7442 						mask);
7443 		}
7444 	}
7445 
7446 	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
7447 		perf_output_put(handle, data->phys_addr);
7448 
7449 	if (sample_type & PERF_SAMPLE_CGROUP)
7450 		perf_output_put(handle, data->cgroup);
7451 
7452 	if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
7453 		perf_output_put(handle, data->data_page_size);
7454 
7455 	if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
7456 		perf_output_put(handle, data->code_page_size);
7457 
7458 	if (sample_type & PERF_SAMPLE_AUX) {
7459 		perf_output_put(handle, data->aux_size);
7460 
7461 		if (data->aux_size)
7462 			perf_aux_sample_output(event, handle, data);
7463 	}
7464 
7465 	if (!event->attr.watermark) {
7466 		int wakeup_events = event->attr.wakeup_events;
7467 
7468 		if (wakeup_events) {
7469 			struct perf_buffer *rb = handle->rb;
7470 			int events = local_inc_return(&rb->events);
7471 
7472 			if (events >= wakeup_events) {
7473 				local_sub(wakeup_events, &rb->events);
7474 				local_inc(&rb->wakeup);
7475 			}
7476 		}
7477 	}
7478 }
7479 
perf_virt_to_phys(u64 virt)7480 static u64 perf_virt_to_phys(u64 virt)
7481 {
7482 	u64 phys_addr = 0;
7483 
7484 	if (!virt)
7485 		return 0;
7486 
7487 	if (virt >= TASK_SIZE) {
7488 		/* If it's vmalloc()d memory, leave phys_addr as 0 */
7489 		if (virt_addr_valid((void *)(uintptr_t)virt) &&
7490 		    !(virt >= VMALLOC_START && virt < VMALLOC_END))
7491 			phys_addr = (u64)virt_to_phys((void *)(uintptr_t)virt);
7492 	} else {
7493 		/*
7494 		 * Walking the pages tables for user address.
7495 		 * Interrupts are disabled, so it prevents any tear down
7496 		 * of the page tables.
7497 		 * Try IRQ-safe get_user_page_fast_only first.
7498 		 * If failed, leave phys_addr as 0.
7499 		 */
7500 		if (current->mm != NULL) {
7501 			struct page *p;
7502 
7503 			pagefault_disable();
7504 			if (get_user_page_fast_only(virt, 0, &p)) {
7505 				phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
7506 				put_page(p);
7507 			}
7508 			pagefault_enable();
7509 		}
7510 	}
7511 
7512 	return phys_addr;
7513 }
7514 
7515 /*
7516  * Return the pagetable size of a given virtual address.
7517  */
perf_get_pgtable_size(struct mm_struct * mm,unsigned long addr)7518 static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr)
7519 {
7520 	u64 size = 0;
7521 
7522 #ifdef CONFIG_HAVE_FAST_GUP
7523 	pgd_t *pgdp, pgd;
7524 	p4d_t *p4dp, p4d;
7525 	pud_t *pudp, pud;
7526 	pmd_t *pmdp, pmd;
7527 	pte_t *ptep, pte;
7528 
7529 	pgdp = pgd_offset(mm, addr);
7530 	pgd = READ_ONCE(*pgdp);
7531 	if (pgd_none(pgd))
7532 		return 0;
7533 
7534 	if (pgd_leaf(pgd))
7535 		return pgd_leaf_size(pgd);
7536 
7537 	p4dp = p4d_offset_lockless(pgdp, pgd, addr);
7538 	p4d = READ_ONCE(*p4dp);
7539 	if (!p4d_present(p4d))
7540 		return 0;
7541 
7542 	if (p4d_leaf(p4d))
7543 		return p4d_leaf_size(p4d);
7544 
7545 	pudp = pud_offset_lockless(p4dp, p4d, addr);
7546 	pud = READ_ONCE(*pudp);
7547 	if (!pud_present(pud))
7548 		return 0;
7549 
7550 	if (pud_leaf(pud))
7551 		return pud_leaf_size(pud);
7552 
7553 	pmdp = pmd_offset_lockless(pudp, pud, addr);
7554 again:
7555 	pmd = pmdp_get_lockless(pmdp);
7556 	if (!pmd_present(pmd))
7557 		return 0;
7558 
7559 	if (pmd_leaf(pmd))
7560 		return pmd_leaf_size(pmd);
7561 
7562 	ptep = pte_offset_map(&pmd, addr);
7563 	if (!ptep)
7564 		goto again;
7565 
7566 	pte = ptep_get_lockless(ptep);
7567 	if (pte_present(pte))
7568 		size = pte_leaf_size(pte);
7569 	pte_unmap(ptep);
7570 #endif /* CONFIG_HAVE_FAST_GUP */
7571 
7572 	return size;
7573 }
7574 
perf_get_page_size(unsigned long addr)7575 static u64 perf_get_page_size(unsigned long addr)
7576 {
7577 	struct mm_struct *mm;
7578 	unsigned long flags;
7579 	u64 size;
7580 
7581 	if (!addr)
7582 		return 0;
7583 
7584 	/*
7585 	 * Software page-table walkers must disable IRQs,
7586 	 * which prevents any tear down of the page tables.
7587 	 */
7588 	local_irq_save(flags);
7589 
7590 	mm = current->mm;
7591 	if (!mm) {
7592 		/*
7593 		 * For kernel threads and the like, use init_mm so that
7594 		 * we can find kernel memory.
7595 		 */
7596 		mm = &init_mm;
7597 	}
7598 
7599 	size = perf_get_pgtable_size(mm, addr);
7600 
7601 	local_irq_restore(flags);
7602 
7603 	return size;
7604 }
7605 
7606 static struct perf_callchain_entry __empty_callchain = { .nr = 0, };
7607 
7608 struct perf_callchain_entry *
perf_callchain(struct perf_event * event,struct pt_regs * regs)7609 perf_callchain(struct perf_event *event, struct pt_regs *regs)
7610 {
7611 	bool kernel = !event->attr.exclude_callchain_kernel;
7612 	bool user   = !event->attr.exclude_callchain_user;
7613 	/* Disallow cross-task user callchains. */
7614 	bool crosstask = event->ctx->task && event->ctx->task != current;
7615 	const u32 max_stack = event->attr.sample_max_stack;
7616 	struct perf_callchain_entry *callchain;
7617 
7618 	if (!kernel && !user)
7619 		return &__empty_callchain;
7620 
7621 	callchain = get_perf_callchain(regs, 0, kernel, user,
7622 				       max_stack, crosstask, true);
7623 	return callchain ?: &__empty_callchain;
7624 }
7625 
__cond_set(u64 flags,u64 s,u64 d)7626 static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d)
7627 {
7628 	return d * !!(flags & s);
7629 }
7630 
perf_prepare_sample(struct perf_sample_data * data,struct perf_event * event,struct pt_regs * regs)7631 void perf_prepare_sample(struct perf_sample_data *data,
7632 			 struct perf_event *event,
7633 			 struct pt_regs *regs)
7634 {
7635 	u64 sample_type = event->attr.sample_type;
7636 	u64 filtered_sample_type;
7637 
7638 	/*
7639 	 * Add the sample flags that are dependent to others.  And clear the
7640 	 * sample flags that have already been done by the PMU driver.
7641 	 */
7642 	filtered_sample_type = sample_type;
7643 	filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_CODE_PAGE_SIZE,
7644 					   PERF_SAMPLE_IP);
7645 	filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_DATA_PAGE_SIZE |
7646 					   PERF_SAMPLE_PHYS_ADDR, PERF_SAMPLE_ADDR);
7647 	filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_STACK_USER,
7648 					   PERF_SAMPLE_REGS_USER);
7649 	filtered_sample_type &= ~data->sample_flags;
7650 
7651 	if (filtered_sample_type == 0) {
7652 		/* Make sure it has the correct data->type for output */
7653 		data->type = event->attr.sample_type;
7654 		return;
7655 	}
7656 
7657 	__perf_event_header__init_id(data, event, filtered_sample_type);
7658 
7659 	if (filtered_sample_type & PERF_SAMPLE_IP) {
7660 		data->ip = perf_instruction_pointer(regs);
7661 		data->sample_flags |= PERF_SAMPLE_IP;
7662 	}
7663 
7664 	if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN)
7665 		perf_sample_save_callchain(data, event, regs);
7666 
7667 	if (filtered_sample_type & PERF_SAMPLE_RAW) {
7668 		data->raw = NULL;
7669 		data->dyn_size += sizeof(u64);
7670 		data->sample_flags |= PERF_SAMPLE_RAW;
7671 	}
7672 
7673 	if (filtered_sample_type & PERF_SAMPLE_BRANCH_STACK) {
7674 		data->br_stack = NULL;
7675 		data->dyn_size += sizeof(u64);
7676 		data->sample_flags |= PERF_SAMPLE_BRANCH_STACK;
7677 	}
7678 
7679 	if (filtered_sample_type & PERF_SAMPLE_REGS_USER)
7680 		perf_sample_regs_user(&data->regs_user, regs);
7681 
7682 	/*
7683 	 * It cannot use the filtered_sample_type here as REGS_USER can be set
7684 	 * by STACK_USER (using __cond_set() above) and we don't want to update
7685 	 * the dyn_size if it's not requested by users.
7686 	 */
7687 	if ((sample_type & ~data->sample_flags) & PERF_SAMPLE_REGS_USER) {
7688 		/* regs dump ABI info */
7689 		int size = sizeof(u64);
7690 
7691 		if (data->regs_user.regs) {
7692 			u64 mask = event->attr.sample_regs_user;
7693 			size += hweight64(mask) * sizeof(u64);
7694 		}
7695 
7696 		data->dyn_size += size;
7697 		data->sample_flags |= PERF_SAMPLE_REGS_USER;
7698 	}
7699 
7700 	if (filtered_sample_type & PERF_SAMPLE_STACK_USER) {
7701 		/*
7702 		 * Either we need PERF_SAMPLE_STACK_USER bit to be always
7703 		 * processed as the last one or have additional check added
7704 		 * in case new sample type is added, because we could eat
7705 		 * up the rest of the sample size.
7706 		 */
7707 		u16 stack_size = event->attr.sample_stack_user;
7708 		u16 header_size = perf_sample_data_size(data, event);
7709 		u16 size = sizeof(u64);
7710 
7711 		stack_size = perf_sample_ustack_size(stack_size, header_size,
7712 						     data->regs_user.regs);
7713 
7714 		/*
7715 		 * If there is something to dump, add space for the dump
7716 		 * itself and for the field that tells the dynamic size,
7717 		 * which is how many have been actually dumped.
7718 		 */
7719 		if (stack_size)
7720 			size += sizeof(u64) + stack_size;
7721 
7722 		data->stack_user_size = stack_size;
7723 		data->dyn_size += size;
7724 		data->sample_flags |= PERF_SAMPLE_STACK_USER;
7725 	}
7726 
7727 	if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE) {
7728 		data->weight.full = 0;
7729 		data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE;
7730 	}
7731 
7732 	if (filtered_sample_type & PERF_SAMPLE_DATA_SRC) {
7733 		data->data_src.val = PERF_MEM_NA;
7734 		data->sample_flags |= PERF_SAMPLE_DATA_SRC;
7735 	}
7736 
7737 	if (filtered_sample_type & PERF_SAMPLE_TRANSACTION) {
7738 		data->txn = 0;
7739 		data->sample_flags |= PERF_SAMPLE_TRANSACTION;
7740 	}
7741 
7742 	if (filtered_sample_type & PERF_SAMPLE_ADDR) {
7743 		data->addr = 0;
7744 		data->sample_flags |= PERF_SAMPLE_ADDR;
7745 	}
7746 
7747 	if (filtered_sample_type & PERF_SAMPLE_REGS_INTR) {
7748 		/* regs dump ABI info */
7749 		int size = sizeof(u64);
7750 
7751 		perf_sample_regs_intr(&data->regs_intr, regs);
7752 
7753 		if (data->regs_intr.regs) {
7754 			u64 mask = event->attr.sample_regs_intr;
7755 
7756 			size += hweight64(mask) * sizeof(u64);
7757 		}
7758 
7759 		data->dyn_size += size;
7760 		data->sample_flags |= PERF_SAMPLE_REGS_INTR;
7761 	}
7762 
7763 	if (filtered_sample_type & PERF_SAMPLE_PHYS_ADDR) {
7764 		data->phys_addr = perf_virt_to_phys(data->addr);
7765 		data->sample_flags |= PERF_SAMPLE_PHYS_ADDR;
7766 	}
7767 
7768 #ifdef CONFIG_CGROUP_PERF
7769 	if (filtered_sample_type & PERF_SAMPLE_CGROUP) {
7770 		struct cgroup *cgrp;
7771 
7772 		/* protected by RCU */
7773 		cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup;
7774 		data->cgroup = cgroup_id(cgrp);
7775 		data->sample_flags |= PERF_SAMPLE_CGROUP;
7776 	}
7777 #endif
7778 
7779 	/*
7780 	 * PERF_DATA_PAGE_SIZE requires PERF_SAMPLE_ADDR. If the user doesn't
7781 	 * require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr,
7782 	 * but the value will not dump to the userspace.
7783 	 */
7784 	if (filtered_sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) {
7785 		data->data_page_size = perf_get_page_size(data->addr);
7786 		data->sample_flags |= PERF_SAMPLE_DATA_PAGE_SIZE;
7787 	}
7788 
7789 	if (filtered_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) {
7790 		data->code_page_size = perf_get_page_size(data->ip);
7791 		data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE;
7792 	}
7793 
7794 	if (filtered_sample_type & PERF_SAMPLE_AUX) {
7795 		u64 size;
7796 		u16 header_size = perf_sample_data_size(data, event);
7797 
7798 		header_size += sizeof(u64); /* size */
7799 
7800 		/*
7801 		 * Given the 16bit nature of header::size, an AUX sample can
7802 		 * easily overflow it, what with all the preceding sample bits.
7803 		 * Make sure this doesn't happen by using up to U16_MAX bytes
7804 		 * per sample in total (rounded down to 8 byte boundary).
7805 		 */
7806 		size = min_t(size_t, U16_MAX - header_size,
7807 			     event->attr.aux_sample_size);
7808 		size = rounddown(size, 8);
7809 		size = perf_prepare_sample_aux(event, data, size);
7810 
7811 		WARN_ON_ONCE(size + header_size > U16_MAX);
7812 		data->dyn_size += size + sizeof(u64); /* size above */
7813 		data->sample_flags |= PERF_SAMPLE_AUX;
7814 	}
7815 }
7816 
perf_prepare_header(struct perf_event_header * header,struct perf_sample_data * data,struct perf_event * event,struct pt_regs * regs)7817 void perf_prepare_header(struct perf_event_header *header,
7818 			 struct perf_sample_data *data,
7819 			 struct perf_event *event,
7820 			 struct pt_regs *regs)
7821 {
7822 	header->type = PERF_RECORD_SAMPLE;
7823 	header->size = perf_sample_data_size(data, event);
7824 	header->misc = perf_misc_flags(regs);
7825 
7826 	/*
7827 	 * If you're adding more sample types here, you likely need to do
7828 	 * something about the overflowing header::size, like repurpose the
7829 	 * lowest 3 bits of size, which should be always zero at the moment.
7830 	 * This raises a more important question, do we really need 512k sized
7831 	 * samples and why, so good argumentation is in order for whatever you
7832 	 * do here next.
7833 	 */
7834 	WARN_ON_ONCE(header->size & 7);
7835 }
7836 
7837 static __always_inline int
__perf_event_output(struct perf_event * event,struct perf_sample_data * data,struct pt_regs * regs,int (* output_begin)(struct perf_output_handle *,struct perf_sample_data *,struct perf_event *,unsigned int))7838 __perf_event_output(struct perf_event *event,
7839 		    struct perf_sample_data *data,
7840 		    struct pt_regs *regs,
7841 		    int (*output_begin)(struct perf_output_handle *,
7842 					struct perf_sample_data *,
7843 					struct perf_event *,
7844 					unsigned int))
7845 {
7846 	struct perf_output_handle handle;
7847 	struct perf_event_header header;
7848 	int err;
7849 
7850 	/* protect the callchain buffers */
7851 	rcu_read_lock();
7852 
7853 	perf_prepare_sample(data, event, regs);
7854 	perf_prepare_header(&header, data, event, regs);
7855 
7856 	err = output_begin(&handle, data, event, header.size);
7857 	if (err)
7858 		goto exit;
7859 
7860 	perf_output_sample(&handle, &header, data, event);
7861 
7862 	perf_output_end(&handle);
7863 
7864 exit:
7865 	rcu_read_unlock();
7866 	return err;
7867 }
7868 
7869 void
perf_event_output_forward(struct perf_event * event,struct perf_sample_data * data,struct pt_regs * regs)7870 perf_event_output_forward(struct perf_event *event,
7871 			 struct perf_sample_data *data,
7872 			 struct pt_regs *regs)
7873 {
7874 	__perf_event_output(event, data, regs, perf_output_begin_forward);
7875 }
7876 
7877 void
perf_event_output_backward(struct perf_event * event,struct perf_sample_data * data,struct pt_regs * regs)7878 perf_event_output_backward(struct perf_event *event,
7879 			   struct perf_sample_data *data,
7880 			   struct pt_regs *regs)
7881 {
7882 	__perf_event_output(event, data, regs, perf_output_begin_backward);
7883 }
7884 
7885 int
perf_event_output(struct perf_event * event,struct perf_sample_data * data,struct pt_regs * regs)7886 perf_event_output(struct perf_event *event,
7887 		  struct perf_sample_data *data,
7888 		  struct pt_regs *regs)
7889 {
7890 	return __perf_event_output(event, data, regs, perf_output_begin);
7891 }
7892 
7893 /*
7894  * read event_id
7895  */
7896 
7897 struct perf_read_event {
7898 	struct perf_event_header	header;
7899 
7900 	u32				pid;
7901 	u32				tid;
7902 };
7903 
7904 static void
perf_event_read_event(struct perf_event * event,struct task_struct * task)7905 perf_event_read_event(struct perf_event *event,
7906 			struct task_struct *task)
7907 {
7908 	struct perf_output_handle handle;
7909 	struct perf_sample_data sample;
7910 	struct perf_read_event read_event = {
7911 		.header = {
7912 			.type = PERF_RECORD_READ,
7913 			.misc = 0,
7914 			.size = sizeof(read_event) + event->read_size,
7915 		},
7916 		.pid = perf_event_pid(event, task),
7917 		.tid = perf_event_tid(event, task),
7918 	};
7919 	int ret;
7920 
7921 	perf_event_header__init_id(&read_event.header, &sample, event);
7922 	ret = perf_output_begin(&handle, &sample, event, read_event.header.size);
7923 	if (ret)
7924 		return;
7925 
7926 	perf_output_put(&handle, read_event);
7927 	perf_output_read(&handle, event);
7928 	perf_event__output_id_sample(event, &handle, &sample);
7929 
7930 	perf_output_end(&handle);
7931 }
7932 
7933 typedef void (perf_iterate_f)(struct perf_event *event, void *data);
7934 
7935 static void
perf_iterate_ctx(struct perf_event_context * ctx,perf_iterate_f output,void * data,bool all)7936 perf_iterate_ctx(struct perf_event_context *ctx,
7937 		   perf_iterate_f output,
7938 		   void *data, bool all)
7939 {
7940 	struct perf_event *event;
7941 
7942 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
7943 		if (!all) {
7944 			if (event->state < PERF_EVENT_STATE_INACTIVE)
7945 				continue;
7946 			if (!event_filter_match(event))
7947 				continue;
7948 		}
7949 
7950 		output(event, data);
7951 	}
7952 }
7953 
perf_iterate_sb_cpu(perf_iterate_f output,void * data)7954 static void perf_iterate_sb_cpu(perf_iterate_f output, void *data)
7955 {
7956 	struct pmu_event_list *pel = this_cpu_ptr(&pmu_sb_events);
7957 	struct perf_event *event;
7958 
7959 	list_for_each_entry_rcu(event, &pel->list, sb_list) {
7960 		/*
7961 		 * Skip events that are not fully formed yet; ensure that
7962 		 * if we observe event->ctx, both event and ctx will be
7963 		 * complete enough. See perf_install_in_context().
7964 		 */
7965 		if (!smp_load_acquire(&event->ctx))
7966 			continue;
7967 
7968 		if (event->state < PERF_EVENT_STATE_INACTIVE)
7969 			continue;
7970 		if (!event_filter_match(event))
7971 			continue;
7972 		output(event, data);
7973 	}
7974 }
7975 
7976 /*
7977  * Iterate all events that need to receive side-band events.
7978  *
7979  * For new callers; ensure that account_pmu_sb_event() includes
7980  * your event, otherwise it might not get delivered.
7981  */
7982 static void
perf_iterate_sb(perf_iterate_f output,void * data,struct perf_event_context * task_ctx)7983 perf_iterate_sb(perf_iterate_f output, void *data,
7984 	       struct perf_event_context *task_ctx)
7985 {
7986 	struct perf_event_context *ctx;
7987 
7988 	rcu_read_lock();
7989 	preempt_disable();
7990 
7991 	/*
7992 	 * If we have task_ctx != NULL we only notify the task context itself.
7993 	 * The task_ctx is set only for EXIT events before releasing task
7994 	 * context.
7995 	 */
7996 	if (task_ctx) {
7997 		perf_iterate_ctx(task_ctx, output, data, false);
7998 		goto done;
7999 	}
8000 
8001 	perf_iterate_sb_cpu(output, data);
8002 
8003 	ctx = rcu_dereference(current->perf_event_ctxp);
8004 	if (ctx)
8005 		perf_iterate_ctx(ctx, output, data, false);
8006 done:
8007 	preempt_enable();
8008 	rcu_read_unlock();
8009 }
8010 
8011 /*
8012  * Clear all file-based filters at exec, they'll have to be
8013  * re-instated when/if these objects are mmapped again.
8014  */
perf_event_addr_filters_exec(struct perf_event * event,void * data)8015 static void perf_event_addr_filters_exec(struct perf_event *event, void *data)
8016 {
8017 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
8018 	struct perf_addr_filter *filter;
8019 	unsigned int restart = 0, count = 0;
8020 	unsigned long flags;
8021 
8022 	if (!has_addr_filter(event))
8023 		return;
8024 
8025 	raw_spin_lock_irqsave(&ifh->lock, flags);
8026 	list_for_each_entry(filter, &ifh->list, entry) {
8027 		if (filter->path.dentry) {
8028 			event->addr_filter_ranges[count].start = 0;
8029 			event->addr_filter_ranges[count].size = 0;
8030 			restart++;
8031 		}
8032 
8033 		count++;
8034 	}
8035 
8036 	if (restart)
8037 		event->addr_filters_gen++;
8038 	raw_spin_unlock_irqrestore(&ifh->lock, flags);
8039 
8040 	if (restart)
8041 		perf_event_stop(event, 1);
8042 }
8043 
perf_event_exec(void)8044 void perf_event_exec(void)
8045 {
8046 	struct perf_event_context *ctx;
8047 
8048 	ctx = perf_pin_task_context(current);
8049 	if (!ctx)
8050 		return;
8051 
8052 	perf_event_enable_on_exec(ctx);
8053 	perf_event_remove_on_exec(ctx);
8054 	perf_iterate_ctx(ctx, perf_event_addr_filters_exec, NULL, true);
8055 
8056 	perf_unpin_context(ctx);
8057 	put_ctx(ctx);
8058 }
8059 
8060 struct remote_output {
8061 	struct perf_buffer	*rb;
8062 	int			err;
8063 };
8064 
__perf_event_output_stop(struct perf_event * event,void * data)8065 static void __perf_event_output_stop(struct perf_event *event, void *data)
8066 {
8067 	struct perf_event *parent = event->parent;
8068 	struct remote_output *ro = data;
8069 	struct perf_buffer *rb = ro->rb;
8070 	struct stop_event_data sd = {
8071 		.event	= event,
8072 	};
8073 
8074 	if (!has_aux(event))
8075 		return;
8076 
8077 	if (!parent)
8078 		parent = event;
8079 
8080 	/*
8081 	 * In case of inheritance, it will be the parent that links to the
8082 	 * ring-buffer, but it will be the child that's actually using it.
8083 	 *
8084 	 * We are using event::rb to determine if the event should be stopped,
8085 	 * however this may race with ring_buffer_attach() (through set_output),
8086 	 * which will make us skip the event that actually needs to be stopped.
8087 	 * So ring_buffer_attach() has to stop an aux event before re-assigning
8088 	 * its rb pointer.
8089 	 */
8090 	if (rcu_dereference(parent->rb) == rb)
8091 		ro->err = __perf_event_stop(&sd);
8092 }
8093 
__perf_pmu_output_stop(void * info)8094 static int __perf_pmu_output_stop(void *info)
8095 {
8096 	struct perf_event *event = info;
8097 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
8098 	struct remote_output ro = {
8099 		.rb	= event->rb,
8100 	};
8101 
8102 	rcu_read_lock();
8103 	perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false);
8104 	if (cpuctx->task_ctx)
8105 		perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop,
8106 				   &ro, false);
8107 	rcu_read_unlock();
8108 
8109 	return ro.err;
8110 }
8111 
perf_pmu_output_stop(struct perf_event * event)8112 static void perf_pmu_output_stop(struct perf_event *event)
8113 {
8114 	struct perf_event *iter;
8115 	int err, cpu;
8116 
8117 restart:
8118 	rcu_read_lock();
8119 	list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) {
8120 		/*
8121 		 * For per-CPU events, we need to make sure that neither they
8122 		 * nor their children are running; for cpu==-1 events it's
8123 		 * sufficient to stop the event itself if it's active, since
8124 		 * it can't have children.
8125 		 */
8126 		cpu = iter->cpu;
8127 		if (cpu == -1)
8128 			cpu = READ_ONCE(iter->oncpu);
8129 
8130 		if (cpu == -1)
8131 			continue;
8132 
8133 		err = cpu_function_call(cpu, __perf_pmu_output_stop, event);
8134 		if (err == -EAGAIN) {
8135 			rcu_read_unlock();
8136 			goto restart;
8137 		}
8138 	}
8139 	rcu_read_unlock();
8140 }
8141 
8142 /*
8143  * task tracking -- fork/exit
8144  *
8145  * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
8146  */
8147 
8148 struct perf_task_event {
8149 	struct task_struct		*task;
8150 	struct perf_event_context	*task_ctx;
8151 
8152 	struct {
8153 		struct perf_event_header	header;
8154 
8155 		u32				pid;
8156 		u32				ppid;
8157 		u32				tid;
8158 		u32				ptid;
8159 		u64				time;
8160 	} event_id;
8161 };
8162 
perf_event_task_match(struct perf_event * event)8163 static int perf_event_task_match(struct perf_event *event)
8164 {
8165 	return event->attr.comm  || event->attr.mmap ||
8166 	       event->attr.mmap2 || event->attr.mmap_data ||
8167 	       event->attr.task;
8168 }
8169 
perf_event_task_output(struct perf_event * event,void * data)8170 static void perf_event_task_output(struct perf_event *event,
8171 				   void *data)
8172 {
8173 	struct perf_task_event *task_event = data;
8174 	struct perf_output_handle handle;
8175 	struct perf_sample_data	sample;
8176 	struct task_struct *task = task_event->task;
8177 	int ret, size = task_event->event_id.header.size;
8178 
8179 	if (!perf_event_task_match(event))
8180 		return;
8181 
8182 	perf_event_header__init_id(&task_event->event_id.header, &sample, event);
8183 
8184 	ret = perf_output_begin(&handle, &sample, event,
8185 				task_event->event_id.header.size);
8186 	if (ret)
8187 		goto out;
8188 
8189 	task_event->event_id.pid = perf_event_pid(event, task);
8190 	task_event->event_id.tid = perf_event_tid(event, task);
8191 
8192 	if (task_event->event_id.header.type == PERF_RECORD_EXIT) {
8193 		task_event->event_id.ppid = perf_event_pid(event,
8194 							task->real_parent);
8195 		task_event->event_id.ptid = perf_event_pid(event,
8196 							task->real_parent);
8197 	} else {  /* PERF_RECORD_FORK */
8198 		task_event->event_id.ppid = perf_event_pid(event, current);
8199 		task_event->event_id.ptid = perf_event_tid(event, current);
8200 	}
8201 
8202 	task_event->event_id.time = perf_event_clock(event);
8203 
8204 	perf_output_put(&handle, task_event->event_id);
8205 
8206 	perf_event__output_id_sample(event, &handle, &sample);
8207 
8208 	perf_output_end(&handle);
8209 out:
8210 	task_event->event_id.header.size = size;
8211 }
8212 
perf_event_task(struct task_struct * task,struct perf_event_context * task_ctx,int new)8213 static void perf_event_task(struct task_struct *task,
8214 			      struct perf_event_context *task_ctx,
8215 			      int new)
8216 {
8217 	struct perf_task_event task_event;
8218 
8219 	if (!atomic_read(&nr_comm_events) &&
8220 	    !atomic_read(&nr_mmap_events) &&
8221 	    !atomic_read(&nr_task_events))
8222 		return;
8223 
8224 	task_event = (struct perf_task_event){
8225 		.task	  = task,
8226 		.task_ctx = task_ctx,
8227 		.event_id    = {
8228 			.header = {
8229 				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
8230 				.misc = 0,
8231 				.size = sizeof(task_event.event_id),
8232 			},
8233 			/* .pid  */
8234 			/* .ppid */
8235 			/* .tid  */
8236 			/* .ptid */
8237 			/* .time */
8238 		},
8239 	};
8240 
8241 	perf_iterate_sb(perf_event_task_output,
8242 		       &task_event,
8243 		       task_ctx);
8244 }
8245 
perf_event_fork(struct task_struct * task)8246 void perf_event_fork(struct task_struct *task)
8247 {
8248 	perf_event_task(task, NULL, 1);
8249 	perf_event_namespaces(task);
8250 }
8251 
8252 /*
8253  * comm tracking
8254  */
8255 
8256 struct perf_comm_event {
8257 	struct task_struct	*task;
8258 	char			*comm;
8259 	int			comm_size;
8260 
8261 	struct {
8262 		struct perf_event_header	header;
8263 
8264 		u32				pid;
8265 		u32				tid;
8266 	} event_id;
8267 };
8268 
perf_event_comm_match(struct perf_event * event)8269 static int perf_event_comm_match(struct perf_event *event)
8270 {
8271 	return event->attr.comm;
8272 }
8273 
perf_event_comm_output(struct perf_event * event,void * data)8274 static void perf_event_comm_output(struct perf_event *event,
8275 				   void *data)
8276 {
8277 	struct perf_comm_event *comm_event = data;
8278 	struct perf_output_handle handle;
8279 	struct perf_sample_data sample;
8280 	int size = comm_event->event_id.header.size;
8281 	int ret;
8282 
8283 	if (!perf_event_comm_match(event))
8284 		return;
8285 
8286 	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
8287 	ret = perf_output_begin(&handle, &sample, event,
8288 				comm_event->event_id.header.size);
8289 
8290 	if (ret)
8291 		goto out;
8292 
8293 	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
8294 	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
8295 
8296 	perf_output_put(&handle, comm_event->event_id);
8297 	__output_copy(&handle, comm_event->comm,
8298 				   comm_event->comm_size);
8299 
8300 	perf_event__output_id_sample(event, &handle, &sample);
8301 
8302 	perf_output_end(&handle);
8303 out:
8304 	comm_event->event_id.header.size = size;
8305 }
8306 
perf_event_comm_event(struct perf_comm_event * comm_event)8307 static void perf_event_comm_event(struct perf_comm_event *comm_event)
8308 {
8309 	char comm[TASK_COMM_LEN];
8310 	unsigned int size;
8311 
8312 	memset(comm, 0, sizeof(comm));
8313 	strscpy(comm, comm_event->task->comm, sizeof(comm));
8314 	size = ALIGN(strlen(comm)+1, sizeof(u64));
8315 
8316 	comm_event->comm = comm;
8317 	comm_event->comm_size = size;
8318 
8319 	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
8320 
8321 	perf_iterate_sb(perf_event_comm_output,
8322 		       comm_event,
8323 		       NULL);
8324 }
8325 
perf_event_comm(struct task_struct * task,bool exec)8326 void perf_event_comm(struct task_struct *task, bool exec)
8327 {
8328 	struct perf_comm_event comm_event;
8329 
8330 	if (!atomic_read(&nr_comm_events))
8331 		return;
8332 
8333 	comm_event = (struct perf_comm_event){
8334 		.task	= task,
8335 		/* .comm      */
8336 		/* .comm_size */
8337 		.event_id  = {
8338 			.header = {
8339 				.type = PERF_RECORD_COMM,
8340 				.misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
8341 				/* .size */
8342 			},
8343 			/* .pid */
8344 			/* .tid */
8345 		},
8346 	};
8347 
8348 	perf_event_comm_event(&comm_event);
8349 }
8350 
8351 /*
8352  * namespaces tracking
8353  */
8354 
8355 struct perf_namespaces_event {
8356 	struct task_struct		*task;
8357 
8358 	struct {
8359 		struct perf_event_header	header;
8360 
8361 		u32				pid;
8362 		u32				tid;
8363 		u64				nr_namespaces;
8364 		struct perf_ns_link_info	link_info[NR_NAMESPACES];
8365 	} event_id;
8366 };
8367 
perf_event_namespaces_match(struct perf_event * event)8368 static int perf_event_namespaces_match(struct perf_event *event)
8369 {
8370 	return event->attr.namespaces;
8371 }
8372 
perf_event_namespaces_output(struct perf_event * event,void * data)8373 static void perf_event_namespaces_output(struct perf_event *event,
8374 					 void *data)
8375 {
8376 	struct perf_namespaces_event *namespaces_event = data;
8377 	struct perf_output_handle handle;
8378 	struct perf_sample_data sample;
8379 	u16 header_size = namespaces_event->event_id.header.size;
8380 	int ret;
8381 
8382 	if (!perf_event_namespaces_match(event))
8383 		return;
8384 
8385 	perf_event_header__init_id(&namespaces_event->event_id.header,
8386 				   &sample, event);
8387 	ret = perf_output_begin(&handle, &sample, event,
8388 				namespaces_event->event_id.header.size);
8389 	if (ret)
8390 		goto out;
8391 
8392 	namespaces_event->event_id.pid = perf_event_pid(event,
8393 							namespaces_event->task);
8394 	namespaces_event->event_id.tid = perf_event_tid(event,
8395 							namespaces_event->task);
8396 
8397 	perf_output_put(&handle, namespaces_event->event_id);
8398 
8399 	perf_event__output_id_sample(event, &handle, &sample);
8400 
8401 	perf_output_end(&handle);
8402 out:
8403 	namespaces_event->event_id.header.size = header_size;
8404 }
8405 
perf_fill_ns_link_info(struct perf_ns_link_info * ns_link_info,struct task_struct * task,const struct proc_ns_operations * ns_ops)8406 static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info,
8407 				   struct task_struct *task,
8408 				   const struct proc_ns_operations *ns_ops)
8409 {
8410 	struct path ns_path;
8411 	struct inode *ns_inode;
8412 	int error;
8413 
8414 	error = ns_get_path(&ns_path, task, ns_ops);
8415 	if (!error) {
8416 		ns_inode = ns_path.dentry->d_inode;
8417 		ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev);
8418 		ns_link_info->ino = ns_inode->i_ino;
8419 		path_put(&ns_path);
8420 	}
8421 }
8422 
perf_event_namespaces(struct task_struct * task)8423 void perf_event_namespaces(struct task_struct *task)
8424 {
8425 	struct perf_namespaces_event namespaces_event;
8426 	struct perf_ns_link_info *ns_link_info;
8427 
8428 	if (!atomic_read(&nr_namespaces_events))
8429 		return;
8430 
8431 	namespaces_event = (struct perf_namespaces_event){
8432 		.task	= task,
8433 		.event_id  = {
8434 			.header = {
8435 				.type = PERF_RECORD_NAMESPACES,
8436 				.misc = 0,
8437 				.size = sizeof(namespaces_event.event_id),
8438 			},
8439 			/* .pid */
8440 			/* .tid */
8441 			.nr_namespaces = NR_NAMESPACES,
8442 			/* .link_info[NR_NAMESPACES] */
8443 		},
8444 	};
8445 
8446 	ns_link_info = namespaces_event.event_id.link_info;
8447 
8448 	perf_fill_ns_link_info(&ns_link_info[MNT_NS_INDEX],
8449 			       task, &mntns_operations);
8450 
8451 #ifdef CONFIG_USER_NS
8452 	perf_fill_ns_link_info(&ns_link_info[USER_NS_INDEX],
8453 			       task, &userns_operations);
8454 #endif
8455 #ifdef CONFIG_NET_NS
8456 	perf_fill_ns_link_info(&ns_link_info[NET_NS_INDEX],
8457 			       task, &netns_operations);
8458 #endif
8459 #ifdef CONFIG_UTS_NS
8460 	perf_fill_ns_link_info(&ns_link_info[UTS_NS_INDEX],
8461 			       task, &utsns_operations);
8462 #endif
8463 #ifdef CONFIG_IPC_NS
8464 	perf_fill_ns_link_info(&ns_link_info[IPC_NS_INDEX],
8465 			       task, &ipcns_operations);
8466 #endif
8467 #ifdef CONFIG_PID_NS
8468 	perf_fill_ns_link_info(&ns_link_info[PID_NS_INDEX],
8469 			       task, &pidns_operations);
8470 #endif
8471 #ifdef CONFIG_CGROUPS
8472 	perf_fill_ns_link_info(&ns_link_info[CGROUP_NS_INDEX],
8473 			       task, &cgroupns_operations);
8474 #endif
8475 
8476 	perf_iterate_sb(perf_event_namespaces_output,
8477 			&namespaces_event,
8478 			NULL);
8479 }
8480 
8481 /*
8482  * cgroup tracking
8483  */
8484 #ifdef CONFIG_CGROUP_PERF
8485 
8486 struct perf_cgroup_event {
8487 	char				*path;
8488 	int				path_size;
8489 	struct {
8490 		struct perf_event_header	header;
8491 		u64				id;
8492 		char				path[];
8493 	} event_id;
8494 };
8495 
perf_event_cgroup_match(struct perf_event * event)8496 static int perf_event_cgroup_match(struct perf_event *event)
8497 {
8498 	return event->attr.cgroup;
8499 }
8500 
perf_event_cgroup_output(struct perf_event * event,void * data)8501 static void perf_event_cgroup_output(struct perf_event *event, void *data)
8502 {
8503 	struct perf_cgroup_event *cgroup_event = data;
8504 	struct perf_output_handle handle;
8505 	struct perf_sample_data sample;
8506 	u16 header_size = cgroup_event->event_id.header.size;
8507 	int ret;
8508 
8509 	if (!perf_event_cgroup_match(event))
8510 		return;
8511 
8512 	perf_event_header__init_id(&cgroup_event->event_id.header,
8513 				   &sample, event);
8514 	ret = perf_output_begin(&handle, &sample, event,
8515 				cgroup_event->event_id.header.size);
8516 	if (ret)
8517 		goto out;
8518 
8519 	perf_output_put(&handle, cgroup_event->event_id);
8520 	__output_copy(&handle, cgroup_event->path, cgroup_event->path_size);
8521 
8522 	perf_event__output_id_sample(event, &handle, &sample);
8523 
8524 	perf_output_end(&handle);
8525 out:
8526 	cgroup_event->event_id.header.size = header_size;
8527 }
8528 
perf_event_cgroup(struct cgroup * cgrp)8529 static void perf_event_cgroup(struct cgroup *cgrp)
8530 {
8531 	struct perf_cgroup_event cgroup_event;
8532 	char path_enomem[16] = "//enomem";
8533 	char *pathname;
8534 	size_t size;
8535 
8536 	if (!atomic_read(&nr_cgroup_events))
8537 		return;
8538 
8539 	cgroup_event = (struct perf_cgroup_event){
8540 		.event_id  = {
8541 			.header = {
8542 				.type = PERF_RECORD_CGROUP,
8543 				.misc = 0,
8544 				.size = sizeof(cgroup_event.event_id),
8545 			},
8546 			.id = cgroup_id(cgrp),
8547 		},
8548 	};
8549 
8550 	pathname = kmalloc(PATH_MAX, GFP_KERNEL);
8551 	if (pathname == NULL) {
8552 		cgroup_event.path = path_enomem;
8553 	} else {
8554 		/* just to be sure to have enough space for alignment */
8555 		cgroup_path(cgrp, pathname, PATH_MAX - sizeof(u64));
8556 		cgroup_event.path = pathname;
8557 	}
8558 
8559 	/*
8560 	 * Since our buffer works in 8 byte units we need to align our string
8561 	 * size to a multiple of 8. However, we must guarantee the tail end is
8562 	 * zero'd out to avoid leaking random bits to userspace.
8563 	 */
8564 	size = strlen(cgroup_event.path) + 1;
8565 	while (!IS_ALIGNED(size, sizeof(u64)))
8566 		cgroup_event.path[size++] = '\0';
8567 
8568 	cgroup_event.event_id.header.size += size;
8569 	cgroup_event.path_size = size;
8570 
8571 	perf_iterate_sb(perf_event_cgroup_output,
8572 			&cgroup_event,
8573 			NULL);
8574 
8575 	kfree(pathname);
8576 }
8577 
8578 #endif
8579 
8580 /*
8581  * mmap tracking
8582  */
8583 
8584 struct perf_mmap_event {
8585 	struct vm_area_struct	*vma;
8586 
8587 	const char		*file_name;
8588 	int			file_size;
8589 	int			maj, min;
8590 	u64			ino;
8591 	u64			ino_generation;
8592 	u32			prot, flags;
8593 	u8			build_id[BUILD_ID_SIZE_MAX];
8594 	u32			build_id_size;
8595 
8596 	struct {
8597 		struct perf_event_header	header;
8598 
8599 		u32				pid;
8600 		u32				tid;
8601 		u64				start;
8602 		u64				len;
8603 		u64				pgoff;
8604 	} event_id;
8605 };
8606 
perf_event_mmap_match(struct perf_event * event,void * data)8607 static int perf_event_mmap_match(struct perf_event *event,
8608 				 void *data)
8609 {
8610 	struct perf_mmap_event *mmap_event = data;
8611 	struct vm_area_struct *vma = mmap_event->vma;
8612 	int executable = vma->vm_flags & VM_EXEC;
8613 
8614 	return (!executable && event->attr.mmap_data) ||
8615 	       (executable && (event->attr.mmap || event->attr.mmap2));
8616 }
8617 
perf_event_mmap_output(struct perf_event * event,void * data)8618 static void perf_event_mmap_output(struct perf_event *event,
8619 				   void *data)
8620 {
8621 	struct perf_mmap_event *mmap_event = data;
8622 	struct perf_output_handle handle;
8623 	struct perf_sample_data sample;
8624 	int size = mmap_event->event_id.header.size;
8625 	u32 type = mmap_event->event_id.header.type;
8626 	bool use_build_id;
8627 	int ret;
8628 
8629 	if (!perf_event_mmap_match(event, data))
8630 		return;
8631 
8632 	if (event->attr.mmap2) {
8633 		mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
8634 		mmap_event->event_id.header.size += sizeof(mmap_event->maj);
8635 		mmap_event->event_id.header.size += sizeof(mmap_event->min);
8636 		mmap_event->event_id.header.size += sizeof(mmap_event->ino);
8637 		mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);
8638 		mmap_event->event_id.header.size += sizeof(mmap_event->prot);
8639 		mmap_event->event_id.header.size += sizeof(mmap_event->flags);
8640 	}
8641 
8642 	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
8643 	ret = perf_output_begin(&handle, &sample, event,
8644 				mmap_event->event_id.header.size);
8645 	if (ret)
8646 		goto out;
8647 
8648 	mmap_event->event_id.pid = perf_event_pid(event, current);
8649 	mmap_event->event_id.tid = perf_event_tid(event, current);
8650 
8651 	use_build_id = event->attr.build_id && mmap_event->build_id_size;
8652 
8653 	if (event->attr.mmap2 && use_build_id)
8654 		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID;
8655 
8656 	perf_output_put(&handle, mmap_event->event_id);
8657 
8658 	if (event->attr.mmap2) {
8659 		if (use_build_id) {
8660 			u8 size[4] = { (u8) mmap_event->build_id_size, 0, 0, 0 };
8661 
8662 			__output_copy(&handle, size, 4);
8663 			__output_copy(&handle, mmap_event->build_id, BUILD_ID_SIZE_MAX);
8664 		} else {
8665 			perf_output_put(&handle, mmap_event->maj);
8666 			perf_output_put(&handle, mmap_event->min);
8667 			perf_output_put(&handle, mmap_event->ino);
8668 			perf_output_put(&handle, mmap_event->ino_generation);
8669 		}
8670 		perf_output_put(&handle, mmap_event->prot);
8671 		perf_output_put(&handle, mmap_event->flags);
8672 	}
8673 
8674 	__output_copy(&handle, mmap_event->file_name,
8675 				   mmap_event->file_size);
8676 
8677 	perf_event__output_id_sample(event, &handle, &sample);
8678 
8679 	perf_output_end(&handle);
8680 out:
8681 	mmap_event->event_id.header.size = size;
8682 	mmap_event->event_id.header.type = type;
8683 }
8684 
perf_event_mmap_event(struct perf_mmap_event * mmap_event)8685 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
8686 {
8687 	struct vm_area_struct *vma = mmap_event->vma;
8688 	struct file *file = vma->vm_file;
8689 	int maj = 0, min = 0;
8690 	u64 ino = 0, gen = 0;
8691 	u32 prot = 0, flags = 0;
8692 	unsigned int size;
8693 	char tmp[16];
8694 	char *buf = NULL;
8695 	char *name = NULL;
8696 
8697 	if (vma->vm_flags & VM_READ)
8698 		prot |= PROT_READ;
8699 	if (vma->vm_flags & VM_WRITE)
8700 		prot |= PROT_WRITE;
8701 	if (vma->vm_flags & VM_EXEC)
8702 		prot |= PROT_EXEC;
8703 
8704 	if (vma->vm_flags & VM_MAYSHARE)
8705 		flags = MAP_SHARED;
8706 	else
8707 		flags = MAP_PRIVATE;
8708 
8709 	if (vma->vm_flags & VM_LOCKED)
8710 		flags |= MAP_LOCKED;
8711 	if (is_vm_hugetlb_page(vma))
8712 		flags |= MAP_HUGETLB;
8713 
8714 	if (file) {
8715 		struct inode *inode;
8716 		dev_t dev;
8717 
8718 		buf = kmalloc(PATH_MAX, GFP_KERNEL);
8719 		if (!buf) {
8720 			name = "//enomem";
8721 			goto cpy_name;
8722 		}
8723 		/*
8724 		 * d_path() works from the end of the rb backwards, so we
8725 		 * need to add enough zero bytes after the string to handle
8726 		 * the 64bit alignment we do later.
8727 		 */
8728 		name = file_path(file, buf, PATH_MAX - sizeof(u64));
8729 		if (IS_ERR(name)) {
8730 			name = "//toolong";
8731 			goto cpy_name;
8732 		}
8733 		inode = file_inode(vma->vm_file);
8734 		dev = inode->i_sb->s_dev;
8735 		ino = inode->i_ino;
8736 		gen = inode->i_generation;
8737 		maj = MAJOR(dev);
8738 		min = MINOR(dev);
8739 
8740 		goto got_name;
8741 	} else {
8742 		if (vma->vm_ops && vma->vm_ops->name)
8743 			name = (char *) vma->vm_ops->name(vma);
8744 		if (!name)
8745 			name = (char *)arch_vma_name(vma);
8746 		if (!name) {
8747 			if (vma_is_initial_heap(vma))
8748 				name = "[heap]";
8749 			else if (vma_is_initial_stack(vma))
8750 				name = "[stack]";
8751 			else
8752 				name = "//anon";
8753 		}
8754 	}
8755 
8756 cpy_name:
8757 	strscpy(tmp, name, sizeof(tmp));
8758 	name = tmp;
8759 got_name:
8760 	/*
8761 	 * Since our buffer works in 8 byte units we need to align our string
8762 	 * size to a multiple of 8. However, we must guarantee the tail end is
8763 	 * zero'd out to avoid leaking random bits to userspace.
8764 	 */
8765 	size = strlen(name)+1;
8766 	while (!IS_ALIGNED(size, sizeof(u64)))
8767 		name[size++] = '\0';
8768 
8769 	mmap_event->file_name = name;
8770 	mmap_event->file_size = size;
8771 	mmap_event->maj = maj;
8772 	mmap_event->min = min;
8773 	mmap_event->ino = ino;
8774 	mmap_event->ino_generation = gen;
8775 	mmap_event->prot = prot;
8776 	mmap_event->flags = flags;
8777 
8778 	if (!(vma->vm_flags & VM_EXEC))
8779 		mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
8780 
8781 	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
8782 
8783 	if (atomic_read(&nr_build_id_events))
8784 		build_id_parse(vma, mmap_event->build_id, &mmap_event->build_id_size);
8785 
8786 	perf_iterate_sb(perf_event_mmap_output,
8787 		       mmap_event,
8788 		       NULL);
8789 
8790 	kfree(buf);
8791 }
8792 
8793 /*
8794  * Check whether inode and address range match filter criteria.
8795  */
perf_addr_filter_match(struct perf_addr_filter * filter,struct file * file,unsigned long offset,unsigned long size)8796 static bool perf_addr_filter_match(struct perf_addr_filter *filter,
8797 				     struct file *file, unsigned long offset,
8798 				     unsigned long size)
8799 {
8800 	/* d_inode(NULL) won't be equal to any mapped user-space file */
8801 	if (!filter->path.dentry)
8802 		return false;
8803 
8804 	if (d_inode(filter->path.dentry) != file_inode(file))
8805 		return false;
8806 
8807 	if (filter->offset > offset + size)
8808 		return false;
8809 
8810 	if (filter->offset + filter->size < offset)
8811 		return false;
8812 
8813 	return true;
8814 }
8815 
perf_addr_filter_vma_adjust(struct perf_addr_filter * filter,struct vm_area_struct * vma,struct perf_addr_filter_range * fr)8816 static bool perf_addr_filter_vma_adjust(struct perf_addr_filter *filter,
8817 					struct vm_area_struct *vma,
8818 					struct perf_addr_filter_range *fr)
8819 {
8820 	unsigned long vma_size = vma->vm_end - vma->vm_start;
8821 	unsigned long off = vma->vm_pgoff << PAGE_SHIFT;
8822 	struct file *file = vma->vm_file;
8823 
8824 	if (!perf_addr_filter_match(filter, file, off, vma_size))
8825 		return false;
8826 
8827 	if (filter->offset < off) {
8828 		fr->start = vma->vm_start;
8829 		fr->size = min(vma_size, filter->size - (off - filter->offset));
8830 	} else {
8831 		fr->start = vma->vm_start + filter->offset - off;
8832 		fr->size = min(vma->vm_end - fr->start, filter->size);
8833 	}
8834 
8835 	return true;
8836 }
8837 
__perf_addr_filters_adjust(struct perf_event * event,void * data)8838 static void __perf_addr_filters_adjust(struct perf_event *event, void *data)
8839 {
8840 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
8841 	struct vm_area_struct *vma = data;
8842 	struct perf_addr_filter *filter;
8843 	unsigned int restart = 0, count = 0;
8844 	unsigned long flags;
8845 
8846 	if (!has_addr_filter(event))
8847 		return;
8848 
8849 	if (!vma->vm_file)
8850 		return;
8851 
8852 	raw_spin_lock_irqsave(&ifh->lock, flags);
8853 	list_for_each_entry(filter, &ifh->list, entry) {
8854 		if (perf_addr_filter_vma_adjust(filter, vma,
8855 						&event->addr_filter_ranges[count]))
8856 			restart++;
8857 
8858 		count++;
8859 	}
8860 
8861 	if (restart)
8862 		event->addr_filters_gen++;
8863 	raw_spin_unlock_irqrestore(&ifh->lock, flags);
8864 
8865 	if (restart)
8866 		perf_event_stop(event, 1);
8867 }
8868 
8869 /*
8870  * Adjust all task's events' filters to the new vma
8871  */
perf_addr_filters_adjust(struct vm_area_struct * vma)8872 static void perf_addr_filters_adjust(struct vm_area_struct *vma)
8873 {
8874 	struct perf_event_context *ctx;
8875 
8876 	/*
8877 	 * Data tracing isn't supported yet and as such there is no need
8878 	 * to keep track of anything that isn't related to executable code:
8879 	 */
8880 	if (!(vma->vm_flags & VM_EXEC))
8881 		return;
8882 
8883 	rcu_read_lock();
8884 	ctx = rcu_dereference(current->perf_event_ctxp);
8885 	if (ctx)
8886 		perf_iterate_ctx(ctx, __perf_addr_filters_adjust, vma, true);
8887 	rcu_read_unlock();
8888 }
8889 
perf_event_mmap(struct vm_area_struct * vma)8890 void perf_event_mmap(struct vm_area_struct *vma)
8891 {
8892 	struct perf_mmap_event mmap_event;
8893 
8894 	if (!atomic_read(&nr_mmap_events))
8895 		return;
8896 
8897 	mmap_event = (struct perf_mmap_event){
8898 		.vma	= vma,
8899 		/* .file_name */
8900 		/* .file_size */
8901 		.event_id  = {
8902 			.header = {
8903 				.type = PERF_RECORD_MMAP,
8904 				.misc = PERF_RECORD_MISC_USER,
8905 				/* .size */
8906 			},
8907 			/* .pid */
8908 			/* .tid */
8909 			.start  = vma->vm_start,
8910 			.len    = vma->vm_end - vma->vm_start,
8911 			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
8912 		},
8913 		/* .maj (attr_mmap2 only) */
8914 		/* .min (attr_mmap2 only) */
8915 		/* .ino (attr_mmap2 only) */
8916 		/* .ino_generation (attr_mmap2 only) */
8917 		/* .prot (attr_mmap2 only) */
8918 		/* .flags (attr_mmap2 only) */
8919 	};
8920 
8921 	perf_addr_filters_adjust(vma);
8922 	perf_event_mmap_event(&mmap_event);
8923 }
8924 
perf_event_aux_event(struct perf_event * event,unsigned long head,unsigned long size,u64 flags)8925 void perf_event_aux_event(struct perf_event *event, unsigned long head,
8926 			  unsigned long size, u64 flags)
8927 {
8928 	struct perf_output_handle handle;
8929 	struct perf_sample_data sample;
8930 	struct perf_aux_event {
8931 		struct perf_event_header	header;
8932 		u64				offset;
8933 		u64				size;
8934 		u64				flags;
8935 	} rec = {
8936 		.header = {
8937 			.type = PERF_RECORD_AUX,
8938 			.misc = 0,
8939 			.size = sizeof(rec),
8940 		},
8941 		.offset		= head,
8942 		.size		= size,
8943 		.flags		= flags,
8944 	};
8945 	int ret;
8946 
8947 	perf_event_header__init_id(&rec.header, &sample, event);
8948 	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
8949 
8950 	if (ret)
8951 		return;
8952 
8953 	perf_output_put(&handle, rec);
8954 	perf_event__output_id_sample(event, &handle, &sample);
8955 
8956 	perf_output_end(&handle);
8957 }
8958 
8959 /*
8960  * Lost/dropped samples logging
8961  */
perf_log_lost_samples(struct perf_event * event,u64 lost)8962 void perf_log_lost_samples(struct perf_event *event, u64 lost)
8963 {
8964 	struct perf_output_handle handle;
8965 	struct perf_sample_data sample;
8966 	int ret;
8967 
8968 	struct {
8969 		struct perf_event_header	header;
8970 		u64				lost;
8971 	} lost_samples_event = {
8972 		.header = {
8973 			.type = PERF_RECORD_LOST_SAMPLES,
8974 			.misc = 0,
8975 			.size = sizeof(lost_samples_event),
8976 		},
8977 		.lost		= lost,
8978 	};
8979 
8980 	perf_event_header__init_id(&lost_samples_event.header, &sample, event);
8981 
8982 	ret = perf_output_begin(&handle, &sample, event,
8983 				lost_samples_event.header.size);
8984 	if (ret)
8985 		return;
8986 
8987 	perf_output_put(&handle, lost_samples_event);
8988 	perf_event__output_id_sample(event, &handle, &sample);
8989 	perf_output_end(&handle);
8990 }
8991 
8992 /*
8993  * context_switch tracking
8994  */
8995 
8996 struct perf_switch_event {
8997 	struct task_struct	*task;
8998 	struct task_struct	*next_prev;
8999 
9000 	struct {
9001 		struct perf_event_header	header;
9002 		u32				next_prev_pid;
9003 		u32				next_prev_tid;
9004 	} event_id;
9005 };
9006 
perf_event_switch_match(struct perf_event * event)9007 static int perf_event_switch_match(struct perf_event *event)
9008 {
9009 	return event->attr.context_switch;
9010 }
9011 
perf_event_switch_output(struct perf_event * event,void * data)9012 static void perf_event_switch_output(struct perf_event *event, void *data)
9013 {
9014 	struct perf_switch_event *se = data;
9015 	struct perf_output_handle handle;
9016 	struct perf_sample_data sample;
9017 	int ret;
9018 
9019 	if (!perf_event_switch_match(event))
9020 		return;
9021 
9022 	/* Only CPU-wide events are allowed to see next/prev pid/tid */
9023 	if (event->ctx->task) {
9024 		se->event_id.header.type = PERF_RECORD_SWITCH;
9025 		se->event_id.header.size = sizeof(se->event_id.header);
9026 	} else {
9027 		se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE;
9028 		se->event_id.header.size = sizeof(se->event_id);
9029 		se->event_id.next_prev_pid =
9030 					perf_event_pid(event, se->next_prev);
9031 		se->event_id.next_prev_tid =
9032 					perf_event_tid(event, se->next_prev);
9033 	}
9034 
9035 	perf_event_header__init_id(&se->event_id.header, &sample, event);
9036 
9037 	ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);
9038 	if (ret)
9039 		return;
9040 
9041 	if (event->ctx->task)
9042 		perf_output_put(&handle, se->event_id.header);
9043 	else
9044 		perf_output_put(&handle, se->event_id);
9045 
9046 	perf_event__output_id_sample(event, &handle, &sample);
9047 
9048 	perf_output_end(&handle);
9049 }
9050 
perf_event_switch(struct task_struct * task,struct task_struct * next_prev,bool sched_in)9051 static void perf_event_switch(struct task_struct *task,
9052 			      struct task_struct *next_prev, bool sched_in)
9053 {
9054 	struct perf_switch_event switch_event;
9055 
9056 	/* N.B. caller checks nr_switch_events != 0 */
9057 
9058 	switch_event = (struct perf_switch_event){
9059 		.task		= task,
9060 		.next_prev	= next_prev,
9061 		.event_id	= {
9062 			.header = {
9063 				/* .type */
9064 				.misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT,
9065 				/* .size */
9066 			},
9067 			/* .next_prev_pid */
9068 			/* .next_prev_tid */
9069 		},
9070 	};
9071 
9072 	if (!sched_in && task->on_rq) {
9073 		switch_event.event_id.header.misc |=
9074 				PERF_RECORD_MISC_SWITCH_OUT_PREEMPT;
9075 	}
9076 
9077 	perf_iterate_sb(perf_event_switch_output, &switch_event, NULL);
9078 }
9079 
9080 /*
9081  * IRQ throttle logging
9082  */
9083 
perf_log_throttle(struct perf_event * event,int enable)9084 static void perf_log_throttle(struct perf_event *event, int enable)
9085 {
9086 	struct perf_output_handle handle;
9087 	struct perf_sample_data sample;
9088 	int ret;
9089 
9090 	struct {
9091 		struct perf_event_header	header;
9092 		u64				time;
9093 		u64				id;
9094 		u64				stream_id;
9095 	} throttle_event = {
9096 		.header = {
9097 			.type = PERF_RECORD_THROTTLE,
9098 			.misc = 0,
9099 			.size = sizeof(throttle_event),
9100 		},
9101 		.time		= perf_event_clock(event),
9102 		.id		= primary_event_id(event),
9103 		.stream_id	= event->id,
9104 	};
9105 
9106 	if (enable)
9107 		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
9108 
9109 	perf_event_header__init_id(&throttle_event.header, &sample, event);
9110 
9111 	ret = perf_output_begin(&handle, &sample, event,
9112 				throttle_event.header.size);
9113 	if (ret)
9114 		return;
9115 
9116 	perf_output_put(&handle, throttle_event);
9117 	perf_event__output_id_sample(event, &handle, &sample);
9118 	perf_output_end(&handle);
9119 }
9120 
9121 /*
9122  * ksymbol register/unregister tracking
9123  */
9124 
9125 struct perf_ksymbol_event {
9126 	const char	*name;
9127 	int		name_len;
9128 	struct {
9129 		struct perf_event_header        header;
9130 		u64				addr;
9131 		u32				len;
9132 		u16				ksym_type;
9133 		u16				flags;
9134 	} event_id;
9135 };
9136 
perf_event_ksymbol_match(struct perf_event * event)9137 static int perf_event_ksymbol_match(struct perf_event *event)
9138 {
9139 	return event->attr.ksymbol;
9140 }
9141 
perf_event_ksymbol_output(struct perf_event * event,void * data)9142 static void perf_event_ksymbol_output(struct perf_event *event, void *data)
9143 {
9144 	struct perf_ksymbol_event *ksymbol_event = data;
9145 	struct perf_output_handle handle;
9146 	struct perf_sample_data sample;
9147 	int ret;
9148 
9149 	if (!perf_event_ksymbol_match(event))
9150 		return;
9151 
9152 	perf_event_header__init_id(&ksymbol_event->event_id.header,
9153 				   &sample, event);
9154 	ret = perf_output_begin(&handle, &sample, event,
9155 				ksymbol_event->event_id.header.size);
9156 	if (ret)
9157 		return;
9158 
9159 	perf_output_put(&handle, ksymbol_event->event_id);
9160 	__output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len);
9161 	perf_event__output_id_sample(event, &handle, &sample);
9162 
9163 	perf_output_end(&handle);
9164 }
9165 
perf_event_ksymbol(u16 ksym_type,u64 addr,u32 len,bool unregister,const char * sym)9166 void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister,
9167 			const char *sym)
9168 {
9169 	struct perf_ksymbol_event ksymbol_event;
9170 	char name[KSYM_NAME_LEN];
9171 	u16 flags = 0;
9172 	int name_len;
9173 
9174 	if (!atomic_read(&nr_ksymbol_events))
9175 		return;
9176 
9177 	if (ksym_type >= PERF_RECORD_KSYMBOL_TYPE_MAX ||
9178 	    ksym_type == PERF_RECORD_KSYMBOL_TYPE_UNKNOWN)
9179 		goto err;
9180 
9181 	strscpy(name, sym, KSYM_NAME_LEN);
9182 	name_len = strlen(name) + 1;
9183 	while (!IS_ALIGNED(name_len, sizeof(u64)))
9184 		name[name_len++] = '\0';
9185 	BUILD_BUG_ON(KSYM_NAME_LEN % sizeof(u64));
9186 
9187 	if (unregister)
9188 		flags |= PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER;
9189 
9190 	ksymbol_event = (struct perf_ksymbol_event){
9191 		.name = name,
9192 		.name_len = name_len,
9193 		.event_id = {
9194 			.header = {
9195 				.type = PERF_RECORD_KSYMBOL,
9196 				.size = sizeof(ksymbol_event.event_id) +
9197 					name_len,
9198 			},
9199 			.addr = addr,
9200 			.len = len,
9201 			.ksym_type = ksym_type,
9202 			.flags = flags,
9203 		},
9204 	};
9205 
9206 	perf_iterate_sb(perf_event_ksymbol_output, &ksymbol_event, NULL);
9207 	return;
9208 err:
9209 	WARN_ONCE(1, "%s: Invalid KSYMBOL type 0x%x\n", __func__, ksym_type);
9210 }
9211 
9212 /*
9213  * bpf program load/unload tracking
9214  */
9215 
9216 struct perf_bpf_event {
9217 	struct bpf_prog	*prog;
9218 	struct {
9219 		struct perf_event_header        header;
9220 		u16				type;
9221 		u16				flags;
9222 		u32				id;
9223 		u8				tag[BPF_TAG_SIZE];
9224 	} event_id;
9225 };
9226 
perf_event_bpf_match(struct perf_event * event)9227 static int perf_event_bpf_match(struct perf_event *event)
9228 {
9229 	return event->attr.bpf_event;
9230 }
9231 
perf_event_bpf_output(struct perf_event * event,void * data)9232 static void perf_event_bpf_output(struct perf_event *event, void *data)
9233 {
9234 	struct perf_bpf_event *bpf_event = data;
9235 	struct perf_output_handle handle;
9236 	struct perf_sample_data sample;
9237 	int ret;
9238 
9239 	if (!perf_event_bpf_match(event))
9240 		return;
9241 
9242 	perf_event_header__init_id(&bpf_event->event_id.header,
9243 				   &sample, event);
9244 	ret = perf_output_begin(&handle, &sample, event,
9245 				bpf_event->event_id.header.size);
9246 	if (ret)
9247 		return;
9248 
9249 	perf_output_put(&handle, bpf_event->event_id);
9250 	perf_event__output_id_sample(event, &handle, &sample);
9251 
9252 	perf_output_end(&handle);
9253 }
9254 
perf_event_bpf_emit_ksymbols(struct bpf_prog * prog,enum perf_bpf_event_type type)9255 static void perf_event_bpf_emit_ksymbols(struct bpf_prog *prog,
9256 					 enum perf_bpf_event_type type)
9257 {
9258 	bool unregister = type == PERF_BPF_EVENT_PROG_UNLOAD;
9259 	int i;
9260 
9261 	if (prog->aux->func_cnt == 0) {
9262 		perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF,
9263 				   (u64)(unsigned long)prog->bpf_func,
9264 				   prog->jited_len, unregister,
9265 				   prog->aux->ksym.name);
9266 	} else {
9267 		for (i = 0; i < prog->aux->func_cnt; i++) {
9268 			struct bpf_prog *subprog = prog->aux->func[i];
9269 
9270 			perf_event_ksymbol(
9271 				PERF_RECORD_KSYMBOL_TYPE_BPF,
9272 				(u64)(unsigned long)subprog->bpf_func,
9273 				subprog->jited_len, unregister,
9274 				subprog->aux->ksym.name);
9275 		}
9276 	}
9277 }
9278 
perf_event_bpf_event(struct bpf_prog * prog,enum perf_bpf_event_type type,u16 flags)9279 void perf_event_bpf_event(struct bpf_prog *prog,
9280 			  enum perf_bpf_event_type type,
9281 			  u16 flags)
9282 {
9283 	struct perf_bpf_event bpf_event;
9284 
9285 	if (type <= PERF_BPF_EVENT_UNKNOWN ||
9286 	    type >= PERF_BPF_EVENT_MAX)
9287 		return;
9288 
9289 	switch (type) {
9290 	case PERF_BPF_EVENT_PROG_LOAD:
9291 	case PERF_BPF_EVENT_PROG_UNLOAD:
9292 		if (atomic_read(&nr_ksymbol_events))
9293 			perf_event_bpf_emit_ksymbols(prog, type);
9294 		break;
9295 	default:
9296 		break;
9297 	}
9298 
9299 	if (!atomic_read(&nr_bpf_events))
9300 		return;
9301 
9302 	bpf_event = (struct perf_bpf_event){
9303 		.prog = prog,
9304 		.event_id = {
9305 			.header = {
9306 				.type = PERF_RECORD_BPF_EVENT,
9307 				.size = sizeof(bpf_event.event_id),
9308 			},
9309 			.type = type,
9310 			.flags = flags,
9311 			.id = prog->aux->id,
9312 		},
9313 	};
9314 
9315 	BUILD_BUG_ON(BPF_TAG_SIZE % sizeof(u64));
9316 
9317 	memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE);
9318 	perf_iterate_sb(perf_event_bpf_output, &bpf_event, NULL);
9319 }
9320 
9321 struct perf_text_poke_event {
9322 	const void		*old_bytes;
9323 	const void		*new_bytes;
9324 	size_t			pad;
9325 	u16			old_len;
9326 	u16			new_len;
9327 
9328 	struct {
9329 		struct perf_event_header	header;
9330 
9331 		u64				addr;
9332 	} event_id;
9333 };
9334 
perf_event_text_poke_match(struct perf_event * event)9335 static int perf_event_text_poke_match(struct perf_event *event)
9336 {
9337 	return event->attr.text_poke;
9338 }
9339 
perf_event_text_poke_output(struct perf_event * event,void * data)9340 static void perf_event_text_poke_output(struct perf_event *event, void *data)
9341 {
9342 	struct perf_text_poke_event *text_poke_event = data;
9343 	struct perf_output_handle handle;
9344 	struct perf_sample_data sample;
9345 	u64 padding = 0;
9346 	int ret;
9347 
9348 	if (!perf_event_text_poke_match(event))
9349 		return;
9350 
9351 	perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);
9352 
9353 	ret = perf_output_begin(&handle, &sample, event,
9354 				text_poke_event->event_id.header.size);
9355 	if (ret)
9356 		return;
9357 
9358 	perf_output_put(&handle, text_poke_event->event_id);
9359 	perf_output_put(&handle, text_poke_event->old_len);
9360 	perf_output_put(&handle, text_poke_event->new_len);
9361 
9362 	__output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len);
9363 	__output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len);
9364 
9365 	if (text_poke_event->pad)
9366 		__output_copy(&handle, &padding, text_poke_event->pad);
9367 
9368 	perf_event__output_id_sample(event, &handle, &sample);
9369 
9370 	perf_output_end(&handle);
9371 }
9372 
perf_event_text_poke(const void * addr,const void * old_bytes,size_t old_len,const void * new_bytes,size_t new_len)9373 void perf_event_text_poke(const void *addr, const void *old_bytes,
9374 			  size_t old_len, const void *new_bytes, size_t new_len)
9375 {
9376 	struct perf_text_poke_event text_poke_event;
9377 	size_t tot, pad;
9378 
9379 	if (!atomic_read(&nr_text_poke_events))
9380 		return;
9381 
9382 	tot  = sizeof(text_poke_event.old_len) + old_len;
9383 	tot += sizeof(text_poke_event.new_len) + new_len;
9384 	pad  = ALIGN(tot, sizeof(u64)) - tot;
9385 
9386 	text_poke_event = (struct perf_text_poke_event){
9387 		.old_bytes    = old_bytes,
9388 		.new_bytes    = new_bytes,
9389 		.pad          = pad,
9390 		.old_len      = old_len,
9391 		.new_len      = new_len,
9392 		.event_id  = {
9393 			.header = {
9394 				.type = PERF_RECORD_TEXT_POKE,
9395 				.misc = PERF_RECORD_MISC_KERNEL,
9396 				.size = sizeof(text_poke_event.event_id) + tot + pad,
9397 			},
9398 			.addr = (unsigned long)addr,
9399 		},
9400 	};
9401 
9402 	perf_iterate_sb(perf_event_text_poke_output, &text_poke_event, NULL);
9403 }
9404 
perf_event_itrace_started(struct perf_event * event)9405 void perf_event_itrace_started(struct perf_event *event)
9406 {
9407 	event->attach_state |= PERF_ATTACH_ITRACE;
9408 }
9409 
perf_log_itrace_start(struct perf_event * event)9410 static void perf_log_itrace_start(struct perf_event *event)
9411 {
9412 	struct perf_output_handle handle;
9413 	struct perf_sample_data sample;
9414 	struct perf_aux_event {
9415 		struct perf_event_header        header;
9416 		u32				pid;
9417 		u32				tid;
9418 	} rec;
9419 	int ret;
9420 
9421 	if (event->parent)
9422 		event = event->parent;
9423 
9424 	if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) ||
9425 	    event->attach_state & PERF_ATTACH_ITRACE)
9426 		return;
9427 
9428 	rec.header.type	= PERF_RECORD_ITRACE_START;
9429 	rec.header.misc	= 0;
9430 	rec.header.size	= sizeof(rec);
9431 	rec.pid	= perf_event_pid(event, current);
9432 	rec.tid	= perf_event_tid(event, current);
9433 
9434 	perf_event_header__init_id(&rec.header, &sample, event);
9435 	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
9436 
9437 	if (ret)
9438 		return;
9439 
9440 	perf_output_put(&handle, rec);
9441 	perf_event__output_id_sample(event, &handle, &sample);
9442 
9443 	perf_output_end(&handle);
9444 }
9445 
perf_report_aux_output_id(struct perf_event * event,u64 hw_id)9446 void perf_report_aux_output_id(struct perf_event *event, u64 hw_id)
9447 {
9448 	struct perf_output_handle handle;
9449 	struct perf_sample_data sample;
9450 	struct perf_aux_event {
9451 		struct perf_event_header        header;
9452 		u64				hw_id;
9453 	} rec;
9454 	int ret;
9455 
9456 	if (event->parent)
9457 		event = event->parent;
9458 
9459 	rec.header.type	= PERF_RECORD_AUX_OUTPUT_HW_ID;
9460 	rec.header.misc	= 0;
9461 	rec.header.size	= sizeof(rec);
9462 	rec.hw_id	= hw_id;
9463 
9464 	perf_event_header__init_id(&rec.header, &sample, event);
9465 	ret = perf_output_begin(&handle, &sample, event, rec.header.size);
9466 
9467 	if (ret)
9468 		return;
9469 
9470 	perf_output_put(&handle, rec);
9471 	perf_event__output_id_sample(event, &handle, &sample);
9472 
9473 	perf_output_end(&handle);
9474 }
9475 EXPORT_SYMBOL_GPL(perf_report_aux_output_id);
9476 
9477 static int
__perf_event_account_interrupt(struct perf_event * event,int throttle)9478 __perf_event_account_interrupt(struct perf_event *event, int throttle)
9479 {
9480 	struct hw_perf_event *hwc = &event->hw;
9481 	int ret = 0;
9482 	u64 seq;
9483 
9484 	seq = __this_cpu_read(perf_throttled_seq);
9485 	if (seq != hwc->interrupts_seq) {
9486 		hwc->interrupts_seq = seq;
9487 		hwc->interrupts = 1;
9488 	} else {
9489 		hwc->interrupts++;
9490 		if (unlikely(throttle &&
9491 			     hwc->interrupts > max_samples_per_tick)) {
9492 			__this_cpu_inc(perf_throttled_count);
9493 			tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
9494 			hwc->interrupts = MAX_INTERRUPTS;
9495 			perf_log_throttle(event, 0);
9496 			ret = 1;
9497 		}
9498 	}
9499 
9500 	if (event->attr.freq) {
9501 		u64 now = perf_clock();
9502 		s64 delta = now - hwc->freq_time_stamp;
9503 
9504 		hwc->freq_time_stamp = now;
9505 
9506 		if (delta > 0 && delta < 2*TICK_NSEC)
9507 			perf_adjust_period(event, delta, hwc->last_period, true);
9508 	}
9509 
9510 	return ret;
9511 }
9512 
perf_event_account_interrupt(struct perf_event * event)9513 int perf_event_account_interrupt(struct perf_event *event)
9514 {
9515 	return __perf_event_account_interrupt(event, 1);
9516 }
9517 
sample_is_allowed(struct perf_event * event,struct pt_regs * regs)9518 static inline bool sample_is_allowed(struct perf_event *event, struct pt_regs *regs)
9519 {
9520 	/*
9521 	 * Due to interrupt latency (AKA "skid"), we may enter the
9522 	 * kernel before taking an overflow, even if the PMU is only
9523 	 * counting user events.
9524 	 */
9525 	if (event->attr.exclude_kernel && !user_mode(regs))
9526 		return false;
9527 
9528 	return true;
9529 }
9530 
9531 /*
9532  * Generic event overflow handling, sampling.
9533  */
9534 
__perf_event_overflow(struct perf_event * event,int throttle,struct perf_sample_data * data,struct pt_regs * regs)9535 static int __perf_event_overflow(struct perf_event *event,
9536 				 int throttle, struct perf_sample_data *data,
9537 				 struct pt_regs *regs)
9538 {
9539 	int events = atomic_read(&event->event_limit);
9540 	int ret = 0;
9541 
9542 	/*
9543 	 * Non-sampling counters might still use the PMI to fold short
9544 	 * hardware counters, ignore those.
9545 	 */
9546 	if (unlikely(!is_sampling_event(event)))
9547 		return 0;
9548 
9549 	ret = __perf_event_account_interrupt(event, throttle);
9550 
9551 	/*
9552 	 * XXX event_limit might not quite work as expected on inherited
9553 	 * events
9554 	 */
9555 
9556 	event->pending_kill = POLL_IN;
9557 	if (events && atomic_dec_and_test(&event->event_limit)) {
9558 		ret = 1;
9559 		event->pending_kill = POLL_HUP;
9560 		perf_event_disable_inatomic(event);
9561 	}
9562 
9563 	if (event->attr.sigtrap) {
9564 		/*
9565 		 * The desired behaviour of sigtrap vs invalid samples is a bit
9566 		 * tricky; on the one hand, one should not loose the SIGTRAP if
9567 		 * it is the first event, on the other hand, we should also not
9568 		 * trigger the WARN or override the data address.
9569 		 */
9570 		bool valid_sample = sample_is_allowed(event, regs);
9571 		unsigned int pending_id = 1;
9572 
9573 		if (regs)
9574 			pending_id = hash32_ptr((void *)instruction_pointer(regs)) ?: 1;
9575 		if (!event->pending_sigtrap) {
9576 			event->pending_sigtrap = pending_id;
9577 			local_inc(&event->ctx->nr_pending);
9578 		} else if (event->attr.exclude_kernel && valid_sample) {
9579 			/*
9580 			 * Should not be able to return to user space without
9581 			 * consuming pending_sigtrap; with exceptions:
9582 			 *
9583 			 *  1. Where !exclude_kernel, events can overflow again
9584 			 *     in the kernel without returning to user space.
9585 			 *
9586 			 *  2. Events that can overflow again before the IRQ-
9587 			 *     work without user space progress (e.g. hrtimer).
9588 			 *     To approximate progress (with false negatives),
9589 			 *     check 32-bit hash of the current IP.
9590 			 */
9591 			WARN_ON_ONCE(event->pending_sigtrap != pending_id);
9592 		}
9593 
9594 		event->pending_addr = 0;
9595 		if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR))
9596 			event->pending_addr = data->addr;
9597 		irq_work_queue(&event->pending_irq);
9598 	}
9599 
9600 	READ_ONCE(event->overflow_handler)(event, data, regs);
9601 
9602 	if (*perf_event_fasync(event) && event->pending_kill) {
9603 		event->pending_wakeup = 1;
9604 		irq_work_queue(&event->pending_irq);
9605 	}
9606 
9607 	return ret;
9608 }
9609 
perf_event_overflow(struct perf_event * event,struct perf_sample_data * data,struct pt_regs * regs)9610 int perf_event_overflow(struct perf_event *event,
9611 			struct perf_sample_data *data,
9612 			struct pt_regs *regs)
9613 {
9614 	return __perf_event_overflow(event, 1, data, regs);
9615 }
9616 
9617 /*
9618  * Generic software event infrastructure
9619  */
9620 
9621 struct swevent_htable {
9622 	struct swevent_hlist		*swevent_hlist;
9623 	struct mutex			hlist_mutex;
9624 	int				hlist_refcount;
9625 
9626 	/* Recursion avoidance in each contexts */
9627 	int				recursion[PERF_NR_CONTEXTS];
9628 };
9629 
9630 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
9631 
9632 /*
9633  * We directly increment event->count and keep a second value in
9634  * event->hw.period_left to count intervals. This period event
9635  * is kept in the range [-sample_period, 0] so that we can use the
9636  * sign as trigger.
9637  */
9638 
perf_swevent_set_period(struct perf_event * event)9639 u64 perf_swevent_set_period(struct perf_event *event)
9640 {
9641 	struct hw_perf_event *hwc = &event->hw;
9642 	u64 period = hwc->last_period;
9643 	u64 nr, offset;
9644 	s64 old, val;
9645 
9646 	hwc->last_period = hwc->sample_period;
9647 
9648 	old = local64_read(&hwc->period_left);
9649 	do {
9650 		val = old;
9651 		if (val < 0)
9652 			return 0;
9653 
9654 		nr = div64_u64(period + val, period);
9655 		offset = nr * period;
9656 		val -= offset;
9657 	} while (!local64_try_cmpxchg(&hwc->period_left, &old, val));
9658 
9659 	return nr;
9660 }
9661 
perf_swevent_overflow(struct perf_event * event,u64 overflow,struct perf_sample_data * data,struct pt_regs * regs)9662 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
9663 				    struct perf_sample_data *data,
9664 				    struct pt_regs *regs)
9665 {
9666 	struct hw_perf_event *hwc = &event->hw;
9667 	int throttle = 0;
9668 
9669 	if (!overflow)
9670 		overflow = perf_swevent_set_period(event);
9671 
9672 	if (hwc->interrupts == MAX_INTERRUPTS)
9673 		return;
9674 
9675 	for (; overflow; overflow--) {
9676 		if (__perf_event_overflow(event, throttle,
9677 					    data, regs)) {
9678 			/*
9679 			 * We inhibit the overflow from happening when
9680 			 * hwc->interrupts == MAX_INTERRUPTS.
9681 			 */
9682 			break;
9683 		}
9684 		throttle = 1;
9685 	}
9686 }
9687 
perf_swevent_event(struct perf_event * event,u64 nr,struct perf_sample_data * data,struct pt_regs * regs)9688 static void perf_swevent_event(struct perf_event *event, u64 nr,
9689 			       struct perf_sample_data *data,
9690 			       struct pt_regs *regs)
9691 {
9692 	struct hw_perf_event *hwc = &event->hw;
9693 
9694 	local64_add(nr, &event->count);
9695 
9696 	if (!regs)
9697 		return;
9698 
9699 	if (!is_sampling_event(event))
9700 		return;
9701 
9702 	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
9703 		data->period = nr;
9704 		return perf_swevent_overflow(event, 1, data, regs);
9705 	} else
9706 		data->period = event->hw.last_period;
9707 
9708 	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
9709 		return perf_swevent_overflow(event, 1, data, regs);
9710 
9711 	if (local64_add_negative(nr, &hwc->period_left))
9712 		return;
9713 
9714 	perf_swevent_overflow(event, 0, data, regs);
9715 }
9716 
perf_exclude_event(struct perf_event * event,struct pt_regs * regs)9717 static int perf_exclude_event(struct perf_event *event,
9718 			      struct pt_regs *regs)
9719 {
9720 	if (event->hw.state & PERF_HES_STOPPED)
9721 		return 1;
9722 
9723 	if (regs) {
9724 		if (event->attr.exclude_user && user_mode(regs))
9725 			return 1;
9726 
9727 		if (event->attr.exclude_kernel && !user_mode(regs))
9728 			return 1;
9729 	}
9730 
9731 	return 0;
9732 }
9733 
perf_swevent_match(struct perf_event * event,enum perf_type_id type,u32 event_id,struct perf_sample_data * data,struct pt_regs * regs)9734 static int perf_swevent_match(struct perf_event *event,
9735 				enum perf_type_id type,
9736 				u32 event_id,
9737 				struct perf_sample_data *data,
9738 				struct pt_regs *regs)
9739 {
9740 	if (event->attr.type != type)
9741 		return 0;
9742 
9743 	if (event->attr.config != event_id)
9744 		return 0;
9745 
9746 	if (perf_exclude_event(event, regs))
9747 		return 0;
9748 
9749 	return 1;
9750 }
9751 
swevent_hash(u64 type,u32 event_id)9752 static inline u64 swevent_hash(u64 type, u32 event_id)
9753 {
9754 	u64 val = event_id | (type << 32);
9755 
9756 	return hash_64(val, SWEVENT_HLIST_BITS);
9757 }
9758 
9759 static inline struct hlist_head *
__find_swevent_head(struct swevent_hlist * hlist,u64 type,u32 event_id)9760 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
9761 {
9762 	u64 hash = swevent_hash(type, event_id);
9763 
9764 	return &hlist->heads[hash];
9765 }
9766 
9767 /* For the read side: events when they trigger */
9768 static inline struct hlist_head *
find_swevent_head_rcu(struct swevent_htable * swhash,u64 type,u32 event_id)9769 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
9770 {
9771 	struct swevent_hlist *hlist;
9772 
9773 	hlist = rcu_dereference(swhash->swevent_hlist);
9774 	if (!hlist)
9775 		return NULL;
9776 
9777 	return __find_swevent_head(hlist, type, event_id);
9778 }
9779 
9780 /* For the event head insertion and removal in the hlist */
9781 static inline struct hlist_head *
find_swevent_head(struct swevent_htable * swhash,struct perf_event * event)9782 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
9783 {
9784 	struct swevent_hlist *hlist;
9785 	u32 event_id = event->attr.config;
9786 	u64 type = event->attr.type;
9787 
9788 	/*
9789 	 * Event scheduling is always serialized against hlist allocation
9790 	 * and release. Which makes the protected version suitable here.
9791 	 * The context lock guarantees that.
9792 	 */
9793 	hlist = rcu_dereference_protected(swhash->swevent_hlist,
9794 					  lockdep_is_held(&event->ctx->lock));
9795 	if (!hlist)
9796 		return NULL;
9797 
9798 	return __find_swevent_head(hlist, type, event_id);
9799 }
9800 
do_perf_sw_event(enum perf_type_id type,u32 event_id,u64 nr,struct perf_sample_data * data,struct pt_regs * regs)9801 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
9802 				    u64 nr,
9803 				    struct perf_sample_data *data,
9804 				    struct pt_regs *regs)
9805 {
9806 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
9807 	struct perf_event *event;
9808 	struct hlist_head *head;
9809 
9810 	rcu_read_lock();
9811 	head = find_swevent_head_rcu(swhash, type, event_id);
9812 	if (!head)
9813 		goto end;
9814 
9815 	hlist_for_each_entry_rcu(event, head, hlist_entry) {
9816 		if (perf_swevent_match(event, type, event_id, data, regs))
9817 			perf_swevent_event(event, nr, data, regs);
9818 	}
9819 end:
9820 	rcu_read_unlock();
9821 }
9822 
9823 DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]);
9824 
perf_swevent_get_recursion_context(void)9825 int perf_swevent_get_recursion_context(void)
9826 {
9827 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
9828 
9829 	return get_recursion_context(swhash->recursion);
9830 }
9831 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
9832 
perf_swevent_put_recursion_context(int rctx)9833 void perf_swevent_put_recursion_context(int rctx)
9834 {
9835 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
9836 
9837 	put_recursion_context(swhash->recursion, rctx);
9838 }
9839 
___perf_sw_event(u32 event_id,u64 nr,struct pt_regs * regs,u64 addr)9840 void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
9841 {
9842 	struct perf_sample_data data;
9843 
9844 	if (WARN_ON_ONCE(!regs))
9845 		return;
9846 
9847 	perf_sample_data_init(&data, addr, 0);
9848 	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
9849 }
9850 
__perf_sw_event(u32 event_id,u64 nr,struct pt_regs * regs,u64 addr)9851 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
9852 {
9853 	int rctx;
9854 
9855 	preempt_disable_notrace();
9856 	rctx = perf_swevent_get_recursion_context();
9857 	if (unlikely(rctx < 0))
9858 		goto fail;
9859 
9860 	___perf_sw_event(event_id, nr, regs, addr);
9861 
9862 	perf_swevent_put_recursion_context(rctx);
9863 fail:
9864 	preempt_enable_notrace();
9865 }
9866 
perf_swevent_read(struct perf_event * event)9867 static void perf_swevent_read(struct perf_event *event)
9868 {
9869 }
9870 
perf_swevent_add(struct perf_event * event,int flags)9871 static int perf_swevent_add(struct perf_event *event, int flags)
9872 {
9873 	struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
9874 	struct hw_perf_event *hwc = &event->hw;
9875 	struct hlist_head *head;
9876 
9877 	if (is_sampling_event(event)) {
9878 		hwc->last_period = hwc->sample_period;
9879 		perf_swevent_set_period(event);
9880 	}
9881 
9882 	hwc->state = !(flags & PERF_EF_START);
9883 
9884 	head = find_swevent_head(swhash, event);
9885 	if (WARN_ON_ONCE(!head))
9886 		return -EINVAL;
9887 
9888 	hlist_add_head_rcu(&event->hlist_entry, head);
9889 	perf_event_update_userpage(event);
9890 
9891 	return 0;
9892 }
9893 
perf_swevent_del(struct perf_event * event,int flags)9894 static void perf_swevent_del(struct perf_event *event, int flags)
9895 {
9896 	hlist_del_rcu(&event->hlist_entry);
9897 }
9898 
perf_swevent_start(struct perf_event * event,int flags)9899 static void perf_swevent_start(struct perf_event *event, int flags)
9900 {
9901 	event->hw.state = 0;
9902 }
9903 
perf_swevent_stop(struct perf_event * event,int flags)9904 static void perf_swevent_stop(struct perf_event *event, int flags)
9905 {
9906 	event->hw.state = PERF_HES_STOPPED;
9907 }
9908 
9909 /* Deref the hlist from the update side */
9910 static inline struct swevent_hlist *
swevent_hlist_deref(struct swevent_htable * swhash)9911 swevent_hlist_deref(struct swevent_htable *swhash)
9912 {
9913 	return rcu_dereference_protected(swhash->swevent_hlist,
9914 					 lockdep_is_held(&swhash->hlist_mutex));
9915 }
9916 
swevent_hlist_release(struct swevent_htable * swhash)9917 static void swevent_hlist_release(struct swevent_htable *swhash)
9918 {
9919 	struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
9920 
9921 	if (!hlist)
9922 		return;
9923 
9924 	RCU_INIT_POINTER(swhash->swevent_hlist, NULL);
9925 	kfree_rcu(hlist, rcu_head);
9926 }
9927 
swevent_hlist_put_cpu(int cpu)9928 static void swevent_hlist_put_cpu(int cpu)
9929 {
9930 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
9931 
9932 	mutex_lock(&swhash->hlist_mutex);
9933 
9934 	if (!--swhash->hlist_refcount)
9935 		swevent_hlist_release(swhash);
9936 
9937 	mutex_unlock(&swhash->hlist_mutex);
9938 }
9939 
swevent_hlist_put(void)9940 static void swevent_hlist_put(void)
9941 {
9942 	int cpu;
9943 
9944 	for_each_possible_cpu(cpu)
9945 		swevent_hlist_put_cpu(cpu);
9946 }
9947 
swevent_hlist_get_cpu(int cpu)9948 static int swevent_hlist_get_cpu(int cpu)
9949 {
9950 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
9951 	int err = 0;
9952 
9953 	mutex_lock(&swhash->hlist_mutex);
9954 	if (!swevent_hlist_deref(swhash) &&
9955 	    cpumask_test_cpu(cpu, perf_online_mask)) {
9956 		struct swevent_hlist *hlist;
9957 
9958 		hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
9959 		if (!hlist) {
9960 			err = -ENOMEM;
9961 			goto exit;
9962 		}
9963 		rcu_assign_pointer(swhash->swevent_hlist, hlist);
9964 	}
9965 	swhash->hlist_refcount++;
9966 exit:
9967 	mutex_unlock(&swhash->hlist_mutex);
9968 
9969 	return err;
9970 }
9971 
swevent_hlist_get(void)9972 static int swevent_hlist_get(void)
9973 {
9974 	int err, cpu, failed_cpu;
9975 
9976 	mutex_lock(&pmus_lock);
9977 	for_each_possible_cpu(cpu) {
9978 		err = swevent_hlist_get_cpu(cpu);
9979 		if (err) {
9980 			failed_cpu = cpu;
9981 			goto fail;
9982 		}
9983 	}
9984 	mutex_unlock(&pmus_lock);
9985 	return 0;
9986 fail:
9987 	for_each_possible_cpu(cpu) {
9988 		if (cpu == failed_cpu)
9989 			break;
9990 		swevent_hlist_put_cpu(cpu);
9991 	}
9992 	mutex_unlock(&pmus_lock);
9993 	return err;
9994 }
9995 
9996 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
9997 
sw_perf_event_destroy(struct perf_event * event)9998 static void sw_perf_event_destroy(struct perf_event *event)
9999 {
10000 	u64 event_id = event->attr.config;
10001 
10002 	WARN_ON(event->parent);
10003 
10004 	static_key_slow_dec(&perf_swevent_enabled[event_id]);
10005 	swevent_hlist_put();
10006 }
10007 
10008 static struct pmu perf_cpu_clock; /* fwd declaration */
10009 static struct pmu perf_task_clock;
10010 
perf_swevent_init(struct perf_event * event)10011 static int perf_swevent_init(struct perf_event *event)
10012 {
10013 	u64 event_id = event->attr.config;
10014 
10015 	if (event->attr.type != PERF_TYPE_SOFTWARE)
10016 		return -ENOENT;
10017 
10018 	/*
10019 	 * no branch sampling for software events
10020 	 */
10021 	if (has_branch_stack(event))
10022 		return -EOPNOTSUPP;
10023 
10024 	switch (event_id) {
10025 	case PERF_COUNT_SW_CPU_CLOCK:
10026 		event->attr.type = perf_cpu_clock.type;
10027 		return -ENOENT;
10028 	case PERF_COUNT_SW_TASK_CLOCK:
10029 		event->attr.type = perf_task_clock.type;
10030 		return -ENOENT;
10031 
10032 	default:
10033 		break;
10034 	}
10035 
10036 	if (event_id >= PERF_COUNT_SW_MAX)
10037 		return -ENOENT;
10038 
10039 	if (!event->parent) {
10040 		int err;
10041 
10042 		err = swevent_hlist_get();
10043 		if (err)
10044 			return err;
10045 
10046 		static_key_slow_inc(&perf_swevent_enabled[event_id]);
10047 		event->destroy = sw_perf_event_destroy;
10048 	}
10049 
10050 	return 0;
10051 }
10052 
10053 static struct pmu perf_swevent = {
10054 	.task_ctx_nr	= perf_sw_context,
10055 
10056 	.capabilities	= PERF_PMU_CAP_NO_NMI,
10057 
10058 	.event_init	= perf_swevent_init,
10059 	.add		= perf_swevent_add,
10060 	.del		= perf_swevent_del,
10061 	.start		= perf_swevent_start,
10062 	.stop		= perf_swevent_stop,
10063 	.read		= perf_swevent_read,
10064 };
10065 
10066 #ifdef CONFIG_EVENT_TRACING
10067 
tp_perf_event_destroy(struct perf_event * event)10068 static void tp_perf_event_destroy(struct perf_event *event)
10069 {
10070 	perf_trace_destroy(event);
10071 }
10072 
perf_tp_event_init(struct perf_event * event)10073 static int perf_tp_event_init(struct perf_event *event)
10074 {
10075 	int err;
10076 
10077 	if (event->attr.type != PERF_TYPE_TRACEPOINT)
10078 		return -ENOENT;
10079 
10080 	/*
10081 	 * no branch sampling for tracepoint events
10082 	 */
10083 	if (has_branch_stack(event))
10084 		return -EOPNOTSUPP;
10085 
10086 	err = perf_trace_init(event);
10087 	if (err)
10088 		return err;
10089 
10090 	event->destroy = tp_perf_event_destroy;
10091 
10092 	return 0;
10093 }
10094 
10095 static struct pmu perf_tracepoint = {
10096 	.task_ctx_nr	= perf_sw_context,
10097 
10098 	.event_init	= perf_tp_event_init,
10099 	.add		= perf_trace_add,
10100 	.del		= perf_trace_del,
10101 	.start		= perf_swevent_start,
10102 	.stop		= perf_swevent_stop,
10103 	.read		= perf_swevent_read,
10104 };
10105 
perf_tp_filter_match(struct perf_event * event,struct perf_sample_data * data)10106 static int perf_tp_filter_match(struct perf_event *event,
10107 				struct perf_sample_data *data)
10108 {
10109 	void *record = data->raw->frag.data;
10110 
10111 	/* only top level events have filters set */
10112 	if (event->parent)
10113 		event = event->parent;
10114 
10115 	if (likely(!event->filter) || filter_match_preds(event->filter, record))
10116 		return 1;
10117 	return 0;
10118 }
10119 
perf_tp_event_match(struct perf_event * event,struct perf_sample_data * data,struct pt_regs * regs)10120 static int perf_tp_event_match(struct perf_event *event,
10121 				struct perf_sample_data *data,
10122 				struct pt_regs *regs)
10123 {
10124 	if (event->hw.state & PERF_HES_STOPPED)
10125 		return 0;
10126 	/*
10127 	 * If exclude_kernel, only trace user-space tracepoints (uprobes)
10128 	 */
10129 	if (event->attr.exclude_kernel && !user_mode(regs))
10130 		return 0;
10131 
10132 	if (!perf_tp_filter_match(event, data))
10133 		return 0;
10134 
10135 	return 1;
10136 }
10137 
perf_trace_run_bpf_submit(void * raw_data,int size,int rctx,struct trace_event_call * call,u64 count,struct pt_regs * regs,struct hlist_head * head,struct task_struct * task)10138 void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
10139 			       struct trace_event_call *call, u64 count,
10140 			       struct pt_regs *regs, struct hlist_head *head,
10141 			       struct task_struct *task)
10142 {
10143 	if (bpf_prog_array_valid(call)) {
10144 		*(struct pt_regs **)raw_data = regs;
10145 		if (!trace_call_bpf(call, raw_data) || hlist_empty(head)) {
10146 			perf_swevent_put_recursion_context(rctx);
10147 			return;
10148 		}
10149 	}
10150 	perf_tp_event(call->event.type, count, raw_data, size, regs, head,
10151 		      rctx, task);
10152 }
10153 EXPORT_SYMBOL_GPL(perf_trace_run_bpf_submit);
10154 
__perf_tp_event_target_task(u64 count,void * record,struct pt_regs * regs,struct perf_sample_data * data,struct perf_event * event)10155 static void __perf_tp_event_target_task(u64 count, void *record,
10156 					struct pt_regs *regs,
10157 					struct perf_sample_data *data,
10158 					struct perf_event *event)
10159 {
10160 	struct trace_entry *entry = record;
10161 
10162 	if (event->attr.config != entry->type)
10163 		return;
10164 	/* Cannot deliver synchronous signal to other task. */
10165 	if (event->attr.sigtrap)
10166 		return;
10167 	if (perf_tp_event_match(event, data, regs))
10168 		perf_swevent_event(event, count, data, regs);
10169 }
10170 
perf_tp_event_target_task(u64 count,void * record,struct pt_regs * regs,struct perf_sample_data * data,struct perf_event_context * ctx)10171 static void perf_tp_event_target_task(u64 count, void *record,
10172 				      struct pt_regs *regs,
10173 				      struct perf_sample_data *data,
10174 				      struct perf_event_context *ctx)
10175 {
10176 	unsigned int cpu = smp_processor_id();
10177 	struct pmu *pmu = &perf_tracepoint;
10178 	struct perf_event *event, *sibling;
10179 
10180 	perf_event_groups_for_cpu_pmu(event, &ctx->pinned_groups, cpu, pmu) {
10181 		__perf_tp_event_target_task(count, record, regs, data, event);
10182 		for_each_sibling_event(sibling, event)
10183 			__perf_tp_event_target_task(count, record, regs, data, sibling);
10184 	}
10185 
10186 	perf_event_groups_for_cpu_pmu(event, &ctx->flexible_groups, cpu, pmu) {
10187 		__perf_tp_event_target_task(count, record, regs, data, event);
10188 		for_each_sibling_event(sibling, event)
10189 			__perf_tp_event_target_task(count, record, regs, data, sibling);
10190 	}
10191 }
10192 
perf_tp_event(u16 event_type,u64 count,void * record,int entry_size,struct pt_regs * regs,struct hlist_head * head,int rctx,struct task_struct * task)10193 void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
10194 		   struct pt_regs *regs, struct hlist_head *head, int rctx,
10195 		   struct task_struct *task)
10196 {
10197 	struct perf_sample_data data;
10198 	struct perf_event *event;
10199 
10200 	struct perf_raw_record raw = {
10201 		.frag = {
10202 			.size = entry_size,
10203 			.data = record,
10204 		},
10205 	};
10206 
10207 	perf_sample_data_init(&data, 0, 0);
10208 	perf_sample_save_raw_data(&data, &raw);
10209 
10210 	perf_trace_buf_update(record, event_type);
10211 
10212 	hlist_for_each_entry_rcu(event, head, hlist_entry) {
10213 		if (perf_tp_event_match(event, &data, regs)) {
10214 			perf_swevent_event(event, count, &data, regs);
10215 
10216 			/*
10217 			 * Here use the same on-stack perf_sample_data,
10218 			 * some members in data are event-specific and
10219 			 * need to be re-computed for different sweveents.
10220 			 * Re-initialize data->sample_flags safely to avoid
10221 			 * the problem that next event skips preparing data
10222 			 * because data->sample_flags is set.
10223 			 */
10224 			perf_sample_data_init(&data, 0, 0);
10225 			perf_sample_save_raw_data(&data, &raw);
10226 		}
10227 	}
10228 
10229 	/*
10230 	 * If we got specified a target task, also iterate its context and
10231 	 * deliver this event there too.
10232 	 */
10233 	if (task && task != current) {
10234 		struct perf_event_context *ctx;
10235 
10236 		rcu_read_lock();
10237 		ctx = rcu_dereference(task->perf_event_ctxp);
10238 		if (!ctx)
10239 			goto unlock;
10240 
10241 		raw_spin_lock(&ctx->lock);
10242 		perf_tp_event_target_task(count, record, regs, &data, ctx);
10243 		raw_spin_unlock(&ctx->lock);
10244 unlock:
10245 		rcu_read_unlock();
10246 	}
10247 
10248 	perf_swevent_put_recursion_context(rctx);
10249 }
10250 EXPORT_SYMBOL_GPL(perf_tp_event);
10251 
10252 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
10253 /*
10254  * Flags in config, used by dynamic PMU kprobe and uprobe
10255  * The flags should match following PMU_FORMAT_ATTR().
10256  *
10257  * PERF_PROBE_CONFIG_IS_RETPROBE if set, create kretprobe/uretprobe
10258  *                               if not set, create kprobe/uprobe
10259  *
10260  * The following values specify a reference counter (or semaphore in the
10261  * terminology of tools like dtrace, systemtap, etc.) Userspace Statically
10262  * Defined Tracepoints (USDT). Currently, we use 40 bit for the offset.
10263  *
10264  * PERF_UPROBE_REF_CTR_OFFSET_BITS	# of bits in config as th offset
10265  * PERF_UPROBE_REF_CTR_OFFSET_SHIFT	# of bits to shift left
10266  */
10267 enum perf_probe_config {
10268 	PERF_PROBE_CONFIG_IS_RETPROBE = 1U << 0,  /* [k,u]retprobe */
10269 	PERF_UPROBE_REF_CTR_OFFSET_BITS = 32,
10270 	PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
10271 };
10272 
10273 PMU_FORMAT_ATTR(retprobe, "config:0");
10274 #endif
10275 
10276 #ifdef CONFIG_KPROBE_EVENTS
10277 static struct attribute *kprobe_attrs[] = {
10278 	&format_attr_retprobe.attr,
10279 	NULL,
10280 };
10281 
10282 static struct attribute_group kprobe_format_group = {
10283 	.name = "format",
10284 	.attrs = kprobe_attrs,
10285 };
10286 
10287 static const struct attribute_group *kprobe_attr_groups[] = {
10288 	&kprobe_format_group,
10289 	NULL,
10290 };
10291 
10292 static int perf_kprobe_event_init(struct perf_event *event);
10293 static struct pmu perf_kprobe = {
10294 	.task_ctx_nr	= perf_sw_context,
10295 	.event_init	= perf_kprobe_event_init,
10296 	.add		= perf_trace_add,
10297 	.del		= perf_trace_del,
10298 	.start		= perf_swevent_start,
10299 	.stop		= perf_swevent_stop,
10300 	.read		= perf_swevent_read,
10301 	.attr_groups	= kprobe_attr_groups,
10302 };
10303 
perf_kprobe_event_init(struct perf_event * event)10304 static int perf_kprobe_event_init(struct perf_event *event)
10305 {
10306 	int err;
10307 	bool is_retprobe;
10308 
10309 	if (event->attr.type != perf_kprobe.type)
10310 		return -ENOENT;
10311 
10312 	if (!perfmon_capable())
10313 		return -EACCES;
10314 
10315 	/*
10316 	 * no branch sampling for probe events
10317 	 */
10318 	if (has_branch_stack(event))
10319 		return -EOPNOTSUPP;
10320 
10321 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
10322 	err = perf_kprobe_init(event, is_retprobe);
10323 	if (err)
10324 		return err;
10325 
10326 	event->destroy = perf_kprobe_destroy;
10327 
10328 	return 0;
10329 }
10330 #endif /* CONFIG_KPROBE_EVENTS */
10331 
10332 #ifdef CONFIG_UPROBE_EVENTS
10333 PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
10334 
10335 static struct attribute *uprobe_attrs[] = {
10336 	&format_attr_retprobe.attr,
10337 	&format_attr_ref_ctr_offset.attr,
10338 	NULL,
10339 };
10340 
10341 static struct attribute_group uprobe_format_group = {
10342 	.name = "format",
10343 	.attrs = uprobe_attrs,
10344 };
10345 
10346 static const struct attribute_group *uprobe_attr_groups[] = {
10347 	&uprobe_format_group,
10348 	NULL,
10349 };
10350 
10351 static int perf_uprobe_event_init(struct perf_event *event);
10352 static struct pmu perf_uprobe = {
10353 	.task_ctx_nr	= perf_sw_context,
10354 	.event_init	= perf_uprobe_event_init,
10355 	.add		= perf_trace_add,
10356 	.del		= perf_trace_del,
10357 	.start		= perf_swevent_start,
10358 	.stop		= perf_swevent_stop,
10359 	.read		= perf_swevent_read,
10360 	.attr_groups	= uprobe_attr_groups,
10361 };
10362 
perf_uprobe_event_init(struct perf_event * event)10363 static int perf_uprobe_event_init(struct perf_event *event)
10364 {
10365 	int err;
10366 	unsigned long ref_ctr_offset;
10367 	bool is_retprobe;
10368 
10369 	if (event->attr.type != perf_uprobe.type)
10370 		return -ENOENT;
10371 
10372 	if (!perfmon_capable())
10373 		return -EACCES;
10374 
10375 	/*
10376 	 * no branch sampling for probe events
10377 	 */
10378 	if (has_branch_stack(event))
10379 		return -EOPNOTSUPP;
10380 
10381 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
10382 	ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
10383 	err = perf_uprobe_init(event, ref_ctr_offset, is_retprobe);
10384 	if (err)
10385 		return err;
10386 
10387 	event->destroy = perf_uprobe_destroy;
10388 
10389 	return 0;
10390 }
10391 #endif /* CONFIG_UPROBE_EVENTS */
10392 
perf_tp_register(void)10393 static inline void perf_tp_register(void)
10394 {
10395 	perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
10396 #ifdef CONFIG_KPROBE_EVENTS
10397 	perf_pmu_register(&perf_kprobe, "kprobe", -1);
10398 #endif
10399 #ifdef CONFIG_UPROBE_EVENTS
10400 	perf_pmu_register(&perf_uprobe, "uprobe", -1);
10401 #endif
10402 }
10403 
perf_event_free_filter(struct perf_event * event)10404 static void perf_event_free_filter(struct perf_event *event)
10405 {
10406 	ftrace_profile_free_filter(event);
10407 }
10408 
10409 #ifdef CONFIG_BPF_SYSCALL
bpf_overflow_handler(struct perf_event * event,struct perf_sample_data * data,struct pt_regs * regs)10410 static void bpf_overflow_handler(struct perf_event *event,
10411 				 struct perf_sample_data *data,
10412 				 struct pt_regs *regs)
10413 {
10414 	struct bpf_perf_event_data_kern ctx = {
10415 		.data = data,
10416 		.event = event,
10417 	};
10418 	struct bpf_prog *prog;
10419 	int ret = 0;
10420 
10421 	ctx.regs = perf_arch_bpf_user_pt_regs(regs);
10422 	if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1))
10423 		goto out;
10424 	rcu_read_lock();
10425 	prog = READ_ONCE(event->prog);
10426 	if (prog) {
10427 		perf_prepare_sample(data, event, regs);
10428 		ret = bpf_prog_run(prog, &ctx);
10429 	}
10430 	rcu_read_unlock();
10431 out:
10432 	__this_cpu_dec(bpf_prog_active);
10433 	if (!ret)
10434 		return;
10435 
10436 	event->orig_overflow_handler(event, data, regs);
10437 }
10438 
perf_event_set_bpf_handler(struct perf_event * event,struct bpf_prog * prog,u64 bpf_cookie)10439 static int perf_event_set_bpf_handler(struct perf_event *event,
10440 				      struct bpf_prog *prog,
10441 				      u64 bpf_cookie)
10442 {
10443 	if (event->overflow_handler_context)
10444 		/* hw breakpoint or kernel counter */
10445 		return -EINVAL;
10446 
10447 	if (event->prog)
10448 		return -EEXIST;
10449 
10450 	if (prog->type != BPF_PROG_TYPE_PERF_EVENT)
10451 		return -EINVAL;
10452 
10453 	if (event->attr.precise_ip &&
10454 	    prog->call_get_stack &&
10455 	    (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) ||
10456 	     event->attr.exclude_callchain_kernel ||
10457 	     event->attr.exclude_callchain_user)) {
10458 		/*
10459 		 * On perf_event with precise_ip, calling bpf_get_stack()
10460 		 * may trigger unwinder warnings and occasional crashes.
10461 		 * bpf_get_[stack|stackid] works around this issue by using
10462 		 * callchain attached to perf_sample_data. If the
10463 		 * perf_event does not full (kernel and user) callchain
10464 		 * attached to perf_sample_data, do not allow attaching BPF
10465 		 * program that calls bpf_get_[stack|stackid].
10466 		 */
10467 		return -EPROTO;
10468 	}
10469 
10470 	event->prog = prog;
10471 	event->bpf_cookie = bpf_cookie;
10472 	event->orig_overflow_handler = READ_ONCE(event->overflow_handler);
10473 	WRITE_ONCE(event->overflow_handler, bpf_overflow_handler);
10474 	return 0;
10475 }
10476 
perf_event_free_bpf_handler(struct perf_event * event)10477 static void perf_event_free_bpf_handler(struct perf_event *event)
10478 {
10479 	struct bpf_prog *prog = event->prog;
10480 
10481 	if (!prog)
10482 		return;
10483 
10484 	WRITE_ONCE(event->overflow_handler, event->orig_overflow_handler);
10485 	event->prog = NULL;
10486 	bpf_prog_put(prog);
10487 }
10488 #else
perf_event_set_bpf_handler(struct perf_event * event,struct bpf_prog * prog,u64 bpf_cookie)10489 static int perf_event_set_bpf_handler(struct perf_event *event,
10490 				      struct bpf_prog *prog,
10491 				      u64 bpf_cookie)
10492 {
10493 	return -EOPNOTSUPP;
10494 }
perf_event_free_bpf_handler(struct perf_event * event)10495 static void perf_event_free_bpf_handler(struct perf_event *event)
10496 {
10497 }
10498 #endif
10499 
10500 /*
10501  * returns true if the event is a tracepoint, or a kprobe/upprobe created
10502  * with perf_event_open()
10503  */
perf_event_is_tracing(struct perf_event * event)10504 static inline bool perf_event_is_tracing(struct perf_event *event)
10505 {
10506 	if (event->pmu == &perf_tracepoint)
10507 		return true;
10508 #ifdef CONFIG_KPROBE_EVENTS
10509 	if (event->pmu == &perf_kprobe)
10510 		return true;
10511 #endif
10512 #ifdef CONFIG_UPROBE_EVENTS
10513 	if (event->pmu == &perf_uprobe)
10514 		return true;
10515 #endif
10516 	return false;
10517 }
10518 
perf_event_set_bpf_prog(struct perf_event * event,struct bpf_prog * prog,u64 bpf_cookie)10519 int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog,
10520 			    u64 bpf_cookie)
10521 {
10522 	bool is_kprobe, is_uprobe, is_tracepoint, is_syscall_tp;
10523 
10524 	if (!perf_event_is_tracing(event))
10525 		return perf_event_set_bpf_handler(event, prog, bpf_cookie);
10526 
10527 	is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE;
10528 	is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE;
10529 	is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;
10530 	is_syscall_tp = is_syscall_trace_event(event->tp_event);
10531 	if (!is_kprobe && !is_uprobe && !is_tracepoint && !is_syscall_tp)
10532 		/* bpf programs can only be attached to u/kprobe or tracepoint */
10533 		return -EINVAL;
10534 
10535 	if (((is_kprobe || is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) ||
10536 	    (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) ||
10537 	    (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT))
10538 		return -EINVAL;
10539 
10540 	if (prog->type == BPF_PROG_TYPE_KPROBE && prog->aux->sleepable && !is_uprobe)
10541 		/* only uprobe programs are allowed to be sleepable */
10542 		return -EINVAL;
10543 
10544 	/* Kprobe override only works for kprobes, not uprobes. */
10545 	if (prog->kprobe_override && !is_kprobe)
10546 		return -EINVAL;
10547 
10548 	if (is_tracepoint || is_syscall_tp) {
10549 		int off = trace_event_get_offsets(event->tp_event);
10550 
10551 		if (prog->aux->max_ctx_offset > off)
10552 			return -EACCES;
10553 	}
10554 
10555 	return perf_event_attach_bpf_prog(event, prog, bpf_cookie);
10556 }
10557 
perf_event_free_bpf_prog(struct perf_event * event)10558 void perf_event_free_bpf_prog(struct perf_event *event)
10559 {
10560 	if (!perf_event_is_tracing(event)) {
10561 		perf_event_free_bpf_handler(event);
10562 		return;
10563 	}
10564 	perf_event_detach_bpf_prog(event);
10565 }
10566 
10567 #else
10568 
perf_tp_register(void)10569 static inline void perf_tp_register(void)
10570 {
10571 }
10572 
perf_event_free_filter(struct perf_event * event)10573 static void perf_event_free_filter(struct perf_event *event)
10574 {
10575 }
10576 
perf_event_set_bpf_prog(struct perf_event * event,struct bpf_prog * prog,u64 bpf_cookie)10577 int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog,
10578 			    u64 bpf_cookie)
10579 {
10580 	return -ENOENT;
10581 }
10582 
perf_event_free_bpf_prog(struct perf_event * event)10583 void perf_event_free_bpf_prog(struct perf_event *event)
10584 {
10585 }
10586 #endif /* CONFIG_EVENT_TRACING */
10587 
10588 #ifdef CONFIG_HAVE_HW_BREAKPOINT
perf_bp_event(struct perf_event * bp,void * data)10589 void perf_bp_event(struct perf_event *bp, void *data)
10590 {
10591 	struct perf_sample_data sample;
10592 	struct pt_regs *regs = data;
10593 
10594 	perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
10595 
10596 	if (!bp->hw.state && !perf_exclude_event(bp, regs))
10597 		perf_swevent_event(bp, 1, &sample, regs);
10598 }
10599 #endif
10600 
10601 /*
10602  * Allocate a new address filter
10603  */
10604 static struct perf_addr_filter *
perf_addr_filter_new(struct perf_event * event,struct list_head * filters)10605 perf_addr_filter_new(struct perf_event *event, struct list_head *filters)
10606 {
10607 	int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu);
10608 	struct perf_addr_filter *filter;
10609 
10610 	filter = kzalloc_node(sizeof(*filter), GFP_KERNEL, node);
10611 	if (!filter)
10612 		return NULL;
10613 
10614 	INIT_LIST_HEAD(&filter->entry);
10615 	list_add_tail(&filter->entry, filters);
10616 
10617 	return filter;
10618 }
10619 
free_filters_list(struct list_head * filters)10620 static void free_filters_list(struct list_head *filters)
10621 {
10622 	struct perf_addr_filter *filter, *iter;
10623 
10624 	list_for_each_entry_safe(filter, iter, filters, entry) {
10625 		path_put(&filter->path);
10626 		list_del(&filter->entry);
10627 		kfree(filter);
10628 	}
10629 }
10630 
10631 /*
10632  * Free existing address filters and optionally install new ones
10633  */
perf_addr_filters_splice(struct perf_event * event,struct list_head * head)10634 static void perf_addr_filters_splice(struct perf_event *event,
10635 				     struct list_head *head)
10636 {
10637 	unsigned long flags;
10638 	LIST_HEAD(list);
10639 
10640 	if (!has_addr_filter(event))
10641 		return;
10642 
10643 	/* don't bother with children, they don't have their own filters */
10644 	if (event->parent)
10645 		return;
10646 
10647 	raw_spin_lock_irqsave(&event->addr_filters.lock, flags);
10648 
10649 	list_splice_init(&event->addr_filters.list, &list);
10650 	if (head)
10651 		list_splice(head, &event->addr_filters.list);
10652 
10653 	raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags);
10654 
10655 	free_filters_list(&list);
10656 }
10657 
10658 /*
10659  * Scan through mm's vmas and see if one of them matches the
10660  * @filter; if so, adjust filter's address range.
10661  * Called with mm::mmap_lock down for reading.
10662  */
perf_addr_filter_apply(struct perf_addr_filter * filter,struct mm_struct * mm,struct perf_addr_filter_range * fr)10663 static void perf_addr_filter_apply(struct perf_addr_filter *filter,
10664 				   struct mm_struct *mm,
10665 				   struct perf_addr_filter_range *fr)
10666 {
10667 	struct vm_area_struct *vma;
10668 	VMA_ITERATOR(vmi, mm, 0);
10669 
10670 	for_each_vma(vmi, vma) {
10671 		if (!vma->vm_file)
10672 			continue;
10673 
10674 		if (perf_addr_filter_vma_adjust(filter, vma, fr))
10675 			return;
10676 	}
10677 }
10678 
10679 /*
10680  * Update event's address range filters based on the
10681  * task's existing mappings, if any.
10682  */
perf_event_addr_filters_apply(struct perf_event * event)10683 static void perf_event_addr_filters_apply(struct perf_event *event)
10684 {
10685 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
10686 	struct task_struct *task = READ_ONCE(event->ctx->task);
10687 	struct perf_addr_filter *filter;
10688 	struct mm_struct *mm = NULL;
10689 	unsigned int count = 0;
10690 	unsigned long flags;
10691 
10692 	/*
10693 	 * We may observe TASK_TOMBSTONE, which means that the event tear-down
10694 	 * will stop on the parent's child_mutex that our caller is also holding
10695 	 */
10696 	if (task == TASK_TOMBSTONE)
10697 		return;
10698 
10699 	if (ifh->nr_file_filters) {
10700 		mm = get_task_mm(task);
10701 		if (!mm)
10702 			goto restart;
10703 
10704 		mmap_read_lock(mm);
10705 	}
10706 
10707 	raw_spin_lock_irqsave(&ifh->lock, flags);
10708 	list_for_each_entry(filter, &ifh->list, entry) {
10709 		if (filter->path.dentry) {
10710 			/*
10711 			 * Adjust base offset if the filter is associated to a
10712 			 * binary that needs to be mapped:
10713 			 */
10714 			event->addr_filter_ranges[count].start = 0;
10715 			event->addr_filter_ranges[count].size = 0;
10716 
10717 			perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]);
10718 		} else {
10719 			event->addr_filter_ranges[count].start = filter->offset;
10720 			event->addr_filter_ranges[count].size  = filter->size;
10721 		}
10722 
10723 		count++;
10724 	}
10725 
10726 	event->addr_filters_gen++;
10727 	raw_spin_unlock_irqrestore(&ifh->lock, flags);
10728 
10729 	if (ifh->nr_file_filters) {
10730 		mmap_read_unlock(mm);
10731 
10732 		mmput(mm);
10733 	}
10734 
10735 restart:
10736 	perf_event_stop(event, 1);
10737 }
10738 
10739 /*
10740  * Address range filtering: limiting the data to certain
10741  * instruction address ranges. Filters are ioctl()ed to us from
10742  * userspace as ascii strings.
10743  *
10744  * Filter string format:
10745  *
10746  * ACTION RANGE_SPEC
10747  * where ACTION is one of the
10748  *  * "filter": limit the trace to this region
10749  *  * "start": start tracing from this address
10750  *  * "stop": stop tracing at this address/region;
10751  * RANGE_SPEC is
10752  *  * for kernel addresses: <start address>[/<size>]
10753  *  * for object files:     <start address>[/<size>]@</path/to/object/file>
10754  *
10755  * if <size> is not specified or is zero, the range is treated as a single
10756  * address; not valid for ACTION=="filter".
10757  */
10758 enum {
10759 	IF_ACT_NONE = -1,
10760 	IF_ACT_FILTER,
10761 	IF_ACT_START,
10762 	IF_ACT_STOP,
10763 	IF_SRC_FILE,
10764 	IF_SRC_KERNEL,
10765 	IF_SRC_FILEADDR,
10766 	IF_SRC_KERNELADDR,
10767 };
10768 
10769 enum {
10770 	IF_STATE_ACTION = 0,
10771 	IF_STATE_SOURCE,
10772 	IF_STATE_END,
10773 };
10774 
10775 static const match_table_t if_tokens = {
10776 	{ IF_ACT_FILTER,	"filter" },
10777 	{ IF_ACT_START,		"start" },
10778 	{ IF_ACT_STOP,		"stop" },
10779 	{ IF_SRC_FILE,		"%u/%u@%s" },
10780 	{ IF_SRC_KERNEL,	"%u/%u" },
10781 	{ IF_SRC_FILEADDR,	"%u@%s" },
10782 	{ IF_SRC_KERNELADDR,	"%u" },
10783 	{ IF_ACT_NONE,		NULL },
10784 };
10785 
10786 /*
10787  * Address filter string parser
10788  */
10789 static int
perf_event_parse_addr_filter(struct perf_event * event,char * fstr,struct list_head * filters)10790 perf_event_parse_addr_filter(struct perf_event *event, char *fstr,
10791 			     struct list_head *filters)
10792 {
10793 	struct perf_addr_filter *filter = NULL;
10794 	char *start, *orig, *filename = NULL;
10795 	substring_t args[MAX_OPT_ARGS];
10796 	int state = IF_STATE_ACTION, token;
10797 	unsigned int kernel = 0;
10798 	int ret = -EINVAL;
10799 
10800 	orig = fstr = kstrdup(fstr, GFP_KERNEL);
10801 	if (!fstr)
10802 		return -ENOMEM;
10803 
10804 	while ((start = strsep(&fstr, " ,\n")) != NULL) {
10805 		static const enum perf_addr_filter_action_t actions[] = {
10806 			[IF_ACT_FILTER]	= PERF_ADDR_FILTER_ACTION_FILTER,
10807 			[IF_ACT_START]	= PERF_ADDR_FILTER_ACTION_START,
10808 			[IF_ACT_STOP]	= PERF_ADDR_FILTER_ACTION_STOP,
10809 		};
10810 		ret = -EINVAL;
10811 
10812 		if (!*start)
10813 			continue;
10814 
10815 		/* filter definition begins */
10816 		if (state == IF_STATE_ACTION) {
10817 			filter = perf_addr_filter_new(event, filters);
10818 			if (!filter)
10819 				goto fail;
10820 		}
10821 
10822 		token = match_token(start, if_tokens, args);
10823 		switch (token) {
10824 		case IF_ACT_FILTER:
10825 		case IF_ACT_START:
10826 		case IF_ACT_STOP:
10827 			if (state != IF_STATE_ACTION)
10828 				goto fail;
10829 
10830 			filter->action = actions[token];
10831 			state = IF_STATE_SOURCE;
10832 			break;
10833 
10834 		case IF_SRC_KERNELADDR:
10835 		case IF_SRC_KERNEL:
10836 			kernel = 1;
10837 			fallthrough;
10838 
10839 		case IF_SRC_FILEADDR:
10840 		case IF_SRC_FILE:
10841 			if (state != IF_STATE_SOURCE)
10842 				goto fail;
10843 
10844 			*args[0].to = 0;
10845 			ret = kstrtoul(args[0].from, 0, &filter->offset);
10846 			if (ret)
10847 				goto fail;
10848 
10849 			if (token == IF_SRC_KERNEL || token == IF_SRC_FILE) {
10850 				*args[1].to = 0;
10851 				ret = kstrtoul(args[1].from, 0, &filter->size);
10852 				if (ret)
10853 					goto fail;
10854 			}
10855 
10856 			if (token == IF_SRC_FILE || token == IF_SRC_FILEADDR) {
10857 				int fpos = token == IF_SRC_FILE ? 2 : 1;
10858 
10859 				kfree(filename);
10860 				filename = match_strdup(&args[fpos]);
10861 				if (!filename) {
10862 					ret = -ENOMEM;
10863 					goto fail;
10864 				}
10865 			}
10866 
10867 			state = IF_STATE_END;
10868 			break;
10869 
10870 		default:
10871 			goto fail;
10872 		}
10873 
10874 		/*
10875 		 * Filter definition is fully parsed, validate and install it.
10876 		 * Make sure that it doesn't contradict itself or the event's
10877 		 * attribute.
10878 		 */
10879 		if (state == IF_STATE_END) {
10880 			ret = -EINVAL;
10881 
10882 			/*
10883 			 * ACTION "filter" must have a non-zero length region
10884 			 * specified.
10885 			 */
10886 			if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER &&
10887 			    !filter->size)
10888 				goto fail;
10889 
10890 			if (!kernel) {
10891 				if (!filename)
10892 					goto fail;
10893 
10894 				/*
10895 				 * For now, we only support file-based filters
10896 				 * in per-task events; doing so for CPU-wide
10897 				 * events requires additional context switching
10898 				 * trickery, since same object code will be
10899 				 * mapped at different virtual addresses in
10900 				 * different processes.
10901 				 */
10902 				ret = -EOPNOTSUPP;
10903 				if (!event->ctx->task)
10904 					goto fail;
10905 
10906 				/* look up the path and grab its inode */
10907 				ret = kern_path(filename, LOOKUP_FOLLOW,
10908 						&filter->path);
10909 				if (ret)
10910 					goto fail;
10911 
10912 				ret = -EINVAL;
10913 				if (!filter->path.dentry ||
10914 				    !S_ISREG(d_inode(filter->path.dentry)
10915 					     ->i_mode))
10916 					goto fail;
10917 
10918 				event->addr_filters.nr_file_filters++;
10919 			}
10920 
10921 			/* ready to consume more filters */
10922 			kfree(filename);
10923 			filename = NULL;
10924 			state = IF_STATE_ACTION;
10925 			filter = NULL;
10926 			kernel = 0;
10927 		}
10928 	}
10929 
10930 	if (state != IF_STATE_ACTION)
10931 		goto fail;
10932 
10933 	kfree(filename);
10934 	kfree(orig);
10935 
10936 	return 0;
10937 
10938 fail:
10939 	kfree(filename);
10940 	free_filters_list(filters);
10941 	kfree(orig);
10942 
10943 	return ret;
10944 }
10945 
10946 static int
perf_event_set_addr_filter(struct perf_event * event,char * filter_str)10947 perf_event_set_addr_filter(struct perf_event *event, char *filter_str)
10948 {
10949 	LIST_HEAD(filters);
10950 	int ret;
10951 
10952 	/*
10953 	 * Since this is called in perf_ioctl() path, we're already holding
10954 	 * ctx::mutex.
10955 	 */
10956 	lockdep_assert_held(&event->ctx->mutex);
10957 
10958 	if (WARN_ON_ONCE(event->parent))
10959 		return -EINVAL;
10960 
10961 	ret = perf_event_parse_addr_filter(event, filter_str, &filters);
10962 	if (ret)
10963 		goto fail_clear_files;
10964 
10965 	ret = event->pmu->addr_filters_validate(&filters);
10966 	if (ret)
10967 		goto fail_free_filters;
10968 
10969 	/* remove existing filters, if any */
10970 	perf_addr_filters_splice(event, &filters);
10971 
10972 	/* install new filters */
10973 	perf_event_for_each_child(event, perf_event_addr_filters_apply);
10974 
10975 	return ret;
10976 
10977 fail_free_filters:
10978 	free_filters_list(&filters);
10979 
10980 fail_clear_files:
10981 	event->addr_filters.nr_file_filters = 0;
10982 
10983 	return ret;
10984 }
10985 
perf_event_set_filter(struct perf_event * event,void __user * arg)10986 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
10987 {
10988 	int ret = -EINVAL;
10989 	char *filter_str;
10990 
10991 	filter_str = strndup_user(arg, PAGE_SIZE);
10992 	if (IS_ERR(filter_str))
10993 		return PTR_ERR(filter_str);
10994 
10995 #ifdef CONFIG_EVENT_TRACING
10996 	if (perf_event_is_tracing(event)) {
10997 		struct perf_event_context *ctx = event->ctx;
10998 
10999 		/*
11000 		 * Beware, here be dragons!!
11001 		 *
11002 		 * the tracepoint muck will deadlock against ctx->mutex, but
11003 		 * the tracepoint stuff does not actually need it. So
11004 		 * temporarily drop ctx->mutex. As per perf_event_ctx_lock() we
11005 		 * already have a reference on ctx.
11006 		 *
11007 		 * This can result in event getting moved to a different ctx,
11008 		 * but that does not affect the tracepoint state.
11009 		 */
11010 		mutex_unlock(&ctx->mutex);
11011 		ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
11012 		mutex_lock(&ctx->mutex);
11013 	} else
11014 #endif
11015 	if (has_addr_filter(event))
11016 		ret = perf_event_set_addr_filter(event, filter_str);
11017 
11018 	kfree(filter_str);
11019 	return ret;
11020 }
11021 
11022 /*
11023  * hrtimer based swevent callback
11024  */
11025 
perf_swevent_hrtimer(struct hrtimer * hrtimer)11026 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
11027 {
11028 	enum hrtimer_restart ret = HRTIMER_RESTART;
11029 	struct perf_sample_data data;
11030 	struct pt_regs *regs;
11031 	struct perf_event *event;
11032 	u64 period;
11033 
11034 	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
11035 
11036 	if (event->state != PERF_EVENT_STATE_ACTIVE)
11037 		return HRTIMER_NORESTART;
11038 
11039 	event->pmu->read(event);
11040 
11041 	perf_sample_data_init(&data, 0, event->hw.last_period);
11042 	regs = get_irq_regs();
11043 
11044 	if (regs && !perf_exclude_event(event, regs)) {
11045 		if (!(event->attr.exclude_idle && is_idle_task(current)))
11046 			if (__perf_event_overflow(event, 1, &data, regs))
11047 				ret = HRTIMER_NORESTART;
11048 	}
11049 
11050 	period = max_t(u64, 10000, event->hw.sample_period);
11051 	hrtimer_forward_now(hrtimer, ns_to_ktime(period));
11052 
11053 	return ret;
11054 }
11055 
perf_swevent_start_hrtimer(struct perf_event * event)11056 static void perf_swevent_start_hrtimer(struct perf_event *event)
11057 {
11058 	struct hw_perf_event *hwc = &event->hw;
11059 	s64 period;
11060 
11061 	if (!is_sampling_event(event))
11062 		return;
11063 
11064 	period = local64_read(&hwc->period_left);
11065 	if (period) {
11066 		if (period < 0)
11067 			period = 10000;
11068 
11069 		local64_set(&hwc->period_left, 0);
11070 	} else {
11071 		period = max_t(u64, 10000, hwc->sample_period);
11072 	}
11073 	hrtimer_start(&hwc->hrtimer, ns_to_ktime(period),
11074 		      HRTIMER_MODE_REL_PINNED_HARD);
11075 }
11076 
perf_swevent_cancel_hrtimer(struct perf_event * event)11077 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
11078 {
11079 	struct hw_perf_event *hwc = &event->hw;
11080 
11081 	if (is_sampling_event(event)) {
11082 		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
11083 		local64_set(&hwc->period_left, ktime_to_ns(remaining));
11084 
11085 		hrtimer_cancel(&hwc->hrtimer);
11086 	}
11087 }
11088 
perf_swevent_init_hrtimer(struct perf_event * event)11089 static void perf_swevent_init_hrtimer(struct perf_event *event)
11090 {
11091 	struct hw_perf_event *hwc = &event->hw;
11092 
11093 	if (!is_sampling_event(event))
11094 		return;
11095 
11096 	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
11097 	hwc->hrtimer.function = perf_swevent_hrtimer;
11098 
11099 	/*
11100 	 * Since hrtimers have a fixed rate, we can do a static freq->period
11101 	 * mapping and avoid the whole period adjust feedback stuff.
11102 	 */
11103 	if (event->attr.freq) {
11104 		long freq = event->attr.sample_freq;
11105 
11106 		event->attr.sample_period = NSEC_PER_SEC / freq;
11107 		hwc->sample_period = event->attr.sample_period;
11108 		local64_set(&hwc->period_left, hwc->sample_period);
11109 		hwc->last_period = hwc->sample_period;
11110 		event->attr.freq = 0;
11111 	}
11112 }
11113 
11114 /*
11115  * Software event: cpu wall time clock
11116  */
11117 
cpu_clock_event_update(struct perf_event * event)11118 static void cpu_clock_event_update(struct perf_event *event)
11119 {
11120 	s64 prev;
11121 	u64 now;
11122 
11123 	now = local_clock();
11124 	prev = local64_xchg(&event->hw.prev_count, now);
11125 	local64_add(now - prev, &event->count);
11126 }
11127 
cpu_clock_event_start(struct perf_event * event,int flags)11128 static void cpu_clock_event_start(struct perf_event *event, int flags)
11129 {
11130 	local64_set(&event->hw.prev_count, local_clock());
11131 	perf_swevent_start_hrtimer(event);
11132 }
11133 
cpu_clock_event_stop(struct perf_event * event,int flags)11134 static void cpu_clock_event_stop(struct perf_event *event, int flags)
11135 {
11136 	perf_swevent_cancel_hrtimer(event);
11137 	cpu_clock_event_update(event);
11138 }
11139 
cpu_clock_event_add(struct perf_event * event,int flags)11140 static int cpu_clock_event_add(struct perf_event *event, int flags)
11141 {
11142 	if (flags & PERF_EF_START)
11143 		cpu_clock_event_start(event, flags);
11144 	perf_event_update_userpage(event);
11145 
11146 	return 0;
11147 }
11148 
cpu_clock_event_del(struct perf_event * event,int flags)11149 static void cpu_clock_event_del(struct perf_event *event, int flags)
11150 {
11151 	cpu_clock_event_stop(event, flags);
11152 }
11153 
cpu_clock_event_read(struct perf_event * event)11154 static void cpu_clock_event_read(struct perf_event *event)
11155 {
11156 	cpu_clock_event_update(event);
11157 }
11158 
cpu_clock_event_init(struct perf_event * event)11159 static int cpu_clock_event_init(struct perf_event *event)
11160 {
11161 	if (event->attr.type != perf_cpu_clock.type)
11162 		return -ENOENT;
11163 
11164 	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
11165 		return -ENOENT;
11166 
11167 	/*
11168 	 * no branch sampling for software events
11169 	 */
11170 	if (has_branch_stack(event))
11171 		return -EOPNOTSUPP;
11172 
11173 	perf_swevent_init_hrtimer(event);
11174 
11175 	return 0;
11176 }
11177 
11178 static struct pmu perf_cpu_clock = {
11179 	.task_ctx_nr	= perf_sw_context,
11180 
11181 	.capabilities	= PERF_PMU_CAP_NO_NMI,
11182 	.dev		= PMU_NULL_DEV,
11183 
11184 	.event_init	= cpu_clock_event_init,
11185 	.add		= cpu_clock_event_add,
11186 	.del		= cpu_clock_event_del,
11187 	.start		= cpu_clock_event_start,
11188 	.stop		= cpu_clock_event_stop,
11189 	.read		= cpu_clock_event_read,
11190 };
11191 
11192 /*
11193  * Software event: task time clock
11194  */
11195 
task_clock_event_update(struct perf_event * event,u64 now)11196 static void task_clock_event_update(struct perf_event *event, u64 now)
11197 {
11198 	u64 prev;
11199 	s64 delta;
11200 
11201 	prev = local64_xchg(&event->hw.prev_count, now);
11202 	delta = now - prev;
11203 	local64_add(delta, &event->count);
11204 }
11205 
task_clock_event_start(struct perf_event * event,int flags)11206 static void task_clock_event_start(struct perf_event *event, int flags)
11207 {
11208 	local64_set(&event->hw.prev_count, event->ctx->time);
11209 	perf_swevent_start_hrtimer(event);
11210 }
11211 
task_clock_event_stop(struct perf_event * event,int flags)11212 static void task_clock_event_stop(struct perf_event *event, int flags)
11213 {
11214 	perf_swevent_cancel_hrtimer(event);
11215 	task_clock_event_update(event, event->ctx->time);
11216 }
11217 
task_clock_event_add(struct perf_event * event,int flags)11218 static int task_clock_event_add(struct perf_event *event, int flags)
11219 {
11220 	if (flags & PERF_EF_START)
11221 		task_clock_event_start(event, flags);
11222 	perf_event_update_userpage(event);
11223 
11224 	return 0;
11225 }
11226 
task_clock_event_del(struct perf_event * event,int flags)11227 static void task_clock_event_del(struct perf_event *event, int flags)
11228 {
11229 	task_clock_event_stop(event, PERF_EF_UPDATE);
11230 }
11231 
task_clock_event_read(struct perf_event * event)11232 static void task_clock_event_read(struct perf_event *event)
11233 {
11234 	u64 now = perf_clock();
11235 	u64 delta = now - event->ctx->timestamp;
11236 	u64 time = event->ctx->time + delta;
11237 
11238 	task_clock_event_update(event, time);
11239 }
11240 
task_clock_event_init(struct perf_event * event)11241 static int task_clock_event_init(struct perf_event *event)
11242 {
11243 	if (event->attr.type != perf_task_clock.type)
11244 		return -ENOENT;
11245 
11246 	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
11247 		return -ENOENT;
11248 
11249 	/*
11250 	 * no branch sampling for software events
11251 	 */
11252 	if (has_branch_stack(event))
11253 		return -EOPNOTSUPP;
11254 
11255 	perf_swevent_init_hrtimer(event);
11256 
11257 	return 0;
11258 }
11259 
11260 static struct pmu perf_task_clock = {
11261 	.task_ctx_nr	= perf_sw_context,
11262 
11263 	.capabilities	= PERF_PMU_CAP_NO_NMI,
11264 	.dev		= PMU_NULL_DEV,
11265 
11266 	.event_init	= task_clock_event_init,
11267 	.add		= task_clock_event_add,
11268 	.del		= task_clock_event_del,
11269 	.start		= task_clock_event_start,
11270 	.stop		= task_clock_event_stop,
11271 	.read		= task_clock_event_read,
11272 };
11273 
perf_pmu_nop_void(struct pmu * pmu)11274 static void perf_pmu_nop_void(struct pmu *pmu)
11275 {
11276 }
11277 
perf_pmu_nop_txn(struct pmu * pmu,unsigned int flags)11278 static void perf_pmu_nop_txn(struct pmu *pmu, unsigned int flags)
11279 {
11280 }
11281 
perf_pmu_nop_int(struct pmu * pmu)11282 static int perf_pmu_nop_int(struct pmu *pmu)
11283 {
11284 	return 0;
11285 }
11286 
perf_event_nop_int(struct perf_event * event,u64 value)11287 static int perf_event_nop_int(struct perf_event *event, u64 value)
11288 {
11289 	return 0;
11290 }
11291 
11292 static DEFINE_PER_CPU(unsigned int, nop_txn_flags);
11293 
perf_pmu_start_txn(struct pmu * pmu,unsigned int flags)11294 static void perf_pmu_start_txn(struct pmu *pmu, unsigned int flags)
11295 {
11296 	__this_cpu_write(nop_txn_flags, flags);
11297 
11298 	if (flags & ~PERF_PMU_TXN_ADD)
11299 		return;
11300 
11301 	perf_pmu_disable(pmu);
11302 }
11303 
perf_pmu_commit_txn(struct pmu * pmu)11304 static int perf_pmu_commit_txn(struct pmu *pmu)
11305 {
11306 	unsigned int flags = __this_cpu_read(nop_txn_flags);
11307 
11308 	__this_cpu_write(nop_txn_flags, 0);
11309 
11310 	if (flags & ~PERF_PMU_TXN_ADD)
11311 		return 0;
11312 
11313 	perf_pmu_enable(pmu);
11314 	return 0;
11315 }
11316 
perf_pmu_cancel_txn(struct pmu * pmu)11317 static void perf_pmu_cancel_txn(struct pmu *pmu)
11318 {
11319 	unsigned int flags =  __this_cpu_read(nop_txn_flags);
11320 
11321 	__this_cpu_write(nop_txn_flags, 0);
11322 
11323 	if (flags & ~PERF_PMU_TXN_ADD)
11324 		return;
11325 
11326 	perf_pmu_enable(pmu);
11327 }
11328 
perf_event_idx_default(struct perf_event * event)11329 static int perf_event_idx_default(struct perf_event *event)
11330 {
11331 	return 0;
11332 }
11333 
free_pmu_context(struct pmu * pmu)11334 static void free_pmu_context(struct pmu *pmu)
11335 {
11336 	free_percpu(pmu->cpu_pmu_context);
11337 }
11338 
11339 /*
11340  * Let userspace know that this PMU supports address range filtering:
11341  */
nr_addr_filters_show(struct device * dev,struct device_attribute * attr,char * page)11342 static ssize_t nr_addr_filters_show(struct device *dev,
11343 				    struct device_attribute *attr,
11344 				    char *page)
11345 {
11346 	struct pmu *pmu = dev_get_drvdata(dev);
11347 
11348 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->nr_addr_filters);
11349 }
11350 DEVICE_ATTR_RO(nr_addr_filters);
11351 
11352 static struct idr pmu_idr;
11353 
11354 static ssize_t
type_show(struct device * dev,struct device_attribute * attr,char * page)11355 type_show(struct device *dev, struct device_attribute *attr, char *page)
11356 {
11357 	struct pmu *pmu = dev_get_drvdata(dev);
11358 
11359 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->type);
11360 }
11361 static DEVICE_ATTR_RO(type);
11362 
11363 static ssize_t
perf_event_mux_interval_ms_show(struct device * dev,struct device_attribute * attr,char * page)11364 perf_event_mux_interval_ms_show(struct device *dev,
11365 				struct device_attribute *attr,
11366 				char *page)
11367 {
11368 	struct pmu *pmu = dev_get_drvdata(dev);
11369 
11370 	return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->hrtimer_interval_ms);
11371 }
11372 
11373 static DEFINE_MUTEX(mux_interval_mutex);
11374 
11375 static ssize_t
perf_event_mux_interval_ms_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)11376 perf_event_mux_interval_ms_store(struct device *dev,
11377 				 struct device_attribute *attr,
11378 				 const char *buf, size_t count)
11379 {
11380 	struct pmu *pmu = dev_get_drvdata(dev);
11381 	int timer, cpu, ret;
11382 
11383 	ret = kstrtoint(buf, 0, &timer);
11384 	if (ret)
11385 		return ret;
11386 
11387 	if (timer < 1)
11388 		return -EINVAL;
11389 
11390 	/* same value, noting to do */
11391 	if (timer == pmu->hrtimer_interval_ms)
11392 		return count;
11393 
11394 	mutex_lock(&mux_interval_mutex);
11395 	pmu->hrtimer_interval_ms = timer;
11396 
11397 	/* update all cpuctx for this PMU */
11398 	cpus_read_lock();
11399 	for_each_online_cpu(cpu) {
11400 		struct perf_cpu_pmu_context *cpc;
11401 		cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu);
11402 		cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
11403 
11404 		cpu_function_call(cpu, perf_mux_hrtimer_restart_ipi, cpc);
11405 	}
11406 	cpus_read_unlock();
11407 	mutex_unlock(&mux_interval_mutex);
11408 
11409 	return count;
11410 }
11411 static DEVICE_ATTR_RW(perf_event_mux_interval_ms);
11412 
11413 static struct attribute *pmu_dev_attrs[] = {
11414 	&dev_attr_type.attr,
11415 	&dev_attr_perf_event_mux_interval_ms.attr,
11416 	&dev_attr_nr_addr_filters.attr,
11417 	NULL,
11418 };
11419 
pmu_dev_is_visible(struct kobject * kobj,struct attribute * a,int n)11420 static umode_t pmu_dev_is_visible(struct kobject *kobj, struct attribute *a, int n)
11421 {
11422 	struct device *dev = kobj_to_dev(kobj);
11423 	struct pmu *pmu = dev_get_drvdata(dev);
11424 
11425 	if (n == 2 && !pmu->nr_addr_filters)
11426 		return 0;
11427 
11428 	return a->mode;
11429 }
11430 
11431 static struct attribute_group pmu_dev_attr_group = {
11432 	.is_visible = pmu_dev_is_visible,
11433 	.attrs = pmu_dev_attrs,
11434 };
11435 
11436 static const struct attribute_group *pmu_dev_groups[] = {
11437 	&pmu_dev_attr_group,
11438 	NULL,
11439 };
11440 
11441 static int pmu_bus_running;
11442 static struct bus_type pmu_bus = {
11443 	.name		= "event_source",
11444 	.dev_groups	= pmu_dev_groups,
11445 };
11446 
pmu_dev_release(struct device * dev)11447 static void pmu_dev_release(struct device *dev)
11448 {
11449 	kfree(dev);
11450 }
11451 
pmu_dev_alloc(struct pmu * pmu)11452 static int pmu_dev_alloc(struct pmu *pmu)
11453 {
11454 	int ret = -ENOMEM;
11455 
11456 	pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
11457 	if (!pmu->dev)
11458 		goto out;
11459 
11460 	pmu->dev->groups = pmu->attr_groups;
11461 	device_initialize(pmu->dev);
11462 
11463 	dev_set_drvdata(pmu->dev, pmu);
11464 	pmu->dev->bus = &pmu_bus;
11465 	pmu->dev->parent = pmu->parent;
11466 	pmu->dev->release = pmu_dev_release;
11467 
11468 	ret = dev_set_name(pmu->dev, "%s", pmu->name);
11469 	if (ret)
11470 		goto free_dev;
11471 
11472 	ret = device_add(pmu->dev);
11473 	if (ret)
11474 		goto free_dev;
11475 
11476 	if (pmu->attr_update) {
11477 		ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update);
11478 		if (ret)
11479 			goto del_dev;
11480 	}
11481 
11482 out:
11483 	return ret;
11484 
11485 del_dev:
11486 	device_del(pmu->dev);
11487 
11488 free_dev:
11489 	put_device(pmu->dev);
11490 	goto out;
11491 }
11492 
11493 static struct lock_class_key cpuctx_mutex;
11494 static struct lock_class_key cpuctx_lock;
11495 
perf_pmu_register(struct pmu * pmu,const char * name,int type)11496 int perf_pmu_register(struct pmu *pmu, const char *name, int type)
11497 {
11498 	int cpu, ret, max = PERF_TYPE_MAX;
11499 
11500 	mutex_lock(&pmus_lock);
11501 	ret = -ENOMEM;
11502 	pmu->pmu_disable_count = alloc_percpu(int);
11503 	if (!pmu->pmu_disable_count)
11504 		goto unlock;
11505 
11506 	pmu->type = -1;
11507 	if (WARN_ONCE(!name, "Can not register anonymous pmu.\n")) {
11508 		ret = -EINVAL;
11509 		goto free_pdc;
11510 	}
11511 
11512 	pmu->name = name;
11513 
11514 	if (type >= 0)
11515 		max = type;
11516 
11517 	ret = idr_alloc(&pmu_idr, pmu, max, 0, GFP_KERNEL);
11518 	if (ret < 0)
11519 		goto free_pdc;
11520 
11521 	WARN_ON(type >= 0 && ret != type);
11522 
11523 	type = ret;
11524 	pmu->type = type;
11525 
11526 	if (pmu_bus_running && !pmu->dev) {
11527 		ret = pmu_dev_alloc(pmu);
11528 		if (ret)
11529 			goto free_idr;
11530 	}
11531 
11532 	ret = -ENOMEM;
11533 	pmu->cpu_pmu_context = alloc_percpu(struct perf_cpu_pmu_context);
11534 	if (!pmu->cpu_pmu_context)
11535 		goto free_dev;
11536 
11537 	for_each_possible_cpu(cpu) {
11538 		struct perf_cpu_pmu_context *cpc;
11539 
11540 		cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu);
11541 		__perf_init_event_pmu_context(&cpc->epc, pmu);
11542 		__perf_mux_hrtimer_init(cpc, cpu);
11543 	}
11544 
11545 	if (!pmu->start_txn) {
11546 		if (pmu->pmu_enable) {
11547 			/*
11548 			 * If we have pmu_enable/pmu_disable calls, install
11549 			 * transaction stubs that use that to try and batch
11550 			 * hardware accesses.
11551 			 */
11552 			pmu->start_txn  = perf_pmu_start_txn;
11553 			pmu->commit_txn = perf_pmu_commit_txn;
11554 			pmu->cancel_txn = perf_pmu_cancel_txn;
11555 		} else {
11556 			pmu->start_txn  = perf_pmu_nop_txn;
11557 			pmu->commit_txn = perf_pmu_nop_int;
11558 			pmu->cancel_txn = perf_pmu_nop_void;
11559 		}
11560 	}
11561 
11562 	if (!pmu->pmu_enable) {
11563 		pmu->pmu_enable  = perf_pmu_nop_void;
11564 		pmu->pmu_disable = perf_pmu_nop_void;
11565 	}
11566 
11567 	if (!pmu->check_period)
11568 		pmu->check_period = perf_event_nop_int;
11569 
11570 	if (!pmu->event_idx)
11571 		pmu->event_idx = perf_event_idx_default;
11572 
11573 	list_add_rcu(&pmu->entry, &pmus);
11574 	atomic_set(&pmu->exclusive_cnt, 0);
11575 	ret = 0;
11576 unlock:
11577 	mutex_unlock(&pmus_lock);
11578 
11579 	return ret;
11580 
11581 free_dev:
11582 	if (pmu->dev && pmu->dev != PMU_NULL_DEV) {
11583 		device_del(pmu->dev);
11584 		put_device(pmu->dev);
11585 	}
11586 
11587 free_idr:
11588 	idr_remove(&pmu_idr, pmu->type);
11589 
11590 free_pdc:
11591 	free_percpu(pmu->pmu_disable_count);
11592 	goto unlock;
11593 }
11594 EXPORT_SYMBOL_GPL(perf_pmu_register);
11595 
perf_pmu_unregister(struct pmu * pmu)11596 void perf_pmu_unregister(struct pmu *pmu)
11597 {
11598 	mutex_lock(&pmus_lock);
11599 	list_del_rcu(&pmu->entry);
11600 
11601 	/*
11602 	 * We dereference the pmu list under both SRCU and regular RCU, so
11603 	 * synchronize against both of those.
11604 	 */
11605 	synchronize_srcu(&pmus_srcu);
11606 	synchronize_rcu();
11607 
11608 	free_percpu(pmu->pmu_disable_count);
11609 	idr_remove(&pmu_idr, pmu->type);
11610 	if (pmu_bus_running && pmu->dev && pmu->dev != PMU_NULL_DEV) {
11611 		if (pmu->nr_addr_filters)
11612 			device_remove_file(pmu->dev, &dev_attr_nr_addr_filters);
11613 		device_del(pmu->dev);
11614 		put_device(pmu->dev);
11615 	}
11616 	free_pmu_context(pmu);
11617 	mutex_unlock(&pmus_lock);
11618 }
11619 EXPORT_SYMBOL_GPL(perf_pmu_unregister);
11620 
has_extended_regs(struct perf_event * event)11621 static inline bool has_extended_regs(struct perf_event *event)
11622 {
11623 	return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) ||
11624 	       (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK);
11625 }
11626 
perf_try_init_event(struct pmu * pmu,struct perf_event * event)11627 static int perf_try_init_event(struct pmu *pmu, struct perf_event *event)
11628 {
11629 	struct perf_event_context *ctx = NULL;
11630 	int ret;
11631 
11632 	if (!try_module_get(pmu->module))
11633 		return -ENODEV;
11634 
11635 	/*
11636 	 * A number of pmu->event_init() methods iterate the sibling_list to,
11637 	 * for example, validate if the group fits on the PMU. Therefore,
11638 	 * if this is a sibling event, acquire the ctx->mutex to protect
11639 	 * the sibling_list.
11640 	 */
11641 	if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) {
11642 		/*
11643 		 * This ctx->mutex can nest when we're called through
11644 		 * inheritance. See the perf_event_ctx_lock_nested() comment.
11645 		 */
11646 		ctx = perf_event_ctx_lock_nested(event->group_leader,
11647 						 SINGLE_DEPTH_NESTING);
11648 		BUG_ON(!ctx);
11649 	}
11650 
11651 	event->pmu = pmu;
11652 	ret = pmu->event_init(event);
11653 
11654 	if (ctx)
11655 		perf_event_ctx_unlock(event->group_leader, ctx);
11656 
11657 	if (!ret) {
11658 		if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) &&
11659 		    has_extended_regs(event))
11660 			ret = -EOPNOTSUPP;
11661 
11662 		if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE &&
11663 		    event_has_any_exclude_flag(event))
11664 			ret = -EINVAL;
11665 
11666 		if (ret && event->destroy)
11667 			event->destroy(event);
11668 	}
11669 
11670 	if (ret)
11671 		module_put(pmu->module);
11672 
11673 	return ret;
11674 }
11675 
perf_init_event(struct perf_event * event)11676 static struct pmu *perf_init_event(struct perf_event *event)
11677 {
11678 	bool extended_type = false;
11679 	int idx, type, ret;
11680 	struct pmu *pmu;
11681 
11682 	idx = srcu_read_lock(&pmus_srcu);
11683 
11684 	/*
11685 	 * Save original type before calling pmu->event_init() since certain
11686 	 * pmus overwrites event->attr.type to forward event to another pmu.
11687 	 */
11688 	event->orig_type = event->attr.type;
11689 
11690 	/* Try parent's PMU first: */
11691 	if (event->parent && event->parent->pmu) {
11692 		pmu = event->parent->pmu;
11693 		ret = perf_try_init_event(pmu, event);
11694 		if (!ret)
11695 			goto unlock;
11696 	}
11697 
11698 	/*
11699 	 * PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
11700 	 * are often aliases for PERF_TYPE_RAW.
11701 	 */
11702 	type = event->attr.type;
11703 	if (type == PERF_TYPE_HARDWARE || type == PERF_TYPE_HW_CACHE) {
11704 		type = event->attr.config >> PERF_PMU_TYPE_SHIFT;
11705 		if (!type) {
11706 			type = PERF_TYPE_RAW;
11707 		} else {
11708 			extended_type = true;
11709 			event->attr.config &= PERF_HW_EVENT_MASK;
11710 		}
11711 	}
11712 
11713 again:
11714 	rcu_read_lock();
11715 	pmu = idr_find(&pmu_idr, type);
11716 	rcu_read_unlock();
11717 	if (pmu) {
11718 		if (event->attr.type != type && type != PERF_TYPE_RAW &&
11719 		    !(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE))
11720 			goto fail;
11721 
11722 		ret = perf_try_init_event(pmu, event);
11723 		if (ret == -ENOENT && event->attr.type != type && !extended_type) {
11724 			type = event->attr.type;
11725 			goto again;
11726 		}
11727 
11728 		if (ret)
11729 			pmu = ERR_PTR(ret);
11730 
11731 		goto unlock;
11732 	}
11733 
11734 	list_for_each_entry_rcu(pmu, &pmus, entry, lockdep_is_held(&pmus_srcu)) {
11735 		ret = perf_try_init_event(pmu, event);
11736 		if (!ret)
11737 			goto unlock;
11738 
11739 		if (ret != -ENOENT) {
11740 			pmu = ERR_PTR(ret);
11741 			goto unlock;
11742 		}
11743 	}
11744 fail:
11745 	pmu = ERR_PTR(-ENOENT);
11746 unlock:
11747 	srcu_read_unlock(&pmus_srcu, idx);
11748 
11749 	return pmu;
11750 }
11751 
attach_sb_event(struct perf_event * event)11752 static void attach_sb_event(struct perf_event *event)
11753 {
11754 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
11755 
11756 	raw_spin_lock(&pel->lock);
11757 	list_add_rcu(&event->sb_list, &pel->list);
11758 	raw_spin_unlock(&pel->lock);
11759 }
11760 
11761 /*
11762  * We keep a list of all !task (and therefore per-cpu) events
11763  * that need to receive side-band records.
11764  *
11765  * This avoids having to scan all the various PMU per-cpu contexts
11766  * looking for them.
11767  */
account_pmu_sb_event(struct perf_event * event)11768 static void account_pmu_sb_event(struct perf_event *event)
11769 {
11770 	if (is_sb_event(event))
11771 		attach_sb_event(event);
11772 }
11773 
11774 /* Freq events need the tick to stay alive (see perf_event_task_tick). */
account_freq_event_nohz(void)11775 static void account_freq_event_nohz(void)
11776 {
11777 #ifdef CONFIG_NO_HZ_FULL
11778 	/* Lock so we don't race with concurrent unaccount */
11779 	spin_lock(&nr_freq_lock);
11780 	if (atomic_inc_return(&nr_freq_events) == 1)
11781 		tick_nohz_dep_set(TICK_DEP_BIT_PERF_EVENTS);
11782 	spin_unlock(&nr_freq_lock);
11783 #endif
11784 }
11785 
account_freq_event(void)11786 static void account_freq_event(void)
11787 {
11788 	if (tick_nohz_full_enabled())
11789 		account_freq_event_nohz();
11790 	else
11791 		atomic_inc(&nr_freq_events);
11792 }
11793 
11794 
account_event(struct perf_event * event)11795 static void account_event(struct perf_event *event)
11796 {
11797 	bool inc = false;
11798 
11799 	if (event->parent)
11800 		return;
11801 
11802 	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
11803 		inc = true;
11804 	if (event->attr.mmap || event->attr.mmap_data)
11805 		atomic_inc(&nr_mmap_events);
11806 	if (event->attr.build_id)
11807 		atomic_inc(&nr_build_id_events);
11808 	if (event->attr.comm)
11809 		atomic_inc(&nr_comm_events);
11810 	if (event->attr.namespaces)
11811 		atomic_inc(&nr_namespaces_events);
11812 	if (event->attr.cgroup)
11813 		atomic_inc(&nr_cgroup_events);
11814 	if (event->attr.task)
11815 		atomic_inc(&nr_task_events);
11816 	if (event->attr.freq)
11817 		account_freq_event();
11818 	if (event->attr.context_switch) {
11819 		atomic_inc(&nr_switch_events);
11820 		inc = true;
11821 	}
11822 	if (has_branch_stack(event))
11823 		inc = true;
11824 	if (is_cgroup_event(event))
11825 		inc = true;
11826 	if (event->attr.ksymbol)
11827 		atomic_inc(&nr_ksymbol_events);
11828 	if (event->attr.bpf_event)
11829 		atomic_inc(&nr_bpf_events);
11830 	if (event->attr.text_poke)
11831 		atomic_inc(&nr_text_poke_events);
11832 
11833 	if (inc) {
11834 		/*
11835 		 * We need the mutex here because static_branch_enable()
11836 		 * must complete *before* the perf_sched_count increment
11837 		 * becomes visible.
11838 		 */
11839 		if (atomic_inc_not_zero(&perf_sched_count))
11840 			goto enabled;
11841 
11842 		mutex_lock(&perf_sched_mutex);
11843 		if (!atomic_read(&perf_sched_count)) {
11844 			static_branch_enable(&perf_sched_events);
11845 			/*
11846 			 * Guarantee that all CPUs observe they key change and
11847 			 * call the perf scheduling hooks before proceeding to
11848 			 * install events that need them.
11849 			 */
11850 			synchronize_rcu();
11851 		}
11852 		/*
11853 		 * Now that we have waited for the sync_sched(), allow further
11854 		 * increments to by-pass the mutex.
11855 		 */
11856 		atomic_inc(&perf_sched_count);
11857 		mutex_unlock(&perf_sched_mutex);
11858 	}
11859 enabled:
11860 
11861 	account_pmu_sb_event(event);
11862 }
11863 
11864 /*
11865  * Allocate and initialize an event structure
11866  */
11867 static struct perf_event *
perf_event_alloc(struct perf_event_attr * attr,int cpu,struct task_struct * task,struct perf_event * group_leader,struct perf_event * parent_event,perf_overflow_handler_t overflow_handler,void * context,int cgroup_fd)11868 perf_event_alloc(struct perf_event_attr *attr, int cpu,
11869 		 struct task_struct *task,
11870 		 struct perf_event *group_leader,
11871 		 struct perf_event *parent_event,
11872 		 perf_overflow_handler_t overflow_handler,
11873 		 void *context, int cgroup_fd)
11874 {
11875 	struct pmu *pmu;
11876 	struct perf_event *event;
11877 	struct hw_perf_event *hwc;
11878 	long err = -EINVAL;
11879 	int node;
11880 
11881 	if ((unsigned)cpu >= nr_cpu_ids) {
11882 		if (!task || cpu != -1)
11883 			return ERR_PTR(-EINVAL);
11884 	}
11885 	if (attr->sigtrap && !task) {
11886 		/* Requires a task: avoid signalling random tasks. */
11887 		return ERR_PTR(-EINVAL);
11888 	}
11889 
11890 	node = (cpu >= 0) ? cpu_to_node(cpu) : -1;
11891 	event = kmem_cache_alloc_node(perf_event_cache, GFP_KERNEL | __GFP_ZERO,
11892 				      node);
11893 	if (!event)
11894 		return ERR_PTR(-ENOMEM);
11895 
11896 	/*
11897 	 * Single events are their own group leaders, with an
11898 	 * empty sibling list:
11899 	 */
11900 	if (!group_leader)
11901 		group_leader = event;
11902 
11903 	mutex_init(&event->child_mutex);
11904 	INIT_LIST_HEAD(&event->child_list);
11905 
11906 	INIT_LIST_HEAD(&event->event_entry);
11907 	INIT_LIST_HEAD(&event->sibling_list);
11908 	INIT_LIST_HEAD(&event->active_list);
11909 	init_event_group(event);
11910 	INIT_LIST_HEAD(&event->rb_entry);
11911 	INIT_LIST_HEAD(&event->active_entry);
11912 	INIT_LIST_HEAD(&event->addr_filters.list);
11913 	INIT_HLIST_NODE(&event->hlist_entry);
11914 
11915 
11916 	init_waitqueue_head(&event->waitq);
11917 	init_irq_work(&event->pending_irq, perf_pending_irq);
11918 	init_task_work(&event->pending_task, perf_pending_task);
11919 
11920 	mutex_init(&event->mmap_mutex);
11921 	raw_spin_lock_init(&event->addr_filters.lock);
11922 
11923 	atomic_long_set(&event->refcount, 1);
11924 	event->cpu		= cpu;
11925 	event->attr		= *attr;
11926 	event->group_leader	= group_leader;
11927 	event->pmu		= NULL;
11928 	event->oncpu		= -1;
11929 
11930 	event->parent		= parent_event;
11931 
11932 	event->ns		= get_pid_ns(task_active_pid_ns(current));
11933 	event->id		= atomic64_inc_return(&perf_event_id);
11934 
11935 	event->state		= PERF_EVENT_STATE_INACTIVE;
11936 
11937 	if (parent_event)
11938 		event->event_caps = parent_event->event_caps;
11939 
11940 	if (task) {
11941 		event->attach_state = PERF_ATTACH_TASK;
11942 		/*
11943 		 * XXX pmu::event_init needs to know what task to account to
11944 		 * and we cannot use the ctx information because we need the
11945 		 * pmu before we get a ctx.
11946 		 */
11947 		event->hw.target = get_task_struct(task);
11948 	}
11949 
11950 	event->clock = &local_clock;
11951 	if (parent_event)
11952 		event->clock = parent_event->clock;
11953 
11954 	if (!overflow_handler && parent_event) {
11955 		overflow_handler = parent_event->overflow_handler;
11956 		context = parent_event->overflow_handler_context;
11957 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING)
11958 		if (overflow_handler == bpf_overflow_handler) {
11959 			struct bpf_prog *prog = parent_event->prog;
11960 
11961 			bpf_prog_inc(prog);
11962 			event->prog = prog;
11963 			event->orig_overflow_handler =
11964 				parent_event->orig_overflow_handler;
11965 		}
11966 #endif
11967 	}
11968 
11969 	if (overflow_handler) {
11970 		event->overflow_handler	= overflow_handler;
11971 		event->overflow_handler_context = context;
11972 	} else if (is_write_backward(event)){
11973 		event->overflow_handler = perf_event_output_backward;
11974 		event->overflow_handler_context = NULL;
11975 	} else {
11976 		event->overflow_handler = perf_event_output_forward;
11977 		event->overflow_handler_context = NULL;
11978 	}
11979 
11980 	perf_event__state_init(event);
11981 
11982 	pmu = NULL;
11983 
11984 	hwc = &event->hw;
11985 	hwc->sample_period = attr->sample_period;
11986 	if (attr->freq && attr->sample_freq)
11987 		hwc->sample_period = 1;
11988 	hwc->last_period = hwc->sample_period;
11989 
11990 	local64_set(&hwc->period_left, hwc->sample_period);
11991 
11992 	/*
11993 	 * We currently do not support PERF_SAMPLE_READ on inherited events.
11994 	 * See perf_output_read().
11995 	 */
11996 	if (attr->inherit && (attr->sample_type & PERF_SAMPLE_READ))
11997 		goto err_ns;
11998 
11999 	if (!has_branch_stack(event))
12000 		event->attr.branch_sample_type = 0;
12001 
12002 	pmu = perf_init_event(event);
12003 	if (IS_ERR(pmu)) {
12004 		err = PTR_ERR(pmu);
12005 		goto err_ns;
12006 	}
12007 
12008 	/*
12009 	 * Disallow uncore-task events. Similarly, disallow uncore-cgroup
12010 	 * events (they don't make sense as the cgroup will be different
12011 	 * on other CPUs in the uncore mask).
12012 	 */
12013 	if (pmu->task_ctx_nr == perf_invalid_context && (task || cgroup_fd != -1)) {
12014 		err = -EINVAL;
12015 		goto err_pmu;
12016 	}
12017 
12018 	if (event->attr.aux_output &&
12019 	    !(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT)) {
12020 		err = -EOPNOTSUPP;
12021 		goto err_pmu;
12022 	}
12023 
12024 	if (cgroup_fd != -1) {
12025 		err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader);
12026 		if (err)
12027 			goto err_pmu;
12028 	}
12029 
12030 	err = exclusive_event_init(event);
12031 	if (err)
12032 		goto err_pmu;
12033 
12034 	if (has_addr_filter(event)) {
12035 		event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters,
12036 						    sizeof(struct perf_addr_filter_range),
12037 						    GFP_KERNEL);
12038 		if (!event->addr_filter_ranges) {
12039 			err = -ENOMEM;
12040 			goto err_per_task;
12041 		}
12042 
12043 		/*
12044 		 * Clone the parent's vma offsets: they are valid until exec()
12045 		 * even if the mm is not shared with the parent.
12046 		 */
12047 		if (event->parent) {
12048 			struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
12049 
12050 			raw_spin_lock_irq(&ifh->lock);
12051 			memcpy(event->addr_filter_ranges,
12052 			       event->parent->addr_filter_ranges,
12053 			       pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range));
12054 			raw_spin_unlock_irq(&ifh->lock);
12055 		}
12056 
12057 		/* force hw sync on the address filters */
12058 		event->addr_filters_gen = 1;
12059 	}
12060 
12061 	if (!event->parent) {
12062 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
12063 			err = get_callchain_buffers(attr->sample_max_stack);
12064 			if (err)
12065 				goto err_addr_filters;
12066 		}
12067 	}
12068 
12069 	err = security_perf_event_alloc(event);
12070 	if (err)
12071 		goto err_callchain_buffer;
12072 
12073 	/* symmetric to unaccount_event() in _free_event() */
12074 	account_event(event);
12075 
12076 	return event;
12077 
12078 err_callchain_buffer:
12079 	if (!event->parent) {
12080 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
12081 			put_callchain_buffers();
12082 	}
12083 err_addr_filters:
12084 	kfree(event->addr_filter_ranges);
12085 
12086 err_per_task:
12087 	exclusive_event_destroy(event);
12088 
12089 err_pmu:
12090 	if (is_cgroup_event(event))
12091 		perf_detach_cgroup(event);
12092 	if (event->destroy)
12093 		event->destroy(event);
12094 	module_put(pmu->module);
12095 err_ns:
12096 	if (event->hw.target)
12097 		put_task_struct(event->hw.target);
12098 	call_rcu(&event->rcu_head, free_event_rcu);
12099 
12100 	return ERR_PTR(err);
12101 }
12102 
perf_copy_attr(struct perf_event_attr __user * uattr,struct perf_event_attr * attr)12103 static int perf_copy_attr(struct perf_event_attr __user *uattr,
12104 			  struct perf_event_attr *attr)
12105 {
12106 	u32 size;
12107 	int ret;
12108 
12109 	/* Zero the full structure, so that a short copy will be nice. */
12110 	memset(attr, 0, sizeof(*attr));
12111 
12112 	ret = get_user(size, &uattr->size);
12113 	if (ret)
12114 		return ret;
12115 
12116 	/* ABI compatibility quirk: */
12117 	if (!size)
12118 		size = PERF_ATTR_SIZE_VER0;
12119 	if (size < PERF_ATTR_SIZE_VER0 || size > PAGE_SIZE)
12120 		goto err_size;
12121 
12122 	ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size);
12123 	if (ret) {
12124 		if (ret == -E2BIG)
12125 			goto err_size;
12126 		return ret;
12127 	}
12128 
12129 	attr->size = size;
12130 
12131 	if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)
12132 		return -EINVAL;
12133 
12134 	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
12135 		return -EINVAL;
12136 
12137 	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
12138 		return -EINVAL;
12139 
12140 	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
12141 		u64 mask = attr->branch_sample_type;
12142 
12143 		/* only using defined bits */
12144 		if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
12145 			return -EINVAL;
12146 
12147 		/* at least one branch bit must be set */
12148 		if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
12149 			return -EINVAL;
12150 
12151 		/* propagate priv level, when not set for branch */
12152 		if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
12153 
12154 			/* exclude_kernel checked on syscall entry */
12155 			if (!attr->exclude_kernel)
12156 				mask |= PERF_SAMPLE_BRANCH_KERNEL;
12157 
12158 			if (!attr->exclude_user)
12159 				mask |= PERF_SAMPLE_BRANCH_USER;
12160 
12161 			if (!attr->exclude_hv)
12162 				mask |= PERF_SAMPLE_BRANCH_HV;
12163 			/*
12164 			 * adjust user setting (for HW filter setup)
12165 			 */
12166 			attr->branch_sample_type = mask;
12167 		}
12168 		/* privileged levels capture (kernel, hv): check permissions */
12169 		if (mask & PERF_SAMPLE_BRANCH_PERM_PLM) {
12170 			ret = perf_allow_kernel(attr);
12171 			if (ret)
12172 				return ret;
12173 		}
12174 	}
12175 
12176 	if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
12177 		ret = perf_reg_validate(attr->sample_regs_user);
12178 		if (ret)
12179 			return ret;
12180 	}
12181 
12182 	if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
12183 		if (!arch_perf_have_user_stack_dump())
12184 			return -ENOSYS;
12185 
12186 		/*
12187 		 * We have __u32 type for the size, but so far
12188 		 * we can only use __u16 as maximum due to the
12189 		 * __u16 sample size limit.
12190 		 */
12191 		if (attr->sample_stack_user >= USHRT_MAX)
12192 			return -EINVAL;
12193 		else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
12194 			return -EINVAL;
12195 	}
12196 
12197 	if (!attr->sample_max_stack)
12198 		attr->sample_max_stack = sysctl_perf_event_max_stack;
12199 
12200 	if (attr->sample_type & PERF_SAMPLE_REGS_INTR)
12201 		ret = perf_reg_validate(attr->sample_regs_intr);
12202 
12203 #ifndef CONFIG_CGROUP_PERF
12204 	if (attr->sample_type & PERF_SAMPLE_CGROUP)
12205 		return -EINVAL;
12206 #endif
12207 	if ((attr->sample_type & PERF_SAMPLE_WEIGHT) &&
12208 	    (attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT))
12209 		return -EINVAL;
12210 
12211 	if (!attr->inherit && attr->inherit_thread)
12212 		return -EINVAL;
12213 
12214 	if (attr->remove_on_exec && attr->enable_on_exec)
12215 		return -EINVAL;
12216 
12217 	if (attr->sigtrap && !attr->remove_on_exec)
12218 		return -EINVAL;
12219 
12220 out:
12221 	return ret;
12222 
12223 err_size:
12224 	put_user(sizeof(*attr), &uattr->size);
12225 	ret = -E2BIG;
12226 	goto out;
12227 }
12228 
mutex_lock_double(struct mutex * a,struct mutex * b)12229 static void mutex_lock_double(struct mutex *a, struct mutex *b)
12230 {
12231 	if (b < a)
12232 		swap(a, b);
12233 
12234 	mutex_lock(a);
12235 	mutex_lock_nested(b, SINGLE_DEPTH_NESTING);
12236 }
12237 
12238 static int
perf_event_set_output(struct perf_event * event,struct perf_event * output_event)12239 perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
12240 {
12241 	struct perf_buffer *rb = NULL;
12242 	int ret = -EINVAL;
12243 
12244 	if (!output_event) {
12245 		mutex_lock(&event->mmap_mutex);
12246 		goto set;
12247 	}
12248 
12249 	/* don't allow circular references */
12250 	if (event == output_event)
12251 		goto out;
12252 
12253 	/*
12254 	 * Don't allow cross-cpu buffers
12255 	 */
12256 	if (output_event->cpu != event->cpu)
12257 		goto out;
12258 
12259 	/*
12260 	 * If its not a per-cpu rb, it must be the same task.
12261 	 */
12262 	if (output_event->cpu == -1 && output_event->hw.target != event->hw.target)
12263 		goto out;
12264 
12265 	/*
12266 	 * Mixing clocks in the same buffer is trouble you don't need.
12267 	 */
12268 	if (output_event->clock != event->clock)
12269 		goto out;
12270 
12271 	/*
12272 	 * Either writing ring buffer from beginning or from end.
12273 	 * Mixing is not allowed.
12274 	 */
12275 	if (is_write_backward(output_event) != is_write_backward(event))
12276 		goto out;
12277 
12278 	/*
12279 	 * If both events generate aux data, they must be on the same PMU
12280 	 */
12281 	if (has_aux(event) && has_aux(output_event) &&
12282 	    event->pmu != output_event->pmu)
12283 		goto out;
12284 
12285 	/*
12286 	 * Hold both mmap_mutex to serialize against perf_mmap_close().  Since
12287 	 * output_event is already on rb->event_list, and the list iteration
12288 	 * restarts after every removal, it is guaranteed this new event is
12289 	 * observed *OR* if output_event is already removed, it's guaranteed we
12290 	 * observe !rb->mmap_count.
12291 	 */
12292 	mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex);
12293 set:
12294 	/* Can't redirect output if we've got an active mmap() */
12295 	if (atomic_read(&event->mmap_count))
12296 		goto unlock;
12297 
12298 	if (output_event) {
12299 		/* get the rb we want to redirect to */
12300 		rb = ring_buffer_get(output_event);
12301 		if (!rb)
12302 			goto unlock;
12303 
12304 		/* did we race against perf_mmap_close() */
12305 		if (!atomic_read(&rb->mmap_count)) {
12306 			ring_buffer_put(rb);
12307 			goto unlock;
12308 		}
12309 	}
12310 
12311 	ring_buffer_attach(event, rb);
12312 
12313 	ret = 0;
12314 unlock:
12315 	mutex_unlock(&event->mmap_mutex);
12316 	if (output_event)
12317 		mutex_unlock(&output_event->mmap_mutex);
12318 
12319 out:
12320 	return ret;
12321 }
12322 
perf_event_set_clock(struct perf_event * event,clockid_t clk_id)12323 static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id)
12324 {
12325 	bool nmi_safe = false;
12326 
12327 	switch (clk_id) {
12328 	case CLOCK_MONOTONIC:
12329 		event->clock = &ktime_get_mono_fast_ns;
12330 		nmi_safe = true;
12331 		break;
12332 
12333 	case CLOCK_MONOTONIC_RAW:
12334 		event->clock = &ktime_get_raw_fast_ns;
12335 		nmi_safe = true;
12336 		break;
12337 
12338 	case CLOCK_REALTIME:
12339 		event->clock = &ktime_get_real_ns;
12340 		break;
12341 
12342 	case CLOCK_BOOTTIME:
12343 		event->clock = &ktime_get_boottime_ns;
12344 		break;
12345 
12346 	case CLOCK_TAI:
12347 		event->clock = &ktime_get_clocktai_ns;
12348 		break;
12349 
12350 	default:
12351 		return -EINVAL;
12352 	}
12353 
12354 	if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI))
12355 		return -EINVAL;
12356 
12357 	return 0;
12358 }
12359 
12360 static bool
perf_check_permission(struct perf_event_attr * attr,struct task_struct * task)12361 perf_check_permission(struct perf_event_attr *attr, struct task_struct *task)
12362 {
12363 	unsigned int ptrace_mode = PTRACE_MODE_READ_REALCREDS;
12364 	bool is_capable = perfmon_capable();
12365 
12366 	if (attr->sigtrap) {
12367 		/*
12368 		 * perf_event_attr::sigtrap sends signals to the other task.
12369 		 * Require the current task to also have CAP_KILL.
12370 		 */
12371 		rcu_read_lock();
12372 		is_capable &= ns_capable(__task_cred(task)->user_ns, CAP_KILL);
12373 		rcu_read_unlock();
12374 
12375 		/*
12376 		 * If the required capabilities aren't available, checks for
12377 		 * ptrace permissions: upgrade to ATTACH, since sending signals
12378 		 * can effectively change the target task.
12379 		 */
12380 		ptrace_mode = PTRACE_MODE_ATTACH_REALCREDS;
12381 	}
12382 
12383 	/*
12384 	 * Preserve ptrace permission check for backwards compatibility. The
12385 	 * ptrace check also includes checks that the current task and other
12386 	 * task have matching uids, and is therefore not done here explicitly.
12387 	 */
12388 	return is_capable || ptrace_may_access(task, ptrace_mode);
12389 }
12390 
12391 /**
12392  * sys_perf_event_open - open a performance event, associate it to a task/cpu
12393  *
12394  * @attr_uptr:	event_id type attributes for monitoring/sampling
12395  * @pid:		target pid
12396  * @cpu:		target cpu
12397  * @group_fd:		group leader event fd
12398  * @flags:		perf event open flags
12399  */
SYSCALL_DEFINE5(perf_event_open,struct perf_event_attr __user *,attr_uptr,pid_t,pid,int,cpu,int,group_fd,unsigned long,flags)12400 SYSCALL_DEFINE5(perf_event_open,
12401 		struct perf_event_attr __user *, attr_uptr,
12402 		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
12403 {
12404 	struct perf_event *group_leader = NULL, *output_event = NULL;
12405 	struct perf_event_pmu_context *pmu_ctx;
12406 	struct perf_event *event, *sibling;
12407 	struct perf_event_attr attr;
12408 	struct perf_event_context *ctx;
12409 	struct file *event_file = NULL;
12410 	struct fd group = {NULL, 0};
12411 	struct task_struct *task = NULL;
12412 	struct pmu *pmu;
12413 	int event_fd;
12414 	int move_group = 0;
12415 	int err;
12416 	int f_flags = O_RDWR;
12417 	int cgroup_fd = -1;
12418 
12419 	/* for future expandability... */
12420 	if (flags & ~PERF_FLAG_ALL)
12421 		return -EINVAL;
12422 
12423 	err = perf_copy_attr(attr_uptr, &attr);
12424 	if (err)
12425 		return err;
12426 
12427 	/* Do we allow access to perf_event_open(2) ? */
12428 	err = security_perf_event_open(&attr, PERF_SECURITY_OPEN);
12429 	if (err)
12430 		return err;
12431 
12432 	if (!attr.exclude_kernel) {
12433 		err = perf_allow_kernel(&attr);
12434 		if (err)
12435 			return err;
12436 	}
12437 
12438 	if (attr.namespaces) {
12439 		if (!perfmon_capable())
12440 			return -EACCES;
12441 	}
12442 
12443 	if (attr.freq) {
12444 		if (attr.sample_freq > sysctl_perf_event_sample_rate)
12445 			return -EINVAL;
12446 	} else {
12447 		if (attr.sample_period & (1ULL << 63))
12448 			return -EINVAL;
12449 	}
12450 
12451 	/* Only privileged users can get physical addresses */
12452 	if ((attr.sample_type & PERF_SAMPLE_PHYS_ADDR)) {
12453 		err = perf_allow_kernel(&attr);
12454 		if (err)
12455 			return err;
12456 	}
12457 
12458 	/* REGS_INTR can leak data, lockdown must prevent this */
12459 	if (attr.sample_type & PERF_SAMPLE_REGS_INTR) {
12460 		err = security_locked_down(LOCKDOWN_PERF);
12461 		if (err)
12462 			return err;
12463 	}
12464 
12465 	/*
12466 	 * In cgroup mode, the pid argument is used to pass the fd
12467 	 * opened to the cgroup directory in cgroupfs. The cpu argument
12468 	 * designates the cpu on which to monitor threads from that
12469 	 * cgroup.
12470 	 */
12471 	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
12472 		return -EINVAL;
12473 
12474 	if (flags & PERF_FLAG_FD_CLOEXEC)
12475 		f_flags |= O_CLOEXEC;
12476 
12477 	event_fd = get_unused_fd_flags(f_flags);
12478 	if (event_fd < 0)
12479 		return event_fd;
12480 
12481 	if (group_fd != -1) {
12482 		err = perf_fget_light(group_fd, &group);
12483 		if (err)
12484 			goto err_fd;
12485 		group_leader = group.file->private_data;
12486 		if (flags & PERF_FLAG_FD_OUTPUT)
12487 			output_event = group_leader;
12488 		if (flags & PERF_FLAG_FD_NO_GROUP)
12489 			group_leader = NULL;
12490 	}
12491 
12492 	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
12493 		task = find_lively_task_by_vpid(pid);
12494 		if (IS_ERR(task)) {
12495 			err = PTR_ERR(task);
12496 			goto err_group_fd;
12497 		}
12498 	}
12499 
12500 	if (task && group_leader &&
12501 	    group_leader->attr.inherit != attr.inherit) {
12502 		err = -EINVAL;
12503 		goto err_task;
12504 	}
12505 
12506 	if (flags & PERF_FLAG_PID_CGROUP)
12507 		cgroup_fd = pid;
12508 
12509 	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
12510 				 NULL, NULL, cgroup_fd);
12511 	if (IS_ERR(event)) {
12512 		err = PTR_ERR(event);
12513 		goto err_task;
12514 	}
12515 
12516 	if (is_sampling_event(event)) {
12517 		if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {
12518 			err = -EOPNOTSUPP;
12519 			goto err_alloc;
12520 		}
12521 	}
12522 
12523 	/*
12524 	 * Special case software events and allow them to be part of
12525 	 * any hardware group.
12526 	 */
12527 	pmu = event->pmu;
12528 
12529 	if (attr.use_clockid) {
12530 		err = perf_event_set_clock(event, attr.clockid);
12531 		if (err)
12532 			goto err_alloc;
12533 	}
12534 
12535 	if (pmu->task_ctx_nr == perf_sw_context)
12536 		event->event_caps |= PERF_EV_CAP_SOFTWARE;
12537 
12538 	if (task) {
12539 		err = down_read_interruptible(&task->signal->exec_update_lock);
12540 		if (err)
12541 			goto err_alloc;
12542 
12543 		/*
12544 		 * We must hold exec_update_lock across this and any potential
12545 		 * perf_install_in_context() call for this new event to
12546 		 * serialize against exec() altering our credentials (and the
12547 		 * perf_event_exit_task() that could imply).
12548 		 */
12549 		err = -EACCES;
12550 		if (!perf_check_permission(&attr, task))
12551 			goto err_cred;
12552 	}
12553 
12554 	/*
12555 	 * Get the target context (task or percpu):
12556 	 */
12557 	ctx = find_get_context(task, event);
12558 	if (IS_ERR(ctx)) {
12559 		err = PTR_ERR(ctx);
12560 		goto err_cred;
12561 	}
12562 
12563 	mutex_lock(&ctx->mutex);
12564 
12565 	if (ctx->task == TASK_TOMBSTONE) {
12566 		err = -ESRCH;
12567 		goto err_locked;
12568 	}
12569 
12570 	if (!task) {
12571 		/*
12572 		 * Check if the @cpu we're creating an event for is online.
12573 		 *
12574 		 * We use the perf_cpu_context::ctx::mutex to serialize against
12575 		 * the hotplug notifiers. See perf_event_{init,exit}_cpu().
12576 		 */
12577 		struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);
12578 
12579 		if (!cpuctx->online) {
12580 			err = -ENODEV;
12581 			goto err_locked;
12582 		}
12583 	}
12584 
12585 	if (group_leader) {
12586 		err = -EINVAL;
12587 
12588 		/*
12589 		 * Do not allow a recursive hierarchy (this new sibling
12590 		 * becoming part of another group-sibling):
12591 		 */
12592 		if (group_leader->group_leader != group_leader)
12593 			goto err_locked;
12594 
12595 		/* All events in a group should have the same clock */
12596 		if (group_leader->clock != event->clock)
12597 			goto err_locked;
12598 
12599 		/*
12600 		 * Make sure we're both events for the same CPU;
12601 		 * grouping events for different CPUs is broken; since
12602 		 * you can never concurrently schedule them anyhow.
12603 		 */
12604 		if (group_leader->cpu != event->cpu)
12605 			goto err_locked;
12606 
12607 		/*
12608 		 * Make sure we're both on the same context; either task or cpu.
12609 		 */
12610 		if (group_leader->ctx != ctx)
12611 			goto err_locked;
12612 
12613 		/*
12614 		 * Only a group leader can be exclusive or pinned
12615 		 */
12616 		if (attr.exclusive || attr.pinned)
12617 			goto err_locked;
12618 
12619 		if (is_software_event(event) &&
12620 		    !in_software_context(group_leader)) {
12621 			/*
12622 			 * If the event is a sw event, but the group_leader
12623 			 * is on hw context.
12624 			 *
12625 			 * Allow the addition of software events to hw
12626 			 * groups, this is safe because software events
12627 			 * never fail to schedule.
12628 			 *
12629 			 * Note the comment that goes with struct
12630 			 * perf_event_pmu_context.
12631 			 */
12632 			pmu = group_leader->pmu_ctx->pmu;
12633 		} else if (!is_software_event(event)) {
12634 			if (is_software_event(group_leader) &&
12635 			    (group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) {
12636 				/*
12637 				 * In case the group is a pure software group, and we
12638 				 * try to add a hardware event, move the whole group to
12639 				 * the hardware context.
12640 				 */
12641 				move_group = 1;
12642 			}
12643 
12644 			/* Don't allow group of multiple hw events from different pmus */
12645 			if (!in_software_context(group_leader) &&
12646 			    group_leader->pmu_ctx->pmu != pmu)
12647 				goto err_locked;
12648 		}
12649 	}
12650 
12651 	/*
12652 	 * Now that we're certain of the pmu; find the pmu_ctx.
12653 	 */
12654 	pmu_ctx = find_get_pmu_context(pmu, ctx, event);
12655 	if (IS_ERR(pmu_ctx)) {
12656 		err = PTR_ERR(pmu_ctx);
12657 		goto err_locked;
12658 	}
12659 	event->pmu_ctx = pmu_ctx;
12660 
12661 	if (output_event) {
12662 		err = perf_event_set_output(event, output_event);
12663 		if (err)
12664 			goto err_context;
12665 	}
12666 
12667 	if (!perf_event_validate_size(event)) {
12668 		err = -E2BIG;
12669 		goto err_context;
12670 	}
12671 
12672 	if (perf_need_aux_event(event) && !perf_get_aux_event(event, group_leader)) {
12673 		err = -EINVAL;
12674 		goto err_context;
12675 	}
12676 
12677 	/*
12678 	 * Must be under the same ctx::mutex as perf_install_in_context(),
12679 	 * because we need to serialize with concurrent event creation.
12680 	 */
12681 	if (!exclusive_event_installable(event, ctx)) {
12682 		err = -EBUSY;
12683 		goto err_context;
12684 	}
12685 
12686 	WARN_ON_ONCE(ctx->parent_ctx);
12687 
12688 	event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, f_flags);
12689 	if (IS_ERR(event_file)) {
12690 		err = PTR_ERR(event_file);
12691 		event_file = NULL;
12692 		goto err_context;
12693 	}
12694 
12695 	/*
12696 	 * This is the point on no return; we cannot fail hereafter. This is
12697 	 * where we start modifying current state.
12698 	 */
12699 
12700 	if (move_group) {
12701 		perf_remove_from_context(group_leader, 0);
12702 		put_pmu_ctx(group_leader->pmu_ctx);
12703 
12704 		for_each_sibling_event(sibling, group_leader) {
12705 			perf_remove_from_context(sibling, 0);
12706 			put_pmu_ctx(sibling->pmu_ctx);
12707 		}
12708 
12709 		/*
12710 		 * Install the group siblings before the group leader.
12711 		 *
12712 		 * Because a group leader will try and install the entire group
12713 		 * (through the sibling list, which is still in-tact), we can
12714 		 * end up with siblings installed in the wrong context.
12715 		 *
12716 		 * By installing siblings first we NO-OP because they're not
12717 		 * reachable through the group lists.
12718 		 */
12719 		for_each_sibling_event(sibling, group_leader) {
12720 			sibling->pmu_ctx = pmu_ctx;
12721 			get_pmu_ctx(pmu_ctx);
12722 			perf_event__state_init(sibling);
12723 			perf_install_in_context(ctx, sibling, sibling->cpu);
12724 		}
12725 
12726 		/*
12727 		 * Removing from the context ends up with disabled
12728 		 * event. What we want here is event in the initial
12729 		 * startup state, ready to be add into new context.
12730 		 */
12731 		group_leader->pmu_ctx = pmu_ctx;
12732 		get_pmu_ctx(pmu_ctx);
12733 		perf_event__state_init(group_leader);
12734 		perf_install_in_context(ctx, group_leader, group_leader->cpu);
12735 	}
12736 
12737 	/*
12738 	 * Precalculate sample_data sizes; do while holding ctx::mutex such
12739 	 * that we're serialized against further additions and before
12740 	 * perf_install_in_context() which is the point the event is active and
12741 	 * can use these values.
12742 	 */
12743 	perf_event__header_size(event);
12744 	perf_event__id_header_size(event);
12745 
12746 	event->owner = current;
12747 
12748 	perf_install_in_context(ctx, event, event->cpu);
12749 	perf_unpin_context(ctx);
12750 
12751 	mutex_unlock(&ctx->mutex);
12752 
12753 	if (task) {
12754 		up_read(&task->signal->exec_update_lock);
12755 		put_task_struct(task);
12756 	}
12757 
12758 	mutex_lock(&current->perf_event_mutex);
12759 	list_add_tail(&event->owner_entry, &current->perf_event_list);
12760 	mutex_unlock(&current->perf_event_mutex);
12761 
12762 	/*
12763 	 * Drop the reference on the group_event after placing the
12764 	 * new event on the sibling_list. This ensures destruction
12765 	 * of the group leader will find the pointer to itself in
12766 	 * perf_group_detach().
12767 	 */
12768 	fdput(group);
12769 	fd_install(event_fd, event_file);
12770 	return event_fd;
12771 
12772 err_context:
12773 	put_pmu_ctx(event->pmu_ctx);
12774 	event->pmu_ctx = NULL; /* _free_event() */
12775 err_locked:
12776 	mutex_unlock(&ctx->mutex);
12777 	perf_unpin_context(ctx);
12778 	put_ctx(ctx);
12779 err_cred:
12780 	if (task)
12781 		up_read(&task->signal->exec_update_lock);
12782 err_alloc:
12783 	free_event(event);
12784 err_task:
12785 	if (task)
12786 		put_task_struct(task);
12787 err_group_fd:
12788 	fdput(group);
12789 err_fd:
12790 	put_unused_fd(event_fd);
12791 	return err;
12792 }
12793 
12794 /**
12795  * perf_event_create_kernel_counter
12796  *
12797  * @attr: attributes of the counter to create
12798  * @cpu: cpu in which the counter is bound
12799  * @task: task to profile (NULL for percpu)
12800  * @overflow_handler: callback to trigger when we hit the event
12801  * @context: context data could be used in overflow_handler callback
12802  */
12803 struct perf_event *
perf_event_create_kernel_counter(struct perf_event_attr * attr,int cpu,struct task_struct * task,perf_overflow_handler_t overflow_handler,void * context)12804 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
12805 				 struct task_struct *task,
12806 				 perf_overflow_handler_t overflow_handler,
12807 				 void *context)
12808 {
12809 	struct perf_event_pmu_context *pmu_ctx;
12810 	struct perf_event_context *ctx;
12811 	struct perf_event *event;
12812 	struct pmu *pmu;
12813 	int err;
12814 
12815 	/*
12816 	 * Grouping is not supported for kernel events, neither is 'AUX',
12817 	 * make sure the caller's intentions are adjusted.
12818 	 */
12819 	if (attr->aux_output)
12820 		return ERR_PTR(-EINVAL);
12821 
12822 	event = perf_event_alloc(attr, cpu, task, NULL, NULL,
12823 				 overflow_handler, context, -1);
12824 	if (IS_ERR(event)) {
12825 		err = PTR_ERR(event);
12826 		goto err;
12827 	}
12828 
12829 	/* Mark owner so we could distinguish it from user events. */
12830 	event->owner = TASK_TOMBSTONE;
12831 	pmu = event->pmu;
12832 
12833 	if (pmu->task_ctx_nr == perf_sw_context)
12834 		event->event_caps |= PERF_EV_CAP_SOFTWARE;
12835 
12836 	/*
12837 	 * Get the target context (task or percpu):
12838 	 */
12839 	ctx = find_get_context(task, event);
12840 	if (IS_ERR(ctx)) {
12841 		err = PTR_ERR(ctx);
12842 		goto err_alloc;
12843 	}
12844 
12845 	WARN_ON_ONCE(ctx->parent_ctx);
12846 	mutex_lock(&ctx->mutex);
12847 	if (ctx->task == TASK_TOMBSTONE) {
12848 		err = -ESRCH;
12849 		goto err_unlock;
12850 	}
12851 
12852 	pmu_ctx = find_get_pmu_context(pmu, ctx, event);
12853 	if (IS_ERR(pmu_ctx)) {
12854 		err = PTR_ERR(pmu_ctx);
12855 		goto err_unlock;
12856 	}
12857 	event->pmu_ctx = pmu_ctx;
12858 
12859 	if (!task) {
12860 		/*
12861 		 * Check if the @cpu we're creating an event for is online.
12862 		 *
12863 		 * We use the perf_cpu_context::ctx::mutex to serialize against
12864 		 * the hotplug notifiers. See perf_event_{init,exit}_cpu().
12865 		 */
12866 		struct perf_cpu_context *cpuctx =
12867 			container_of(ctx, struct perf_cpu_context, ctx);
12868 		if (!cpuctx->online) {
12869 			err = -ENODEV;
12870 			goto err_pmu_ctx;
12871 		}
12872 	}
12873 
12874 	if (!exclusive_event_installable(event, ctx)) {
12875 		err = -EBUSY;
12876 		goto err_pmu_ctx;
12877 	}
12878 
12879 	perf_install_in_context(ctx, event, event->cpu);
12880 	perf_unpin_context(ctx);
12881 	mutex_unlock(&ctx->mutex);
12882 
12883 	return event;
12884 
12885 err_pmu_ctx:
12886 	put_pmu_ctx(pmu_ctx);
12887 	event->pmu_ctx = NULL; /* _free_event() */
12888 err_unlock:
12889 	mutex_unlock(&ctx->mutex);
12890 	perf_unpin_context(ctx);
12891 	put_ctx(ctx);
12892 err_alloc:
12893 	free_event(event);
12894 err:
12895 	return ERR_PTR(err);
12896 }
12897 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
12898 
__perf_pmu_remove(struct perf_event_context * ctx,int cpu,struct pmu * pmu,struct perf_event_groups * groups,struct list_head * events)12899 static void __perf_pmu_remove(struct perf_event_context *ctx,
12900 			      int cpu, struct pmu *pmu,
12901 			      struct perf_event_groups *groups,
12902 			      struct list_head *events)
12903 {
12904 	struct perf_event *event, *sibling;
12905 
12906 	perf_event_groups_for_cpu_pmu(event, groups, cpu, pmu) {
12907 		perf_remove_from_context(event, 0);
12908 		put_pmu_ctx(event->pmu_ctx);
12909 		list_add(&event->migrate_entry, events);
12910 
12911 		for_each_sibling_event(sibling, event) {
12912 			perf_remove_from_context(sibling, 0);
12913 			put_pmu_ctx(sibling->pmu_ctx);
12914 			list_add(&sibling->migrate_entry, events);
12915 		}
12916 	}
12917 }
12918 
__perf_pmu_install_event(struct pmu * pmu,struct perf_event_context * ctx,int cpu,struct perf_event * event)12919 static void __perf_pmu_install_event(struct pmu *pmu,
12920 				     struct perf_event_context *ctx,
12921 				     int cpu, struct perf_event *event)
12922 {
12923 	struct perf_event_pmu_context *epc;
12924 	struct perf_event_context *old_ctx = event->ctx;
12925 
12926 	get_ctx(ctx); /* normally find_get_context() */
12927 
12928 	event->cpu = cpu;
12929 	epc = find_get_pmu_context(pmu, ctx, event);
12930 	event->pmu_ctx = epc;
12931 
12932 	if (event->state >= PERF_EVENT_STATE_OFF)
12933 		event->state = PERF_EVENT_STATE_INACTIVE;
12934 	perf_install_in_context(ctx, event, cpu);
12935 
12936 	/*
12937 	 * Now that event->ctx is updated and visible, put the old ctx.
12938 	 */
12939 	put_ctx(old_ctx);
12940 }
12941 
__perf_pmu_install(struct perf_event_context * ctx,int cpu,struct pmu * pmu,struct list_head * events)12942 static void __perf_pmu_install(struct perf_event_context *ctx,
12943 			       int cpu, struct pmu *pmu, struct list_head *events)
12944 {
12945 	struct perf_event *event, *tmp;
12946 
12947 	/*
12948 	 * Re-instate events in 2 passes.
12949 	 *
12950 	 * Skip over group leaders and only install siblings on this first
12951 	 * pass, siblings will not get enabled without a leader, however a
12952 	 * leader will enable its siblings, even if those are still on the old
12953 	 * context.
12954 	 */
12955 	list_for_each_entry_safe(event, tmp, events, migrate_entry) {
12956 		if (event->group_leader == event)
12957 			continue;
12958 
12959 		list_del(&event->migrate_entry);
12960 		__perf_pmu_install_event(pmu, ctx, cpu, event);
12961 	}
12962 
12963 	/*
12964 	 * Once all the siblings are setup properly, install the group leaders
12965 	 * to make it go.
12966 	 */
12967 	list_for_each_entry_safe(event, tmp, events, migrate_entry) {
12968 		list_del(&event->migrate_entry);
12969 		__perf_pmu_install_event(pmu, ctx, cpu, event);
12970 	}
12971 }
12972 
perf_pmu_migrate_context(struct pmu * pmu,int src_cpu,int dst_cpu)12973 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
12974 {
12975 	struct perf_event_context *src_ctx, *dst_ctx;
12976 	LIST_HEAD(events);
12977 
12978 	/*
12979 	 * Since per-cpu context is persistent, no need to grab an extra
12980 	 * reference.
12981 	 */
12982 	src_ctx = &per_cpu_ptr(&perf_cpu_context, src_cpu)->ctx;
12983 	dst_ctx = &per_cpu_ptr(&perf_cpu_context, dst_cpu)->ctx;
12984 
12985 	/*
12986 	 * See perf_event_ctx_lock() for comments on the details
12987 	 * of swizzling perf_event::ctx.
12988 	 */
12989 	mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex);
12990 
12991 	__perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->pinned_groups, &events);
12992 	__perf_pmu_remove(src_ctx, src_cpu, pmu, &src_ctx->flexible_groups, &events);
12993 
12994 	if (!list_empty(&events)) {
12995 		/*
12996 		 * Wait for the events to quiesce before re-instating them.
12997 		 */
12998 		synchronize_rcu();
12999 
13000 		__perf_pmu_install(dst_ctx, dst_cpu, pmu, &events);
13001 	}
13002 
13003 	mutex_unlock(&dst_ctx->mutex);
13004 	mutex_unlock(&src_ctx->mutex);
13005 }
13006 EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
13007 
sync_child_event(struct perf_event * child_event)13008 static void sync_child_event(struct perf_event *child_event)
13009 {
13010 	struct perf_event *parent_event = child_event->parent;
13011 	u64 child_val;
13012 
13013 	if (child_event->attr.inherit_stat) {
13014 		struct task_struct *task = child_event->ctx->task;
13015 
13016 		if (task && task != TASK_TOMBSTONE)
13017 			perf_event_read_event(child_event, task);
13018 	}
13019 
13020 	child_val = perf_event_count(child_event);
13021 
13022 	/*
13023 	 * Add back the child's count to the parent's count:
13024 	 */
13025 	atomic64_add(child_val, &parent_event->child_count);
13026 	atomic64_add(child_event->total_time_enabled,
13027 		     &parent_event->child_total_time_enabled);
13028 	atomic64_add(child_event->total_time_running,
13029 		     &parent_event->child_total_time_running);
13030 }
13031 
13032 static void
perf_event_exit_event(struct perf_event * event,struct perf_event_context * ctx)13033 perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx)
13034 {
13035 	struct perf_event *parent_event = event->parent;
13036 	unsigned long detach_flags = 0;
13037 
13038 	if (parent_event) {
13039 		/*
13040 		 * Do not destroy the 'original' grouping; because of the
13041 		 * context switch optimization the original events could've
13042 		 * ended up in a random child task.
13043 		 *
13044 		 * If we were to destroy the original group, all group related
13045 		 * operations would cease to function properly after this
13046 		 * random child dies.
13047 		 *
13048 		 * Do destroy all inherited groups, we don't care about those
13049 		 * and being thorough is better.
13050 		 */
13051 		detach_flags = DETACH_GROUP | DETACH_CHILD;
13052 		mutex_lock(&parent_event->child_mutex);
13053 	}
13054 
13055 	perf_remove_from_context(event, detach_flags);
13056 
13057 	raw_spin_lock_irq(&ctx->lock);
13058 	if (event->state > PERF_EVENT_STATE_EXIT)
13059 		perf_event_set_state(event, PERF_EVENT_STATE_EXIT);
13060 	raw_spin_unlock_irq(&ctx->lock);
13061 
13062 	/*
13063 	 * Child events can be freed.
13064 	 */
13065 	if (parent_event) {
13066 		mutex_unlock(&parent_event->child_mutex);
13067 		/*
13068 		 * Kick perf_poll() for is_event_hup();
13069 		 */
13070 		perf_event_wakeup(parent_event);
13071 		free_event(event);
13072 		put_event(parent_event);
13073 		return;
13074 	}
13075 
13076 	/*
13077 	 * Parent events are governed by their filedesc, retain them.
13078 	 */
13079 	perf_event_wakeup(event);
13080 }
13081 
perf_event_exit_task_context(struct task_struct * child)13082 static void perf_event_exit_task_context(struct task_struct *child)
13083 {
13084 	struct perf_event_context *child_ctx, *clone_ctx = NULL;
13085 	struct perf_event *child_event, *next;
13086 
13087 	WARN_ON_ONCE(child != current);
13088 
13089 	child_ctx = perf_pin_task_context(child);
13090 	if (!child_ctx)
13091 		return;
13092 
13093 	/*
13094 	 * In order to reduce the amount of tricky in ctx tear-down, we hold
13095 	 * ctx::mutex over the entire thing. This serializes against almost
13096 	 * everything that wants to access the ctx.
13097 	 *
13098 	 * The exception is sys_perf_event_open() /
13099 	 * perf_event_create_kernel_count() which does find_get_context()
13100 	 * without ctx::mutex (it cannot because of the move_group double mutex
13101 	 * lock thing). See the comments in perf_install_in_context().
13102 	 */
13103 	mutex_lock(&child_ctx->mutex);
13104 
13105 	/*
13106 	 * In a single ctx::lock section, de-schedule the events and detach the
13107 	 * context from the task such that we cannot ever get it scheduled back
13108 	 * in.
13109 	 */
13110 	raw_spin_lock_irq(&child_ctx->lock);
13111 	task_ctx_sched_out(child_ctx, EVENT_ALL);
13112 
13113 	/*
13114 	 * Now that the context is inactive, destroy the task <-> ctx relation
13115 	 * and mark the context dead.
13116 	 */
13117 	RCU_INIT_POINTER(child->perf_event_ctxp, NULL);
13118 	put_ctx(child_ctx); /* cannot be last */
13119 	WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE);
13120 	put_task_struct(current); /* cannot be last */
13121 
13122 	clone_ctx = unclone_ctx(child_ctx);
13123 	raw_spin_unlock_irq(&child_ctx->lock);
13124 
13125 	if (clone_ctx)
13126 		put_ctx(clone_ctx);
13127 
13128 	/*
13129 	 * Report the task dead after unscheduling the events so that we
13130 	 * won't get any samples after PERF_RECORD_EXIT. We can however still
13131 	 * get a few PERF_RECORD_READ events.
13132 	 */
13133 	perf_event_task(child, child_ctx, 0);
13134 
13135 	list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)
13136 		perf_event_exit_event(child_event, child_ctx);
13137 
13138 	mutex_unlock(&child_ctx->mutex);
13139 
13140 	put_ctx(child_ctx);
13141 }
13142 
13143 /*
13144  * When a child task exits, feed back event values to parent events.
13145  *
13146  * Can be called with exec_update_lock held when called from
13147  * setup_new_exec().
13148  */
perf_event_exit_task(struct task_struct * child)13149 void perf_event_exit_task(struct task_struct *child)
13150 {
13151 	struct perf_event *event, *tmp;
13152 
13153 	mutex_lock(&child->perf_event_mutex);
13154 	list_for_each_entry_safe(event, tmp, &child->perf_event_list,
13155 				 owner_entry) {
13156 		list_del_init(&event->owner_entry);
13157 
13158 		/*
13159 		 * Ensure the list deletion is visible before we clear
13160 		 * the owner, closes a race against perf_release() where
13161 		 * we need to serialize on the owner->perf_event_mutex.
13162 		 */
13163 		smp_store_release(&event->owner, NULL);
13164 	}
13165 	mutex_unlock(&child->perf_event_mutex);
13166 
13167 	perf_event_exit_task_context(child);
13168 
13169 	/*
13170 	 * The perf_event_exit_task_context calls perf_event_task
13171 	 * with child's task_ctx, which generates EXIT events for
13172 	 * child contexts and sets child->perf_event_ctxp[] to NULL.
13173 	 * At this point we need to send EXIT events to cpu contexts.
13174 	 */
13175 	perf_event_task(child, NULL, 0);
13176 }
13177 
perf_free_event(struct perf_event * event,struct perf_event_context * ctx)13178 static void perf_free_event(struct perf_event *event,
13179 			    struct perf_event_context *ctx)
13180 {
13181 	struct perf_event *parent = event->parent;
13182 
13183 	if (WARN_ON_ONCE(!parent))
13184 		return;
13185 
13186 	mutex_lock(&parent->child_mutex);
13187 	list_del_init(&event->child_list);
13188 	mutex_unlock(&parent->child_mutex);
13189 
13190 	put_event(parent);
13191 
13192 	raw_spin_lock_irq(&ctx->lock);
13193 	perf_group_detach(event);
13194 	list_del_event(event, ctx);
13195 	raw_spin_unlock_irq(&ctx->lock);
13196 	free_event(event);
13197 }
13198 
13199 /*
13200  * Free a context as created by inheritance by perf_event_init_task() below,
13201  * used by fork() in case of fail.
13202  *
13203  * Even though the task has never lived, the context and events have been
13204  * exposed through the child_list, so we must take care tearing it all down.
13205  */
perf_event_free_task(struct task_struct * task)13206 void perf_event_free_task(struct task_struct *task)
13207 {
13208 	struct perf_event_context *ctx;
13209 	struct perf_event *event, *tmp;
13210 
13211 	ctx = rcu_access_pointer(task->perf_event_ctxp);
13212 	if (!ctx)
13213 		return;
13214 
13215 	mutex_lock(&ctx->mutex);
13216 	raw_spin_lock_irq(&ctx->lock);
13217 	/*
13218 	 * Destroy the task <-> ctx relation and mark the context dead.
13219 	 *
13220 	 * This is important because even though the task hasn't been
13221 	 * exposed yet the context has been (through child_list).
13222 	 */
13223 	RCU_INIT_POINTER(task->perf_event_ctxp, NULL);
13224 	WRITE_ONCE(ctx->task, TASK_TOMBSTONE);
13225 	put_task_struct(task); /* cannot be last */
13226 	raw_spin_unlock_irq(&ctx->lock);
13227 
13228 
13229 	list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry)
13230 		perf_free_event(event, ctx);
13231 
13232 	mutex_unlock(&ctx->mutex);
13233 
13234 	/*
13235 	 * perf_event_release_kernel() could've stolen some of our
13236 	 * child events and still have them on its free_list. In that
13237 	 * case we must wait for these events to have been freed (in
13238 	 * particular all their references to this task must've been
13239 	 * dropped).
13240 	 *
13241 	 * Without this copy_process() will unconditionally free this
13242 	 * task (irrespective of its reference count) and
13243 	 * _free_event()'s put_task_struct(event->hw.target) will be a
13244 	 * use-after-free.
13245 	 *
13246 	 * Wait for all events to drop their context reference.
13247 	 */
13248 	wait_var_event(&ctx->refcount, refcount_read(&ctx->refcount) == 1);
13249 	put_ctx(ctx); /* must be last */
13250 }
13251 
perf_event_delayed_put(struct task_struct * task)13252 void perf_event_delayed_put(struct task_struct *task)
13253 {
13254 	WARN_ON_ONCE(task->perf_event_ctxp);
13255 }
13256 
perf_event_get(unsigned int fd)13257 struct file *perf_event_get(unsigned int fd)
13258 {
13259 	struct file *file = fget(fd);
13260 	if (!file)
13261 		return ERR_PTR(-EBADF);
13262 
13263 	if (file->f_op != &perf_fops) {
13264 		fput(file);
13265 		return ERR_PTR(-EBADF);
13266 	}
13267 
13268 	return file;
13269 }
13270 
perf_get_event(struct file * file)13271 const struct perf_event *perf_get_event(struct file *file)
13272 {
13273 	if (file->f_op != &perf_fops)
13274 		return ERR_PTR(-EINVAL);
13275 
13276 	return file->private_data;
13277 }
13278 
perf_event_attrs(struct perf_event * event)13279 const struct perf_event_attr *perf_event_attrs(struct perf_event *event)
13280 {
13281 	if (!event)
13282 		return ERR_PTR(-EINVAL);
13283 
13284 	return &event->attr;
13285 }
13286 
13287 /*
13288  * Inherit an event from parent task to child task.
13289  *
13290  * Returns:
13291  *  - valid pointer on success
13292  *  - NULL for orphaned events
13293  *  - IS_ERR() on error
13294  */
13295 static struct perf_event *
inherit_event(struct perf_event * parent_event,struct task_struct * parent,struct perf_event_context * parent_ctx,struct task_struct * child,struct perf_event * group_leader,struct perf_event_context * child_ctx)13296 inherit_event(struct perf_event *parent_event,
13297 	      struct task_struct *parent,
13298 	      struct perf_event_context *parent_ctx,
13299 	      struct task_struct *child,
13300 	      struct perf_event *group_leader,
13301 	      struct perf_event_context *child_ctx)
13302 {
13303 	enum perf_event_state parent_state = parent_event->state;
13304 	struct perf_event_pmu_context *pmu_ctx;
13305 	struct perf_event *child_event;
13306 	unsigned long flags;
13307 
13308 	/*
13309 	 * Instead of creating recursive hierarchies of events,
13310 	 * we link inherited events back to the original parent,
13311 	 * which has a filp for sure, which we use as the reference
13312 	 * count:
13313 	 */
13314 	if (parent_event->parent)
13315 		parent_event = parent_event->parent;
13316 
13317 	child_event = perf_event_alloc(&parent_event->attr,
13318 					   parent_event->cpu,
13319 					   child,
13320 					   group_leader, parent_event,
13321 					   NULL, NULL, -1);
13322 	if (IS_ERR(child_event))
13323 		return child_event;
13324 
13325 	pmu_ctx = find_get_pmu_context(child_event->pmu, child_ctx, child_event);
13326 	if (IS_ERR(pmu_ctx)) {
13327 		free_event(child_event);
13328 		return ERR_CAST(pmu_ctx);
13329 	}
13330 	child_event->pmu_ctx = pmu_ctx;
13331 
13332 	/*
13333 	 * is_orphaned_event() and list_add_tail(&parent_event->child_list)
13334 	 * must be under the same lock in order to serialize against
13335 	 * perf_event_release_kernel(), such that either we must observe
13336 	 * is_orphaned_event() or they will observe us on the child_list.
13337 	 */
13338 	mutex_lock(&parent_event->child_mutex);
13339 	if (is_orphaned_event(parent_event) ||
13340 	    !atomic_long_inc_not_zero(&parent_event->refcount)) {
13341 		mutex_unlock(&parent_event->child_mutex);
13342 		/* task_ctx_data is freed with child_ctx */
13343 		free_event(child_event);
13344 		return NULL;
13345 	}
13346 
13347 	get_ctx(child_ctx);
13348 
13349 	/*
13350 	 * Make the child state follow the state of the parent event,
13351 	 * not its attr.disabled bit.  We hold the parent's mutex,
13352 	 * so we won't race with perf_event_{en, dis}able_family.
13353 	 */
13354 	if (parent_state >= PERF_EVENT_STATE_INACTIVE)
13355 		child_event->state = PERF_EVENT_STATE_INACTIVE;
13356 	else
13357 		child_event->state = PERF_EVENT_STATE_OFF;
13358 
13359 	if (parent_event->attr.freq) {
13360 		u64 sample_period = parent_event->hw.sample_period;
13361 		struct hw_perf_event *hwc = &child_event->hw;
13362 
13363 		hwc->sample_period = sample_period;
13364 		hwc->last_period   = sample_period;
13365 
13366 		local64_set(&hwc->period_left, sample_period);
13367 	}
13368 
13369 	child_event->ctx = child_ctx;
13370 	child_event->overflow_handler = parent_event->overflow_handler;
13371 	child_event->overflow_handler_context
13372 		= parent_event->overflow_handler_context;
13373 
13374 	/*
13375 	 * Precalculate sample_data sizes
13376 	 */
13377 	perf_event__header_size(child_event);
13378 	perf_event__id_header_size(child_event);
13379 
13380 	/*
13381 	 * Link it up in the child's context:
13382 	 */
13383 	raw_spin_lock_irqsave(&child_ctx->lock, flags);
13384 	add_event_to_ctx(child_event, child_ctx);
13385 	child_event->attach_state |= PERF_ATTACH_CHILD;
13386 	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
13387 
13388 	/*
13389 	 * Link this into the parent event's child list
13390 	 */
13391 	list_add_tail(&child_event->child_list, &parent_event->child_list);
13392 	mutex_unlock(&parent_event->child_mutex);
13393 
13394 	return child_event;
13395 }
13396 
13397 /*
13398  * Inherits an event group.
13399  *
13400  * This will quietly suppress orphaned events; !inherit_event() is not an error.
13401  * This matches with perf_event_release_kernel() removing all child events.
13402  *
13403  * Returns:
13404  *  - 0 on success
13405  *  - <0 on error
13406  */
inherit_group(struct perf_event * parent_event,struct task_struct * parent,struct perf_event_context * parent_ctx,struct task_struct * child,struct perf_event_context * child_ctx)13407 static int inherit_group(struct perf_event *parent_event,
13408 	      struct task_struct *parent,
13409 	      struct perf_event_context *parent_ctx,
13410 	      struct task_struct *child,
13411 	      struct perf_event_context *child_ctx)
13412 {
13413 	struct perf_event *leader;
13414 	struct perf_event *sub;
13415 	struct perf_event *child_ctr;
13416 
13417 	leader = inherit_event(parent_event, parent, parent_ctx,
13418 				 child, NULL, child_ctx);
13419 	if (IS_ERR(leader))
13420 		return PTR_ERR(leader);
13421 	/*
13422 	 * @leader can be NULL here because of is_orphaned_event(). In this
13423 	 * case inherit_event() will create individual events, similar to what
13424 	 * perf_group_detach() would do anyway.
13425 	 */
13426 	for_each_sibling_event(sub, parent_event) {
13427 		child_ctr = inherit_event(sub, parent, parent_ctx,
13428 					    child, leader, child_ctx);
13429 		if (IS_ERR(child_ctr))
13430 			return PTR_ERR(child_ctr);
13431 
13432 		if (sub->aux_event == parent_event && child_ctr &&
13433 		    !perf_get_aux_event(child_ctr, leader))
13434 			return -EINVAL;
13435 	}
13436 	if (leader)
13437 		leader->group_generation = parent_event->group_generation;
13438 	return 0;
13439 }
13440 
13441 /*
13442  * Creates the child task context and tries to inherit the event-group.
13443  *
13444  * Clears @inherited_all on !attr.inherited or error. Note that we'll leave
13445  * inherited_all set when we 'fail' to inherit an orphaned event; this is
13446  * consistent with perf_event_release_kernel() removing all child events.
13447  *
13448  * Returns:
13449  *  - 0 on success
13450  *  - <0 on error
13451  */
13452 static int
inherit_task_group(struct perf_event * event,struct task_struct * parent,struct perf_event_context * parent_ctx,struct task_struct * child,u64 clone_flags,int * inherited_all)13453 inherit_task_group(struct perf_event *event, struct task_struct *parent,
13454 		   struct perf_event_context *parent_ctx,
13455 		   struct task_struct *child,
13456 		   u64 clone_flags, int *inherited_all)
13457 {
13458 	struct perf_event_context *child_ctx;
13459 	int ret;
13460 
13461 	if (!event->attr.inherit ||
13462 	    (event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) ||
13463 	    /* Do not inherit if sigtrap and signal handlers were cleared. */
13464 	    (event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) {
13465 		*inherited_all = 0;
13466 		return 0;
13467 	}
13468 
13469 	child_ctx = child->perf_event_ctxp;
13470 	if (!child_ctx) {
13471 		/*
13472 		 * This is executed from the parent task context, so
13473 		 * inherit events that have been marked for cloning.
13474 		 * First allocate and initialize a context for the
13475 		 * child.
13476 		 */
13477 		child_ctx = alloc_perf_context(child);
13478 		if (!child_ctx)
13479 			return -ENOMEM;
13480 
13481 		child->perf_event_ctxp = child_ctx;
13482 	}
13483 
13484 	ret = inherit_group(event, parent, parent_ctx, child, child_ctx);
13485 	if (ret)
13486 		*inherited_all = 0;
13487 
13488 	return ret;
13489 }
13490 
13491 /*
13492  * Initialize the perf_event context in task_struct
13493  */
perf_event_init_context(struct task_struct * child,u64 clone_flags)13494 static int perf_event_init_context(struct task_struct *child, u64 clone_flags)
13495 {
13496 	struct perf_event_context *child_ctx, *parent_ctx;
13497 	struct perf_event_context *cloned_ctx;
13498 	struct perf_event *event;
13499 	struct task_struct *parent = current;
13500 	int inherited_all = 1;
13501 	unsigned long flags;
13502 	int ret = 0;
13503 
13504 	if (likely(!parent->perf_event_ctxp))
13505 		return 0;
13506 
13507 	/*
13508 	 * If the parent's context is a clone, pin it so it won't get
13509 	 * swapped under us.
13510 	 */
13511 	parent_ctx = perf_pin_task_context(parent);
13512 	if (!parent_ctx)
13513 		return 0;
13514 
13515 	/*
13516 	 * No need to check if parent_ctx != NULL here; since we saw
13517 	 * it non-NULL earlier, the only reason for it to become NULL
13518 	 * is if we exit, and since we're currently in the middle of
13519 	 * a fork we can't be exiting at the same time.
13520 	 */
13521 
13522 	/*
13523 	 * Lock the parent list. No need to lock the child - not PID
13524 	 * hashed yet and not running, so nobody can access it.
13525 	 */
13526 	mutex_lock(&parent_ctx->mutex);
13527 
13528 	/*
13529 	 * We dont have to disable NMIs - we are only looking at
13530 	 * the list, not manipulating it:
13531 	 */
13532 	perf_event_groups_for_each(event, &parent_ctx->pinned_groups) {
13533 		ret = inherit_task_group(event, parent, parent_ctx,
13534 					 child, clone_flags, &inherited_all);
13535 		if (ret)
13536 			goto out_unlock;
13537 	}
13538 
13539 	/*
13540 	 * We can't hold ctx->lock when iterating the ->flexible_group list due
13541 	 * to allocations, but we need to prevent rotation because
13542 	 * rotate_ctx() will change the list from interrupt context.
13543 	 */
13544 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
13545 	parent_ctx->rotate_disable = 1;
13546 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
13547 
13548 	perf_event_groups_for_each(event, &parent_ctx->flexible_groups) {
13549 		ret = inherit_task_group(event, parent, parent_ctx,
13550 					 child, clone_flags, &inherited_all);
13551 		if (ret)
13552 			goto out_unlock;
13553 	}
13554 
13555 	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
13556 	parent_ctx->rotate_disable = 0;
13557 
13558 	child_ctx = child->perf_event_ctxp;
13559 
13560 	if (child_ctx && inherited_all) {
13561 		/*
13562 		 * Mark the child context as a clone of the parent
13563 		 * context, or of whatever the parent is a clone of.
13564 		 *
13565 		 * Note that if the parent is a clone, the holding of
13566 		 * parent_ctx->lock avoids it from being uncloned.
13567 		 */
13568 		cloned_ctx = parent_ctx->parent_ctx;
13569 		if (cloned_ctx) {
13570 			child_ctx->parent_ctx = cloned_ctx;
13571 			child_ctx->parent_gen = parent_ctx->parent_gen;
13572 		} else {
13573 			child_ctx->parent_ctx = parent_ctx;
13574 			child_ctx->parent_gen = parent_ctx->generation;
13575 		}
13576 		get_ctx(child_ctx->parent_ctx);
13577 	}
13578 
13579 	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
13580 out_unlock:
13581 	mutex_unlock(&parent_ctx->mutex);
13582 
13583 	perf_unpin_context(parent_ctx);
13584 	put_ctx(parent_ctx);
13585 
13586 	return ret;
13587 }
13588 
13589 /*
13590  * Initialize the perf_event context in task_struct
13591  */
perf_event_init_task(struct task_struct * child,u64 clone_flags)13592 int perf_event_init_task(struct task_struct *child, u64 clone_flags)
13593 {
13594 	int ret;
13595 
13596 	child->perf_event_ctxp = NULL;
13597 	mutex_init(&child->perf_event_mutex);
13598 	INIT_LIST_HEAD(&child->perf_event_list);
13599 
13600 	ret = perf_event_init_context(child, clone_flags);
13601 	if (ret) {
13602 		perf_event_free_task(child);
13603 		return ret;
13604 	}
13605 
13606 	return 0;
13607 }
13608 
perf_event_init_all_cpus(void)13609 static void __init perf_event_init_all_cpus(void)
13610 {
13611 	struct swevent_htable *swhash;
13612 	struct perf_cpu_context *cpuctx;
13613 	int cpu;
13614 
13615 	zalloc_cpumask_var(&perf_online_mask, GFP_KERNEL);
13616 
13617 	for_each_possible_cpu(cpu) {
13618 		swhash = &per_cpu(swevent_htable, cpu);
13619 		mutex_init(&swhash->hlist_mutex);
13620 
13621 		INIT_LIST_HEAD(&per_cpu(pmu_sb_events.list, cpu));
13622 		raw_spin_lock_init(&per_cpu(pmu_sb_events.lock, cpu));
13623 
13624 		INIT_LIST_HEAD(&per_cpu(sched_cb_list, cpu));
13625 
13626 		cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
13627 		__perf_event_init_context(&cpuctx->ctx);
13628 		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
13629 		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
13630 		cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask);
13631 		cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default);
13632 		cpuctx->heap = cpuctx->heap_default;
13633 	}
13634 }
13635 
perf_swevent_init_cpu(unsigned int cpu)13636 static void perf_swevent_init_cpu(unsigned int cpu)
13637 {
13638 	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
13639 
13640 	mutex_lock(&swhash->hlist_mutex);
13641 	if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) {
13642 		struct swevent_hlist *hlist;
13643 
13644 		hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
13645 		WARN_ON(!hlist);
13646 		rcu_assign_pointer(swhash->swevent_hlist, hlist);
13647 	}
13648 	mutex_unlock(&swhash->hlist_mutex);
13649 }
13650 
13651 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC_CORE
__perf_event_exit_context(void * __info)13652 static void __perf_event_exit_context(void *__info)
13653 {
13654 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
13655 	struct perf_event_context *ctx = __info;
13656 	struct perf_event *event;
13657 
13658 	raw_spin_lock(&ctx->lock);
13659 	ctx_sched_out(ctx, EVENT_TIME);
13660 	list_for_each_entry(event, &ctx->event_list, event_entry)
13661 		__perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP);
13662 	raw_spin_unlock(&ctx->lock);
13663 }
13664 
perf_event_exit_cpu_context(int cpu)13665 static void perf_event_exit_cpu_context(int cpu)
13666 {
13667 	struct perf_cpu_context *cpuctx;
13668 	struct perf_event_context *ctx;
13669 
13670 	// XXX simplify cpuctx->online
13671 	mutex_lock(&pmus_lock);
13672 	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
13673 	ctx = &cpuctx->ctx;
13674 
13675 	mutex_lock(&ctx->mutex);
13676 	smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
13677 	cpuctx->online = 0;
13678 	mutex_unlock(&ctx->mutex);
13679 	cpumask_clear_cpu(cpu, perf_online_mask);
13680 	mutex_unlock(&pmus_lock);
13681 }
13682 #else
13683 
perf_event_exit_cpu_context(int cpu)13684 static void perf_event_exit_cpu_context(int cpu) { }
13685 
13686 #endif
13687 
perf_event_init_cpu(unsigned int cpu)13688 int perf_event_init_cpu(unsigned int cpu)
13689 {
13690 	struct perf_cpu_context *cpuctx;
13691 	struct perf_event_context *ctx;
13692 
13693 	perf_swevent_init_cpu(cpu);
13694 
13695 	mutex_lock(&pmus_lock);
13696 	cpumask_set_cpu(cpu, perf_online_mask);
13697 	cpuctx = per_cpu_ptr(&perf_cpu_context, cpu);
13698 	ctx = &cpuctx->ctx;
13699 
13700 	mutex_lock(&ctx->mutex);
13701 	cpuctx->online = 1;
13702 	mutex_unlock(&ctx->mutex);
13703 	mutex_unlock(&pmus_lock);
13704 
13705 	return 0;
13706 }
13707 
perf_event_exit_cpu(unsigned int cpu)13708 int perf_event_exit_cpu(unsigned int cpu)
13709 {
13710 	perf_event_exit_cpu_context(cpu);
13711 	return 0;
13712 }
13713 
13714 static int
perf_reboot(struct notifier_block * notifier,unsigned long val,void * v)13715 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
13716 {
13717 	int cpu;
13718 
13719 	for_each_online_cpu(cpu)
13720 		perf_event_exit_cpu(cpu);
13721 
13722 	return NOTIFY_OK;
13723 }
13724 
13725 /*
13726  * Run the perf reboot notifier at the very last possible moment so that
13727  * the generic watchdog code runs as long as possible.
13728  */
13729 static struct notifier_block perf_reboot_notifier = {
13730 	.notifier_call = perf_reboot,
13731 	.priority = INT_MIN,
13732 };
13733 
perf_event_init(void)13734 void __init perf_event_init(void)
13735 {
13736 	int ret;
13737 
13738 	idr_init(&pmu_idr);
13739 
13740 	perf_event_init_all_cpus();
13741 	init_srcu_struct(&pmus_srcu);
13742 	perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
13743 	perf_pmu_register(&perf_cpu_clock, "cpu_clock", -1);
13744 	perf_pmu_register(&perf_task_clock, "task_clock", -1);
13745 	perf_tp_register();
13746 	perf_event_init_cpu(smp_processor_id());
13747 	register_reboot_notifier(&perf_reboot_notifier);
13748 
13749 	ret = init_hw_breakpoint();
13750 	WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
13751 
13752 	perf_event_cache = KMEM_CACHE(perf_event, SLAB_PANIC);
13753 
13754 	/*
13755 	 * Build time assertion that we keep the data_head at the intended
13756 	 * location.  IOW, validation we got the __reserved[] size right.
13757 	 */
13758 	BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
13759 		     != 1024);
13760 }
13761 
perf_event_sysfs_show(struct device * dev,struct device_attribute * attr,char * page)13762 ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr,
13763 			      char *page)
13764 {
13765 	struct perf_pmu_events_attr *pmu_attr =
13766 		container_of(attr, struct perf_pmu_events_attr, attr);
13767 
13768 	if (pmu_attr->event_str)
13769 		return sprintf(page, "%s\n", pmu_attr->event_str);
13770 
13771 	return 0;
13772 }
13773 EXPORT_SYMBOL_GPL(perf_event_sysfs_show);
13774 
perf_event_sysfs_init(void)13775 static int __init perf_event_sysfs_init(void)
13776 {
13777 	struct pmu *pmu;
13778 	int ret;
13779 
13780 	mutex_lock(&pmus_lock);
13781 
13782 	ret = bus_register(&pmu_bus);
13783 	if (ret)
13784 		goto unlock;
13785 
13786 	list_for_each_entry(pmu, &pmus, entry) {
13787 		if (pmu->dev)
13788 			continue;
13789 
13790 		ret = pmu_dev_alloc(pmu);
13791 		WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
13792 	}
13793 	pmu_bus_running = 1;
13794 	ret = 0;
13795 
13796 unlock:
13797 	mutex_unlock(&pmus_lock);
13798 
13799 	return ret;
13800 }
13801 device_initcall(perf_event_sysfs_init);
13802 
13803 #ifdef CONFIG_CGROUP_PERF
13804 static struct cgroup_subsys_state *
perf_cgroup_css_alloc(struct cgroup_subsys_state * parent_css)13805 perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
13806 {
13807 	struct perf_cgroup *jc;
13808 
13809 	jc = kzalloc(sizeof(*jc), GFP_KERNEL);
13810 	if (!jc)
13811 		return ERR_PTR(-ENOMEM);
13812 
13813 	jc->info = alloc_percpu(struct perf_cgroup_info);
13814 	if (!jc->info) {
13815 		kfree(jc);
13816 		return ERR_PTR(-ENOMEM);
13817 	}
13818 
13819 	return &jc->css;
13820 }
13821 
perf_cgroup_css_free(struct cgroup_subsys_state * css)13822 static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
13823 {
13824 	struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
13825 
13826 	free_percpu(jc->info);
13827 	kfree(jc);
13828 }
13829 
perf_cgroup_css_online(struct cgroup_subsys_state * css)13830 static int perf_cgroup_css_online(struct cgroup_subsys_state *css)
13831 {
13832 	perf_event_cgroup(css->cgroup);
13833 	return 0;
13834 }
13835 
__perf_cgroup_move(void * info)13836 static int __perf_cgroup_move(void *info)
13837 {
13838 	struct task_struct *task = info;
13839 
13840 	preempt_disable();
13841 	perf_cgroup_switch(task);
13842 	preempt_enable();
13843 
13844 	return 0;
13845 }
13846 
perf_cgroup_attach(struct cgroup_taskset * tset)13847 static void perf_cgroup_attach(struct cgroup_taskset *tset)
13848 {
13849 	struct task_struct *task;
13850 	struct cgroup_subsys_state *css;
13851 
13852 	cgroup_taskset_for_each(task, css, tset)
13853 		task_function_call(task, __perf_cgroup_move, task);
13854 }
13855 
13856 struct cgroup_subsys perf_event_cgrp_subsys = {
13857 	.css_alloc	= perf_cgroup_css_alloc,
13858 	.css_free	= perf_cgroup_css_free,
13859 	.css_online	= perf_cgroup_css_online,
13860 	.attach		= perf_cgroup_attach,
13861 	/*
13862 	 * Implicitly enable on dfl hierarchy so that perf events can
13863 	 * always be filtered by cgroup2 path as long as perf_event
13864 	 * controller is not mounted on a legacy hierarchy.
13865 	 */
13866 	.implicit_on_dfl = true,
13867 	.threaded	= true,
13868 };
13869 #endif /* CONFIG_CGROUP_PERF */
13870 
13871 DEFINE_STATIC_CALL_RET0(perf_snapshot_branch_stack, perf_snapshot_branch_stack_t);
13872