1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com>
4  *
5  * Parts came from builtin-{top,stat,record}.c, see those files for further
6  * copyright notes.
7  */
8 
9 #include <byteswap.h>
10 #include <errno.h>
11 #include <inttypes.h>
12 #include <linux/bitops.h>
13 #include <api/fs/fs.h>
14 #include <api/fs/tracing_path.h>
15 #include <traceevent/event-parse.h>
16 #include <linux/hw_breakpoint.h>
17 #include <linux/perf_event.h>
18 #include <linux/compiler.h>
19 #include <linux/err.h>
20 #include <linux/zalloc.h>
21 #include <sys/ioctl.h>
22 #include <sys/resource.h>
23 #include <sys/types.h>
24 #include <dirent.h>
25 #include <stdlib.h>
26 #include <perf/evsel.h>
27 #include "asm/bug.h"
28 #include "bpf_counter.h"
29 #include "callchain.h"
30 #include "cgroup.h"
31 #include "counts.h"
32 #include "event.h"
33 #include "evsel.h"
34 #include "util/env.h"
35 #include "util/evsel_config.h"
36 #include "util/evsel_fprintf.h"
37 #include "evlist.h"
38 #include <perf/cpumap.h>
39 #include "thread_map.h"
40 #include "target.h"
41 #include "perf_regs.h"
42 #include "record.h"
43 #include "debug.h"
44 #include "trace-event.h"
45 #include "stat.h"
46 #include "string2.h"
47 #include "memswap.h"
48 #include "util.h"
49 #ifdef HAVE_LIBBPF_SUPPORT
50 #include <bpf/hashmap.h>
51 #else
52 #include "util/hashmap.h"
53 #endif
54 #include "pmu-hybrid.h"
55 #include "off_cpu.h"
56 #include "../perf-sys.h"
57 #include "util/parse-branch-options.h"
58 #include <internal/xyarray.h>
59 #include <internal/lib.h>
60 
61 #include <linux/ctype.h>
62 
63 struct perf_missing_features perf_missing_features;
64 
65 static clockid_t clockid;
66 
67 static const char *const perf_tool_event__tool_names[PERF_TOOL_MAX] = {
68 	NULL,
69 	"duration_time",
70 	"user_time",
71 	"system_time",
72 };
73 
perf_tool_event__to_str(enum perf_tool_event ev)74 const char *perf_tool_event__to_str(enum perf_tool_event ev)
75 {
76 	if (ev > PERF_TOOL_NONE && ev < PERF_TOOL_MAX)
77 		return perf_tool_event__tool_names[ev];
78 
79 	return NULL;
80 }
81 
perf_tool_event__from_str(const char * str)82 enum perf_tool_event perf_tool_event__from_str(const char *str)
83 {
84 	int i;
85 
86 	perf_tool_event__for_each_event(i) {
87 		if (!strcmp(str, perf_tool_event__tool_names[i]))
88 			return i;
89 	}
90 	return PERF_TOOL_NONE;
91 }
92 
93 
evsel__no_extra_init(struct evsel * evsel __maybe_unused)94 static int evsel__no_extra_init(struct evsel *evsel __maybe_unused)
95 {
96 	return 0;
97 }
98 
test_attr__ready(void)99 void __weak test_attr__ready(void) { }
100 
evsel__no_extra_fini(struct evsel * evsel __maybe_unused)101 static void evsel__no_extra_fini(struct evsel *evsel __maybe_unused)
102 {
103 }
104 
105 static struct {
106 	size_t	size;
107 	int	(*init)(struct evsel *evsel);
108 	void	(*fini)(struct evsel *evsel);
109 } perf_evsel__object = {
110 	.size = sizeof(struct evsel),
111 	.init = evsel__no_extra_init,
112 	.fini = evsel__no_extra_fini,
113 };
114 
evsel__object_config(size_t object_size,int (* init)(struct evsel * evsel),void (* fini)(struct evsel * evsel))115 int evsel__object_config(size_t object_size, int (*init)(struct evsel *evsel),
116 			 void (*fini)(struct evsel *evsel))
117 {
118 
119 	if (object_size == 0)
120 		goto set_methods;
121 
122 	if (perf_evsel__object.size > object_size)
123 		return -EINVAL;
124 
125 	perf_evsel__object.size = object_size;
126 
127 set_methods:
128 	if (init != NULL)
129 		perf_evsel__object.init = init;
130 
131 	if (fini != NULL)
132 		perf_evsel__object.fini = fini;
133 
134 	return 0;
135 }
136 
137 #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y))
138 
__evsel__sample_size(u64 sample_type)139 int __evsel__sample_size(u64 sample_type)
140 {
141 	u64 mask = sample_type & PERF_SAMPLE_MASK;
142 	int size = 0;
143 	int i;
144 
145 	for (i = 0; i < 64; i++) {
146 		if (mask & (1ULL << i))
147 			size++;
148 	}
149 
150 	size *= sizeof(u64);
151 
152 	return size;
153 }
154 
155 /**
156  * __perf_evsel__calc_id_pos - calculate id_pos.
157  * @sample_type: sample type
158  *
159  * This function returns the position of the event id (PERF_SAMPLE_ID or
160  * PERF_SAMPLE_IDENTIFIER) in a sample event i.e. in the array of struct
161  * perf_record_sample.
162  */
__perf_evsel__calc_id_pos(u64 sample_type)163 static int __perf_evsel__calc_id_pos(u64 sample_type)
164 {
165 	int idx = 0;
166 
167 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
168 		return 0;
169 
170 	if (!(sample_type & PERF_SAMPLE_ID))
171 		return -1;
172 
173 	if (sample_type & PERF_SAMPLE_IP)
174 		idx += 1;
175 
176 	if (sample_type & PERF_SAMPLE_TID)
177 		idx += 1;
178 
179 	if (sample_type & PERF_SAMPLE_TIME)
180 		idx += 1;
181 
182 	if (sample_type & PERF_SAMPLE_ADDR)
183 		idx += 1;
184 
185 	return idx;
186 }
187 
188 /**
189  * __perf_evsel__calc_is_pos - calculate is_pos.
190  * @sample_type: sample type
191  *
192  * This function returns the position (counting backwards) of the event id
193  * (PERF_SAMPLE_ID or PERF_SAMPLE_IDENTIFIER) in a non-sample event i.e. if
194  * sample_id_all is used there is an id sample appended to non-sample events.
195  */
__perf_evsel__calc_is_pos(u64 sample_type)196 static int __perf_evsel__calc_is_pos(u64 sample_type)
197 {
198 	int idx = 1;
199 
200 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
201 		return 1;
202 
203 	if (!(sample_type & PERF_SAMPLE_ID))
204 		return -1;
205 
206 	if (sample_type & PERF_SAMPLE_CPU)
207 		idx += 1;
208 
209 	if (sample_type & PERF_SAMPLE_STREAM_ID)
210 		idx += 1;
211 
212 	return idx;
213 }
214 
evsel__calc_id_pos(struct evsel * evsel)215 void evsel__calc_id_pos(struct evsel *evsel)
216 {
217 	evsel->id_pos = __perf_evsel__calc_id_pos(evsel->core.attr.sample_type);
218 	evsel->is_pos = __perf_evsel__calc_is_pos(evsel->core.attr.sample_type);
219 }
220 
__evsel__set_sample_bit(struct evsel * evsel,enum perf_event_sample_format bit)221 void __evsel__set_sample_bit(struct evsel *evsel,
222 				  enum perf_event_sample_format bit)
223 {
224 	if (!(evsel->core.attr.sample_type & bit)) {
225 		evsel->core.attr.sample_type |= bit;
226 		evsel->sample_size += sizeof(u64);
227 		evsel__calc_id_pos(evsel);
228 	}
229 }
230 
__evsel__reset_sample_bit(struct evsel * evsel,enum perf_event_sample_format bit)231 void __evsel__reset_sample_bit(struct evsel *evsel,
232 				    enum perf_event_sample_format bit)
233 {
234 	if (evsel->core.attr.sample_type & bit) {
235 		evsel->core.attr.sample_type &= ~bit;
236 		evsel->sample_size -= sizeof(u64);
237 		evsel__calc_id_pos(evsel);
238 	}
239 }
240 
evsel__set_sample_id(struct evsel * evsel,bool can_sample_identifier)241 void evsel__set_sample_id(struct evsel *evsel,
242 			       bool can_sample_identifier)
243 {
244 	if (can_sample_identifier) {
245 		evsel__reset_sample_bit(evsel, ID);
246 		evsel__set_sample_bit(evsel, IDENTIFIER);
247 	} else {
248 		evsel__set_sample_bit(evsel, ID);
249 	}
250 	evsel->core.attr.read_format |= PERF_FORMAT_ID;
251 }
252 
253 /**
254  * evsel__is_function_event - Return whether given evsel is a function
255  * trace event
256  *
257  * @evsel - evsel selector to be tested
258  *
259  * Return %true if event is function trace event
260  */
evsel__is_function_event(struct evsel * evsel)261 bool evsel__is_function_event(struct evsel *evsel)
262 {
263 #define FUNCTION_EVENT "ftrace:function"
264 
265 	return evsel->name &&
266 	       !strncmp(FUNCTION_EVENT, evsel->name, sizeof(FUNCTION_EVENT));
267 
268 #undef FUNCTION_EVENT
269 }
270 
evsel__init(struct evsel * evsel,struct perf_event_attr * attr,int idx)271 void evsel__init(struct evsel *evsel,
272 		 struct perf_event_attr *attr, int idx)
273 {
274 	perf_evsel__init(&evsel->core, attr, idx);
275 	evsel->tracking	   = !idx;
276 	evsel->unit	   = strdup("");
277 	evsel->scale	   = 1.0;
278 	evsel->max_events  = ULONG_MAX;
279 	evsel->evlist	   = NULL;
280 	evsel->bpf_obj	   = NULL;
281 	evsel->bpf_fd	   = -1;
282 	INIT_LIST_HEAD(&evsel->config_terms);
283 	INIT_LIST_HEAD(&evsel->bpf_counter_list);
284 	perf_evsel__object.init(evsel);
285 	evsel->sample_size = __evsel__sample_size(attr->sample_type);
286 	evsel__calc_id_pos(evsel);
287 	evsel->cmdline_group_boundary = false;
288 	evsel->metric_expr   = NULL;
289 	evsel->metric_name   = NULL;
290 	evsel->metric_events = NULL;
291 	evsel->per_pkg_mask  = NULL;
292 	evsel->collect_stat  = false;
293 	evsel->pmu_name      = NULL;
294 }
295 
evsel__new_idx(struct perf_event_attr * attr,int idx)296 struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx)
297 {
298 	struct evsel *evsel = zalloc(perf_evsel__object.size);
299 
300 	if (!evsel)
301 		return NULL;
302 	evsel__init(evsel, attr, idx);
303 
304 	if (evsel__is_bpf_output(evsel) && !attr->sample_type) {
305 		evsel->core.attr.sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
306 					    PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
307 		evsel->core.attr.sample_period = 1;
308 	}
309 
310 	if (evsel__is_clock(evsel)) {
311 		free((char *)evsel->unit);
312 		evsel->unit = strdup("msec");
313 		evsel->scale = 1e-6;
314 	}
315 
316 	return evsel;
317 }
318 
perf_event_can_profile_kernel(void)319 static bool perf_event_can_profile_kernel(void)
320 {
321 	return perf_event_paranoid_check(1);
322 }
323 
evsel__new_cycles(bool precise __maybe_unused,__u32 type,__u64 config)324 struct evsel *evsel__new_cycles(bool precise __maybe_unused, __u32 type, __u64 config)
325 {
326 	struct perf_event_attr attr = {
327 		.type	= type,
328 		.config	= config,
329 		.exclude_kernel	= !perf_event_can_profile_kernel(),
330 	};
331 	struct evsel *evsel;
332 
333 	event_attr_init(&attr);
334 
335 	/*
336 	 * Now let the usual logic to set up the perf_event_attr defaults
337 	 * to kick in when we return and before perf_evsel__open() is called.
338 	 */
339 	evsel = evsel__new(&attr);
340 	if (evsel == NULL)
341 		goto out;
342 
343 	arch_evsel__fixup_new_cycles(&evsel->core.attr);
344 
345 	evsel->precise_max = true;
346 
347 	/* use asprintf() because free(evsel) assumes name is allocated */
348 	if (asprintf(&evsel->name, "cycles%s%s%.*s",
349 		     (attr.precise_ip || attr.exclude_kernel) ? ":" : "",
350 		     attr.exclude_kernel ? "u" : "",
351 		     attr.precise_ip ? attr.precise_ip + 1 : 0, "ppp") < 0)
352 		goto error_free;
353 out:
354 	return evsel;
355 error_free:
356 	evsel__delete(evsel);
357 	evsel = NULL;
358 	goto out;
359 }
360 
copy_config_terms(struct list_head * dst,struct list_head * src)361 int copy_config_terms(struct list_head *dst, struct list_head *src)
362 {
363 	struct evsel_config_term *pos, *tmp;
364 
365 	list_for_each_entry(pos, src, list) {
366 		tmp = malloc(sizeof(*tmp));
367 		if (tmp == NULL)
368 			return -ENOMEM;
369 
370 		*tmp = *pos;
371 		if (tmp->free_str) {
372 			tmp->val.str = strdup(pos->val.str);
373 			if (tmp->val.str == NULL) {
374 				free(tmp);
375 				return -ENOMEM;
376 			}
377 		}
378 		list_add_tail(&tmp->list, dst);
379 	}
380 	return 0;
381 }
382 
evsel__copy_config_terms(struct evsel * dst,struct evsel * src)383 static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src)
384 {
385 	return copy_config_terms(&dst->config_terms, &src->config_terms);
386 }
387 
388 /**
389  * evsel__clone - create a new evsel copied from @orig
390  * @orig: original evsel
391  *
392  * The assumption is that @orig is not configured nor opened yet.
393  * So we only care about the attributes that can be set while it's parsed.
394  */
evsel__clone(struct evsel * orig)395 struct evsel *evsel__clone(struct evsel *orig)
396 {
397 	struct evsel *evsel;
398 
399 	BUG_ON(orig->core.fd);
400 	BUG_ON(orig->counts);
401 	BUG_ON(orig->priv);
402 	BUG_ON(orig->per_pkg_mask);
403 
404 	/* cannot handle BPF objects for now */
405 	if (orig->bpf_obj)
406 		return NULL;
407 
408 	evsel = evsel__new(&orig->core.attr);
409 	if (evsel == NULL)
410 		return NULL;
411 
412 	evsel->core.cpus = perf_cpu_map__get(orig->core.cpus);
413 	evsel->core.own_cpus = perf_cpu_map__get(orig->core.own_cpus);
414 	evsel->core.threads = perf_thread_map__get(orig->core.threads);
415 	evsel->core.nr_members = orig->core.nr_members;
416 	evsel->core.system_wide = orig->core.system_wide;
417 	evsel->core.requires_cpu = orig->core.requires_cpu;
418 
419 	if (orig->name) {
420 		evsel->name = strdup(orig->name);
421 		if (evsel->name == NULL)
422 			goto out_err;
423 	}
424 	if (orig->group_name) {
425 		evsel->group_name = strdup(orig->group_name);
426 		if (evsel->group_name == NULL)
427 			goto out_err;
428 	}
429 	if (orig->pmu_name) {
430 		evsel->pmu_name = strdup(orig->pmu_name);
431 		if (evsel->pmu_name == NULL)
432 			goto out_err;
433 	}
434 	if (orig->filter) {
435 		evsel->filter = strdup(orig->filter);
436 		if (evsel->filter == NULL)
437 			goto out_err;
438 	}
439 	if (orig->metric_id) {
440 		evsel->metric_id = strdup(orig->metric_id);
441 		if (evsel->metric_id == NULL)
442 			goto out_err;
443 	}
444 	evsel->cgrp = cgroup__get(orig->cgrp);
445 	evsel->tp_format = orig->tp_format;
446 	evsel->handler = orig->handler;
447 	evsel->core.leader = orig->core.leader;
448 
449 	evsel->max_events = orig->max_events;
450 	evsel->tool_event = orig->tool_event;
451 	free((char *)evsel->unit);
452 	evsel->unit = strdup(orig->unit);
453 	if (evsel->unit == NULL)
454 		goto out_err;
455 
456 	evsel->scale = orig->scale;
457 	evsel->snapshot = orig->snapshot;
458 	evsel->per_pkg = orig->per_pkg;
459 	evsel->percore = orig->percore;
460 	evsel->precise_max = orig->precise_max;
461 	evsel->use_uncore_alias = orig->use_uncore_alias;
462 	evsel->is_libpfm_event = orig->is_libpfm_event;
463 
464 	evsel->exclude_GH = orig->exclude_GH;
465 	evsel->sample_read = orig->sample_read;
466 	evsel->auto_merge_stats = orig->auto_merge_stats;
467 	evsel->collect_stat = orig->collect_stat;
468 	evsel->weak_group = orig->weak_group;
469 	evsel->use_config_name = orig->use_config_name;
470 
471 	if (evsel__copy_config_terms(evsel, orig) < 0)
472 		goto out_err;
473 
474 	return evsel;
475 
476 out_err:
477 	evsel__delete(evsel);
478 	return NULL;
479 }
480 
481 /*
482  * Returns pointer with encoded error via <linux/err.h> interface.
483  */
evsel__newtp_idx(const char * sys,const char * name,int idx)484 struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx)
485 {
486 	struct evsel *evsel = zalloc(perf_evsel__object.size);
487 	int err = -ENOMEM;
488 
489 	if (evsel == NULL) {
490 		goto out_err;
491 	} else {
492 		struct perf_event_attr attr = {
493 			.type	       = PERF_TYPE_TRACEPOINT,
494 			.sample_type   = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
495 					  PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
496 		};
497 
498 		if (asprintf(&evsel->name, "%s:%s", sys, name) < 0)
499 			goto out_free;
500 
501 		evsel->tp_format = trace_event__tp_format(sys, name);
502 		if (IS_ERR(evsel->tp_format)) {
503 			err = PTR_ERR(evsel->tp_format);
504 			goto out_free;
505 		}
506 
507 		event_attr_init(&attr);
508 		attr.config = evsel->tp_format->id;
509 		attr.sample_period = 1;
510 		evsel__init(evsel, &attr, idx);
511 	}
512 
513 	return evsel;
514 
515 out_free:
516 	zfree(&evsel->name);
517 	free(evsel);
518 out_err:
519 	return ERR_PTR(err);
520 }
521 
522 const char *const evsel__hw_names[PERF_COUNT_HW_MAX] = {
523 	"cycles",
524 	"instructions",
525 	"cache-references",
526 	"cache-misses",
527 	"branches",
528 	"branch-misses",
529 	"bus-cycles",
530 	"stalled-cycles-frontend",
531 	"stalled-cycles-backend",
532 	"ref-cycles",
533 };
534 
535 char *evsel__bpf_counter_events;
536 
evsel__match_bpf_counter_events(const char * name)537 bool evsel__match_bpf_counter_events(const char *name)
538 {
539 	int name_len;
540 	bool match;
541 	char *ptr;
542 
543 	if (!evsel__bpf_counter_events)
544 		return false;
545 
546 	ptr = strstr(evsel__bpf_counter_events, name);
547 	name_len = strlen(name);
548 
549 	/* check name matches a full token in evsel__bpf_counter_events */
550 	match = (ptr != NULL) &&
551 		((ptr == evsel__bpf_counter_events) || (*(ptr - 1) == ',')) &&
552 		((*(ptr + name_len) == ',') || (*(ptr + name_len) == '\0'));
553 
554 	return match;
555 }
556 
__evsel__hw_name(u64 config)557 static const char *__evsel__hw_name(u64 config)
558 {
559 	if (config < PERF_COUNT_HW_MAX && evsel__hw_names[config])
560 		return evsel__hw_names[config];
561 
562 	return "unknown-hardware";
563 }
564 
evsel__add_modifiers(struct evsel * evsel,char * bf,size_t size)565 static int evsel__add_modifiers(struct evsel *evsel, char *bf, size_t size)
566 {
567 	int colon = 0, r = 0;
568 	struct perf_event_attr *attr = &evsel->core.attr;
569 	bool exclude_guest_default = false;
570 
571 #define MOD_PRINT(context, mod)	do {					\
572 		if (!attr->exclude_##context) {				\
573 			if (!colon) colon = ++r;			\
574 			r += scnprintf(bf + r, size - r, "%c", mod);	\
575 		} } while(0)
576 
577 	if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) {
578 		MOD_PRINT(kernel, 'k');
579 		MOD_PRINT(user, 'u');
580 		MOD_PRINT(hv, 'h');
581 		exclude_guest_default = true;
582 	}
583 
584 	if (attr->precise_ip) {
585 		if (!colon)
586 			colon = ++r;
587 		r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp");
588 		exclude_guest_default = true;
589 	}
590 
591 	if (attr->exclude_host || attr->exclude_guest == exclude_guest_default) {
592 		MOD_PRINT(host, 'H');
593 		MOD_PRINT(guest, 'G');
594 	}
595 #undef MOD_PRINT
596 	if (colon)
597 		bf[colon - 1] = ':';
598 	return r;
599 }
600 
arch_evsel__hw_name(struct evsel * evsel,char * bf,size_t size)601 int __weak arch_evsel__hw_name(struct evsel *evsel, char *bf, size_t size)
602 {
603 	return scnprintf(bf, size, "%s", __evsel__hw_name(evsel->core.attr.config));
604 }
605 
evsel__hw_name(struct evsel * evsel,char * bf,size_t size)606 static int evsel__hw_name(struct evsel *evsel, char *bf, size_t size)
607 {
608 	int r = arch_evsel__hw_name(evsel, bf, size);
609 	return r + evsel__add_modifiers(evsel, bf + r, size - r);
610 }
611 
612 const char *const evsel__sw_names[PERF_COUNT_SW_MAX] = {
613 	"cpu-clock",
614 	"task-clock",
615 	"page-faults",
616 	"context-switches",
617 	"cpu-migrations",
618 	"minor-faults",
619 	"major-faults",
620 	"alignment-faults",
621 	"emulation-faults",
622 	"dummy",
623 };
624 
__evsel__sw_name(u64 config)625 static const char *__evsel__sw_name(u64 config)
626 {
627 	if (config < PERF_COUNT_SW_MAX && evsel__sw_names[config])
628 		return evsel__sw_names[config];
629 	return "unknown-software";
630 }
631 
evsel__sw_name(struct evsel * evsel,char * bf,size_t size)632 static int evsel__sw_name(struct evsel *evsel, char *bf, size_t size)
633 {
634 	int r = scnprintf(bf, size, "%s", __evsel__sw_name(evsel->core.attr.config));
635 	return r + evsel__add_modifiers(evsel, bf + r, size - r);
636 }
637 
evsel__tool_name(enum perf_tool_event ev,char * bf,size_t size)638 static int evsel__tool_name(enum perf_tool_event ev, char *bf, size_t size)
639 {
640 	return scnprintf(bf, size, "%s", perf_tool_event__to_str(ev));
641 }
642 
__evsel__bp_name(char * bf,size_t size,u64 addr,u64 type)643 static int __evsel__bp_name(char *bf, size_t size, u64 addr, u64 type)
644 {
645 	int r;
646 
647 	r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr);
648 
649 	if (type & HW_BREAKPOINT_R)
650 		r += scnprintf(bf + r, size - r, "r");
651 
652 	if (type & HW_BREAKPOINT_W)
653 		r += scnprintf(bf + r, size - r, "w");
654 
655 	if (type & HW_BREAKPOINT_X)
656 		r += scnprintf(bf + r, size - r, "x");
657 
658 	return r;
659 }
660 
evsel__bp_name(struct evsel * evsel,char * bf,size_t size)661 static int evsel__bp_name(struct evsel *evsel, char *bf, size_t size)
662 {
663 	struct perf_event_attr *attr = &evsel->core.attr;
664 	int r = __evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type);
665 	return r + evsel__add_modifiers(evsel, bf + r, size - r);
666 }
667 
668 const char *const evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX][EVSEL__MAX_ALIASES] = {
669  { "L1-dcache",	"l1-d",		"l1d",		"L1-data",		},
670  { "L1-icache",	"l1-i",		"l1i",		"L1-instruction",	},
671  { "LLC",	"L2",							},
672  { "dTLB",	"d-tlb",	"Data-TLB",				},
673  { "iTLB",	"i-tlb",	"Instruction-TLB",			},
674  { "branch",	"branches",	"bpu",		"btb",		"bpc",	},
675  { "node",								},
676 };
677 
678 const char *const evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX][EVSEL__MAX_ALIASES] = {
679  { "load",	"loads",	"read",					},
680  { "store",	"stores",	"write",				},
681  { "prefetch",	"prefetches",	"speculative-read", "speculative-load",	},
682 };
683 
684 const char *const evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX][EVSEL__MAX_ALIASES] = {
685  { "refs",	"Reference",	"ops",		"access",		},
686  { "misses",	"miss",							},
687 };
688 
689 #define C(x)		PERF_COUNT_HW_CACHE_##x
690 #define CACHE_READ	(1 << C(OP_READ))
691 #define CACHE_WRITE	(1 << C(OP_WRITE))
692 #define CACHE_PREFETCH	(1 << C(OP_PREFETCH))
693 #define COP(x)		(1 << x)
694 
695 /*
696  * cache operation stat
697  * L1I : Read and prefetch only
698  * ITLB and BPU : Read-only
699  */
700 static const unsigned long evsel__hw_cache_stat[C(MAX)] = {
701  [C(L1D)]	= (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
702  [C(L1I)]	= (CACHE_READ | CACHE_PREFETCH),
703  [C(LL)]	= (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
704  [C(DTLB)]	= (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
705  [C(ITLB)]	= (CACHE_READ),
706  [C(BPU)]	= (CACHE_READ),
707  [C(NODE)]	= (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
708 };
709 
evsel__is_cache_op_valid(u8 type,u8 op)710 bool evsel__is_cache_op_valid(u8 type, u8 op)
711 {
712 	if (evsel__hw_cache_stat[type] & COP(op))
713 		return true;	/* valid */
714 	else
715 		return false;	/* invalid */
716 }
717 
__evsel__hw_cache_type_op_res_name(u8 type,u8 op,u8 result,char * bf,size_t size)718 int __evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result, char *bf, size_t size)
719 {
720 	if (result) {
721 		return scnprintf(bf, size, "%s-%s-%s", evsel__hw_cache[type][0],
722 				 evsel__hw_cache_op[op][0],
723 				 evsel__hw_cache_result[result][0]);
724 	}
725 
726 	return scnprintf(bf, size, "%s-%s", evsel__hw_cache[type][0],
727 			 evsel__hw_cache_op[op][1]);
728 }
729 
__evsel__hw_cache_name(u64 config,char * bf,size_t size)730 static int __evsel__hw_cache_name(u64 config, char *bf, size_t size)
731 {
732 	u8 op, result, type = (config >>  0) & 0xff;
733 	const char *err = "unknown-ext-hardware-cache-type";
734 
735 	if (type >= PERF_COUNT_HW_CACHE_MAX)
736 		goto out_err;
737 
738 	op = (config >>  8) & 0xff;
739 	err = "unknown-ext-hardware-cache-op";
740 	if (op >= PERF_COUNT_HW_CACHE_OP_MAX)
741 		goto out_err;
742 
743 	result = (config >> 16) & 0xff;
744 	err = "unknown-ext-hardware-cache-result";
745 	if (result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
746 		goto out_err;
747 
748 	err = "invalid-cache";
749 	if (!evsel__is_cache_op_valid(type, op))
750 		goto out_err;
751 
752 	return __evsel__hw_cache_type_op_res_name(type, op, result, bf, size);
753 out_err:
754 	return scnprintf(bf, size, "%s", err);
755 }
756 
evsel__hw_cache_name(struct evsel * evsel,char * bf,size_t size)757 static int evsel__hw_cache_name(struct evsel *evsel, char *bf, size_t size)
758 {
759 	int ret = __evsel__hw_cache_name(evsel->core.attr.config, bf, size);
760 	return ret + evsel__add_modifiers(evsel, bf + ret, size - ret);
761 }
762 
evsel__raw_name(struct evsel * evsel,char * bf,size_t size)763 static int evsel__raw_name(struct evsel *evsel, char *bf, size_t size)
764 {
765 	int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->core.attr.config);
766 	return ret + evsel__add_modifiers(evsel, bf + ret, size - ret);
767 }
768 
evsel__name(struct evsel * evsel)769 const char *evsel__name(struct evsel *evsel)
770 {
771 	char bf[128];
772 
773 	if (!evsel)
774 		goto out_unknown;
775 
776 	if (evsel->name)
777 		return evsel->name;
778 
779 	switch (evsel->core.attr.type) {
780 	case PERF_TYPE_RAW:
781 		evsel__raw_name(evsel, bf, sizeof(bf));
782 		break;
783 
784 	case PERF_TYPE_HARDWARE:
785 		evsel__hw_name(evsel, bf, sizeof(bf));
786 		break;
787 
788 	case PERF_TYPE_HW_CACHE:
789 		evsel__hw_cache_name(evsel, bf, sizeof(bf));
790 		break;
791 
792 	case PERF_TYPE_SOFTWARE:
793 		if (evsel__is_tool(evsel))
794 			evsel__tool_name(evsel->tool_event, bf, sizeof(bf));
795 		else
796 			evsel__sw_name(evsel, bf, sizeof(bf));
797 		break;
798 
799 	case PERF_TYPE_TRACEPOINT:
800 		scnprintf(bf, sizeof(bf), "%s", "unknown tracepoint");
801 		break;
802 
803 	case PERF_TYPE_BREAKPOINT:
804 		evsel__bp_name(evsel, bf, sizeof(bf));
805 		break;
806 
807 	default:
808 		scnprintf(bf, sizeof(bf), "unknown attr type: %d",
809 			  evsel->core.attr.type);
810 		break;
811 	}
812 
813 	evsel->name = strdup(bf);
814 
815 	if (evsel->name)
816 		return evsel->name;
817 out_unknown:
818 	return "unknown";
819 }
820 
evsel__metric_id(const struct evsel * evsel)821 const char *evsel__metric_id(const struct evsel *evsel)
822 {
823 	if (evsel->metric_id)
824 		return evsel->metric_id;
825 
826 	if (evsel__is_tool(evsel))
827 		return perf_tool_event__to_str(evsel->tool_event);
828 
829 	return "unknown";
830 }
831 
evsel__group_name(struct evsel * evsel)832 const char *evsel__group_name(struct evsel *evsel)
833 {
834 	return evsel->group_name ?: "anon group";
835 }
836 
837 /*
838  * Returns the group details for the specified leader,
839  * with following rules.
840  *
841  *  For record -e '{cycles,instructions}'
842  *    'anon group { cycles:u, instructions:u }'
843  *
844  *  For record -e 'cycles,instructions' and report --group
845  *    'cycles:u, instructions:u'
846  */
evsel__group_desc(struct evsel * evsel,char * buf,size_t size)847 int evsel__group_desc(struct evsel *evsel, char *buf, size_t size)
848 {
849 	int ret = 0;
850 	struct evsel *pos;
851 	const char *group_name = evsel__group_name(evsel);
852 
853 	if (!evsel->forced_leader)
854 		ret = scnprintf(buf, size, "%s { ", group_name);
855 
856 	ret += scnprintf(buf + ret, size - ret, "%s", evsel__name(evsel));
857 
858 	for_each_group_member(pos, evsel)
859 		ret += scnprintf(buf + ret, size - ret, ", %s", evsel__name(pos));
860 
861 	if (!evsel->forced_leader)
862 		ret += scnprintf(buf + ret, size - ret, " }");
863 
864 	return ret;
865 }
866 
__evsel__config_callchain(struct evsel * evsel,struct record_opts * opts,struct callchain_param * param)867 static void __evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
868 				      struct callchain_param *param)
869 {
870 	bool function = evsel__is_function_event(evsel);
871 	struct perf_event_attr *attr = &evsel->core.attr;
872 
873 	evsel__set_sample_bit(evsel, CALLCHAIN);
874 
875 	attr->sample_max_stack = param->max_stack;
876 
877 	if (opts->kernel_callchains)
878 		attr->exclude_callchain_user = 1;
879 	if (opts->user_callchains)
880 		attr->exclude_callchain_kernel = 1;
881 	if (param->record_mode == CALLCHAIN_LBR) {
882 		if (!opts->branch_stack) {
883 			if (attr->exclude_user) {
884 				pr_warning("LBR callstack option is only available "
885 					   "to get user callchain information. "
886 					   "Falling back to framepointers.\n");
887 			} else {
888 				evsel__set_sample_bit(evsel, BRANCH_STACK);
889 				attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER |
890 							PERF_SAMPLE_BRANCH_CALL_STACK |
891 							PERF_SAMPLE_BRANCH_NO_CYCLES |
892 							PERF_SAMPLE_BRANCH_NO_FLAGS |
893 							PERF_SAMPLE_BRANCH_HW_INDEX;
894 			}
895 		} else
896 			 pr_warning("Cannot use LBR callstack with branch stack. "
897 				    "Falling back to framepointers.\n");
898 	}
899 
900 	if (param->record_mode == CALLCHAIN_DWARF) {
901 		if (!function) {
902 			evsel__set_sample_bit(evsel, REGS_USER);
903 			evsel__set_sample_bit(evsel, STACK_USER);
904 			if (opts->sample_user_regs && DWARF_MINIMAL_REGS != PERF_REGS_MASK) {
905 				attr->sample_regs_user |= DWARF_MINIMAL_REGS;
906 				pr_warning("WARNING: The use of --call-graph=dwarf may require all the user registers, "
907 					   "specifying a subset with --user-regs may render DWARF unwinding unreliable, "
908 					   "so the minimal registers set (IP, SP) is explicitly forced.\n");
909 			} else {
910 				attr->sample_regs_user |= arch__user_reg_mask();
911 			}
912 			attr->sample_stack_user = param->dump_size;
913 			attr->exclude_callchain_user = 1;
914 		} else {
915 			pr_info("Cannot use DWARF unwind for function trace event,"
916 				" falling back to framepointers.\n");
917 		}
918 	}
919 
920 	if (function) {
921 		pr_info("Disabling user space callchains for function trace event.\n");
922 		attr->exclude_callchain_user = 1;
923 	}
924 }
925 
evsel__config_callchain(struct evsel * evsel,struct record_opts * opts,struct callchain_param * param)926 void evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
927 			     struct callchain_param *param)
928 {
929 	if (param->enabled)
930 		return __evsel__config_callchain(evsel, opts, param);
931 }
932 
evsel__reset_callgraph(struct evsel * evsel,struct callchain_param * param)933 static void evsel__reset_callgraph(struct evsel *evsel, struct callchain_param *param)
934 {
935 	struct perf_event_attr *attr = &evsel->core.attr;
936 
937 	evsel__reset_sample_bit(evsel, CALLCHAIN);
938 	if (param->record_mode == CALLCHAIN_LBR) {
939 		evsel__reset_sample_bit(evsel, BRANCH_STACK);
940 		attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER |
941 					      PERF_SAMPLE_BRANCH_CALL_STACK |
942 					      PERF_SAMPLE_BRANCH_HW_INDEX);
943 	}
944 	if (param->record_mode == CALLCHAIN_DWARF) {
945 		evsel__reset_sample_bit(evsel, REGS_USER);
946 		evsel__reset_sample_bit(evsel, STACK_USER);
947 	}
948 }
949 
evsel__apply_config_terms(struct evsel * evsel,struct record_opts * opts,bool track)950 static void evsel__apply_config_terms(struct evsel *evsel,
951 				      struct record_opts *opts, bool track)
952 {
953 	struct evsel_config_term *term;
954 	struct list_head *config_terms = &evsel->config_terms;
955 	struct perf_event_attr *attr = &evsel->core.attr;
956 	/* callgraph default */
957 	struct callchain_param param = {
958 		.record_mode = callchain_param.record_mode,
959 	};
960 	u32 dump_size = 0;
961 	int max_stack = 0;
962 	const char *callgraph_buf = NULL;
963 
964 	list_for_each_entry(term, config_terms, list) {
965 		switch (term->type) {
966 		case EVSEL__CONFIG_TERM_PERIOD:
967 			if (!(term->weak && opts->user_interval != ULLONG_MAX)) {
968 				attr->sample_period = term->val.period;
969 				attr->freq = 0;
970 				evsel__reset_sample_bit(evsel, PERIOD);
971 			}
972 			break;
973 		case EVSEL__CONFIG_TERM_FREQ:
974 			if (!(term->weak && opts->user_freq != UINT_MAX)) {
975 				attr->sample_freq = term->val.freq;
976 				attr->freq = 1;
977 				evsel__set_sample_bit(evsel, PERIOD);
978 			}
979 			break;
980 		case EVSEL__CONFIG_TERM_TIME:
981 			if (term->val.time)
982 				evsel__set_sample_bit(evsel, TIME);
983 			else
984 				evsel__reset_sample_bit(evsel, TIME);
985 			break;
986 		case EVSEL__CONFIG_TERM_CALLGRAPH:
987 			callgraph_buf = term->val.str;
988 			break;
989 		case EVSEL__CONFIG_TERM_BRANCH:
990 			if (term->val.str && strcmp(term->val.str, "no")) {
991 				evsel__set_sample_bit(evsel, BRANCH_STACK);
992 				parse_branch_str(term->val.str,
993 						 &attr->branch_sample_type);
994 			} else
995 				evsel__reset_sample_bit(evsel, BRANCH_STACK);
996 			break;
997 		case EVSEL__CONFIG_TERM_STACK_USER:
998 			dump_size = term->val.stack_user;
999 			break;
1000 		case EVSEL__CONFIG_TERM_MAX_STACK:
1001 			max_stack = term->val.max_stack;
1002 			break;
1003 		case EVSEL__CONFIG_TERM_MAX_EVENTS:
1004 			evsel->max_events = term->val.max_events;
1005 			break;
1006 		case EVSEL__CONFIG_TERM_INHERIT:
1007 			/*
1008 			 * attr->inherit should has already been set by
1009 			 * evsel__config. If user explicitly set
1010 			 * inherit using config terms, override global
1011 			 * opt->no_inherit setting.
1012 			 */
1013 			attr->inherit = term->val.inherit ? 1 : 0;
1014 			break;
1015 		case EVSEL__CONFIG_TERM_OVERWRITE:
1016 			attr->write_backward = term->val.overwrite ? 1 : 0;
1017 			break;
1018 		case EVSEL__CONFIG_TERM_DRV_CFG:
1019 			break;
1020 		case EVSEL__CONFIG_TERM_PERCORE:
1021 			break;
1022 		case EVSEL__CONFIG_TERM_AUX_OUTPUT:
1023 			attr->aux_output = term->val.aux_output ? 1 : 0;
1024 			break;
1025 		case EVSEL__CONFIG_TERM_AUX_SAMPLE_SIZE:
1026 			/* Already applied by auxtrace */
1027 			break;
1028 		case EVSEL__CONFIG_TERM_CFG_CHG:
1029 			break;
1030 		default:
1031 			break;
1032 		}
1033 	}
1034 
1035 	/* User explicitly set per-event callgraph, clear the old setting and reset. */
1036 	if ((callgraph_buf != NULL) || (dump_size > 0) || max_stack) {
1037 		bool sample_address = false;
1038 
1039 		if (max_stack) {
1040 			param.max_stack = max_stack;
1041 			if (callgraph_buf == NULL)
1042 				callgraph_buf = "fp";
1043 		}
1044 
1045 		/* parse callgraph parameters */
1046 		if (callgraph_buf != NULL) {
1047 			if (!strcmp(callgraph_buf, "no")) {
1048 				param.enabled = false;
1049 				param.record_mode = CALLCHAIN_NONE;
1050 			} else {
1051 				param.enabled = true;
1052 				if (parse_callchain_record(callgraph_buf, &param)) {
1053 					pr_err("per-event callgraph setting for %s failed. "
1054 					       "Apply callgraph global setting for it\n",
1055 					       evsel->name);
1056 					return;
1057 				}
1058 				if (param.record_mode == CALLCHAIN_DWARF)
1059 					sample_address = true;
1060 			}
1061 		}
1062 		if (dump_size > 0) {
1063 			dump_size = round_up(dump_size, sizeof(u64));
1064 			param.dump_size = dump_size;
1065 		}
1066 
1067 		/* If global callgraph set, clear it */
1068 		if (callchain_param.enabled)
1069 			evsel__reset_callgraph(evsel, &callchain_param);
1070 
1071 		/* set perf-event callgraph */
1072 		if (param.enabled) {
1073 			if (sample_address) {
1074 				evsel__set_sample_bit(evsel, ADDR);
1075 				evsel__set_sample_bit(evsel, DATA_SRC);
1076 				evsel->core.attr.mmap_data = track;
1077 			}
1078 			evsel__config_callchain(evsel, opts, &param);
1079 		}
1080 	}
1081 }
1082 
__evsel__get_config_term(struct evsel * evsel,enum evsel_term_type type)1083 struct evsel_config_term *__evsel__get_config_term(struct evsel *evsel, enum evsel_term_type type)
1084 {
1085 	struct evsel_config_term *term, *found_term = NULL;
1086 
1087 	list_for_each_entry(term, &evsel->config_terms, list) {
1088 		if (term->type == type)
1089 			found_term = term;
1090 	}
1091 
1092 	return found_term;
1093 }
1094 
arch_evsel__set_sample_weight(struct evsel * evsel)1095 void __weak arch_evsel__set_sample_weight(struct evsel *evsel)
1096 {
1097 	evsel__set_sample_bit(evsel, WEIGHT);
1098 }
1099 
arch_evsel__fixup_new_cycles(struct perf_event_attr * attr __maybe_unused)1100 void __weak arch_evsel__fixup_new_cycles(struct perf_event_attr *attr __maybe_unused)
1101 {
1102 }
1103 
arch__post_evsel_config(struct evsel * evsel __maybe_unused,struct perf_event_attr * attr __maybe_unused)1104 void __weak arch__post_evsel_config(struct evsel *evsel __maybe_unused,
1105 				    struct perf_event_attr *attr __maybe_unused)
1106 {
1107 }
1108 
evsel__set_default_freq_period(struct record_opts * opts,struct perf_event_attr * attr)1109 static void evsel__set_default_freq_period(struct record_opts *opts,
1110 					   struct perf_event_attr *attr)
1111 {
1112 	if (opts->freq) {
1113 		attr->freq = 1;
1114 		attr->sample_freq = opts->freq;
1115 	} else {
1116 		attr->sample_period = opts->default_interval;
1117 	}
1118 }
1119 
evsel__is_offcpu_event(struct evsel * evsel)1120 static bool evsel__is_offcpu_event(struct evsel *evsel)
1121 {
1122 	return evsel__is_bpf_output(evsel) && !strcmp(evsel->name, OFFCPU_EVENT);
1123 }
1124 
1125 /*
1126  * The enable_on_exec/disabled value strategy:
1127  *
1128  *  1) For any type of traced program:
1129  *    - all independent events and group leaders are disabled
1130  *    - all group members are enabled
1131  *
1132  *     Group members are ruled by group leaders. They need to
1133  *     be enabled, because the group scheduling relies on that.
1134  *
1135  *  2) For traced programs executed by perf:
1136  *     - all independent events and group leaders have
1137  *       enable_on_exec set
1138  *     - we don't specifically enable or disable any event during
1139  *       the record command
1140  *
1141  *     Independent events and group leaders are initially disabled
1142  *     and get enabled by exec. Group members are ruled by group
1143  *     leaders as stated in 1).
1144  *
1145  *  3) For traced programs attached by perf (pid/tid):
1146  *     - we specifically enable or disable all events during
1147  *       the record command
1148  *
1149  *     When attaching events to already running traced we
1150  *     enable/disable events specifically, as there's no
1151  *     initial traced exec call.
1152  */
evsel__config(struct evsel * evsel,struct record_opts * opts,struct callchain_param * callchain)1153 void evsel__config(struct evsel *evsel, struct record_opts *opts,
1154 		   struct callchain_param *callchain)
1155 {
1156 	struct evsel *leader = evsel__leader(evsel);
1157 	struct perf_event_attr *attr = &evsel->core.attr;
1158 	int track = evsel->tracking;
1159 	bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread;
1160 
1161 	attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1;
1162 	attr->inherit	    = !opts->no_inherit;
1163 	attr->write_backward = opts->overwrite ? 1 : 0;
1164 	attr->read_format   = PERF_FORMAT_LOST;
1165 
1166 	evsel__set_sample_bit(evsel, IP);
1167 	evsel__set_sample_bit(evsel, TID);
1168 
1169 	if (evsel->sample_read) {
1170 		evsel__set_sample_bit(evsel, READ);
1171 
1172 		/*
1173 		 * We need ID even in case of single event, because
1174 		 * PERF_SAMPLE_READ process ID specific data.
1175 		 */
1176 		evsel__set_sample_id(evsel, false);
1177 
1178 		/*
1179 		 * Apply group format only if we belong to group
1180 		 * with more than one members.
1181 		 */
1182 		if (leader->core.nr_members > 1) {
1183 			attr->read_format |= PERF_FORMAT_GROUP;
1184 			attr->inherit = 0;
1185 		}
1186 	}
1187 
1188 	/*
1189 	 * We default some events to have a default interval. But keep
1190 	 * it a weak assumption overridable by the user.
1191 	 */
1192 	if ((evsel->is_libpfm_event && !attr->sample_period) ||
1193 	    (!evsel->is_libpfm_event && (!attr->sample_period ||
1194 					 opts->user_freq != UINT_MAX ||
1195 					 opts->user_interval != ULLONG_MAX)))
1196 		evsel__set_default_freq_period(opts, attr);
1197 
1198 	/*
1199 	 * If attr->freq was set (here or earlier), ask for period
1200 	 * to be sampled.
1201 	 */
1202 	if (attr->freq)
1203 		evsel__set_sample_bit(evsel, PERIOD);
1204 
1205 	if (opts->no_samples)
1206 		attr->sample_freq = 0;
1207 
1208 	if (opts->inherit_stat) {
1209 		evsel->core.attr.read_format |=
1210 			PERF_FORMAT_TOTAL_TIME_ENABLED |
1211 			PERF_FORMAT_TOTAL_TIME_RUNNING |
1212 			PERF_FORMAT_ID;
1213 		attr->inherit_stat = 1;
1214 	}
1215 
1216 	if (opts->sample_address) {
1217 		evsel__set_sample_bit(evsel, ADDR);
1218 		attr->mmap_data = track;
1219 	}
1220 
1221 	/*
1222 	 * We don't allow user space callchains for  function trace
1223 	 * event, due to issues with page faults while tracing page
1224 	 * fault handler and its overall trickiness nature.
1225 	 */
1226 	if (evsel__is_function_event(evsel))
1227 		evsel->core.attr.exclude_callchain_user = 1;
1228 
1229 	if (callchain && callchain->enabled && !evsel->no_aux_samples)
1230 		evsel__config_callchain(evsel, opts, callchain);
1231 
1232 	if (opts->sample_intr_regs && !evsel->no_aux_samples &&
1233 	    !evsel__is_dummy_event(evsel)) {
1234 		attr->sample_regs_intr = opts->sample_intr_regs;
1235 		evsel__set_sample_bit(evsel, REGS_INTR);
1236 	}
1237 
1238 	if (opts->sample_user_regs && !evsel->no_aux_samples &&
1239 	    !evsel__is_dummy_event(evsel)) {
1240 		attr->sample_regs_user |= opts->sample_user_regs;
1241 		evsel__set_sample_bit(evsel, REGS_USER);
1242 	}
1243 
1244 	if (target__has_cpu(&opts->target) || opts->sample_cpu)
1245 		evsel__set_sample_bit(evsel, CPU);
1246 
1247 	/*
1248 	 * When the user explicitly disabled time don't force it here.
1249 	 */
1250 	if (opts->sample_time &&
1251 	    (!perf_missing_features.sample_id_all &&
1252 	    (!opts->no_inherit || target__has_cpu(&opts->target) || per_cpu ||
1253 	     opts->sample_time_set)))
1254 		evsel__set_sample_bit(evsel, TIME);
1255 
1256 	if (opts->raw_samples && !evsel->no_aux_samples) {
1257 		evsel__set_sample_bit(evsel, TIME);
1258 		evsel__set_sample_bit(evsel, RAW);
1259 		evsel__set_sample_bit(evsel, CPU);
1260 	}
1261 
1262 	if (opts->sample_address)
1263 		evsel__set_sample_bit(evsel, DATA_SRC);
1264 
1265 	if (opts->sample_phys_addr)
1266 		evsel__set_sample_bit(evsel, PHYS_ADDR);
1267 
1268 	if (opts->no_buffering) {
1269 		attr->watermark = 0;
1270 		attr->wakeup_events = 1;
1271 	}
1272 	if (opts->branch_stack && !evsel->no_aux_samples) {
1273 		evsel__set_sample_bit(evsel, BRANCH_STACK);
1274 		attr->branch_sample_type = opts->branch_stack;
1275 	}
1276 
1277 	if (opts->sample_weight)
1278 		arch_evsel__set_sample_weight(evsel);
1279 
1280 	attr->task     = track;
1281 	attr->mmap     = track;
1282 	attr->mmap2    = track && !perf_missing_features.mmap2;
1283 	attr->comm     = track;
1284 	attr->build_id = track && opts->build_id;
1285 
1286 	/*
1287 	 * ksymbol is tracked separately with text poke because it needs to be
1288 	 * system wide and enabled immediately.
1289 	 */
1290 	if (!opts->text_poke)
1291 		attr->ksymbol = track && !perf_missing_features.ksymbol;
1292 	attr->bpf_event = track && !opts->no_bpf_event && !perf_missing_features.bpf;
1293 
1294 	if (opts->record_namespaces)
1295 		attr->namespaces  = track;
1296 
1297 	if (opts->record_cgroup) {
1298 		attr->cgroup = track && !perf_missing_features.cgroup;
1299 		evsel__set_sample_bit(evsel, CGROUP);
1300 	}
1301 
1302 	if (opts->sample_data_page_size)
1303 		evsel__set_sample_bit(evsel, DATA_PAGE_SIZE);
1304 
1305 	if (opts->sample_code_page_size)
1306 		evsel__set_sample_bit(evsel, CODE_PAGE_SIZE);
1307 
1308 	if (opts->record_switch_events)
1309 		attr->context_switch = track;
1310 
1311 	if (opts->sample_transaction)
1312 		evsel__set_sample_bit(evsel, TRANSACTION);
1313 
1314 	if (opts->running_time) {
1315 		evsel->core.attr.read_format |=
1316 			PERF_FORMAT_TOTAL_TIME_ENABLED |
1317 			PERF_FORMAT_TOTAL_TIME_RUNNING;
1318 	}
1319 
1320 	/*
1321 	 * XXX see the function comment above
1322 	 *
1323 	 * Disabling only independent events or group leaders,
1324 	 * keeping group members enabled.
1325 	 */
1326 	if (evsel__is_group_leader(evsel))
1327 		attr->disabled = 1;
1328 
1329 	/*
1330 	 * Setting enable_on_exec for independent events and
1331 	 * group leaders for traced executed by perf.
1332 	 */
1333 	if (target__none(&opts->target) && evsel__is_group_leader(evsel) &&
1334 	    !opts->initial_delay)
1335 		attr->enable_on_exec = 1;
1336 
1337 	if (evsel->immediate) {
1338 		attr->disabled = 0;
1339 		attr->enable_on_exec = 0;
1340 	}
1341 
1342 	clockid = opts->clockid;
1343 	if (opts->use_clockid) {
1344 		attr->use_clockid = 1;
1345 		attr->clockid = opts->clockid;
1346 	}
1347 
1348 	if (evsel->precise_max)
1349 		attr->precise_ip = 3;
1350 
1351 	if (opts->all_user) {
1352 		attr->exclude_kernel = 1;
1353 		attr->exclude_user   = 0;
1354 	}
1355 
1356 	if (opts->all_kernel) {
1357 		attr->exclude_kernel = 0;
1358 		attr->exclude_user   = 1;
1359 	}
1360 
1361 	if (evsel->core.own_cpus || evsel->unit)
1362 		evsel->core.attr.read_format |= PERF_FORMAT_ID;
1363 
1364 	/*
1365 	 * Apply event specific term settings,
1366 	 * it overloads any global configuration.
1367 	 */
1368 	evsel__apply_config_terms(evsel, opts, track);
1369 
1370 	evsel->ignore_missing_thread = opts->ignore_missing_thread;
1371 
1372 	/* The --period option takes the precedence. */
1373 	if (opts->period_set) {
1374 		if (opts->period)
1375 			evsel__set_sample_bit(evsel, PERIOD);
1376 		else
1377 			evsel__reset_sample_bit(evsel, PERIOD);
1378 	}
1379 
1380 	/*
1381 	 * A dummy event never triggers any actual counter and therefore
1382 	 * cannot be used with branch_stack.
1383 	 *
1384 	 * For initial_delay, a dummy event is added implicitly.
1385 	 * The software event will trigger -EOPNOTSUPP error out,
1386 	 * if BRANCH_STACK bit is set.
1387 	 */
1388 	if (evsel__is_dummy_event(evsel))
1389 		evsel__reset_sample_bit(evsel, BRANCH_STACK);
1390 
1391 	if (evsel__is_offcpu_event(evsel))
1392 		evsel->core.attr.sample_type &= OFFCPU_SAMPLE_TYPES;
1393 
1394 	arch__post_evsel_config(evsel, attr);
1395 }
1396 
evsel__set_filter(struct evsel * evsel,const char * filter)1397 int evsel__set_filter(struct evsel *evsel, const char *filter)
1398 {
1399 	char *new_filter = strdup(filter);
1400 
1401 	if (new_filter != NULL) {
1402 		free(evsel->filter);
1403 		evsel->filter = new_filter;
1404 		return 0;
1405 	}
1406 
1407 	return -1;
1408 }
1409 
evsel__append_filter(struct evsel * evsel,const char * fmt,const char * filter)1410 static int evsel__append_filter(struct evsel *evsel, const char *fmt, const char *filter)
1411 {
1412 	char *new_filter;
1413 
1414 	if (evsel->filter == NULL)
1415 		return evsel__set_filter(evsel, filter);
1416 
1417 	if (asprintf(&new_filter, fmt, evsel->filter, filter) > 0) {
1418 		free(evsel->filter);
1419 		evsel->filter = new_filter;
1420 		return 0;
1421 	}
1422 
1423 	return -1;
1424 }
1425 
evsel__append_tp_filter(struct evsel * evsel,const char * filter)1426 int evsel__append_tp_filter(struct evsel *evsel, const char *filter)
1427 {
1428 	return evsel__append_filter(evsel, "(%s) && (%s)", filter);
1429 }
1430 
evsel__append_addr_filter(struct evsel * evsel,const char * filter)1431 int evsel__append_addr_filter(struct evsel *evsel, const char *filter)
1432 {
1433 	return evsel__append_filter(evsel, "%s,%s", filter);
1434 }
1435 
1436 /* Caller has to clear disabled after going through all CPUs. */
evsel__enable_cpu(struct evsel * evsel,int cpu_map_idx)1437 int evsel__enable_cpu(struct evsel *evsel, int cpu_map_idx)
1438 {
1439 	return perf_evsel__enable_cpu(&evsel->core, cpu_map_idx);
1440 }
1441 
evsel__enable(struct evsel * evsel)1442 int evsel__enable(struct evsel *evsel)
1443 {
1444 	int err = perf_evsel__enable(&evsel->core);
1445 
1446 	if (!err)
1447 		evsel->disabled = false;
1448 	return err;
1449 }
1450 
1451 /* Caller has to set disabled after going through all CPUs. */
evsel__disable_cpu(struct evsel * evsel,int cpu_map_idx)1452 int evsel__disable_cpu(struct evsel *evsel, int cpu_map_idx)
1453 {
1454 	return perf_evsel__disable_cpu(&evsel->core, cpu_map_idx);
1455 }
1456 
evsel__disable(struct evsel * evsel)1457 int evsel__disable(struct evsel *evsel)
1458 {
1459 	int err = perf_evsel__disable(&evsel->core);
1460 	/*
1461 	 * We mark it disabled here so that tools that disable a event can
1462 	 * ignore events after they disable it. I.e. the ring buffer may have
1463 	 * already a few more events queued up before the kernel got the stop
1464 	 * request.
1465 	 */
1466 	if (!err)
1467 		evsel->disabled = true;
1468 
1469 	return err;
1470 }
1471 
free_config_terms(struct list_head * config_terms)1472 void free_config_terms(struct list_head *config_terms)
1473 {
1474 	struct evsel_config_term *term, *h;
1475 
1476 	list_for_each_entry_safe(term, h, config_terms, list) {
1477 		list_del_init(&term->list);
1478 		if (term->free_str)
1479 			zfree(&term->val.str);
1480 		free(term);
1481 	}
1482 }
1483 
evsel__free_config_terms(struct evsel * evsel)1484 static void evsel__free_config_terms(struct evsel *evsel)
1485 {
1486 	free_config_terms(&evsel->config_terms);
1487 }
1488 
evsel__exit(struct evsel * evsel)1489 void evsel__exit(struct evsel *evsel)
1490 {
1491 	assert(list_empty(&evsel->core.node));
1492 	assert(evsel->evlist == NULL);
1493 	bpf_counter__destroy(evsel);
1494 	evsel__free_counts(evsel);
1495 	perf_evsel__free_fd(&evsel->core);
1496 	perf_evsel__free_id(&evsel->core);
1497 	evsel__free_config_terms(evsel);
1498 	cgroup__put(evsel->cgrp);
1499 	perf_cpu_map__put(evsel->core.cpus);
1500 	perf_cpu_map__put(evsel->core.own_cpus);
1501 	perf_thread_map__put(evsel->core.threads);
1502 	zfree(&evsel->group_name);
1503 	zfree(&evsel->name);
1504 	zfree(&evsel->pmu_name);
1505 	zfree(&evsel->unit);
1506 	zfree(&evsel->metric_id);
1507 	evsel__zero_per_pkg(evsel);
1508 	hashmap__free(evsel->per_pkg_mask);
1509 	evsel->per_pkg_mask = NULL;
1510 	zfree(&evsel->metric_events);
1511 	perf_evsel__object.fini(evsel);
1512 }
1513 
evsel__delete(struct evsel * evsel)1514 void evsel__delete(struct evsel *evsel)
1515 {
1516 	evsel__exit(evsel);
1517 	free(evsel);
1518 }
1519 
evsel__compute_deltas(struct evsel * evsel,int cpu_map_idx,int thread,struct perf_counts_values * count)1520 void evsel__compute_deltas(struct evsel *evsel, int cpu_map_idx, int thread,
1521 			   struct perf_counts_values *count)
1522 {
1523 	struct perf_counts_values tmp;
1524 
1525 	if (!evsel->prev_raw_counts)
1526 		return;
1527 
1528 	if (cpu_map_idx == -1) {
1529 		tmp = evsel->prev_raw_counts->aggr;
1530 		evsel->prev_raw_counts->aggr = *count;
1531 	} else {
1532 		tmp = *perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread);
1533 		*perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread) = *count;
1534 	}
1535 
1536 	count->val = count->val - tmp.val;
1537 	count->ena = count->ena - tmp.ena;
1538 	count->run = count->run - tmp.run;
1539 }
1540 
evsel__read_one(struct evsel * evsel,int cpu_map_idx,int thread)1541 static int evsel__read_one(struct evsel *evsel, int cpu_map_idx, int thread)
1542 {
1543 	struct perf_counts_values *count = perf_counts(evsel->counts, cpu_map_idx, thread);
1544 
1545 	return perf_evsel__read(&evsel->core, cpu_map_idx, thread, count);
1546 }
1547 
evsel__set_count(struct evsel * counter,int cpu_map_idx,int thread,u64 val,u64 ena,u64 run,u64 lost)1548 static void evsel__set_count(struct evsel *counter, int cpu_map_idx, int thread,
1549 			     u64 val, u64 ena, u64 run, u64 lost)
1550 {
1551 	struct perf_counts_values *count;
1552 
1553 	count = perf_counts(counter->counts, cpu_map_idx, thread);
1554 
1555 	count->val    = val;
1556 	count->ena    = ena;
1557 	count->run    = run;
1558 	count->lost   = lost;
1559 
1560 	perf_counts__set_loaded(counter->counts, cpu_map_idx, thread, true);
1561 }
1562 
evsel__process_group_data(struct evsel * leader,int cpu_map_idx,int thread,u64 * data)1563 static int evsel__process_group_data(struct evsel *leader, int cpu_map_idx, int thread, u64 *data)
1564 {
1565 	u64 read_format = leader->core.attr.read_format;
1566 	struct sample_read_value *v;
1567 	u64 nr, ena = 0, run = 0, lost = 0;
1568 
1569 	nr = *data++;
1570 
1571 	if (nr != (u64) leader->core.nr_members)
1572 		return -EINVAL;
1573 
1574 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1575 		ena = *data++;
1576 
1577 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1578 		run = *data++;
1579 
1580 	v = (void *)data;
1581 	sample_read_group__for_each(v, nr, read_format) {
1582 		struct evsel *counter;
1583 
1584 		counter = evlist__id2evsel(leader->evlist, v->id);
1585 		if (!counter)
1586 			return -EINVAL;
1587 
1588 		if (read_format & PERF_FORMAT_LOST)
1589 			lost = v->lost;
1590 
1591 		evsel__set_count(counter, cpu_map_idx, thread, v->value, ena, run, lost);
1592 	}
1593 
1594 	return 0;
1595 }
1596 
evsel__read_group(struct evsel * leader,int cpu_map_idx,int thread)1597 static int evsel__read_group(struct evsel *leader, int cpu_map_idx, int thread)
1598 {
1599 	struct perf_stat_evsel *ps = leader->stats;
1600 	u64 read_format = leader->core.attr.read_format;
1601 	int size = perf_evsel__read_size(&leader->core);
1602 	u64 *data = ps->group_data;
1603 
1604 	if (!(read_format & PERF_FORMAT_ID))
1605 		return -EINVAL;
1606 
1607 	if (!evsel__is_group_leader(leader))
1608 		return -EINVAL;
1609 
1610 	if (!data) {
1611 		data = zalloc(size);
1612 		if (!data)
1613 			return -ENOMEM;
1614 
1615 		ps->group_data = data;
1616 	}
1617 
1618 	if (FD(leader, cpu_map_idx, thread) < 0)
1619 		return -EINVAL;
1620 
1621 	if (readn(FD(leader, cpu_map_idx, thread), data, size) <= 0)
1622 		return -errno;
1623 
1624 	return evsel__process_group_data(leader, cpu_map_idx, thread, data);
1625 }
1626 
evsel__read_counter(struct evsel * evsel,int cpu_map_idx,int thread)1627 int evsel__read_counter(struct evsel *evsel, int cpu_map_idx, int thread)
1628 {
1629 	u64 read_format = evsel->core.attr.read_format;
1630 
1631 	if (read_format & PERF_FORMAT_GROUP)
1632 		return evsel__read_group(evsel, cpu_map_idx, thread);
1633 
1634 	return evsel__read_one(evsel, cpu_map_idx, thread);
1635 }
1636 
__evsel__read_on_cpu(struct evsel * evsel,int cpu_map_idx,int thread,bool scale)1637 int __evsel__read_on_cpu(struct evsel *evsel, int cpu_map_idx, int thread, bool scale)
1638 {
1639 	struct perf_counts_values count;
1640 	size_t nv = scale ? 3 : 1;
1641 
1642 	if (FD(evsel, cpu_map_idx, thread) < 0)
1643 		return -EINVAL;
1644 
1645 	if (evsel->counts == NULL && evsel__alloc_counts(evsel) < 0)
1646 		return -ENOMEM;
1647 
1648 	if (readn(FD(evsel, cpu_map_idx, thread), &count, nv * sizeof(u64)) <= 0)
1649 		return -errno;
1650 
1651 	evsel__compute_deltas(evsel, cpu_map_idx, thread, &count);
1652 	perf_counts_values__scale(&count, scale, NULL);
1653 	*perf_counts(evsel->counts, cpu_map_idx, thread) = count;
1654 	return 0;
1655 }
1656 
evsel__match_other_cpu(struct evsel * evsel,struct evsel * other,int cpu_map_idx)1657 static int evsel__match_other_cpu(struct evsel *evsel, struct evsel *other,
1658 				  int cpu_map_idx)
1659 {
1660 	struct perf_cpu cpu;
1661 
1662 	cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx);
1663 	return perf_cpu_map__idx(other->core.cpus, cpu);
1664 }
1665 
evsel__hybrid_group_cpu_map_idx(struct evsel * evsel,int cpu_map_idx)1666 static int evsel__hybrid_group_cpu_map_idx(struct evsel *evsel, int cpu_map_idx)
1667 {
1668 	struct evsel *leader = evsel__leader(evsel);
1669 
1670 	if ((evsel__is_hybrid(evsel) && !evsel__is_hybrid(leader)) ||
1671 	    (!evsel__is_hybrid(evsel) && evsel__is_hybrid(leader))) {
1672 		return evsel__match_other_cpu(evsel, leader, cpu_map_idx);
1673 	}
1674 
1675 	return cpu_map_idx;
1676 }
1677 
get_group_fd(struct evsel * evsel,int cpu_map_idx,int thread)1678 static int get_group_fd(struct evsel *evsel, int cpu_map_idx, int thread)
1679 {
1680 	struct evsel *leader = evsel__leader(evsel);
1681 	int fd;
1682 
1683 	if (evsel__is_group_leader(evsel))
1684 		return -1;
1685 
1686 	/*
1687 	 * Leader must be already processed/open,
1688 	 * if not it's a bug.
1689 	 */
1690 	BUG_ON(!leader->core.fd);
1691 
1692 	cpu_map_idx = evsel__hybrid_group_cpu_map_idx(evsel, cpu_map_idx);
1693 	if (cpu_map_idx == -1)
1694 		return -1;
1695 
1696 	fd = FD(leader, cpu_map_idx, thread);
1697 	BUG_ON(fd == -1);
1698 
1699 	return fd;
1700 }
1701 
evsel__remove_fd(struct evsel * pos,int nr_cpus,int nr_threads,int thread_idx)1702 static void evsel__remove_fd(struct evsel *pos, int nr_cpus, int nr_threads, int thread_idx)
1703 {
1704 	for (int cpu = 0; cpu < nr_cpus; cpu++)
1705 		for (int thread = thread_idx; thread < nr_threads - 1; thread++)
1706 			FD(pos, cpu, thread) = FD(pos, cpu, thread + 1);
1707 }
1708 
update_fds(struct evsel * evsel,int nr_cpus,int cpu_map_idx,int nr_threads,int thread_idx)1709 static int update_fds(struct evsel *evsel,
1710 		      int nr_cpus, int cpu_map_idx,
1711 		      int nr_threads, int thread_idx)
1712 {
1713 	struct evsel *pos;
1714 
1715 	if (cpu_map_idx >= nr_cpus || thread_idx >= nr_threads)
1716 		return -EINVAL;
1717 
1718 	evlist__for_each_entry(evsel->evlist, pos) {
1719 		nr_cpus = pos != evsel ? nr_cpus : cpu_map_idx;
1720 
1721 		evsel__remove_fd(pos, nr_cpus, nr_threads, thread_idx);
1722 
1723 		/*
1724 		 * Since fds for next evsel has not been created,
1725 		 * there is no need to iterate whole event list.
1726 		 */
1727 		if (pos == evsel)
1728 			break;
1729 	}
1730 	return 0;
1731 }
1732 
evsel__ignore_missing_thread(struct evsel * evsel,int nr_cpus,int cpu_map_idx,struct perf_thread_map * threads,int thread,int err)1733 static bool evsel__ignore_missing_thread(struct evsel *evsel,
1734 					 int nr_cpus, int cpu_map_idx,
1735 					 struct perf_thread_map *threads,
1736 					 int thread, int err)
1737 {
1738 	pid_t ignore_pid = perf_thread_map__pid(threads, thread);
1739 
1740 	if (!evsel->ignore_missing_thread)
1741 		return false;
1742 
1743 	/* The system wide setup does not work with threads. */
1744 	if (evsel->core.system_wide)
1745 		return false;
1746 
1747 	/* The -ESRCH is perf event syscall errno for pid's not found. */
1748 	if (err != -ESRCH)
1749 		return false;
1750 
1751 	/* If there's only one thread, let it fail. */
1752 	if (threads->nr == 1)
1753 		return false;
1754 
1755 	/*
1756 	 * We should remove fd for missing_thread first
1757 	 * because thread_map__remove() will decrease threads->nr.
1758 	 */
1759 	if (update_fds(evsel, nr_cpus, cpu_map_idx, threads->nr, thread))
1760 		return false;
1761 
1762 	if (thread_map__remove(threads, thread))
1763 		return false;
1764 
1765 	pr_warning("WARNING: Ignored open failure for pid %d\n",
1766 		   ignore_pid);
1767 	return true;
1768 }
1769 
__open_attr__fprintf(FILE * fp,const char * name,const char * val,void * priv __maybe_unused)1770 static int __open_attr__fprintf(FILE *fp, const char *name, const char *val,
1771 				void *priv __maybe_unused)
1772 {
1773 	return fprintf(fp, "  %-32s %s\n", name, val);
1774 }
1775 
display_attr(struct perf_event_attr * attr)1776 static void display_attr(struct perf_event_attr *attr)
1777 {
1778 	if (verbose >= 2 || debug_peo_args) {
1779 		fprintf(stderr, "%.60s\n", graph_dotted_line);
1780 		fprintf(stderr, "perf_event_attr:\n");
1781 		perf_event_attr__fprintf(stderr, attr, __open_attr__fprintf, NULL);
1782 		fprintf(stderr, "%.60s\n", graph_dotted_line);
1783 	}
1784 }
1785 
evsel__precise_ip_fallback(struct evsel * evsel)1786 bool evsel__precise_ip_fallback(struct evsel *evsel)
1787 {
1788 	/* Do not try less precise if not requested. */
1789 	if (!evsel->precise_max)
1790 		return false;
1791 
1792 	/*
1793 	 * We tried all the precise_ip values, and it's
1794 	 * still failing, so leave it to standard fallback.
1795 	 */
1796 	if (!evsel->core.attr.precise_ip) {
1797 		evsel->core.attr.precise_ip = evsel->precise_ip_original;
1798 		return false;
1799 	}
1800 
1801 	if (!evsel->precise_ip_original)
1802 		evsel->precise_ip_original = evsel->core.attr.precise_ip;
1803 
1804 	evsel->core.attr.precise_ip--;
1805 	pr_debug2_peo("decreasing precise_ip by one (%d)\n", evsel->core.attr.precise_ip);
1806 	display_attr(&evsel->core.attr);
1807 	return true;
1808 }
1809 
1810 static struct perf_cpu_map *empty_cpu_map;
1811 static struct perf_thread_map *empty_thread_map;
1812 
__evsel__prepare_open(struct evsel * evsel,struct perf_cpu_map * cpus,struct perf_thread_map * threads)1813 static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus,
1814 		struct perf_thread_map *threads)
1815 {
1816 	int nthreads = perf_thread_map__nr(threads);
1817 
1818 	if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) ||
1819 	    (perf_missing_features.aux_output     && evsel->core.attr.aux_output))
1820 		return -EINVAL;
1821 
1822 	if (cpus == NULL) {
1823 		if (empty_cpu_map == NULL) {
1824 			empty_cpu_map = perf_cpu_map__dummy_new();
1825 			if (empty_cpu_map == NULL)
1826 				return -ENOMEM;
1827 		}
1828 
1829 		cpus = empty_cpu_map;
1830 	}
1831 
1832 	if (threads == NULL) {
1833 		if (empty_thread_map == NULL) {
1834 			empty_thread_map = thread_map__new_by_tid(-1);
1835 			if (empty_thread_map == NULL)
1836 				return -ENOMEM;
1837 		}
1838 
1839 		threads = empty_thread_map;
1840 	}
1841 
1842 	if (evsel->core.fd == NULL &&
1843 	    perf_evsel__alloc_fd(&evsel->core, perf_cpu_map__nr(cpus), nthreads) < 0)
1844 		return -ENOMEM;
1845 
1846 	evsel->open_flags = PERF_FLAG_FD_CLOEXEC;
1847 	if (evsel->cgrp)
1848 		evsel->open_flags |= PERF_FLAG_PID_CGROUP;
1849 
1850 	return 0;
1851 }
1852 
evsel__disable_missing_features(struct evsel * evsel)1853 static void evsel__disable_missing_features(struct evsel *evsel)
1854 {
1855 	if (perf_missing_features.read_lost)
1856 		evsel->core.attr.read_format &= ~PERF_FORMAT_LOST;
1857 	if (perf_missing_features.weight_struct) {
1858 		evsel__set_sample_bit(evsel, WEIGHT);
1859 		evsel__reset_sample_bit(evsel, WEIGHT_STRUCT);
1860 	}
1861 	if (perf_missing_features.clockid_wrong)
1862 		evsel->core.attr.clockid = CLOCK_MONOTONIC; /* should always work */
1863 	if (perf_missing_features.clockid) {
1864 		evsel->core.attr.use_clockid = 0;
1865 		evsel->core.attr.clockid = 0;
1866 	}
1867 	if (perf_missing_features.cloexec)
1868 		evsel->open_flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC;
1869 	if (perf_missing_features.mmap2)
1870 		evsel->core.attr.mmap2 = 0;
1871 	if (evsel->pmu && evsel->pmu->missing_features.exclude_guest)
1872 		evsel->core.attr.exclude_guest = evsel->core.attr.exclude_host = 0;
1873 	if (perf_missing_features.lbr_flags)
1874 		evsel->core.attr.branch_sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS |
1875 				     PERF_SAMPLE_BRANCH_NO_CYCLES);
1876 	if (perf_missing_features.group_read && evsel->core.attr.inherit)
1877 		evsel->core.attr.read_format &= ~(PERF_FORMAT_GROUP|PERF_FORMAT_ID);
1878 	if (perf_missing_features.ksymbol)
1879 		evsel->core.attr.ksymbol = 0;
1880 	if (perf_missing_features.bpf)
1881 		evsel->core.attr.bpf_event = 0;
1882 	if (perf_missing_features.branch_hw_idx)
1883 		evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_HW_INDEX;
1884 	if (perf_missing_features.sample_id_all)
1885 		evsel->core.attr.sample_id_all = 0;
1886 }
1887 
evsel__prepare_open(struct evsel * evsel,struct perf_cpu_map * cpus,struct perf_thread_map * threads)1888 int evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus,
1889 			struct perf_thread_map *threads)
1890 {
1891 	int err;
1892 
1893 	err = __evsel__prepare_open(evsel, cpus, threads);
1894 	if (err)
1895 		return err;
1896 
1897 	evsel__disable_missing_features(evsel);
1898 
1899 	return err;
1900 }
1901 
evsel__detect_missing_features(struct evsel * evsel)1902 bool evsel__detect_missing_features(struct evsel *evsel)
1903 {
1904 	/*
1905 	 * Must probe features in the order they were added to the
1906 	 * perf_event_attr interface.
1907 	 */
1908 	if (!perf_missing_features.read_lost &&
1909 	    (evsel->core.attr.read_format & PERF_FORMAT_LOST)) {
1910 		perf_missing_features.read_lost = true;
1911 		pr_debug2("switching off PERF_FORMAT_LOST support\n");
1912 		return true;
1913 	} else if (!perf_missing_features.weight_struct &&
1914 	    (evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) {
1915 		perf_missing_features.weight_struct = true;
1916 		pr_debug2("switching off weight struct support\n");
1917 		return true;
1918 	} else if (!perf_missing_features.code_page_size &&
1919 	    (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)) {
1920 		perf_missing_features.code_page_size = true;
1921 		pr_debug2_peo("Kernel has no PERF_SAMPLE_CODE_PAGE_SIZE support, bailing out\n");
1922 		return false;
1923 	} else if (!perf_missing_features.data_page_size &&
1924 	    (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)) {
1925 		perf_missing_features.data_page_size = true;
1926 		pr_debug2_peo("Kernel has no PERF_SAMPLE_DATA_PAGE_SIZE support, bailing out\n");
1927 		return false;
1928 	} else if (!perf_missing_features.cgroup && evsel->core.attr.cgroup) {
1929 		perf_missing_features.cgroup = true;
1930 		pr_debug2_peo("Kernel has no cgroup sampling support, bailing out\n");
1931 		return false;
1932 	} else if (!perf_missing_features.branch_hw_idx &&
1933 	    (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX)) {
1934 		perf_missing_features.branch_hw_idx = true;
1935 		pr_debug2("switching off branch HW index support\n");
1936 		return true;
1937 	} else if (!perf_missing_features.aux_output && evsel->core.attr.aux_output) {
1938 		perf_missing_features.aux_output = true;
1939 		pr_debug2_peo("Kernel has no attr.aux_output support, bailing out\n");
1940 		return false;
1941 	} else if (!perf_missing_features.bpf && evsel->core.attr.bpf_event) {
1942 		perf_missing_features.bpf = true;
1943 		pr_debug2_peo("switching off bpf_event\n");
1944 		return true;
1945 	} else if (!perf_missing_features.ksymbol && evsel->core.attr.ksymbol) {
1946 		perf_missing_features.ksymbol = true;
1947 		pr_debug2_peo("switching off ksymbol\n");
1948 		return true;
1949 	} else if (!perf_missing_features.write_backward && evsel->core.attr.write_backward) {
1950 		perf_missing_features.write_backward = true;
1951 		pr_debug2_peo("switching off write_backward\n");
1952 		return false;
1953 	} else if (!perf_missing_features.clockid_wrong && evsel->core.attr.use_clockid) {
1954 		perf_missing_features.clockid_wrong = true;
1955 		pr_debug2_peo("switching off clockid\n");
1956 		return true;
1957 	} else if (!perf_missing_features.clockid && evsel->core.attr.use_clockid) {
1958 		perf_missing_features.clockid = true;
1959 		pr_debug2_peo("switching off use_clockid\n");
1960 		return true;
1961 	} else if (!perf_missing_features.cloexec && (evsel->open_flags & PERF_FLAG_FD_CLOEXEC)) {
1962 		perf_missing_features.cloexec = true;
1963 		pr_debug2_peo("switching off cloexec flag\n");
1964 		return true;
1965 	} else if (!perf_missing_features.mmap2 && evsel->core.attr.mmap2) {
1966 		perf_missing_features.mmap2 = true;
1967 		pr_debug2_peo("switching off mmap2\n");
1968 		return true;
1969 	} else if ((evsel->core.attr.exclude_guest || evsel->core.attr.exclude_host) &&
1970 		   (evsel->pmu == NULL || evsel->pmu->missing_features.exclude_guest)) {
1971 		if (evsel->pmu == NULL) {
1972 			evsel->pmu = evsel__find_pmu(evsel);
1973 			if (evsel->pmu)
1974 				evsel->pmu->missing_features.exclude_guest = true;
1975 			else {
1976 				/* we cannot find PMU, disable attrs now */
1977 				evsel->core.attr.exclude_host = false;
1978 				evsel->core.attr.exclude_guest = false;
1979 			}
1980 		}
1981 
1982 		if (evsel->exclude_GH) {
1983 			pr_debug2_peo("PMU has no exclude_host/guest support, bailing out\n");
1984 			return false;
1985 		}
1986 		if (!perf_missing_features.exclude_guest) {
1987 			perf_missing_features.exclude_guest = true;
1988 			pr_debug2_peo("switching off exclude_guest, exclude_host\n");
1989 		}
1990 		return true;
1991 	} else if (!perf_missing_features.sample_id_all) {
1992 		perf_missing_features.sample_id_all = true;
1993 		pr_debug2_peo("switching off sample_id_all\n");
1994 		return true;
1995 	} else if (!perf_missing_features.lbr_flags &&
1996 			(evsel->core.attr.branch_sample_type &
1997 			 (PERF_SAMPLE_BRANCH_NO_CYCLES |
1998 			  PERF_SAMPLE_BRANCH_NO_FLAGS))) {
1999 		perf_missing_features.lbr_flags = true;
2000 		pr_debug2_peo("switching off branch sample type no (cycles/flags)\n");
2001 		return true;
2002 	} else if (!perf_missing_features.group_read &&
2003 		    evsel->core.attr.inherit &&
2004 		   (evsel->core.attr.read_format & PERF_FORMAT_GROUP) &&
2005 		   evsel__is_group_leader(evsel)) {
2006 		perf_missing_features.group_read = true;
2007 		pr_debug2_peo("switching off group read\n");
2008 		return true;
2009 	} else {
2010 		return false;
2011 	}
2012 }
2013 
evsel__increase_rlimit(enum rlimit_action * set_rlimit)2014 bool evsel__increase_rlimit(enum rlimit_action *set_rlimit)
2015 {
2016 	int old_errno;
2017 	struct rlimit l;
2018 
2019 	if (*set_rlimit < INCREASED_MAX) {
2020 		old_errno = errno;
2021 
2022 		if (getrlimit(RLIMIT_NOFILE, &l) == 0) {
2023 			if (*set_rlimit == NO_CHANGE) {
2024 				l.rlim_cur = l.rlim_max;
2025 			} else {
2026 				l.rlim_cur = l.rlim_max + 1000;
2027 				l.rlim_max = l.rlim_cur;
2028 			}
2029 			if (setrlimit(RLIMIT_NOFILE, &l) == 0) {
2030 				(*set_rlimit) += 1;
2031 				errno = old_errno;
2032 				return true;
2033 			}
2034 		}
2035 		errno = old_errno;
2036 	}
2037 
2038 	return false;
2039 }
2040 
evsel__open_cpu(struct evsel * evsel,struct perf_cpu_map * cpus,struct perf_thread_map * threads,int start_cpu_map_idx,int end_cpu_map_idx)2041 static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus,
2042 		struct perf_thread_map *threads,
2043 		int start_cpu_map_idx, int end_cpu_map_idx)
2044 {
2045 	int idx, thread, nthreads;
2046 	int pid = -1, err, old_errno;
2047 	enum rlimit_action set_rlimit = NO_CHANGE;
2048 
2049 	err = __evsel__prepare_open(evsel, cpus, threads);
2050 	if (err)
2051 		return err;
2052 
2053 	if (cpus == NULL)
2054 		cpus = empty_cpu_map;
2055 
2056 	if (threads == NULL)
2057 		threads = empty_thread_map;
2058 
2059 	nthreads = perf_thread_map__nr(threads);
2060 
2061 	if (evsel->cgrp)
2062 		pid = evsel->cgrp->fd;
2063 
2064 fallback_missing_features:
2065 	evsel__disable_missing_features(evsel);
2066 
2067 	display_attr(&evsel->core.attr);
2068 
2069 	for (idx = start_cpu_map_idx; idx < end_cpu_map_idx; idx++) {
2070 
2071 		for (thread = 0; thread < nthreads; thread++) {
2072 			int fd, group_fd;
2073 retry_open:
2074 			if (thread >= nthreads)
2075 				break;
2076 
2077 			if (!evsel->cgrp && !evsel->core.system_wide)
2078 				pid = perf_thread_map__pid(threads, thread);
2079 
2080 			group_fd = get_group_fd(evsel, idx, thread);
2081 
2082 			test_attr__ready();
2083 
2084 			/* Debug message used by test scripts */
2085 			pr_debug2_peo("sys_perf_event_open: pid %d  cpu %d  group_fd %d  flags %#lx",
2086 				pid, perf_cpu_map__cpu(cpus, idx).cpu, group_fd, evsel->open_flags);
2087 
2088 			fd = sys_perf_event_open(&evsel->core.attr, pid,
2089 						perf_cpu_map__cpu(cpus, idx).cpu,
2090 						group_fd, evsel->open_flags);
2091 
2092 			FD(evsel, idx, thread) = fd;
2093 
2094 			if (fd < 0) {
2095 				err = -errno;
2096 
2097 				pr_debug2_peo("\nsys_perf_event_open failed, error %d\n",
2098 					  err);
2099 				goto try_fallback;
2100 			}
2101 
2102 			bpf_counter__install_pe(evsel, idx, fd);
2103 
2104 			if (unlikely(test_attr__enabled)) {
2105 				test_attr__open(&evsel->core.attr, pid,
2106 						perf_cpu_map__cpu(cpus, idx),
2107 						fd, group_fd, evsel->open_flags);
2108 			}
2109 
2110 			/* Debug message used by test scripts */
2111 			pr_debug2_peo(" = %d\n", fd);
2112 
2113 			if (evsel->bpf_fd >= 0) {
2114 				int evt_fd = fd;
2115 				int bpf_fd = evsel->bpf_fd;
2116 
2117 				err = ioctl(evt_fd,
2118 					    PERF_EVENT_IOC_SET_BPF,
2119 					    bpf_fd);
2120 				if (err && errno != EEXIST) {
2121 					pr_err("failed to attach bpf fd %d: %s\n",
2122 					       bpf_fd, strerror(errno));
2123 					err = -EINVAL;
2124 					goto out_close;
2125 				}
2126 			}
2127 
2128 			set_rlimit = NO_CHANGE;
2129 
2130 			/*
2131 			 * If we succeeded but had to kill clockid, fail and
2132 			 * have evsel__open_strerror() print us a nice error.
2133 			 */
2134 			if (perf_missing_features.clockid ||
2135 			    perf_missing_features.clockid_wrong) {
2136 				err = -EINVAL;
2137 				goto out_close;
2138 			}
2139 		}
2140 	}
2141 
2142 	return 0;
2143 
2144 try_fallback:
2145 	if (evsel__precise_ip_fallback(evsel))
2146 		goto retry_open;
2147 
2148 	if (evsel__ignore_missing_thread(evsel, perf_cpu_map__nr(cpus),
2149 					 idx, threads, thread, err)) {
2150 		/* We just removed 1 thread, so lower the upper nthreads limit. */
2151 		nthreads--;
2152 
2153 		/* ... and pretend like nothing have happened. */
2154 		err = 0;
2155 		goto retry_open;
2156 	}
2157 	/*
2158 	 * perf stat needs between 5 and 22 fds per CPU. When we run out
2159 	 * of them try to increase the limits.
2160 	 */
2161 	if (err == -EMFILE && evsel__increase_rlimit(&set_rlimit))
2162 		goto retry_open;
2163 
2164 	if (err != -EINVAL || idx > 0 || thread > 0)
2165 		goto out_close;
2166 
2167 	if (evsel__detect_missing_features(evsel))
2168 		goto fallback_missing_features;
2169 out_close:
2170 	if (err)
2171 		threads->err_thread = thread;
2172 
2173 	old_errno = errno;
2174 	do {
2175 		while (--thread >= 0) {
2176 			if (FD(evsel, idx, thread) >= 0)
2177 				close(FD(evsel, idx, thread));
2178 			FD(evsel, idx, thread) = -1;
2179 		}
2180 		thread = nthreads;
2181 	} while (--idx >= 0);
2182 	errno = old_errno;
2183 	return err;
2184 }
2185 
evsel__open(struct evsel * evsel,struct perf_cpu_map * cpus,struct perf_thread_map * threads)2186 int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus,
2187 		struct perf_thread_map *threads)
2188 {
2189 	return evsel__open_cpu(evsel, cpus, threads, 0, perf_cpu_map__nr(cpus));
2190 }
2191 
evsel__close(struct evsel * evsel)2192 void evsel__close(struct evsel *evsel)
2193 {
2194 	perf_evsel__close(&evsel->core);
2195 	perf_evsel__free_id(&evsel->core);
2196 }
2197 
evsel__open_per_cpu(struct evsel * evsel,struct perf_cpu_map * cpus,int cpu_map_idx)2198 int evsel__open_per_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int cpu_map_idx)
2199 {
2200 	if (cpu_map_idx == -1)
2201 		return evsel__open_cpu(evsel, cpus, NULL, 0, perf_cpu_map__nr(cpus));
2202 
2203 	return evsel__open_cpu(evsel, cpus, NULL, cpu_map_idx, cpu_map_idx + 1);
2204 }
2205 
evsel__open_per_thread(struct evsel * evsel,struct perf_thread_map * threads)2206 int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads)
2207 {
2208 	return evsel__open(evsel, NULL, threads);
2209 }
2210 
perf_evsel__parse_id_sample(const struct evsel * evsel,const union perf_event * event,struct perf_sample * sample)2211 static int perf_evsel__parse_id_sample(const struct evsel *evsel,
2212 				       const union perf_event *event,
2213 				       struct perf_sample *sample)
2214 {
2215 	u64 type = evsel->core.attr.sample_type;
2216 	const __u64 *array = event->sample.array;
2217 	bool swapped = evsel->needs_swap;
2218 	union u64_swap u;
2219 
2220 	array += ((event->header.size -
2221 		   sizeof(event->header)) / sizeof(u64)) - 1;
2222 
2223 	if (type & PERF_SAMPLE_IDENTIFIER) {
2224 		sample->id = *array;
2225 		array--;
2226 	}
2227 
2228 	if (type & PERF_SAMPLE_CPU) {
2229 		u.val64 = *array;
2230 		if (swapped) {
2231 			/* undo swap of u64, then swap on individual u32s */
2232 			u.val64 = bswap_64(u.val64);
2233 			u.val32[0] = bswap_32(u.val32[0]);
2234 		}
2235 
2236 		sample->cpu = u.val32[0];
2237 		array--;
2238 	}
2239 
2240 	if (type & PERF_SAMPLE_STREAM_ID) {
2241 		sample->stream_id = *array;
2242 		array--;
2243 	}
2244 
2245 	if (type & PERF_SAMPLE_ID) {
2246 		sample->id = *array;
2247 		array--;
2248 	}
2249 
2250 	if (type & PERF_SAMPLE_TIME) {
2251 		sample->time = *array;
2252 		array--;
2253 	}
2254 
2255 	if (type & PERF_SAMPLE_TID) {
2256 		u.val64 = *array;
2257 		if (swapped) {
2258 			/* undo swap of u64, then swap on individual u32s */
2259 			u.val64 = bswap_64(u.val64);
2260 			u.val32[0] = bswap_32(u.val32[0]);
2261 			u.val32[1] = bswap_32(u.val32[1]);
2262 		}
2263 
2264 		sample->pid = u.val32[0];
2265 		sample->tid = u.val32[1];
2266 		array--;
2267 	}
2268 
2269 	return 0;
2270 }
2271 
overflow(const void * endp,u16 max_size,const void * offset,u64 size)2272 static inline bool overflow(const void *endp, u16 max_size, const void *offset,
2273 			    u64 size)
2274 {
2275 	return size > max_size || offset + size > endp;
2276 }
2277 
2278 #define OVERFLOW_CHECK(offset, size, max_size)				\
2279 	do {								\
2280 		if (overflow(endp, (max_size), (offset), (size)))	\
2281 			return -EFAULT;					\
2282 	} while (0)
2283 
2284 #define OVERFLOW_CHECK_u64(offset) \
2285 	OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64))
2286 
2287 static int
perf_event__check_size(union perf_event * event,unsigned int sample_size)2288 perf_event__check_size(union perf_event *event, unsigned int sample_size)
2289 {
2290 	/*
2291 	 * The evsel's sample_size is based on PERF_SAMPLE_MASK which includes
2292 	 * up to PERF_SAMPLE_PERIOD.  After that overflow() must be used to
2293 	 * check the format does not go past the end of the event.
2294 	 */
2295 	if (sample_size + sizeof(event->header) > event->header.size)
2296 		return -EFAULT;
2297 
2298 	return 0;
2299 }
2300 
arch_perf_parse_sample_weight(struct perf_sample * data,const __u64 * array,u64 type __maybe_unused)2301 void __weak arch_perf_parse_sample_weight(struct perf_sample *data,
2302 					  const __u64 *array,
2303 					  u64 type __maybe_unused)
2304 {
2305 	data->weight = *array;
2306 }
2307 
evsel__bitfield_swap_branch_flags(u64 value)2308 u64 evsel__bitfield_swap_branch_flags(u64 value)
2309 {
2310 	u64 new_val = 0;
2311 
2312 	/*
2313 	 * branch_flags
2314 	 * union {
2315 	 * 	u64 values;
2316 	 * 	struct {
2317 	 * 		mispred:1	//target mispredicted
2318 	 * 		predicted:1	//target predicted
2319 	 * 		in_tx:1		//in transaction
2320 	 * 		abort:1		//transaction abort
2321 	 * 		cycles:16	//cycle count to last branch
2322 	 * 		type:4		//branch type
2323 	 * 		reserved:40
2324 	 * 	}
2325 	 * }
2326 	 *
2327 	 * Avoid bswap64() the entire branch_flag.value,
2328 	 * as it has variable bit-field sizes. Instead the
2329 	 * macro takes the bit-field position/size,
2330 	 * swaps it based on the host endianness.
2331 	 *
2332 	 * tep_is_bigendian() is used here instead of
2333 	 * bigendian() to avoid python test fails.
2334 	 */
2335 	if (tep_is_bigendian()) {
2336 		new_val = bitfield_swap(value, 0, 1);
2337 		new_val |= bitfield_swap(value, 1, 1);
2338 		new_val |= bitfield_swap(value, 2, 1);
2339 		new_val |= bitfield_swap(value, 3, 1);
2340 		new_val |= bitfield_swap(value, 4, 16);
2341 		new_val |= bitfield_swap(value, 20, 4);
2342 		new_val |= bitfield_swap(value, 24, 40);
2343 	} else {
2344 		new_val = bitfield_swap(value, 63, 1);
2345 		new_val |= bitfield_swap(value, 62, 1);
2346 		new_val |= bitfield_swap(value, 61, 1);
2347 		new_val |= bitfield_swap(value, 60, 1);
2348 		new_val |= bitfield_swap(value, 44, 16);
2349 		new_val |= bitfield_swap(value, 40, 4);
2350 		new_val |= bitfield_swap(value, 0, 40);
2351 	}
2352 
2353 	return new_val;
2354 }
2355 
evsel__parse_sample(struct evsel * evsel,union perf_event * event,struct perf_sample * data)2356 int evsel__parse_sample(struct evsel *evsel, union perf_event *event,
2357 			struct perf_sample *data)
2358 {
2359 	u64 type = evsel->core.attr.sample_type;
2360 	bool swapped = evsel->needs_swap;
2361 	const __u64 *array;
2362 	u16 max_size = event->header.size;
2363 	const void *endp = (void *)event + max_size;
2364 	u64 sz;
2365 
2366 	/*
2367 	 * used for cross-endian analysis. See git commit 65014ab3
2368 	 * for why this goofiness is needed.
2369 	 */
2370 	union u64_swap u;
2371 
2372 	memset(data, 0, sizeof(*data));
2373 	data->cpu = data->pid = data->tid = -1;
2374 	data->stream_id = data->id = data->time = -1ULL;
2375 	data->period = evsel->core.attr.sample_period;
2376 	data->cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
2377 	data->misc    = event->header.misc;
2378 	data->id = -1ULL;
2379 	data->data_src = PERF_MEM_DATA_SRC_NONE;
2380 	data->vcpu = -1;
2381 
2382 	if (event->header.type != PERF_RECORD_SAMPLE) {
2383 		if (!evsel->core.attr.sample_id_all)
2384 			return 0;
2385 		return perf_evsel__parse_id_sample(evsel, event, data);
2386 	}
2387 
2388 	array = event->sample.array;
2389 
2390 	if (perf_event__check_size(event, evsel->sample_size))
2391 		return -EFAULT;
2392 
2393 	if (type & PERF_SAMPLE_IDENTIFIER) {
2394 		data->id = *array;
2395 		array++;
2396 	}
2397 
2398 	if (type & PERF_SAMPLE_IP) {
2399 		data->ip = *array;
2400 		array++;
2401 	}
2402 
2403 	if (type & PERF_SAMPLE_TID) {
2404 		u.val64 = *array;
2405 		if (swapped) {
2406 			/* undo swap of u64, then swap on individual u32s */
2407 			u.val64 = bswap_64(u.val64);
2408 			u.val32[0] = bswap_32(u.val32[0]);
2409 			u.val32[1] = bswap_32(u.val32[1]);
2410 		}
2411 
2412 		data->pid = u.val32[0];
2413 		data->tid = u.val32[1];
2414 		array++;
2415 	}
2416 
2417 	if (type & PERF_SAMPLE_TIME) {
2418 		data->time = *array;
2419 		array++;
2420 	}
2421 
2422 	if (type & PERF_SAMPLE_ADDR) {
2423 		data->addr = *array;
2424 		array++;
2425 	}
2426 
2427 	if (type & PERF_SAMPLE_ID) {
2428 		data->id = *array;
2429 		array++;
2430 	}
2431 
2432 	if (type & PERF_SAMPLE_STREAM_ID) {
2433 		data->stream_id = *array;
2434 		array++;
2435 	}
2436 
2437 	if (type & PERF_SAMPLE_CPU) {
2438 
2439 		u.val64 = *array;
2440 		if (swapped) {
2441 			/* undo swap of u64, then swap on individual u32s */
2442 			u.val64 = bswap_64(u.val64);
2443 			u.val32[0] = bswap_32(u.val32[0]);
2444 		}
2445 
2446 		data->cpu = u.val32[0];
2447 		array++;
2448 	}
2449 
2450 	if (type & PERF_SAMPLE_PERIOD) {
2451 		data->period = *array;
2452 		array++;
2453 	}
2454 
2455 	if (type & PERF_SAMPLE_READ) {
2456 		u64 read_format = evsel->core.attr.read_format;
2457 
2458 		OVERFLOW_CHECK_u64(array);
2459 		if (read_format & PERF_FORMAT_GROUP)
2460 			data->read.group.nr = *array;
2461 		else
2462 			data->read.one.value = *array;
2463 
2464 		array++;
2465 
2466 		if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2467 			OVERFLOW_CHECK_u64(array);
2468 			data->read.time_enabled = *array;
2469 			array++;
2470 		}
2471 
2472 		if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2473 			OVERFLOW_CHECK_u64(array);
2474 			data->read.time_running = *array;
2475 			array++;
2476 		}
2477 
2478 		/* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
2479 		if (read_format & PERF_FORMAT_GROUP) {
2480 			const u64 max_group_nr = UINT64_MAX /
2481 					sizeof(struct sample_read_value);
2482 
2483 			if (data->read.group.nr > max_group_nr)
2484 				return -EFAULT;
2485 
2486 			sz = data->read.group.nr * sample_read_value_size(read_format);
2487 			OVERFLOW_CHECK(array, sz, max_size);
2488 			data->read.group.values =
2489 					(struct sample_read_value *)array;
2490 			array = (void *)array + sz;
2491 		} else {
2492 			OVERFLOW_CHECK_u64(array);
2493 			data->read.one.id = *array;
2494 			array++;
2495 
2496 			if (read_format & PERF_FORMAT_LOST) {
2497 				OVERFLOW_CHECK_u64(array);
2498 				data->read.one.lost = *array;
2499 				array++;
2500 			}
2501 		}
2502 	}
2503 
2504 	if (type & PERF_SAMPLE_CALLCHAIN) {
2505 		const u64 max_callchain_nr = UINT64_MAX / sizeof(u64);
2506 
2507 		OVERFLOW_CHECK_u64(array);
2508 		data->callchain = (struct ip_callchain *)array++;
2509 		if (data->callchain->nr > max_callchain_nr)
2510 			return -EFAULT;
2511 		sz = data->callchain->nr * sizeof(u64);
2512 		OVERFLOW_CHECK(array, sz, max_size);
2513 		array = (void *)array + sz;
2514 	}
2515 
2516 	if (type & PERF_SAMPLE_RAW) {
2517 		OVERFLOW_CHECK_u64(array);
2518 		u.val64 = *array;
2519 
2520 		/*
2521 		 * Undo swap of u64, then swap on individual u32s,
2522 		 * get the size of the raw area and undo all of the
2523 		 * swap. The pevent interface handles endianness by
2524 		 * itself.
2525 		 */
2526 		if (swapped) {
2527 			u.val64 = bswap_64(u.val64);
2528 			u.val32[0] = bswap_32(u.val32[0]);
2529 			u.val32[1] = bswap_32(u.val32[1]);
2530 		}
2531 		data->raw_size = u.val32[0];
2532 
2533 		/*
2534 		 * The raw data is aligned on 64bits including the
2535 		 * u32 size, so it's safe to use mem_bswap_64.
2536 		 */
2537 		if (swapped)
2538 			mem_bswap_64((void *) array, data->raw_size);
2539 
2540 		array = (void *)array + sizeof(u32);
2541 
2542 		OVERFLOW_CHECK(array, data->raw_size, max_size);
2543 		data->raw_data = (void *)array;
2544 		array = (void *)array + data->raw_size;
2545 	}
2546 
2547 	if (type & PERF_SAMPLE_BRANCH_STACK) {
2548 		const u64 max_branch_nr = UINT64_MAX /
2549 					  sizeof(struct branch_entry);
2550 		struct branch_entry *e;
2551 		unsigned int i;
2552 
2553 		OVERFLOW_CHECK_u64(array);
2554 		data->branch_stack = (struct branch_stack *)array++;
2555 
2556 		if (data->branch_stack->nr > max_branch_nr)
2557 			return -EFAULT;
2558 
2559 		sz = data->branch_stack->nr * sizeof(struct branch_entry);
2560 		if (evsel__has_branch_hw_idx(evsel)) {
2561 			sz += sizeof(u64);
2562 			e = &data->branch_stack->entries[0];
2563 		} else {
2564 			data->no_hw_idx = true;
2565 			/*
2566 			 * if the PERF_SAMPLE_BRANCH_HW_INDEX is not applied,
2567 			 * only nr and entries[] will be output by kernel.
2568 			 */
2569 			e = (struct branch_entry *)&data->branch_stack->hw_idx;
2570 		}
2571 
2572 		if (swapped) {
2573 			/*
2574 			 * struct branch_flag does not have endian
2575 			 * specific bit field definition. And bswap
2576 			 * will not resolve the issue, since these
2577 			 * are bit fields.
2578 			 *
2579 			 * evsel__bitfield_swap_branch_flags() uses a
2580 			 * bitfield_swap macro to swap the bit position
2581 			 * based on the host endians.
2582 			 */
2583 			for (i = 0; i < data->branch_stack->nr; i++, e++)
2584 				e->flags.value = evsel__bitfield_swap_branch_flags(e->flags.value);
2585 		}
2586 
2587 		OVERFLOW_CHECK(array, sz, max_size);
2588 		array = (void *)array + sz;
2589 	}
2590 
2591 	if (type & PERF_SAMPLE_REGS_USER) {
2592 		OVERFLOW_CHECK_u64(array);
2593 		data->user_regs.abi = *array;
2594 		array++;
2595 
2596 		if (data->user_regs.abi) {
2597 			u64 mask = evsel->core.attr.sample_regs_user;
2598 
2599 			sz = hweight64(mask) * sizeof(u64);
2600 			OVERFLOW_CHECK(array, sz, max_size);
2601 			data->user_regs.mask = mask;
2602 			data->user_regs.regs = (u64 *)array;
2603 			array = (void *)array + sz;
2604 		}
2605 	}
2606 
2607 	if (type & PERF_SAMPLE_STACK_USER) {
2608 		OVERFLOW_CHECK_u64(array);
2609 		sz = *array++;
2610 
2611 		data->user_stack.offset = ((char *)(array - 1)
2612 					  - (char *) event);
2613 
2614 		if (!sz) {
2615 			data->user_stack.size = 0;
2616 		} else {
2617 			OVERFLOW_CHECK(array, sz, max_size);
2618 			data->user_stack.data = (char *)array;
2619 			array = (void *)array + sz;
2620 			OVERFLOW_CHECK_u64(array);
2621 			data->user_stack.size = *array++;
2622 			if (WARN_ONCE(data->user_stack.size > sz,
2623 				      "user stack dump failure\n"))
2624 				return -EFAULT;
2625 		}
2626 	}
2627 
2628 	if (type & PERF_SAMPLE_WEIGHT_TYPE) {
2629 		OVERFLOW_CHECK_u64(array);
2630 		arch_perf_parse_sample_weight(data, array, type);
2631 		array++;
2632 	}
2633 
2634 	if (type & PERF_SAMPLE_DATA_SRC) {
2635 		OVERFLOW_CHECK_u64(array);
2636 		data->data_src = *array;
2637 		array++;
2638 	}
2639 
2640 	if (type & PERF_SAMPLE_TRANSACTION) {
2641 		OVERFLOW_CHECK_u64(array);
2642 		data->transaction = *array;
2643 		array++;
2644 	}
2645 
2646 	data->intr_regs.abi = PERF_SAMPLE_REGS_ABI_NONE;
2647 	if (type & PERF_SAMPLE_REGS_INTR) {
2648 		OVERFLOW_CHECK_u64(array);
2649 		data->intr_regs.abi = *array;
2650 		array++;
2651 
2652 		if (data->intr_regs.abi != PERF_SAMPLE_REGS_ABI_NONE) {
2653 			u64 mask = evsel->core.attr.sample_regs_intr;
2654 
2655 			sz = hweight64(mask) * sizeof(u64);
2656 			OVERFLOW_CHECK(array, sz, max_size);
2657 			data->intr_regs.mask = mask;
2658 			data->intr_regs.regs = (u64 *)array;
2659 			array = (void *)array + sz;
2660 		}
2661 	}
2662 
2663 	data->phys_addr = 0;
2664 	if (type & PERF_SAMPLE_PHYS_ADDR) {
2665 		data->phys_addr = *array;
2666 		array++;
2667 	}
2668 
2669 	data->cgroup = 0;
2670 	if (type & PERF_SAMPLE_CGROUP) {
2671 		data->cgroup = *array;
2672 		array++;
2673 	}
2674 
2675 	data->data_page_size = 0;
2676 	if (type & PERF_SAMPLE_DATA_PAGE_SIZE) {
2677 		data->data_page_size = *array;
2678 		array++;
2679 	}
2680 
2681 	data->code_page_size = 0;
2682 	if (type & PERF_SAMPLE_CODE_PAGE_SIZE) {
2683 		data->code_page_size = *array;
2684 		array++;
2685 	}
2686 
2687 	if (type & PERF_SAMPLE_AUX) {
2688 		OVERFLOW_CHECK_u64(array);
2689 		sz = *array++;
2690 
2691 		OVERFLOW_CHECK(array, sz, max_size);
2692 		/* Undo swap of data */
2693 		if (swapped)
2694 			mem_bswap_64((char *)array, sz);
2695 		data->aux_sample.size = sz;
2696 		data->aux_sample.data = (char *)array;
2697 		array = (void *)array + sz;
2698 	}
2699 
2700 	return 0;
2701 }
2702 
evsel__parse_sample_timestamp(struct evsel * evsel,union perf_event * event,u64 * timestamp)2703 int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event,
2704 				  u64 *timestamp)
2705 {
2706 	u64 type = evsel->core.attr.sample_type;
2707 	const __u64 *array;
2708 
2709 	if (!(type & PERF_SAMPLE_TIME))
2710 		return -1;
2711 
2712 	if (event->header.type != PERF_RECORD_SAMPLE) {
2713 		struct perf_sample data = {
2714 			.time = -1ULL,
2715 		};
2716 
2717 		if (!evsel->core.attr.sample_id_all)
2718 			return -1;
2719 		if (perf_evsel__parse_id_sample(evsel, event, &data))
2720 			return -1;
2721 
2722 		*timestamp = data.time;
2723 		return 0;
2724 	}
2725 
2726 	array = event->sample.array;
2727 
2728 	if (perf_event__check_size(event, evsel->sample_size))
2729 		return -EFAULT;
2730 
2731 	if (type & PERF_SAMPLE_IDENTIFIER)
2732 		array++;
2733 
2734 	if (type & PERF_SAMPLE_IP)
2735 		array++;
2736 
2737 	if (type & PERF_SAMPLE_TID)
2738 		array++;
2739 
2740 	if (type & PERF_SAMPLE_TIME)
2741 		*timestamp = *array;
2742 
2743 	return 0;
2744 }
2745 
evsel__id_hdr_size(struct evsel * evsel)2746 u16 evsel__id_hdr_size(struct evsel *evsel)
2747 {
2748 	u64 sample_type = evsel->core.attr.sample_type;
2749 	u16 size = 0;
2750 
2751 	if (sample_type & PERF_SAMPLE_TID)
2752 		size += sizeof(u64);
2753 
2754 	if (sample_type & PERF_SAMPLE_TIME)
2755 		size += sizeof(u64);
2756 
2757 	if (sample_type & PERF_SAMPLE_ID)
2758 		size += sizeof(u64);
2759 
2760 	if (sample_type & PERF_SAMPLE_STREAM_ID)
2761 		size += sizeof(u64);
2762 
2763 	if (sample_type & PERF_SAMPLE_CPU)
2764 		size += sizeof(u64);
2765 
2766 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
2767 		size += sizeof(u64);
2768 
2769 	return size;
2770 }
2771 
evsel__field(struct evsel * evsel,const char * name)2772 struct tep_format_field *evsel__field(struct evsel *evsel, const char *name)
2773 {
2774 	return tep_find_field(evsel->tp_format, name);
2775 }
2776 
evsel__rawptr(struct evsel * evsel,struct perf_sample * sample,const char * name)2777 void *evsel__rawptr(struct evsel *evsel, struct perf_sample *sample, const char *name)
2778 {
2779 	struct tep_format_field *field = evsel__field(evsel, name);
2780 	int offset;
2781 
2782 	if (!field)
2783 		return NULL;
2784 
2785 	offset = field->offset;
2786 
2787 	if (field->flags & TEP_FIELD_IS_DYNAMIC) {
2788 		offset = *(int *)(sample->raw_data + field->offset);
2789 		offset &= 0xffff;
2790 		if (field->flags & TEP_FIELD_IS_RELATIVE)
2791 			offset += field->offset + field->size;
2792 	}
2793 
2794 	return sample->raw_data + offset;
2795 }
2796 
format_field__intval(struct tep_format_field * field,struct perf_sample * sample,bool needs_swap)2797 u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample,
2798 			 bool needs_swap)
2799 {
2800 	u64 value;
2801 	void *ptr = sample->raw_data + field->offset;
2802 
2803 	switch (field->size) {
2804 	case 1:
2805 		return *(u8 *)ptr;
2806 	case 2:
2807 		value = *(u16 *)ptr;
2808 		break;
2809 	case 4:
2810 		value = *(u32 *)ptr;
2811 		break;
2812 	case 8:
2813 		memcpy(&value, ptr, sizeof(u64));
2814 		break;
2815 	default:
2816 		return 0;
2817 	}
2818 
2819 	if (!needs_swap)
2820 		return value;
2821 
2822 	switch (field->size) {
2823 	case 2:
2824 		return bswap_16(value);
2825 	case 4:
2826 		return bswap_32(value);
2827 	case 8:
2828 		return bswap_64(value);
2829 	default:
2830 		return 0;
2831 	}
2832 
2833 	return 0;
2834 }
2835 
evsel__intval(struct evsel * evsel,struct perf_sample * sample,const char * name)2836 u64 evsel__intval(struct evsel *evsel, struct perf_sample *sample, const char *name)
2837 {
2838 	struct tep_format_field *field = evsel__field(evsel, name);
2839 
2840 	if (!field)
2841 		return 0;
2842 
2843 	return field ? format_field__intval(field, sample, evsel->needs_swap) : 0;
2844 }
2845 
evsel__fallback(struct evsel * evsel,int err,char * msg,size_t msgsize)2846 bool evsel__fallback(struct evsel *evsel, int err, char *msg, size_t msgsize)
2847 {
2848 	int paranoid;
2849 
2850 	if ((err == ENOENT || err == ENXIO || err == ENODEV) &&
2851 	    evsel->core.attr.type   == PERF_TYPE_HARDWARE &&
2852 	    evsel->core.attr.config == PERF_COUNT_HW_CPU_CYCLES) {
2853 		/*
2854 		 * If it's cycles then fall back to hrtimer based
2855 		 * cpu-clock-tick sw counter, which is always available even if
2856 		 * no PMU support.
2857 		 *
2858 		 * PPC returns ENXIO until 2.6.37 (behavior changed with commit
2859 		 * b0a873e).
2860 		 */
2861 		scnprintf(msg, msgsize, "%s",
2862 "The cycles event is not supported, trying to fall back to cpu-clock-ticks");
2863 
2864 		evsel->core.attr.type   = PERF_TYPE_SOFTWARE;
2865 		evsel->core.attr.config = PERF_COUNT_SW_CPU_CLOCK;
2866 
2867 		zfree(&evsel->name);
2868 		return true;
2869 	} else if (err == EACCES && !evsel->core.attr.exclude_kernel &&
2870 		   (paranoid = perf_event_paranoid()) > 1) {
2871 		const char *name = evsel__name(evsel);
2872 		char *new_name;
2873 		const char *sep = ":";
2874 
2875 		/* If event has exclude user then don't exclude kernel. */
2876 		if (evsel->core.attr.exclude_user)
2877 			return false;
2878 
2879 		/* Is there already the separator in the name. */
2880 		if (strchr(name, '/') ||
2881 		    (strchr(name, ':') && !evsel->is_libpfm_event))
2882 			sep = "";
2883 
2884 		if (asprintf(&new_name, "%s%su", name, sep) < 0)
2885 			return false;
2886 
2887 		if (evsel->name)
2888 			free(evsel->name);
2889 		evsel->name = new_name;
2890 		scnprintf(msg, msgsize, "kernel.perf_event_paranoid=%d, trying "
2891 			  "to fall back to excluding kernel and hypervisor "
2892 			  " samples", paranoid);
2893 		evsel->core.attr.exclude_kernel = 1;
2894 		evsel->core.attr.exclude_hv     = 1;
2895 
2896 		return true;
2897 	}
2898 
2899 	return false;
2900 }
2901 
find_process(const char * name)2902 static bool find_process(const char *name)
2903 {
2904 	size_t len = strlen(name);
2905 	DIR *dir;
2906 	struct dirent *d;
2907 	int ret = -1;
2908 
2909 	dir = opendir(procfs__mountpoint());
2910 	if (!dir)
2911 		return false;
2912 
2913 	/* Walk through the directory. */
2914 	while (ret && (d = readdir(dir)) != NULL) {
2915 		char path[PATH_MAX];
2916 		char *data;
2917 		size_t size;
2918 
2919 		if ((d->d_type != DT_DIR) ||
2920 		     !strcmp(".", d->d_name) ||
2921 		     !strcmp("..", d->d_name))
2922 			continue;
2923 
2924 		scnprintf(path, sizeof(path), "%s/%s/comm",
2925 			  procfs__mountpoint(), d->d_name);
2926 
2927 		if (filename__read_str(path, &data, &size))
2928 			continue;
2929 
2930 		ret = strncmp(name, data, len);
2931 		free(data);
2932 	}
2933 
2934 	closedir(dir);
2935 	return ret ? false : true;
2936 }
2937 
is_amd(const char * arch,const char * cpuid)2938 static bool is_amd(const char *arch, const char *cpuid)
2939 {
2940 	return arch && !strcmp("x86", arch) && cpuid && strstarts(cpuid, "AuthenticAMD");
2941 }
2942 
is_amd_ibs(struct evsel * evsel)2943 static bool is_amd_ibs(struct evsel *evsel)
2944 {
2945 	return evsel->core.attr.precise_ip
2946 	    || (evsel->pmu_name && !strncmp(evsel->pmu_name, "ibs", 3));
2947 }
2948 
evsel__open_strerror(struct evsel * evsel,struct target * target,int err,char * msg,size_t size)2949 int evsel__open_strerror(struct evsel *evsel, struct target *target,
2950 			 int err, char *msg, size_t size)
2951 {
2952 	struct perf_env *env = evsel__env(evsel);
2953 	const char *arch = perf_env__arch(env);
2954 	const char *cpuid = perf_env__cpuid(env);
2955 	char sbuf[STRERR_BUFSIZE];
2956 	int printed = 0, enforced = 0;
2957 
2958 	switch (err) {
2959 	case EPERM:
2960 	case EACCES:
2961 		printed += scnprintf(msg + printed, size - printed,
2962 			"Access to performance monitoring and observability operations is limited.\n");
2963 
2964 		if (!sysfs__read_int("fs/selinux/enforce", &enforced)) {
2965 			if (enforced) {
2966 				printed += scnprintf(msg + printed, size - printed,
2967 					"Enforced MAC policy settings (SELinux) can limit access to performance\n"
2968 					"monitoring and observability operations. Inspect system audit records for\n"
2969 					"more perf_event access control information and adjusting the policy.\n");
2970 			}
2971 		}
2972 
2973 		if (err == EPERM)
2974 			printed += scnprintf(msg, size,
2975 				"No permission to enable %s event.\n\n", evsel__name(evsel));
2976 
2977 		return scnprintf(msg + printed, size - printed,
2978 		 "Consider adjusting /proc/sys/kernel/perf_event_paranoid setting to open\n"
2979 		 "access to performance monitoring and observability operations for processes\n"
2980 		 "without CAP_PERFMON, CAP_SYS_PTRACE or CAP_SYS_ADMIN Linux capability.\n"
2981 		 "More information can be found at 'Perf events and tool security' document:\n"
2982 		 "https://www.kernel.org/doc/html/latest/admin-guide/perf-security.html\n"
2983 		 "perf_event_paranoid setting is %d:\n"
2984 		 "  -1: Allow use of (almost) all events by all users\n"
2985 		 "      Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK\n"
2986 		 ">= 0: Disallow raw and ftrace function tracepoint access\n"
2987 		 ">= 1: Disallow CPU event access\n"
2988 		 ">= 2: Disallow kernel profiling\n"
2989 		 "To make the adjusted perf_event_paranoid setting permanent preserve it\n"
2990 		 "in /etc/sysctl.conf (e.g. kernel.perf_event_paranoid = <setting>)",
2991 		 perf_event_paranoid());
2992 	case ENOENT:
2993 		return scnprintf(msg, size, "The %s event is not supported.", evsel__name(evsel));
2994 	case EMFILE:
2995 		return scnprintf(msg, size, "%s",
2996 			 "Too many events are opened.\n"
2997 			 "Probably the maximum number of open file descriptors has been reached.\n"
2998 			 "Hint: Try again after reducing the number of events.\n"
2999 			 "Hint: Try increasing the limit with 'ulimit -n <limit>'");
3000 	case ENOMEM:
3001 		if (evsel__has_callchain(evsel) &&
3002 		    access("/proc/sys/kernel/perf_event_max_stack", F_OK) == 0)
3003 			return scnprintf(msg, size,
3004 					 "Not enough memory to setup event with callchain.\n"
3005 					 "Hint: Try tweaking /proc/sys/kernel/perf_event_max_stack\n"
3006 					 "Hint: Current value: %d", sysctl__max_stack());
3007 		break;
3008 	case ENODEV:
3009 		if (target->cpu_list)
3010 			return scnprintf(msg, size, "%s",
3011 	 "No such device - did you specify an out-of-range profile CPU?");
3012 		break;
3013 	case EOPNOTSUPP:
3014 		if (evsel->core.attr.sample_type & PERF_SAMPLE_BRANCH_STACK)
3015 			return scnprintf(msg, size,
3016 	"%s: PMU Hardware or event type doesn't support branch stack sampling.",
3017 					 evsel__name(evsel));
3018 		if (evsel->core.attr.aux_output)
3019 			return scnprintf(msg, size,
3020 	"%s: PMU Hardware doesn't support 'aux_output' feature",
3021 					 evsel__name(evsel));
3022 		if (evsel->core.attr.sample_period != 0)
3023 			return scnprintf(msg, size,
3024 	"%s: PMU Hardware doesn't support sampling/overflow-interrupts. Try 'perf stat'",
3025 					 evsel__name(evsel));
3026 		if (evsel->core.attr.precise_ip)
3027 			return scnprintf(msg, size, "%s",
3028 	"\'precise\' request may not be supported. Try removing 'p' modifier.");
3029 #if defined(__i386__) || defined(__x86_64__)
3030 		if (evsel->core.attr.type == PERF_TYPE_HARDWARE)
3031 			return scnprintf(msg, size, "%s",
3032 	"No hardware sampling interrupt available.\n");
3033 #endif
3034 		break;
3035 	case EBUSY:
3036 		if (find_process("oprofiled"))
3037 			return scnprintf(msg, size,
3038 	"The PMU counters are busy/taken by another profiler.\n"
3039 	"We found oprofile daemon running, please stop it and try again.");
3040 		break;
3041 	case EINVAL:
3042 		if (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE && perf_missing_features.code_page_size)
3043 			return scnprintf(msg, size, "Asking for the code page size isn't supported by this kernel.");
3044 		if (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE && perf_missing_features.data_page_size)
3045 			return scnprintf(msg, size, "Asking for the data page size isn't supported by this kernel.");
3046 		if (evsel->core.attr.write_backward && perf_missing_features.write_backward)
3047 			return scnprintf(msg, size, "Reading from overwrite event is not supported by this kernel.");
3048 		if (perf_missing_features.clockid)
3049 			return scnprintf(msg, size, "clockid feature not supported.");
3050 		if (perf_missing_features.clockid_wrong)
3051 			return scnprintf(msg, size, "wrong clockid (%d).", clockid);
3052 		if (perf_missing_features.aux_output)
3053 			return scnprintf(msg, size, "The 'aux_output' feature is not supported, update the kernel.");
3054 		if (!target__has_cpu(target))
3055 			return scnprintf(msg, size,
3056 	"Invalid event (%s) in per-thread mode, enable system wide with '-a'.",
3057 					evsel__name(evsel));
3058 		if (is_amd(arch, cpuid)) {
3059 			if (is_amd_ibs(evsel)) {
3060 				if (evsel->core.attr.exclude_kernel)
3061 					return scnprintf(msg, size,
3062 	"AMD IBS can't exclude kernel events.  Try running at a higher privilege level.");
3063 				if (!evsel->core.system_wide)
3064 					return scnprintf(msg, size,
3065 	"AMD IBS may only be available in system-wide/per-cpu mode.  Try using -a, or -C and workload affinity");
3066 			}
3067 		}
3068 
3069 		break;
3070 	case ENODATA:
3071 		return scnprintf(msg, size, "Cannot collect data source with the load latency event alone. "
3072 				 "Please add an auxiliary event in front of the load latency event.");
3073 	default:
3074 		break;
3075 	}
3076 
3077 	return scnprintf(msg, size,
3078 	"The sys_perf_event_open() syscall returned with %d (%s) for event (%s).\n"
3079 	"/bin/dmesg | grep -i perf may provide additional information.\n",
3080 			 err, str_error_r(err, sbuf, sizeof(sbuf)), evsel__name(evsel));
3081 }
3082 
evsel__env(struct evsel * evsel)3083 struct perf_env *evsel__env(struct evsel *evsel)
3084 {
3085 	if (evsel && evsel->evlist && evsel->evlist->env)
3086 		return evsel->evlist->env;
3087 	return &perf_env;
3088 }
3089 
store_evsel_ids(struct evsel * evsel,struct evlist * evlist)3090 static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist)
3091 {
3092 	int cpu_map_idx, thread;
3093 
3094 	for (cpu_map_idx = 0; cpu_map_idx < xyarray__max_x(evsel->core.fd); cpu_map_idx++) {
3095 		for (thread = 0; thread < xyarray__max_y(evsel->core.fd);
3096 		     thread++) {
3097 			int fd = FD(evsel, cpu_map_idx, thread);
3098 
3099 			if (perf_evlist__id_add_fd(&evlist->core, &evsel->core,
3100 						   cpu_map_idx, thread, fd) < 0)
3101 				return -1;
3102 		}
3103 	}
3104 
3105 	return 0;
3106 }
3107 
evsel__store_ids(struct evsel * evsel,struct evlist * evlist)3108 int evsel__store_ids(struct evsel *evsel, struct evlist *evlist)
3109 {
3110 	struct perf_cpu_map *cpus = evsel->core.cpus;
3111 	struct perf_thread_map *threads = evsel->core.threads;
3112 
3113 	if (perf_evsel__alloc_id(&evsel->core, perf_cpu_map__nr(cpus), threads->nr))
3114 		return -ENOMEM;
3115 
3116 	return store_evsel_ids(evsel, evlist);
3117 }
3118 
evsel__zero_per_pkg(struct evsel * evsel)3119 void evsel__zero_per_pkg(struct evsel *evsel)
3120 {
3121 	struct hashmap_entry *cur;
3122 	size_t bkt;
3123 
3124 	if (evsel->per_pkg_mask) {
3125 		hashmap__for_each_entry(evsel->per_pkg_mask, cur, bkt)
3126 			free((char *)cur->key);
3127 
3128 		hashmap__clear(evsel->per_pkg_mask);
3129 	}
3130 }
3131 
evsel__is_hybrid(struct evsel * evsel)3132 bool evsel__is_hybrid(struct evsel *evsel)
3133 {
3134 	return evsel->pmu_name && perf_pmu__is_hybrid(evsel->pmu_name);
3135 }
3136 
evsel__leader(struct evsel * evsel)3137 struct evsel *evsel__leader(struct evsel *evsel)
3138 {
3139 	return container_of(evsel->core.leader, struct evsel, core);
3140 }
3141 
evsel__has_leader(struct evsel * evsel,struct evsel * leader)3142 bool evsel__has_leader(struct evsel *evsel, struct evsel *leader)
3143 {
3144 	return evsel->core.leader == &leader->core;
3145 }
3146 
evsel__is_leader(struct evsel * evsel)3147 bool evsel__is_leader(struct evsel *evsel)
3148 {
3149 	return evsel__has_leader(evsel, evsel);
3150 }
3151 
evsel__set_leader(struct evsel * evsel,struct evsel * leader)3152 void evsel__set_leader(struct evsel *evsel, struct evsel *leader)
3153 {
3154 	evsel->core.leader = &leader->core;
3155 }
3156 
evsel__source_count(const struct evsel * evsel)3157 int evsel__source_count(const struct evsel *evsel)
3158 {
3159 	struct evsel *pos;
3160 	int count = 0;
3161 
3162 	evlist__for_each_entry(evsel->evlist, pos) {
3163 		if (pos->metric_leader == evsel)
3164 			count++;
3165 	}
3166 	return count;
3167 }
3168 
arch_evsel__must_be_in_group(const struct evsel * evsel __maybe_unused)3169 bool __weak arch_evsel__must_be_in_group(const struct evsel *evsel __maybe_unused)
3170 {
3171 	return false;
3172 }
3173 
3174 /*
3175  * Remove an event from a given group (leader).
3176  * Some events, e.g., perf metrics Topdown events,
3177  * must always be grouped. Ignore the events.
3178  */
evsel__remove_from_group(struct evsel * evsel,struct evsel * leader)3179 void evsel__remove_from_group(struct evsel *evsel, struct evsel *leader)
3180 {
3181 	if (!arch_evsel__must_be_in_group(evsel) && evsel != leader) {
3182 		evsel__set_leader(evsel, evsel);
3183 		evsel->core.nr_members = 0;
3184 		leader->core.nr_members--;
3185 	}
3186 }
3187