1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com>
4  *
5  * Parts came from builtin-{top,stat,record}.c, see those files for further
6  * copyright notes.
7  */
8 
9 #include <byteswap.h>
10 #include <errno.h>
11 #include <inttypes.h>
12 #include <linux/bitops.h>
13 #include <api/fs/fs.h>
14 #include <api/fs/tracing_path.h>
15 #include <linux/hw_breakpoint.h>
16 #include <linux/perf_event.h>
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/zalloc.h>
20 #include <sys/ioctl.h>
21 #include <sys/resource.h>
22 #include <sys/types.h>
23 #include <dirent.h>
24 #include <stdlib.h>
25 #include <perf/evsel.h>
26 #include "asm/bug.h"
27 #include "bpf_counter.h"
28 #include "callchain.h"
29 #include "cgroup.h"
30 #include "counts.h"
31 #include "event.h"
32 #include "evsel.h"
33 #include "util/env.h"
34 #include "util/evsel_config.h"
35 #include "util/evsel_fprintf.h"
36 #include "evlist.h"
37 #include <perf/cpumap.h>
38 #include "thread_map.h"
39 #include "target.h"
40 #include "perf_regs.h"
41 #include "record.h"
42 #include "debug.h"
43 #include "trace-event.h"
44 #include "stat.h"
45 #include "string2.h"
46 #include "memswap.h"
47 #include "util.h"
48 #include "util/hashmap.h"
49 #include "off_cpu.h"
50 #include "pmu.h"
51 #include "pmus.h"
52 #include "../perf-sys.h"
53 #include "util/parse-branch-options.h"
54 #include "util/bpf-filter.h"
55 #include <internal/xyarray.h>
56 #include <internal/lib.h>
57 #include <internal/threadmap.h>
58 
59 #include <linux/ctype.h>
60 
61 #ifdef HAVE_LIBTRACEEVENT
62 #include <traceevent/event-parse.h>
63 #endif
64 
65 struct perf_missing_features perf_missing_features;
66 
67 static clockid_t clockid;
68 
69 static const char *const perf_tool_event__tool_names[PERF_TOOL_MAX] = {
70 	NULL,
71 	"duration_time",
72 	"user_time",
73 	"system_time",
74 };
75 
perf_tool_event__to_str(enum perf_tool_event ev)76 const char *perf_tool_event__to_str(enum perf_tool_event ev)
77 {
78 	if (ev > PERF_TOOL_NONE && ev < PERF_TOOL_MAX)
79 		return perf_tool_event__tool_names[ev];
80 
81 	return NULL;
82 }
83 
perf_tool_event__from_str(const char * str)84 enum perf_tool_event perf_tool_event__from_str(const char *str)
85 {
86 	int i;
87 
88 	perf_tool_event__for_each_event(i) {
89 		if (!strcmp(str, perf_tool_event__tool_names[i]))
90 			return i;
91 	}
92 	return PERF_TOOL_NONE;
93 }
94 
95 
evsel__no_extra_init(struct evsel * evsel __maybe_unused)96 static int evsel__no_extra_init(struct evsel *evsel __maybe_unused)
97 {
98 	return 0;
99 }
100 
test_attr__ready(void)101 void __weak test_attr__ready(void) { }
102 
evsel__no_extra_fini(struct evsel * evsel __maybe_unused)103 static void evsel__no_extra_fini(struct evsel *evsel __maybe_unused)
104 {
105 }
106 
107 static struct {
108 	size_t	size;
109 	int	(*init)(struct evsel *evsel);
110 	void	(*fini)(struct evsel *evsel);
111 } perf_evsel__object = {
112 	.size = sizeof(struct evsel),
113 	.init = evsel__no_extra_init,
114 	.fini = evsel__no_extra_fini,
115 };
116 
evsel__object_config(size_t object_size,int (* init)(struct evsel * evsel),void (* fini)(struct evsel * evsel))117 int evsel__object_config(size_t object_size, int (*init)(struct evsel *evsel),
118 			 void (*fini)(struct evsel *evsel))
119 {
120 
121 	if (object_size == 0)
122 		goto set_methods;
123 
124 	if (perf_evsel__object.size > object_size)
125 		return -EINVAL;
126 
127 	perf_evsel__object.size = object_size;
128 
129 set_methods:
130 	if (init != NULL)
131 		perf_evsel__object.init = init;
132 
133 	if (fini != NULL)
134 		perf_evsel__object.fini = fini;
135 
136 	return 0;
137 }
138 
139 #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y))
140 
__evsel__sample_size(u64 sample_type)141 int __evsel__sample_size(u64 sample_type)
142 {
143 	u64 mask = sample_type & PERF_SAMPLE_MASK;
144 	int size = 0;
145 	int i;
146 
147 	for (i = 0; i < 64; i++) {
148 		if (mask & (1ULL << i))
149 			size++;
150 	}
151 
152 	size *= sizeof(u64);
153 
154 	return size;
155 }
156 
157 /**
158  * __perf_evsel__calc_id_pos - calculate id_pos.
159  * @sample_type: sample type
160  *
161  * This function returns the position of the event id (PERF_SAMPLE_ID or
162  * PERF_SAMPLE_IDENTIFIER) in a sample event i.e. in the array of struct
163  * perf_record_sample.
164  */
__perf_evsel__calc_id_pos(u64 sample_type)165 static int __perf_evsel__calc_id_pos(u64 sample_type)
166 {
167 	int idx = 0;
168 
169 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
170 		return 0;
171 
172 	if (!(sample_type & PERF_SAMPLE_ID))
173 		return -1;
174 
175 	if (sample_type & PERF_SAMPLE_IP)
176 		idx += 1;
177 
178 	if (sample_type & PERF_SAMPLE_TID)
179 		idx += 1;
180 
181 	if (sample_type & PERF_SAMPLE_TIME)
182 		idx += 1;
183 
184 	if (sample_type & PERF_SAMPLE_ADDR)
185 		idx += 1;
186 
187 	return idx;
188 }
189 
190 /**
191  * __perf_evsel__calc_is_pos - calculate is_pos.
192  * @sample_type: sample type
193  *
194  * This function returns the position (counting backwards) of the event id
195  * (PERF_SAMPLE_ID or PERF_SAMPLE_IDENTIFIER) in a non-sample event i.e. if
196  * sample_id_all is used there is an id sample appended to non-sample events.
197  */
__perf_evsel__calc_is_pos(u64 sample_type)198 static int __perf_evsel__calc_is_pos(u64 sample_type)
199 {
200 	int idx = 1;
201 
202 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
203 		return 1;
204 
205 	if (!(sample_type & PERF_SAMPLE_ID))
206 		return -1;
207 
208 	if (sample_type & PERF_SAMPLE_CPU)
209 		idx += 1;
210 
211 	if (sample_type & PERF_SAMPLE_STREAM_ID)
212 		idx += 1;
213 
214 	return idx;
215 }
216 
evsel__calc_id_pos(struct evsel * evsel)217 void evsel__calc_id_pos(struct evsel *evsel)
218 {
219 	evsel->id_pos = __perf_evsel__calc_id_pos(evsel->core.attr.sample_type);
220 	evsel->is_pos = __perf_evsel__calc_is_pos(evsel->core.attr.sample_type);
221 }
222 
__evsel__set_sample_bit(struct evsel * evsel,enum perf_event_sample_format bit)223 void __evsel__set_sample_bit(struct evsel *evsel,
224 				  enum perf_event_sample_format bit)
225 {
226 	if (!(evsel->core.attr.sample_type & bit)) {
227 		evsel->core.attr.sample_type |= bit;
228 		evsel->sample_size += sizeof(u64);
229 		evsel__calc_id_pos(evsel);
230 	}
231 }
232 
__evsel__reset_sample_bit(struct evsel * evsel,enum perf_event_sample_format bit)233 void __evsel__reset_sample_bit(struct evsel *evsel,
234 				    enum perf_event_sample_format bit)
235 {
236 	if (evsel->core.attr.sample_type & bit) {
237 		evsel->core.attr.sample_type &= ~bit;
238 		evsel->sample_size -= sizeof(u64);
239 		evsel__calc_id_pos(evsel);
240 	}
241 }
242 
evsel__set_sample_id(struct evsel * evsel,bool can_sample_identifier)243 void evsel__set_sample_id(struct evsel *evsel,
244 			       bool can_sample_identifier)
245 {
246 	if (can_sample_identifier) {
247 		evsel__reset_sample_bit(evsel, ID);
248 		evsel__set_sample_bit(evsel, IDENTIFIER);
249 	} else {
250 		evsel__set_sample_bit(evsel, ID);
251 	}
252 	evsel->core.attr.read_format |= PERF_FORMAT_ID;
253 }
254 
255 /**
256  * evsel__is_function_event - Return whether given evsel is a function
257  * trace event
258  *
259  * @evsel - evsel selector to be tested
260  *
261  * Return %true if event is function trace event
262  */
evsel__is_function_event(struct evsel * evsel)263 bool evsel__is_function_event(struct evsel *evsel)
264 {
265 #define FUNCTION_EVENT "ftrace:function"
266 
267 	return evsel->name &&
268 	       !strncmp(FUNCTION_EVENT, evsel->name, sizeof(FUNCTION_EVENT));
269 
270 #undef FUNCTION_EVENT
271 }
272 
evsel__init(struct evsel * evsel,struct perf_event_attr * attr,int idx)273 void evsel__init(struct evsel *evsel,
274 		 struct perf_event_attr *attr, int idx)
275 {
276 	perf_evsel__init(&evsel->core, attr, idx);
277 	evsel->tracking	   = !idx;
278 	evsel->unit	   = strdup("");
279 	evsel->scale	   = 1.0;
280 	evsel->max_events  = ULONG_MAX;
281 	evsel->evlist	   = NULL;
282 	evsel->bpf_obj	   = NULL;
283 	evsel->bpf_fd	   = -1;
284 	INIT_LIST_HEAD(&evsel->config_terms);
285 	INIT_LIST_HEAD(&evsel->bpf_counter_list);
286 	INIT_LIST_HEAD(&evsel->bpf_filters);
287 	perf_evsel__object.init(evsel);
288 	evsel->sample_size = __evsel__sample_size(attr->sample_type);
289 	evsel__calc_id_pos(evsel);
290 	evsel->cmdline_group_boundary = false;
291 	evsel->metric_events = NULL;
292 	evsel->per_pkg_mask  = NULL;
293 	evsel->collect_stat  = false;
294 	evsel->pmu_name      = NULL;
295 	evsel->group_pmu_name = NULL;
296 	evsel->skippable     = false;
297 }
298 
evsel__new_idx(struct perf_event_attr * attr,int idx)299 struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx)
300 {
301 	struct evsel *evsel = zalloc(perf_evsel__object.size);
302 
303 	if (!evsel)
304 		return NULL;
305 	evsel__init(evsel, attr, idx);
306 
307 	if (evsel__is_bpf_output(evsel) && !attr->sample_type) {
308 		evsel->core.attr.sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
309 					    PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
310 		evsel->core.attr.sample_period = 1;
311 	}
312 
313 	if (evsel__is_clock(evsel)) {
314 		free((char *)evsel->unit);
315 		evsel->unit = strdup("msec");
316 		evsel->scale = 1e-6;
317 	}
318 
319 	return evsel;
320 }
321 
copy_config_terms(struct list_head * dst,struct list_head * src)322 int copy_config_terms(struct list_head *dst, struct list_head *src)
323 {
324 	struct evsel_config_term *pos, *tmp;
325 
326 	list_for_each_entry(pos, src, list) {
327 		tmp = malloc(sizeof(*tmp));
328 		if (tmp == NULL)
329 			return -ENOMEM;
330 
331 		*tmp = *pos;
332 		if (tmp->free_str) {
333 			tmp->val.str = strdup(pos->val.str);
334 			if (tmp->val.str == NULL) {
335 				free(tmp);
336 				return -ENOMEM;
337 			}
338 		}
339 		list_add_tail(&tmp->list, dst);
340 	}
341 	return 0;
342 }
343 
evsel__copy_config_terms(struct evsel * dst,struct evsel * src)344 static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src)
345 {
346 	return copy_config_terms(&dst->config_terms, &src->config_terms);
347 }
348 
349 /**
350  * evsel__clone - create a new evsel copied from @orig
351  * @orig: original evsel
352  *
353  * The assumption is that @orig is not configured nor opened yet.
354  * So we only care about the attributes that can be set while it's parsed.
355  */
evsel__clone(struct evsel * orig)356 struct evsel *evsel__clone(struct evsel *orig)
357 {
358 	struct evsel *evsel;
359 
360 	BUG_ON(orig->core.fd);
361 	BUG_ON(orig->counts);
362 	BUG_ON(orig->priv);
363 	BUG_ON(orig->per_pkg_mask);
364 
365 	/* cannot handle BPF objects for now */
366 	if (orig->bpf_obj)
367 		return NULL;
368 
369 	evsel = evsel__new(&orig->core.attr);
370 	if (evsel == NULL)
371 		return NULL;
372 
373 	evsel->core.cpus = perf_cpu_map__get(orig->core.cpus);
374 	evsel->core.own_cpus = perf_cpu_map__get(orig->core.own_cpus);
375 	evsel->core.threads = perf_thread_map__get(orig->core.threads);
376 	evsel->core.nr_members = orig->core.nr_members;
377 	evsel->core.system_wide = orig->core.system_wide;
378 	evsel->core.requires_cpu = orig->core.requires_cpu;
379 	evsel->core.is_pmu_core = orig->core.is_pmu_core;
380 
381 	if (orig->name) {
382 		evsel->name = strdup(orig->name);
383 		if (evsel->name == NULL)
384 			goto out_err;
385 	}
386 	if (orig->group_name) {
387 		evsel->group_name = strdup(orig->group_name);
388 		if (evsel->group_name == NULL)
389 			goto out_err;
390 	}
391 	if (orig->pmu_name) {
392 		evsel->pmu_name = strdup(orig->pmu_name);
393 		if (evsel->pmu_name == NULL)
394 			goto out_err;
395 	}
396 	if (orig->group_pmu_name) {
397 		evsel->group_pmu_name = strdup(orig->group_pmu_name);
398 		if (evsel->group_pmu_name == NULL)
399 			goto out_err;
400 	}
401 	if (orig->filter) {
402 		evsel->filter = strdup(orig->filter);
403 		if (evsel->filter == NULL)
404 			goto out_err;
405 	}
406 	if (orig->metric_id) {
407 		evsel->metric_id = strdup(orig->metric_id);
408 		if (evsel->metric_id == NULL)
409 			goto out_err;
410 	}
411 	evsel->cgrp = cgroup__get(orig->cgrp);
412 #ifdef HAVE_LIBTRACEEVENT
413 	evsel->tp_format = orig->tp_format;
414 #endif
415 	evsel->handler = orig->handler;
416 	evsel->core.leader = orig->core.leader;
417 
418 	evsel->max_events = orig->max_events;
419 	evsel->tool_event = orig->tool_event;
420 	free((char *)evsel->unit);
421 	evsel->unit = strdup(orig->unit);
422 	if (evsel->unit == NULL)
423 		goto out_err;
424 
425 	evsel->scale = orig->scale;
426 	evsel->snapshot = orig->snapshot;
427 	evsel->per_pkg = orig->per_pkg;
428 	evsel->percore = orig->percore;
429 	evsel->precise_max = orig->precise_max;
430 	evsel->is_libpfm_event = orig->is_libpfm_event;
431 
432 	evsel->exclude_GH = orig->exclude_GH;
433 	evsel->sample_read = orig->sample_read;
434 	evsel->auto_merge_stats = orig->auto_merge_stats;
435 	evsel->collect_stat = orig->collect_stat;
436 	evsel->weak_group = orig->weak_group;
437 	evsel->use_config_name = orig->use_config_name;
438 	evsel->pmu = orig->pmu;
439 
440 	if (evsel__copy_config_terms(evsel, orig) < 0)
441 		goto out_err;
442 
443 	return evsel;
444 
445 out_err:
446 	evsel__delete(evsel);
447 	return NULL;
448 }
449 
450 /*
451  * Returns pointer with encoded error via <linux/err.h> interface.
452  */
453 #ifdef HAVE_LIBTRACEEVENT
evsel__newtp_idx(const char * sys,const char * name,int idx)454 struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx)
455 {
456 	struct evsel *evsel = zalloc(perf_evsel__object.size);
457 	int err = -ENOMEM;
458 
459 	if (evsel == NULL) {
460 		goto out_err;
461 	} else {
462 		struct perf_event_attr attr = {
463 			.type	       = PERF_TYPE_TRACEPOINT,
464 			.sample_type   = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
465 					  PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
466 		};
467 
468 		if (asprintf(&evsel->name, "%s:%s", sys, name) < 0)
469 			goto out_free;
470 
471 		evsel->tp_format = trace_event__tp_format(sys, name);
472 		if (IS_ERR(evsel->tp_format)) {
473 			err = PTR_ERR(evsel->tp_format);
474 			goto out_free;
475 		}
476 
477 		event_attr_init(&attr);
478 		attr.config = evsel->tp_format->id;
479 		attr.sample_period = 1;
480 		evsel__init(evsel, &attr, idx);
481 	}
482 
483 	return evsel;
484 
485 out_free:
486 	zfree(&evsel->name);
487 	free(evsel);
488 out_err:
489 	return ERR_PTR(err);
490 }
491 #endif
492 
493 const char *const evsel__hw_names[PERF_COUNT_HW_MAX] = {
494 	"cycles",
495 	"instructions",
496 	"cache-references",
497 	"cache-misses",
498 	"branches",
499 	"branch-misses",
500 	"bus-cycles",
501 	"stalled-cycles-frontend",
502 	"stalled-cycles-backend",
503 	"ref-cycles",
504 };
505 
506 char *evsel__bpf_counter_events;
507 
evsel__match_bpf_counter_events(const char * name)508 bool evsel__match_bpf_counter_events(const char *name)
509 {
510 	int name_len;
511 	bool match;
512 	char *ptr;
513 
514 	if (!evsel__bpf_counter_events)
515 		return false;
516 
517 	ptr = strstr(evsel__bpf_counter_events, name);
518 	name_len = strlen(name);
519 
520 	/* check name matches a full token in evsel__bpf_counter_events */
521 	match = (ptr != NULL) &&
522 		((ptr == evsel__bpf_counter_events) || (*(ptr - 1) == ',')) &&
523 		((*(ptr + name_len) == ',') || (*(ptr + name_len) == '\0'));
524 
525 	return match;
526 }
527 
__evsel__hw_name(u64 config)528 static const char *__evsel__hw_name(u64 config)
529 {
530 	if (config < PERF_COUNT_HW_MAX && evsel__hw_names[config])
531 		return evsel__hw_names[config];
532 
533 	return "unknown-hardware";
534 }
535 
evsel__add_modifiers(struct evsel * evsel,char * bf,size_t size)536 static int evsel__add_modifiers(struct evsel *evsel, char *bf, size_t size)
537 {
538 	int colon = 0, r = 0;
539 	struct perf_event_attr *attr = &evsel->core.attr;
540 	bool exclude_guest_default = false;
541 
542 #define MOD_PRINT(context, mod)	do {					\
543 		if (!attr->exclude_##context) {				\
544 			if (!colon) colon = ++r;			\
545 			r += scnprintf(bf + r, size - r, "%c", mod);	\
546 		} } while(0)
547 
548 	if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) {
549 		MOD_PRINT(kernel, 'k');
550 		MOD_PRINT(user, 'u');
551 		MOD_PRINT(hv, 'h');
552 		exclude_guest_default = true;
553 	}
554 
555 	if (attr->precise_ip) {
556 		if (!colon)
557 			colon = ++r;
558 		r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp");
559 		exclude_guest_default = true;
560 	}
561 
562 	if (attr->exclude_host || attr->exclude_guest == exclude_guest_default) {
563 		MOD_PRINT(host, 'H');
564 		MOD_PRINT(guest, 'G');
565 	}
566 #undef MOD_PRINT
567 	if (colon)
568 		bf[colon - 1] = ':';
569 	return r;
570 }
571 
arch_evsel__hw_name(struct evsel * evsel,char * bf,size_t size)572 int __weak arch_evsel__hw_name(struct evsel *evsel, char *bf, size_t size)
573 {
574 	return scnprintf(bf, size, "%s", __evsel__hw_name(evsel->core.attr.config));
575 }
576 
evsel__hw_name(struct evsel * evsel,char * bf,size_t size)577 static int evsel__hw_name(struct evsel *evsel, char *bf, size_t size)
578 {
579 	int r = arch_evsel__hw_name(evsel, bf, size);
580 	return r + evsel__add_modifiers(evsel, bf + r, size - r);
581 }
582 
583 const char *const evsel__sw_names[PERF_COUNT_SW_MAX] = {
584 	"cpu-clock",
585 	"task-clock",
586 	"page-faults",
587 	"context-switches",
588 	"cpu-migrations",
589 	"minor-faults",
590 	"major-faults",
591 	"alignment-faults",
592 	"emulation-faults",
593 	"dummy",
594 };
595 
__evsel__sw_name(u64 config)596 static const char *__evsel__sw_name(u64 config)
597 {
598 	if (config < PERF_COUNT_SW_MAX && evsel__sw_names[config])
599 		return evsel__sw_names[config];
600 	return "unknown-software";
601 }
602 
evsel__sw_name(struct evsel * evsel,char * bf,size_t size)603 static int evsel__sw_name(struct evsel *evsel, char *bf, size_t size)
604 {
605 	int r = scnprintf(bf, size, "%s", __evsel__sw_name(evsel->core.attr.config));
606 	return r + evsel__add_modifiers(evsel, bf + r, size - r);
607 }
608 
evsel__tool_name(enum perf_tool_event ev,char * bf,size_t size)609 static int evsel__tool_name(enum perf_tool_event ev, char *bf, size_t size)
610 {
611 	return scnprintf(bf, size, "%s", perf_tool_event__to_str(ev));
612 }
613 
__evsel__bp_name(char * bf,size_t size,u64 addr,u64 type)614 static int __evsel__bp_name(char *bf, size_t size, u64 addr, u64 type)
615 {
616 	int r;
617 
618 	r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr);
619 
620 	if (type & HW_BREAKPOINT_R)
621 		r += scnprintf(bf + r, size - r, "r");
622 
623 	if (type & HW_BREAKPOINT_W)
624 		r += scnprintf(bf + r, size - r, "w");
625 
626 	if (type & HW_BREAKPOINT_X)
627 		r += scnprintf(bf + r, size - r, "x");
628 
629 	return r;
630 }
631 
evsel__bp_name(struct evsel * evsel,char * bf,size_t size)632 static int evsel__bp_name(struct evsel *evsel, char *bf, size_t size)
633 {
634 	struct perf_event_attr *attr = &evsel->core.attr;
635 	int r = __evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type);
636 	return r + evsel__add_modifiers(evsel, bf + r, size - r);
637 }
638 
639 const char *const evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX][EVSEL__MAX_ALIASES] = {
640  { "L1-dcache",	"l1-d",		"l1d",		"L1-data",		},
641  { "L1-icache",	"l1-i",		"l1i",		"L1-instruction",	},
642  { "LLC",	"L2",							},
643  { "dTLB",	"d-tlb",	"Data-TLB",				},
644  { "iTLB",	"i-tlb",	"Instruction-TLB",			},
645  { "branch",	"branches",	"bpu",		"btb",		"bpc",	},
646  { "node",								},
647 };
648 
649 const char *const evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX][EVSEL__MAX_ALIASES] = {
650  { "load",	"loads",	"read",					},
651  { "store",	"stores",	"write",				},
652  { "prefetch",	"prefetches",	"speculative-read", "speculative-load",	},
653 };
654 
655 const char *const evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX][EVSEL__MAX_ALIASES] = {
656  { "refs",	"Reference",	"ops",		"access",		},
657  { "misses",	"miss",							},
658 };
659 
660 #define C(x)		PERF_COUNT_HW_CACHE_##x
661 #define CACHE_READ	(1 << C(OP_READ))
662 #define CACHE_WRITE	(1 << C(OP_WRITE))
663 #define CACHE_PREFETCH	(1 << C(OP_PREFETCH))
664 #define COP(x)		(1 << x)
665 
666 /*
667  * cache operation stat
668  * L1I : Read and prefetch only
669  * ITLB and BPU : Read-only
670  */
671 static const unsigned long evsel__hw_cache_stat[C(MAX)] = {
672  [C(L1D)]	= (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
673  [C(L1I)]	= (CACHE_READ | CACHE_PREFETCH),
674  [C(LL)]	= (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
675  [C(DTLB)]	= (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
676  [C(ITLB)]	= (CACHE_READ),
677  [C(BPU)]	= (CACHE_READ),
678  [C(NODE)]	= (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
679 };
680 
evsel__is_cache_op_valid(u8 type,u8 op)681 bool evsel__is_cache_op_valid(u8 type, u8 op)
682 {
683 	if (evsel__hw_cache_stat[type] & COP(op))
684 		return true;	/* valid */
685 	else
686 		return false;	/* invalid */
687 }
688 
__evsel__hw_cache_type_op_res_name(u8 type,u8 op,u8 result,char * bf,size_t size)689 int __evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result, char *bf, size_t size)
690 {
691 	if (result) {
692 		return scnprintf(bf, size, "%s-%s-%s", evsel__hw_cache[type][0],
693 				 evsel__hw_cache_op[op][0],
694 				 evsel__hw_cache_result[result][0]);
695 	}
696 
697 	return scnprintf(bf, size, "%s-%s", evsel__hw_cache[type][0],
698 			 evsel__hw_cache_op[op][1]);
699 }
700 
__evsel__hw_cache_name(u64 config,char * bf,size_t size)701 static int __evsel__hw_cache_name(u64 config, char *bf, size_t size)
702 {
703 	u8 op, result, type = (config >>  0) & 0xff;
704 	const char *err = "unknown-ext-hardware-cache-type";
705 
706 	if (type >= PERF_COUNT_HW_CACHE_MAX)
707 		goto out_err;
708 
709 	op = (config >>  8) & 0xff;
710 	err = "unknown-ext-hardware-cache-op";
711 	if (op >= PERF_COUNT_HW_CACHE_OP_MAX)
712 		goto out_err;
713 
714 	result = (config >> 16) & 0xff;
715 	err = "unknown-ext-hardware-cache-result";
716 	if (result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
717 		goto out_err;
718 
719 	err = "invalid-cache";
720 	if (!evsel__is_cache_op_valid(type, op))
721 		goto out_err;
722 
723 	return __evsel__hw_cache_type_op_res_name(type, op, result, bf, size);
724 out_err:
725 	return scnprintf(bf, size, "%s", err);
726 }
727 
evsel__hw_cache_name(struct evsel * evsel,char * bf,size_t size)728 static int evsel__hw_cache_name(struct evsel *evsel, char *bf, size_t size)
729 {
730 	int ret = __evsel__hw_cache_name(evsel->core.attr.config, bf, size);
731 	return ret + evsel__add_modifiers(evsel, bf + ret, size - ret);
732 }
733 
evsel__raw_name(struct evsel * evsel,char * bf,size_t size)734 static int evsel__raw_name(struct evsel *evsel, char *bf, size_t size)
735 {
736 	int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->core.attr.config);
737 	return ret + evsel__add_modifiers(evsel, bf + ret, size - ret);
738 }
739 
evsel__name(struct evsel * evsel)740 const char *evsel__name(struct evsel *evsel)
741 {
742 	char bf[128];
743 
744 	if (!evsel)
745 		goto out_unknown;
746 
747 	if (evsel->name)
748 		return evsel->name;
749 
750 	switch (evsel->core.attr.type) {
751 	case PERF_TYPE_RAW:
752 		evsel__raw_name(evsel, bf, sizeof(bf));
753 		break;
754 
755 	case PERF_TYPE_HARDWARE:
756 		evsel__hw_name(evsel, bf, sizeof(bf));
757 		break;
758 
759 	case PERF_TYPE_HW_CACHE:
760 		evsel__hw_cache_name(evsel, bf, sizeof(bf));
761 		break;
762 
763 	case PERF_TYPE_SOFTWARE:
764 		if (evsel__is_tool(evsel))
765 			evsel__tool_name(evsel->tool_event, bf, sizeof(bf));
766 		else
767 			evsel__sw_name(evsel, bf, sizeof(bf));
768 		break;
769 
770 	case PERF_TYPE_TRACEPOINT:
771 		scnprintf(bf, sizeof(bf), "%s", "unknown tracepoint");
772 		break;
773 
774 	case PERF_TYPE_BREAKPOINT:
775 		evsel__bp_name(evsel, bf, sizeof(bf));
776 		break;
777 
778 	default:
779 		scnprintf(bf, sizeof(bf), "unknown attr type: %d",
780 			  evsel->core.attr.type);
781 		break;
782 	}
783 
784 	evsel->name = strdup(bf);
785 
786 	if (evsel->name)
787 		return evsel->name;
788 out_unknown:
789 	return "unknown";
790 }
791 
evsel__name_is(struct evsel * evsel,const char * name)792 bool evsel__name_is(struct evsel *evsel, const char *name)
793 {
794 	return !strcmp(evsel__name(evsel), name);
795 }
796 
evsel__metric_id(const struct evsel * evsel)797 const char *evsel__metric_id(const struct evsel *evsel)
798 {
799 	if (evsel->metric_id)
800 		return evsel->metric_id;
801 
802 	if (evsel__is_tool(evsel))
803 		return perf_tool_event__to_str(evsel->tool_event);
804 
805 	return "unknown";
806 }
807 
evsel__group_name(struct evsel * evsel)808 const char *evsel__group_name(struct evsel *evsel)
809 {
810 	return evsel->group_name ?: "anon group";
811 }
812 
813 /*
814  * Returns the group details for the specified leader,
815  * with following rules.
816  *
817  *  For record -e '{cycles,instructions}'
818  *    'anon group { cycles:u, instructions:u }'
819  *
820  *  For record -e 'cycles,instructions' and report --group
821  *    'cycles:u, instructions:u'
822  */
evsel__group_desc(struct evsel * evsel,char * buf,size_t size)823 int evsel__group_desc(struct evsel *evsel, char *buf, size_t size)
824 {
825 	int ret = 0;
826 	struct evsel *pos;
827 	const char *group_name = evsel__group_name(evsel);
828 
829 	if (!evsel->forced_leader)
830 		ret = scnprintf(buf, size, "%s { ", group_name);
831 
832 	ret += scnprintf(buf + ret, size - ret, "%s", evsel__name(evsel));
833 
834 	for_each_group_member(pos, evsel)
835 		ret += scnprintf(buf + ret, size - ret, ", %s", evsel__name(pos));
836 
837 	if (!evsel->forced_leader)
838 		ret += scnprintf(buf + ret, size - ret, " }");
839 
840 	return ret;
841 }
842 
__evsel__config_callchain(struct evsel * evsel,struct record_opts * opts,struct callchain_param * param)843 static void __evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
844 				      struct callchain_param *param)
845 {
846 	bool function = evsel__is_function_event(evsel);
847 	struct perf_event_attr *attr = &evsel->core.attr;
848 	const char *arch = perf_env__arch(evsel__env(evsel));
849 
850 	evsel__set_sample_bit(evsel, CALLCHAIN);
851 
852 	attr->sample_max_stack = param->max_stack;
853 
854 	if (opts->kernel_callchains)
855 		attr->exclude_callchain_user = 1;
856 	if (opts->user_callchains)
857 		attr->exclude_callchain_kernel = 1;
858 	if (param->record_mode == CALLCHAIN_LBR) {
859 		if (!opts->branch_stack) {
860 			if (attr->exclude_user) {
861 				pr_warning("LBR callstack option is only available "
862 					   "to get user callchain information. "
863 					   "Falling back to framepointers.\n");
864 			} else {
865 				evsel__set_sample_bit(evsel, BRANCH_STACK);
866 				attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER |
867 							PERF_SAMPLE_BRANCH_CALL_STACK |
868 							PERF_SAMPLE_BRANCH_NO_CYCLES |
869 							PERF_SAMPLE_BRANCH_NO_FLAGS |
870 							PERF_SAMPLE_BRANCH_HW_INDEX;
871 			}
872 		} else
873 			 pr_warning("Cannot use LBR callstack with branch stack. "
874 				    "Falling back to framepointers.\n");
875 	}
876 
877 	if (param->record_mode == CALLCHAIN_DWARF) {
878 		if (!function) {
879 			evsel__set_sample_bit(evsel, REGS_USER);
880 			evsel__set_sample_bit(evsel, STACK_USER);
881 			if (opts->sample_user_regs &&
882 			    DWARF_MINIMAL_REGS(arch) != arch__user_reg_mask()) {
883 				attr->sample_regs_user |= DWARF_MINIMAL_REGS(arch);
884 				pr_warning("WARNING: The use of --call-graph=dwarf may require all the user registers, "
885 					   "specifying a subset with --user-regs may render DWARF unwinding unreliable, "
886 					   "so the minimal registers set (IP, SP) is explicitly forced.\n");
887 			} else {
888 				attr->sample_regs_user |= arch__user_reg_mask();
889 			}
890 			attr->sample_stack_user = param->dump_size;
891 			attr->exclude_callchain_user = 1;
892 		} else {
893 			pr_info("Cannot use DWARF unwind for function trace event,"
894 				" falling back to framepointers.\n");
895 		}
896 	}
897 
898 	if (function) {
899 		pr_info("Disabling user space callchains for function trace event.\n");
900 		attr->exclude_callchain_user = 1;
901 	}
902 }
903 
evsel__config_callchain(struct evsel * evsel,struct record_opts * opts,struct callchain_param * param)904 void evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
905 			     struct callchain_param *param)
906 {
907 	if (param->enabled)
908 		return __evsel__config_callchain(evsel, opts, param);
909 }
910 
evsel__reset_callgraph(struct evsel * evsel,struct callchain_param * param)911 static void evsel__reset_callgraph(struct evsel *evsel, struct callchain_param *param)
912 {
913 	struct perf_event_attr *attr = &evsel->core.attr;
914 
915 	evsel__reset_sample_bit(evsel, CALLCHAIN);
916 	if (param->record_mode == CALLCHAIN_LBR) {
917 		evsel__reset_sample_bit(evsel, BRANCH_STACK);
918 		attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER |
919 					      PERF_SAMPLE_BRANCH_CALL_STACK |
920 					      PERF_SAMPLE_BRANCH_HW_INDEX);
921 	}
922 	if (param->record_mode == CALLCHAIN_DWARF) {
923 		evsel__reset_sample_bit(evsel, REGS_USER);
924 		evsel__reset_sample_bit(evsel, STACK_USER);
925 	}
926 }
927 
evsel__apply_config_terms(struct evsel * evsel,struct record_opts * opts,bool track)928 static void evsel__apply_config_terms(struct evsel *evsel,
929 				      struct record_opts *opts, bool track)
930 {
931 	struct evsel_config_term *term;
932 	struct list_head *config_terms = &evsel->config_terms;
933 	struct perf_event_attr *attr = &evsel->core.attr;
934 	/* callgraph default */
935 	struct callchain_param param = {
936 		.record_mode = callchain_param.record_mode,
937 	};
938 	u32 dump_size = 0;
939 	int max_stack = 0;
940 	const char *callgraph_buf = NULL;
941 
942 	list_for_each_entry(term, config_terms, list) {
943 		switch (term->type) {
944 		case EVSEL__CONFIG_TERM_PERIOD:
945 			if (!(term->weak && opts->user_interval != ULLONG_MAX)) {
946 				attr->sample_period = term->val.period;
947 				attr->freq = 0;
948 				evsel__reset_sample_bit(evsel, PERIOD);
949 			}
950 			break;
951 		case EVSEL__CONFIG_TERM_FREQ:
952 			if (!(term->weak && opts->user_freq != UINT_MAX)) {
953 				attr->sample_freq = term->val.freq;
954 				attr->freq = 1;
955 				evsel__set_sample_bit(evsel, PERIOD);
956 			}
957 			break;
958 		case EVSEL__CONFIG_TERM_TIME:
959 			if (term->val.time)
960 				evsel__set_sample_bit(evsel, TIME);
961 			else
962 				evsel__reset_sample_bit(evsel, TIME);
963 			break;
964 		case EVSEL__CONFIG_TERM_CALLGRAPH:
965 			callgraph_buf = term->val.str;
966 			break;
967 		case EVSEL__CONFIG_TERM_BRANCH:
968 			if (term->val.str && strcmp(term->val.str, "no")) {
969 				evsel__set_sample_bit(evsel, BRANCH_STACK);
970 				parse_branch_str(term->val.str,
971 						 &attr->branch_sample_type);
972 			} else
973 				evsel__reset_sample_bit(evsel, BRANCH_STACK);
974 			break;
975 		case EVSEL__CONFIG_TERM_STACK_USER:
976 			dump_size = term->val.stack_user;
977 			break;
978 		case EVSEL__CONFIG_TERM_MAX_STACK:
979 			max_stack = term->val.max_stack;
980 			break;
981 		case EVSEL__CONFIG_TERM_MAX_EVENTS:
982 			evsel->max_events = term->val.max_events;
983 			break;
984 		case EVSEL__CONFIG_TERM_INHERIT:
985 			/*
986 			 * attr->inherit should has already been set by
987 			 * evsel__config. If user explicitly set
988 			 * inherit using config terms, override global
989 			 * opt->no_inherit setting.
990 			 */
991 			attr->inherit = term->val.inherit ? 1 : 0;
992 			break;
993 		case EVSEL__CONFIG_TERM_OVERWRITE:
994 			attr->write_backward = term->val.overwrite ? 1 : 0;
995 			break;
996 		case EVSEL__CONFIG_TERM_DRV_CFG:
997 			break;
998 		case EVSEL__CONFIG_TERM_PERCORE:
999 			break;
1000 		case EVSEL__CONFIG_TERM_AUX_OUTPUT:
1001 			attr->aux_output = term->val.aux_output ? 1 : 0;
1002 			break;
1003 		case EVSEL__CONFIG_TERM_AUX_SAMPLE_SIZE:
1004 			/* Already applied by auxtrace */
1005 			break;
1006 		case EVSEL__CONFIG_TERM_CFG_CHG:
1007 			break;
1008 		default:
1009 			break;
1010 		}
1011 	}
1012 
1013 	/* User explicitly set per-event callgraph, clear the old setting and reset. */
1014 	if ((callgraph_buf != NULL) || (dump_size > 0) || max_stack) {
1015 		bool sample_address = false;
1016 
1017 		if (max_stack) {
1018 			param.max_stack = max_stack;
1019 			if (callgraph_buf == NULL)
1020 				callgraph_buf = "fp";
1021 		}
1022 
1023 		/* parse callgraph parameters */
1024 		if (callgraph_buf != NULL) {
1025 			if (!strcmp(callgraph_buf, "no")) {
1026 				param.enabled = false;
1027 				param.record_mode = CALLCHAIN_NONE;
1028 			} else {
1029 				param.enabled = true;
1030 				if (parse_callchain_record(callgraph_buf, &param)) {
1031 					pr_err("per-event callgraph setting for %s failed. "
1032 					       "Apply callgraph global setting for it\n",
1033 					       evsel->name);
1034 					return;
1035 				}
1036 				if (param.record_mode == CALLCHAIN_DWARF)
1037 					sample_address = true;
1038 			}
1039 		}
1040 		if (dump_size > 0) {
1041 			dump_size = round_up(dump_size, sizeof(u64));
1042 			param.dump_size = dump_size;
1043 		}
1044 
1045 		/* If global callgraph set, clear it */
1046 		if (callchain_param.enabled)
1047 			evsel__reset_callgraph(evsel, &callchain_param);
1048 
1049 		/* set perf-event callgraph */
1050 		if (param.enabled) {
1051 			if (sample_address) {
1052 				evsel__set_sample_bit(evsel, ADDR);
1053 				evsel__set_sample_bit(evsel, DATA_SRC);
1054 				evsel->core.attr.mmap_data = track;
1055 			}
1056 			evsel__config_callchain(evsel, opts, &param);
1057 		}
1058 	}
1059 }
1060 
__evsel__get_config_term(struct evsel * evsel,enum evsel_term_type type)1061 struct evsel_config_term *__evsel__get_config_term(struct evsel *evsel, enum evsel_term_type type)
1062 {
1063 	struct evsel_config_term *term, *found_term = NULL;
1064 
1065 	list_for_each_entry(term, &evsel->config_terms, list) {
1066 		if (term->type == type)
1067 			found_term = term;
1068 	}
1069 
1070 	return found_term;
1071 }
1072 
arch_evsel__set_sample_weight(struct evsel * evsel)1073 void __weak arch_evsel__set_sample_weight(struct evsel *evsel)
1074 {
1075 	evsel__set_sample_bit(evsel, WEIGHT);
1076 }
1077 
arch__post_evsel_config(struct evsel * evsel __maybe_unused,struct perf_event_attr * attr __maybe_unused)1078 void __weak arch__post_evsel_config(struct evsel *evsel __maybe_unused,
1079 				    struct perf_event_attr *attr __maybe_unused)
1080 {
1081 }
1082 
evsel__set_default_freq_period(struct record_opts * opts,struct perf_event_attr * attr)1083 static void evsel__set_default_freq_period(struct record_opts *opts,
1084 					   struct perf_event_attr *attr)
1085 {
1086 	if (opts->freq) {
1087 		attr->freq = 1;
1088 		attr->sample_freq = opts->freq;
1089 	} else {
1090 		attr->sample_period = opts->default_interval;
1091 	}
1092 }
1093 
evsel__is_offcpu_event(struct evsel * evsel)1094 static bool evsel__is_offcpu_event(struct evsel *evsel)
1095 {
1096 	return evsel__is_bpf_output(evsel) && evsel__name_is(evsel, OFFCPU_EVENT);
1097 }
1098 
1099 /*
1100  * The enable_on_exec/disabled value strategy:
1101  *
1102  *  1) For any type of traced program:
1103  *    - all independent events and group leaders are disabled
1104  *    - all group members are enabled
1105  *
1106  *     Group members are ruled by group leaders. They need to
1107  *     be enabled, because the group scheduling relies on that.
1108  *
1109  *  2) For traced programs executed by perf:
1110  *     - all independent events and group leaders have
1111  *       enable_on_exec set
1112  *     - we don't specifically enable or disable any event during
1113  *       the record command
1114  *
1115  *     Independent events and group leaders are initially disabled
1116  *     and get enabled by exec. Group members are ruled by group
1117  *     leaders as stated in 1).
1118  *
1119  *  3) For traced programs attached by perf (pid/tid):
1120  *     - we specifically enable or disable all events during
1121  *       the record command
1122  *
1123  *     When attaching events to already running traced we
1124  *     enable/disable events specifically, as there's no
1125  *     initial traced exec call.
1126  */
evsel__config(struct evsel * evsel,struct record_opts * opts,struct callchain_param * callchain)1127 void evsel__config(struct evsel *evsel, struct record_opts *opts,
1128 		   struct callchain_param *callchain)
1129 {
1130 	struct evsel *leader = evsel__leader(evsel);
1131 	struct perf_event_attr *attr = &evsel->core.attr;
1132 	int track = evsel->tracking;
1133 	bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread;
1134 
1135 	attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1;
1136 	attr->inherit	    = !opts->no_inherit;
1137 	attr->write_backward = opts->overwrite ? 1 : 0;
1138 	attr->read_format   = PERF_FORMAT_LOST;
1139 
1140 	evsel__set_sample_bit(evsel, IP);
1141 	evsel__set_sample_bit(evsel, TID);
1142 
1143 	if (evsel->sample_read) {
1144 		evsel__set_sample_bit(evsel, READ);
1145 
1146 		/*
1147 		 * We need ID even in case of single event, because
1148 		 * PERF_SAMPLE_READ process ID specific data.
1149 		 */
1150 		evsel__set_sample_id(evsel, false);
1151 
1152 		/*
1153 		 * Apply group format only if we belong to group
1154 		 * with more than one members.
1155 		 */
1156 		if (leader->core.nr_members > 1) {
1157 			attr->read_format |= PERF_FORMAT_GROUP;
1158 			attr->inherit = 0;
1159 		}
1160 	}
1161 
1162 	/*
1163 	 * We default some events to have a default interval. But keep
1164 	 * it a weak assumption overridable by the user.
1165 	 */
1166 	if ((evsel->is_libpfm_event && !attr->sample_period) ||
1167 	    (!evsel->is_libpfm_event && (!attr->sample_period ||
1168 					 opts->user_freq != UINT_MAX ||
1169 					 opts->user_interval != ULLONG_MAX)))
1170 		evsel__set_default_freq_period(opts, attr);
1171 
1172 	/*
1173 	 * If attr->freq was set (here or earlier), ask for period
1174 	 * to be sampled.
1175 	 */
1176 	if (attr->freq)
1177 		evsel__set_sample_bit(evsel, PERIOD);
1178 
1179 	if (opts->no_samples)
1180 		attr->sample_freq = 0;
1181 
1182 	if (opts->inherit_stat) {
1183 		evsel->core.attr.read_format |=
1184 			PERF_FORMAT_TOTAL_TIME_ENABLED |
1185 			PERF_FORMAT_TOTAL_TIME_RUNNING |
1186 			PERF_FORMAT_ID;
1187 		attr->inherit_stat = 1;
1188 	}
1189 
1190 	if (opts->sample_address) {
1191 		evsel__set_sample_bit(evsel, ADDR);
1192 		attr->mmap_data = track;
1193 	}
1194 
1195 	/*
1196 	 * We don't allow user space callchains for  function trace
1197 	 * event, due to issues with page faults while tracing page
1198 	 * fault handler and its overall trickiness nature.
1199 	 */
1200 	if (evsel__is_function_event(evsel))
1201 		evsel->core.attr.exclude_callchain_user = 1;
1202 
1203 	if (callchain && callchain->enabled && !evsel->no_aux_samples)
1204 		evsel__config_callchain(evsel, opts, callchain);
1205 
1206 	if (opts->sample_intr_regs && !evsel->no_aux_samples &&
1207 	    !evsel__is_dummy_event(evsel)) {
1208 		attr->sample_regs_intr = opts->sample_intr_regs;
1209 		evsel__set_sample_bit(evsel, REGS_INTR);
1210 	}
1211 
1212 	if (opts->sample_user_regs && !evsel->no_aux_samples &&
1213 	    !evsel__is_dummy_event(evsel)) {
1214 		attr->sample_regs_user |= opts->sample_user_regs;
1215 		evsel__set_sample_bit(evsel, REGS_USER);
1216 	}
1217 
1218 	if (target__has_cpu(&opts->target) || opts->sample_cpu)
1219 		evsel__set_sample_bit(evsel, CPU);
1220 
1221 	/*
1222 	 * When the user explicitly disabled time don't force it here.
1223 	 */
1224 	if (opts->sample_time &&
1225 	    (!perf_missing_features.sample_id_all &&
1226 	    (!opts->no_inherit || target__has_cpu(&opts->target) || per_cpu ||
1227 	     opts->sample_time_set)))
1228 		evsel__set_sample_bit(evsel, TIME);
1229 
1230 	if (opts->raw_samples && !evsel->no_aux_samples) {
1231 		evsel__set_sample_bit(evsel, TIME);
1232 		evsel__set_sample_bit(evsel, RAW);
1233 		evsel__set_sample_bit(evsel, CPU);
1234 	}
1235 
1236 	if (opts->sample_address)
1237 		evsel__set_sample_bit(evsel, DATA_SRC);
1238 
1239 	if (opts->sample_phys_addr)
1240 		evsel__set_sample_bit(evsel, PHYS_ADDR);
1241 
1242 	if (opts->no_buffering) {
1243 		attr->watermark = 0;
1244 		attr->wakeup_events = 1;
1245 	}
1246 	if (opts->branch_stack && !evsel->no_aux_samples) {
1247 		evsel__set_sample_bit(evsel, BRANCH_STACK);
1248 		attr->branch_sample_type = opts->branch_stack;
1249 	}
1250 
1251 	if (opts->sample_weight)
1252 		arch_evsel__set_sample_weight(evsel);
1253 
1254 	attr->task     = track;
1255 	attr->mmap     = track;
1256 	attr->mmap2    = track && !perf_missing_features.mmap2;
1257 	attr->comm     = track;
1258 	attr->build_id = track && opts->build_id;
1259 
1260 	/*
1261 	 * ksymbol is tracked separately with text poke because it needs to be
1262 	 * system wide and enabled immediately.
1263 	 */
1264 	if (!opts->text_poke)
1265 		attr->ksymbol = track && !perf_missing_features.ksymbol;
1266 	attr->bpf_event = track && !opts->no_bpf_event && !perf_missing_features.bpf;
1267 
1268 	if (opts->record_namespaces)
1269 		attr->namespaces  = track;
1270 
1271 	if (opts->record_cgroup) {
1272 		attr->cgroup = track && !perf_missing_features.cgroup;
1273 		evsel__set_sample_bit(evsel, CGROUP);
1274 	}
1275 
1276 	if (opts->sample_data_page_size)
1277 		evsel__set_sample_bit(evsel, DATA_PAGE_SIZE);
1278 
1279 	if (opts->sample_code_page_size)
1280 		evsel__set_sample_bit(evsel, CODE_PAGE_SIZE);
1281 
1282 	if (opts->record_switch_events)
1283 		attr->context_switch = track;
1284 
1285 	if (opts->sample_transaction)
1286 		evsel__set_sample_bit(evsel, TRANSACTION);
1287 
1288 	if (opts->running_time) {
1289 		evsel->core.attr.read_format |=
1290 			PERF_FORMAT_TOTAL_TIME_ENABLED |
1291 			PERF_FORMAT_TOTAL_TIME_RUNNING;
1292 	}
1293 
1294 	/*
1295 	 * XXX see the function comment above
1296 	 *
1297 	 * Disabling only independent events or group leaders,
1298 	 * keeping group members enabled.
1299 	 */
1300 	if (evsel__is_group_leader(evsel))
1301 		attr->disabled = 1;
1302 
1303 	/*
1304 	 * Setting enable_on_exec for independent events and
1305 	 * group leaders for traced executed by perf.
1306 	 */
1307 	if (target__none(&opts->target) && evsel__is_group_leader(evsel) &&
1308 	    !opts->target.initial_delay)
1309 		attr->enable_on_exec = 1;
1310 
1311 	if (evsel->immediate) {
1312 		attr->disabled = 0;
1313 		attr->enable_on_exec = 0;
1314 	}
1315 
1316 	clockid = opts->clockid;
1317 	if (opts->use_clockid) {
1318 		attr->use_clockid = 1;
1319 		attr->clockid = opts->clockid;
1320 	}
1321 
1322 	if (evsel->precise_max)
1323 		attr->precise_ip = 3;
1324 
1325 	if (opts->all_user) {
1326 		attr->exclude_kernel = 1;
1327 		attr->exclude_user   = 0;
1328 	}
1329 
1330 	if (opts->all_kernel) {
1331 		attr->exclude_kernel = 0;
1332 		attr->exclude_user   = 1;
1333 	}
1334 
1335 	if (evsel->core.own_cpus || evsel->unit)
1336 		evsel->core.attr.read_format |= PERF_FORMAT_ID;
1337 
1338 	/*
1339 	 * Apply event specific term settings,
1340 	 * it overloads any global configuration.
1341 	 */
1342 	evsel__apply_config_terms(evsel, opts, track);
1343 
1344 	evsel->ignore_missing_thread = opts->ignore_missing_thread;
1345 
1346 	/* The --period option takes the precedence. */
1347 	if (opts->period_set) {
1348 		if (opts->period)
1349 			evsel__set_sample_bit(evsel, PERIOD);
1350 		else
1351 			evsel__reset_sample_bit(evsel, PERIOD);
1352 	}
1353 
1354 	/*
1355 	 * A dummy event never triggers any actual counter and therefore
1356 	 * cannot be used with branch_stack.
1357 	 *
1358 	 * For initial_delay, a dummy event is added implicitly.
1359 	 * The software event will trigger -EOPNOTSUPP error out,
1360 	 * if BRANCH_STACK bit is set.
1361 	 */
1362 	if (evsel__is_dummy_event(evsel))
1363 		evsel__reset_sample_bit(evsel, BRANCH_STACK);
1364 
1365 	if (evsel__is_offcpu_event(evsel))
1366 		evsel->core.attr.sample_type &= OFFCPU_SAMPLE_TYPES;
1367 
1368 	arch__post_evsel_config(evsel, attr);
1369 }
1370 
evsel__set_filter(struct evsel * evsel,const char * filter)1371 int evsel__set_filter(struct evsel *evsel, const char *filter)
1372 {
1373 	char *new_filter = strdup(filter);
1374 
1375 	if (new_filter != NULL) {
1376 		free(evsel->filter);
1377 		evsel->filter = new_filter;
1378 		return 0;
1379 	}
1380 
1381 	return -1;
1382 }
1383 
evsel__append_filter(struct evsel * evsel,const char * fmt,const char * filter)1384 static int evsel__append_filter(struct evsel *evsel, const char *fmt, const char *filter)
1385 {
1386 	char *new_filter;
1387 
1388 	if (evsel->filter == NULL)
1389 		return evsel__set_filter(evsel, filter);
1390 
1391 	if (asprintf(&new_filter, fmt, evsel->filter, filter) > 0) {
1392 		free(evsel->filter);
1393 		evsel->filter = new_filter;
1394 		return 0;
1395 	}
1396 
1397 	return -1;
1398 }
1399 
evsel__append_tp_filter(struct evsel * evsel,const char * filter)1400 int evsel__append_tp_filter(struct evsel *evsel, const char *filter)
1401 {
1402 	return evsel__append_filter(evsel, "(%s) && (%s)", filter);
1403 }
1404 
evsel__append_addr_filter(struct evsel * evsel,const char * filter)1405 int evsel__append_addr_filter(struct evsel *evsel, const char *filter)
1406 {
1407 	return evsel__append_filter(evsel, "%s,%s", filter);
1408 }
1409 
1410 /* Caller has to clear disabled after going through all CPUs. */
evsel__enable_cpu(struct evsel * evsel,int cpu_map_idx)1411 int evsel__enable_cpu(struct evsel *evsel, int cpu_map_idx)
1412 {
1413 	return perf_evsel__enable_cpu(&evsel->core, cpu_map_idx);
1414 }
1415 
evsel__enable(struct evsel * evsel)1416 int evsel__enable(struct evsel *evsel)
1417 {
1418 	int err = perf_evsel__enable(&evsel->core);
1419 
1420 	if (!err)
1421 		evsel->disabled = false;
1422 	return err;
1423 }
1424 
1425 /* Caller has to set disabled after going through all CPUs. */
evsel__disable_cpu(struct evsel * evsel,int cpu_map_idx)1426 int evsel__disable_cpu(struct evsel *evsel, int cpu_map_idx)
1427 {
1428 	return perf_evsel__disable_cpu(&evsel->core, cpu_map_idx);
1429 }
1430 
evsel__disable(struct evsel * evsel)1431 int evsel__disable(struct evsel *evsel)
1432 {
1433 	int err = perf_evsel__disable(&evsel->core);
1434 	/*
1435 	 * We mark it disabled here so that tools that disable a event can
1436 	 * ignore events after they disable it. I.e. the ring buffer may have
1437 	 * already a few more events queued up before the kernel got the stop
1438 	 * request.
1439 	 */
1440 	if (!err)
1441 		evsel->disabled = true;
1442 
1443 	return err;
1444 }
1445 
free_config_terms(struct list_head * config_terms)1446 void free_config_terms(struct list_head *config_terms)
1447 {
1448 	struct evsel_config_term *term, *h;
1449 
1450 	list_for_each_entry_safe(term, h, config_terms, list) {
1451 		list_del_init(&term->list);
1452 		if (term->free_str)
1453 			zfree(&term->val.str);
1454 		free(term);
1455 	}
1456 }
1457 
evsel__free_config_terms(struct evsel * evsel)1458 static void evsel__free_config_terms(struct evsel *evsel)
1459 {
1460 	free_config_terms(&evsel->config_terms);
1461 }
1462 
evsel__exit(struct evsel * evsel)1463 void evsel__exit(struct evsel *evsel)
1464 {
1465 	assert(list_empty(&evsel->core.node));
1466 	assert(evsel->evlist == NULL);
1467 	bpf_counter__destroy(evsel);
1468 	perf_bpf_filter__destroy(evsel);
1469 	evsel__free_counts(evsel);
1470 	perf_evsel__free_fd(&evsel->core);
1471 	perf_evsel__free_id(&evsel->core);
1472 	evsel__free_config_terms(evsel);
1473 	cgroup__put(evsel->cgrp);
1474 	perf_cpu_map__put(evsel->core.cpus);
1475 	perf_cpu_map__put(evsel->core.own_cpus);
1476 	perf_thread_map__put(evsel->core.threads);
1477 	zfree(&evsel->group_name);
1478 	zfree(&evsel->name);
1479 	zfree(&evsel->filter);
1480 	zfree(&evsel->pmu_name);
1481 	zfree(&evsel->group_pmu_name);
1482 	zfree(&evsel->unit);
1483 	zfree(&evsel->metric_id);
1484 	evsel__zero_per_pkg(evsel);
1485 	hashmap__free(evsel->per_pkg_mask);
1486 	evsel->per_pkg_mask = NULL;
1487 	zfree(&evsel->metric_events);
1488 	perf_evsel__object.fini(evsel);
1489 }
1490 
evsel__delete(struct evsel * evsel)1491 void evsel__delete(struct evsel *evsel)
1492 {
1493 	if (!evsel)
1494 		return;
1495 
1496 	evsel__exit(evsel);
1497 	free(evsel);
1498 }
1499 
evsel__compute_deltas(struct evsel * evsel,int cpu_map_idx,int thread,struct perf_counts_values * count)1500 void evsel__compute_deltas(struct evsel *evsel, int cpu_map_idx, int thread,
1501 			   struct perf_counts_values *count)
1502 {
1503 	struct perf_counts_values tmp;
1504 
1505 	if (!evsel->prev_raw_counts)
1506 		return;
1507 
1508 	tmp = *perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread);
1509 	*perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread) = *count;
1510 
1511 	count->val = count->val - tmp.val;
1512 	count->ena = count->ena - tmp.ena;
1513 	count->run = count->run - tmp.run;
1514 }
1515 
evsel__read_one(struct evsel * evsel,int cpu_map_idx,int thread)1516 static int evsel__read_one(struct evsel *evsel, int cpu_map_idx, int thread)
1517 {
1518 	struct perf_counts_values *count = perf_counts(evsel->counts, cpu_map_idx, thread);
1519 
1520 	return perf_evsel__read(&evsel->core, cpu_map_idx, thread, count);
1521 }
1522 
evsel__set_count(struct evsel * counter,int cpu_map_idx,int thread,u64 val,u64 ena,u64 run,u64 lost)1523 static void evsel__set_count(struct evsel *counter, int cpu_map_idx, int thread,
1524 			     u64 val, u64 ena, u64 run, u64 lost)
1525 {
1526 	struct perf_counts_values *count;
1527 
1528 	count = perf_counts(counter->counts, cpu_map_idx, thread);
1529 
1530 	count->val    = val;
1531 	count->ena    = ena;
1532 	count->run    = run;
1533 	count->lost   = lost;
1534 
1535 	perf_counts__set_loaded(counter->counts, cpu_map_idx, thread, true);
1536 }
1537 
evsel__process_group_data(struct evsel * leader,int cpu_map_idx,int thread,u64 * data)1538 static int evsel__process_group_data(struct evsel *leader, int cpu_map_idx, int thread, u64 *data)
1539 {
1540 	u64 read_format = leader->core.attr.read_format;
1541 	struct sample_read_value *v;
1542 	u64 nr, ena = 0, run = 0, lost = 0;
1543 
1544 	nr = *data++;
1545 
1546 	if (nr != (u64) leader->core.nr_members)
1547 		return -EINVAL;
1548 
1549 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1550 		ena = *data++;
1551 
1552 	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1553 		run = *data++;
1554 
1555 	v = (void *)data;
1556 	sample_read_group__for_each(v, nr, read_format) {
1557 		struct evsel *counter;
1558 
1559 		counter = evlist__id2evsel(leader->evlist, v->id);
1560 		if (!counter)
1561 			return -EINVAL;
1562 
1563 		if (read_format & PERF_FORMAT_LOST)
1564 			lost = v->lost;
1565 
1566 		evsel__set_count(counter, cpu_map_idx, thread, v->value, ena, run, lost);
1567 	}
1568 
1569 	return 0;
1570 }
1571 
evsel__read_group(struct evsel * leader,int cpu_map_idx,int thread)1572 static int evsel__read_group(struct evsel *leader, int cpu_map_idx, int thread)
1573 {
1574 	struct perf_stat_evsel *ps = leader->stats;
1575 	u64 read_format = leader->core.attr.read_format;
1576 	int size = perf_evsel__read_size(&leader->core);
1577 	u64 *data = ps->group_data;
1578 
1579 	if (!(read_format & PERF_FORMAT_ID))
1580 		return -EINVAL;
1581 
1582 	if (!evsel__is_group_leader(leader))
1583 		return -EINVAL;
1584 
1585 	if (!data) {
1586 		data = zalloc(size);
1587 		if (!data)
1588 			return -ENOMEM;
1589 
1590 		ps->group_data = data;
1591 	}
1592 
1593 	if (FD(leader, cpu_map_idx, thread) < 0)
1594 		return -EINVAL;
1595 
1596 	if (readn(FD(leader, cpu_map_idx, thread), data, size) <= 0)
1597 		return -errno;
1598 
1599 	return evsel__process_group_data(leader, cpu_map_idx, thread, data);
1600 }
1601 
evsel__read_counter(struct evsel * evsel,int cpu_map_idx,int thread)1602 int evsel__read_counter(struct evsel *evsel, int cpu_map_idx, int thread)
1603 {
1604 	u64 read_format = evsel->core.attr.read_format;
1605 
1606 	if (read_format & PERF_FORMAT_GROUP)
1607 		return evsel__read_group(evsel, cpu_map_idx, thread);
1608 
1609 	return evsel__read_one(evsel, cpu_map_idx, thread);
1610 }
1611 
__evsel__read_on_cpu(struct evsel * evsel,int cpu_map_idx,int thread,bool scale)1612 int __evsel__read_on_cpu(struct evsel *evsel, int cpu_map_idx, int thread, bool scale)
1613 {
1614 	struct perf_counts_values count;
1615 	size_t nv = scale ? 3 : 1;
1616 
1617 	if (FD(evsel, cpu_map_idx, thread) < 0)
1618 		return -EINVAL;
1619 
1620 	if (evsel->counts == NULL && evsel__alloc_counts(evsel) < 0)
1621 		return -ENOMEM;
1622 
1623 	if (readn(FD(evsel, cpu_map_idx, thread), &count, nv * sizeof(u64)) <= 0)
1624 		return -errno;
1625 
1626 	evsel__compute_deltas(evsel, cpu_map_idx, thread, &count);
1627 	perf_counts_values__scale(&count, scale, NULL);
1628 	*perf_counts(evsel->counts, cpu_map_idx, thread) = count;
1629 	return 0;
1630 }
1631 
evsel__match_other_cpu(struct evsel * evsel,struct evsel * other,int cpu_map_idx)1632 static int evsel__match_other_cpu(struct evsel *evsel, struct evsel *other,
1633 				  int cpu_map_idx)
1634 {
1635 	struct perf_cpu cpu;
1636 
1637 	cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx);
1638 	return perf_cpu_map__idx(other->core.cpus, cpu);
1639 }
1640 
evsel__hybrid_group_cpu_map_idx(struct evsel * evsel,int cpu_map_idx)1641 static int evsel__hybrid_group_cpu_map_idx(struct evsel *evsel, int cpu_map_idx)
1642 {
1643 	struct evsel *leader = evsel__leader(evsel);
1644 
1645 	if ((evsel__is_hybrid(evsel) && !evsel__is_hybrid(leader)) ||
1646 	    (!evsel__is_hybrid(evsel) && evsel__is_hybrid(leader))) {
1647 		return evsel__match_other_cpu(evsel, leader, cpu_map_idx);
1648 	}
1649 
1650 	return cpu_map_idx;
1651 }
1652 
get_group_fd(struct evsel * evsel,int cpu_map_idx,int thread)1653 static int get_group_fd(struct evsel *evsel, int cpu_map_idx, int thread)
1654 {
1655 	struct evsel *leader = evsel__leader(evsel);
1656 	int fd;
1657 
1658 	if (evsel__is_group_leader(evsel))
1659 		return -1;
1660 
1661 	/*
1662 	 * Leader must be already processed/open,
1663 	 * if not it's a bug.
1664 	 */
1665 	BUG_ON(!leader->core.fd);
1666 
1667 	cpu_map_idx = evsel__hybrid_group_cpu_map_idx(evsel, cpu_map_idx);
1668 	if (cpu_map_idx == -1)
1669 		return -1;
1670 
1671 	fd = FD(leader, cpu_map_idx, thread);
1672 	BUG_ON(fd == -1 && !leader->skippable);
1673 
1674 	/*
1675 	 * When the leader has been skipped, return -2 to distinguish from no
1676 	 * group leader case.
1677 	 */
1678 	return fd == -1 ? -2 : fd;
1679 }
1680 
evsel__remove_fd(struct evsel * pos,int nr_cpus,int nr_threads,int thread_idx)1681 static void evsel__remove_fd(struct evsel *pos, int nr_cpus, int nr_threads, int thread_idx)
1682 {
1683 	for (int cpu = 0; cpu < nr_cpus; cpu++)
1684 		for (int thread = thread_idx; thread < nr_threads - 1; thread++)
1685 			FD(pos, cpu, thread) = FD(pos, cpu, thread + 1);
1686 }
1687 
update_fds(struct evsel * evsel,int nr_cpus,int cpu_map_idx,int nr_threads,int thread_idx)1688 static int update_fds(struct evsel *evsel,
1689 		      int nr_cpus, int cpu_map_idx,
1690 		      int nr_threads, int thread_idx)
1691 {
1692 	struct evsel *pos;
1693 
1694 	if (cpu_map_idx >= nr_cpus || thread_idx >= nr_threads)
1695 		return -EINVAL;
1696 
1697 	evlist__for_each_entry(evsel->evlist, pos) {
1698 		nr_cpus = pos != evsel ? nr_cpus : cpu_map_idx;
1699 
1700 		evsel__remove_fd(pos, nr_cpus, nr_threads, thread_idx);
1701 
1702 		/*
1703 		 * Since fds for next evsel has not been created,
1704 		 * there is no need to iterate whole event list.
1705 		 */
1706 		if (pos == evsel)
1707 			break;
1708 	}
1709 	return 0;
1710 }
1711 
evsel__ignore_missing_thread(struct evsel * evsel,int nr_cpus,int cpu_map_idx,struct perf_thread_map * threads,int thread,int err)1712 static bool evsel__ignore_missing_thread(struct evsel *evsel,
1713 					 int nr_cpus, int cpu_map_idx,
1714 					 struct perf_thread_map *threads,
1715 					 int thread, int err)
1716 {
1717 	pid_t ignore_pid = perf_thread_map__pid(threads, thread);
1718 
1719 	if (!evsel->ignore_missing_thread)
1720 		return false;
1721 
1722 	/* The system wide setup does not work with threads. */
1723 	if (evsel->core.system_wide)
1724 		return false;
1725 
1726 	/* The -ESRCH is perf event syscall errno for pid's not found. */
1727 	if (err != -ESRCH)
1728 		return false;
1729 
1730 	/* If there's only one thread, let it fail. */
1731 	if (threads->nr == 1)
1732 		return false;
1733 
1734 	/*
1735 	 * We should remove fd for missing_thread first
1736 	 * because thread_map__remove() will decrease threads->nr.
1737 	 */
1738 	if (update_fds(evsel, nr_cpus, cpu_map_idx, threads->nr, thread))
1739 		return false;
1740 
1741 	if (thread_map__remove(threads, thread))
1742 		return false;
1743 
1744 	pr_warning("WARNING: Ignored open failure for pid %d\n",
1745 		   ignore_pid);
1746 	return true;
1747 }
1748 
__open_attr__fprintf(FILE * fp,const char * name,const char * val,void * priv __maybe_unused)1749 static int __open_attr__fprintf(FILE *fp, const char *name, const char *val,
1750 				void *priv __maybe_unused)
1751 {
1752 	return fprintf(fp, "  %-32s %s\n", name, val);
1753 }
1754 
display_attr(struct perf_event_attr * attr)1755 static void display_attr(struct perf_event_attr *attr)
1756 {
1757 	if (verbose >= 2 || debug_peo_args) {
1758 		fprintf(stderr, "%.60s\n", graph_dotted_line);
1759 		fprintf(stderr, "perf_event_attr:\n");
1760 		perf_event_attr__fprintf(stderr, attr, __open_attr__fprintf, NULL);
1761 		fprintf(stderr, "%.60s\n", graph_dotted_line);
1762 	}
1763 }
1764 
evsel__precise_ip_fallback(struct evsel * evsel)1765 bool evsel__precise_ip_fallback(struct evsel *evsel)
1766 {
1767 	/* Do not try less precise if not requested. */
1768 	if (!evsel->precise_max)
1769 		return false;
1770 
1771 	/*
1772 	 * We tried all the precise_ip values, and it's
1773 	 * still failing, so leave it to standard fallback.
1774 	 */
1775 	if (!evsel->core.attr.precise_ip) {
1776 		evsel->core.attr.precise_ip = evsel->precise_ip_original;
1777 		return false;
1778 	}
1779 
1780 	if (!evsel->precise_ip_original)
1781 		evsel->precise_ip_original = evsel->core.attr.precise_ip;
1782 
1783 	evsel->core.attr.precise_ip--;
1784 	pr_debug2_peo("decreasing precise_ip by one (%d)\n", evsel->core.attr.precise_ip);
1785 	display_attr(&evsel->core.attr);
1786 	return true;
1787 }
1788 
1789 static struct perf_cpu_map *empty_cpu_map;
1790 static struct perf_thread_map *empty_thread_map;
1791 
__evsel__prepare_open(struct evsel * evsel,struct perf_cpu_map * cpus,struct perf_thread_map * threads)1792 static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus,
1793 		struct perf_thread_map *threads)
1794 {
1795 	int nthreads = perf_thread_map__nr(threads);
1796 
1797 	if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) ||
1798 	    (perf_missing_features.aux_output     && evsel->core.attr.aux_output))
1799 		return -EINVAL;
1800 
1801 	if (cpus == NULL) {
1802 		if (empty_cpu_map == NULL) {
1803 			empty_cpu_map = perf_cpu_map__dummy_new();
1804 			if (empty_cpu_map == NULL)
1805 				return -ENOMEM;
1806 		}
1807 
1808 		cpus = empty_cpu_map;
1809 	}
1810 
1811 	if (threads == NULL) {
1812 		if (empty_thread_map == NULL) {
1813 			empty_thread_map = thread_map__new_by_tid(-1);
1814 			if (empty_thread_map == NULL)
1815 				return -ENOMEM;
1816 		}
1817 
1818 		threads = empty_thread_map;
1819 	}
1820 
1821 	if (evsel->core.fd == NULL &&
1822 	    perf_evsel__alloc_fd(&evsel->core, perf_cpu_map__nr(cpus), nthreads) < 0)
1823 		return -ENOMEM;
1824 
1825 	evsel->open_flags = PERF_FLAG_FD_CLOEXEC;
1826 	if (evsel->cgrp)
1827 		evsel->open_flags |= PERF_FLAG_PID_CGROUP;
1828 
1829 	return 0;
1830 }
1831 
evsel__disable_missing_features(struct evsel * evsel)1832 static void evsel__disable_missing_features(struct evsel *evsel)
1833 {
1834 	if (perf_missing_features.read_lost)
1835 		evsel->core.attr.read_format &= ~PERF_FORMAT_LOST;
1836 	if (perf_missing_features.weight_struct) {
1837 		evsel__set_sample_bit(evsel, WEIGHT);
1838 		evsel__reset_sample_bit(evsel, WEIGHT_STRUCT);
1839 	}
1840 	if (perf_missing_features.clockid_wrong)
1841 		evsel->core.attr.clockid = CLOCK_MONOTONIC; /* should always work */
1842 	if (perf_missing_features.clockid) {
1843 		evsel->core.attr.use_clockid = 0;
1844 		evsel->core.attr.clockid = 0;
1845 	}
1846 	if (perf_missing_features.cloexec)
1847 		evsel->open_flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC;
1848 	if (perf_missing_features.mmap2)
1849 		evsel->core.attr.mmap2 = 0;
1850 	if (evsel->pmu && evsel->pmu->missing_features.exclude_guest)
1851 		evsel->core.attr.exclude_guest = evsel->core.attr.exclude_host = 0;
1852 	if (perf_missing_features.lbr_flags)
1853 		evsel->core.attr.branch_sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS |
1854 				     PERF_SAMPLE_BRANCH_NO_CYCLES);
1855 	if (perf_missing_features.group_read && evsel->core.attr.inherit)
1856 		evsel->core.attr.read_format &= ~(PERF_FORMAT_GROUP|PERF_FORMAT_ID);
1857 	if (perf_missing_features.ksymbol)
1858 		evsel->core.attr.ksymbol = 0;
1859 	if (perf_missing_features.bpf)
1860 		evsel->core.attr.bpf_event = 0;
1861 	if (perf_missing_features.branch_hw_idx)
1862 		evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_HW_INDEX;
1863 	if (perf_missing_features.sample_id_all)
1864 		evsel->core.attr.sample_id_all = 0;
1865 }
1866 
evsel__prepare_open(struct evsel * evsel,struct perf_cpu_map * cpus,struct perf_thread_map * threads)1867 int evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus,
1868 			struct perf_thread_map *threads)
1869 {
1870 	int err;
1871 
1872 	err = __evsel__prepare_open(evsel, cpus, threads);
1873 	if (err)
1874 		return err;
1875 
1876 	evsel__disable_missing_features(evsel);
1877 
1878 	return err;
1879 }
1880 
evsel__detect_missing_features(struct evsel * evsel)1881 bool evsel__detect_missing_features(struct evsel *evsel)
1882 {
1883 	/*
1884 	 * Must probe features in the order they were added to the
1885 	 * perf_event_attr interface.
1886 	 */
1887 	if (!perf_missing_features.read_lost &&
1888 	    (evsel->core.attr.read_format & PERF_FORMAT_LOST)) {
1889 		perf_missing_features.read_lost = true;
1890 		pr_debug2("switching off PERF_FORMAT_LOST support\n");
1891 		return true;
1892 	} else if (!perf_missing_features.weight_struct &&
1893 	    (evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) {
1894 		perf_missing_features.weight_struct = true;
1895 		pr_debug2("switching off weight struct support\n");
1896 		return true;
1897 	} else if (!perf_missing_features.code_page_size &&
1898 	    (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)) {
1899 		perf_missing_features.code_page_size = true;
1900 		pr_debug2_peo("Kernel has no PERF_SAMPLE_CODE_PAGE_SIZE support, bailing out\n");
1901 		return false;
1902 	} else if (!perf_missing_features.data_page_size &&
1903 	    (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)) {
1904 		perf_missing_features.data_page_size = true;
1905 		pr_debug2_peo("Kernel has no PERF_SAMPLE_DATA_PAGE_SIZE support, bailing out\n");
1906 		return false;
1907 	} else if (!perf_missing_features.cgroup && evsel->core.attr.cgroup) {
1908 		perf_missing_features.cgroup = true;
1909 		pr_debug2_peo("Kernel has no cgroup sampling support, bailing out\n");
1910 		return false;
1911 	} else if (!perf_missing_features.branch_hw_idx &&
1912 	    (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX)) {
1913 		perf_missing_features.branch_hw_idx = true;
1914 		pr_debug2("switching off branch HW index support\n");
1915 		return true;
1916 	} else if (!perf_missing_features.aux_output && evsel->core.attr.aux_output) {
1917 		perf_missing_features.aux_output = true;
1918 		pr_debug2_peo("Kernel has no attr.aux_output support, bailing out\n");
1919 		return false;
1920 	} else if (!perf_missing_features.bpf && evsel->core.attr.bpf_event) {
1921 		perf_missing_features.bpf = true;
1922 		pr_debug2_peo("switching off bpf_event\n");
1923 		return true;
1924 	} else if (!perf_missing_features.ksymbol && evsel->core.attr.ksymbol) {
1925 		perf_missing_features.ksymbol = true;
1926 		pr_debug2_peo("switching off ksymbol\n");
1927 		return true;
1928 	} else if (!perf_missing_features.write_backward && evsel->core.attr.write_backward) {
1929 		perf_missing_features.write_backward = true;
1930 		pr_debug2_peo("switching off write_backward\n");
1931 		return false;
1932 	} else if (!perf_missing_features.clockid_wrong && evsel->core.attr.use_clockid) {
1933 		perf_missing_features.clockid_wrong = true;
1934 		pr_debug2_peo("switching off clockid\n");
1935 		return true;
1936 	} else if (!perf_missing_features.clockid && evsel->core.attr.use_clockid) {
1937 		perf_missing_features.clockid = true;
1938 		pr_debug2_peo("switching off use_clockid\n");
1939 		return true;
1940 	} else if (!perf_missing_features.cloexec && (evsel->open_flags & PERF_FLAG_FD_CLOEXEC)) {
1941 		perf_missing_features.cloexec = true;
1942 		pr_debug2_peo("switching off cloexec flag\n");
1943 		return true;
1944 	} else if (!perf_missing_features.mmap2 && evsel->core.attr.mmap2) {
1945 		perf_missing_features.mmap2 = true;
1946 		pr_debug2_peo("switching off mmap2\n");
1947 		return true;
1948 	} else if (evsel->core.attr.exclude_guest || evsel->core.attr.exclude_host) {
1949 		if (evsel->pmu == NULL)
1950 			evsel->pmu = evsel__find_pmu(evsel);
1951 
1952 		if (evsel->pmu)
1953 			evsel->pmu->missing_features.exclude_guest = true;
1954 		else {
1955 			/* we cannot find PMU, disable attrs now */
1956 			evsel->core.attr.exclude_host = false;
1957 			evsel->core.attr.exclude_guest = false;
1958 		}
1959 
1960 		if (evsel->exclude_GH) {
1961 			pr_debug2_peo("PMU has no exclude_host/guest support, bailing out\n");
1962 			return false;
1963 		}
1964 		if (!perf_missing_features.exclude_guest) {
1965 			perf_missing_features.exclude_guest = true;
1966 			pr_debug2_peo("switching off exclude_guest, exclude_host\n");
1967 		}
1968 		return true;
1969 	} else if (!perf_missing_features.sample_id_all) {
1970 		perf_missing_features.sample_id_all = true;
1971 		pr_debug2_peo("switching off sample_id_all\n");
1972 		return true;
1973 	} else if (!perf_missing_features.lbr_flags &&
1974 			(evsel->core.attr.branch_sample_type &
1975 			 (PERF_SAMPLE_BRANCH_NO_CYCLES |
1976 			  PERF_SAMPLE_BRANCH_NO_FLAGS))) {
1977 		perf_missing_features.lbr_flags = true;
1978 		pr_debug2_peo("switching off branch sample type no (cycles/flags)\n");
1979 		return true;
1980 	} else if (!perf_missing_features.group_read &&
1981 		    evsel->core.attr.inherit &&
1982 		   (evsel->core.attr.read_format & PERF_FORMAT_GROUP) &&
1983 		   evsel__is_group_leader(evsel)) {
1984 		perf_missing_features.group_read = true;
1985 		pr_debug2_peo("switching off group read\n");
1986 		return true;
1987 	} else {
1988 		return false;
1989 	}
1990 }
1991 
evsel__increase_rlimit(enum rlimit_action * set_rlimit)1992 bool evsel__increase_rlimit(enum rlimit_action *set_rlimit)
1993 {
1994 	int old_errno;
1995 	struct rlimit l;
1996 
1997 	if (*set_rlimit < INCREASED_MAX) {
1998 		old_errno = errno;
1999 
2000 		if (getrlimit(RLIMIT_NOFILE, &l) == 0) {
2001 			if (*set_rlimit == NO_CHANGE) {
2002 				l.rlim_cur = l.rlim_max;
2003 			} else {
2004 				l.rlim_cur = l.rlim_max + 1000;
2005 				l.rlim_max = l.rlim_cur;
2006 			}
2007 			if (setrlimit(RLIMIT_NOFILE, &l) == 0) {
2008 				(*set_rlimit) += 1;
2009 				errno = old_errno;
2010 				return true;
2011 			}
2012 		}
2013 		errno = old_errno;
2014 	}
2015 
2016 	return false;
2017 }
2018 
evsel__open_cpu(struct evsel * evsel,struct perf_cpu_map * cpus,struct perf_thread_map * threads,int start_cpu_map_idx,int end_cpu_map_idx)2019 static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus,
2020 		struct perf_thread_map *threads,
2021 		int start_cpu_map_idx, int end_cpu_map_idx)
2022 {
2023 	int idx, thread, nthreads;
2024 	int pid = -1, err, old_errno;
2025 	enum rlimit_action set_rlimit = NO_CHANGE;
2026 
2027 	err = __evsel__prepare_open(evsel, cpus, threads);
2028 	if (err)
2029 		return err;
2030 
2031 	if (cpus == NULL)
2032 		cpus = empty_cpu_map;
2033 
2034 	if (threads == NULL)
2035 		threads = empty_thread_map;
2036 
2037 	nthreads = perf_thread_map__nr(threads);
2038 
2039 	if (evsel->cgrp)
2040 		pid = evsel->cgrp->fd;
2041 
2042 fallback_missing_features:
2043 	evsel__disable_missing_features(evsel);
2044 
2045 	pr_debug3("Opening: %s\n", evsel__name(evsel));
2046 	display_attr(&evsel->core.attr);
2047 
2048 	for (idx = start_cpu_map_idx; idx < end_cpu_map_idx; idx++) {
2049 
2050 		for (thread = 0; thread < nthreads; thread++) {
2051 			int fd, group_fd;
2052 retry_open:
2053 			if (thread >= nthreads)
2054 				break;
2055 
2056 			if (!evsel->cgrp && !evsel->core.system_wide)
2057 				pid = perf_thread_map__pid(threads, thread);
2058 
2059 			group_fd = get_group_fd(evsel, idx, thread);
2060 
2061 			if (group_fd == -2) {
2062 				pr_debug("broken group leader for %s\n", evsel->name);
2063 				err = -EINVAL;
2064 				goto out_close;
2065 			}
2066 
2067 			test_attr__ready();
2068 
2069 			/* Debug message used by test scripts */
2070 			pr_debug2_peo("sys_perf_event_open: pid %d  cpu %d  group_fd %d  flags %#lx",
2071 				pid, perf_cpu_map__cpu(cpus, idx).cpu, group_fd, evsel->open_flags);
2072 
2073 			fd = sys_perf_event_open(&evsel->core.attr, pid,
2074 						perf_cpu_map__cpu(cpus, idx).cpu,
2075 						group_fd, evsel->open_flags);
2076 
2077 			FD(evsel, idx, thread) = fd;
2078 
2079 			if (fd < 0) {
2080 				err = -errno;
2081 
2082 				pr_debug2_peo("\nsys_perf_event_open failed, error %d\n",
2083 					  err);
2084 				goto try_fallback;
2085 			}
2086 
2087 			bpf_counter__install_pe(evsel, idx, fd);
2088 
2089 			if (unlikely(test_attr__enabled)) {
2090 				test_attr__open(&evsel->core.attr, pid,
2091 						perf_cpu_map__cpu(cpus, idx),
2092 						fd, group_fd, evsel->open_flags);
2093 			}
2094 
2095 			/* Debug message used by test scripts */
2096 			pr_debug2_peo(" = %d\n", fd);
2097 
2098 			if (evsel->bpf_fd >= 0) {
2099 				int evt_fd = fd;
2100 				int bpf_fd = evsel->bpf_fd;
2101 
2102 				err = ioctl(evt_fd,
2103 					    PERF_EVENT_IOC_SET_BPF,
2104 					    bpf_fd);
2105 				if (err && errno != EEXIST) {
2106 					pr_err("failed to attach bpf fd %d: %s\n",
2107 					       bpf_fd, strerror(errno));
2108 					err = -EINVAL;
2109 					goto out_close;
2110 				}
2111 			}
2112 
2113 			set_rlimit = NO_CHANGE;
2114 
2115 			/*
2116 			 * If we succeeded but had to kill clockid, fail and
2117 			 * have evsel__open_strerror() print us a nice error.
2118 			 */
2119 			if (perf_missing_features.clockid ||
2120 			    perf_missing_features.clockid_wrong) {
2121 				err = -EINVAL;
2122 				goto out_close;
2123 			}
2124 		}
2125 	}
2126 
2127 	return 0;
2128 
2129 try_fallback:
2130 	if (evsel__precise_ip_fallback(evsel))
2131 		goto retry_open;
2132 
2133 	if (evsel__ignore_missing_thread(evsel, perf_cpu_map__nr(cpus),
2134 					 idx, threads, thread, err)) {
2135 		/* We just removed 1 thread, so lower the upper nthreads limit. */
2136 		nthreads--;
2137 
2138 		/* ... and pretend like nothing have happened. */
2139 		err = 0;
2140 		goto retry_open;
2141 	}
2142 	/*
2143 	 * perf stat needs between 5 and 22 fds per CPU. When we run out
2144 	 * of them try to increase the limits.
2145 	 */
2146 	if (err == -EMFILE && evsel__increase_rlimit(&set_rlimit))
2147 		goto retry_open;
2148 
2149 	if (err != -EINVAL || idx > 0 || thread > 0)
2150 		goto out_close;
2151 
2152 	if (evsel__detect_missing_features(evsel))
2153 		goto fallback_missing_features;
2154 out_close:
2155 	if (err)
2156 		threads->err_thread = thread;
2157 
2158 	old_errno = errno;
2159 	do {
2160 		while (--thread >= 0) {
2161 			if (FD(evsel, idx, thread) >= 0)
2162 				close(FD(evsel, idx, thread));
2163 			FD(evsel, idx, thread) = -1;
2164 		}
2165 		thread = nthreads;
2166 	} while (--idx >= 0);
2167 	errno = old_errno;
2168 	return err;
2169 }
2170 
evsel__open(struct evsel * evsel,struct perf_cpu_map * cpus,struct perf_thread_map * threads)2171 int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus,
2172 		struct perf_thread_map *threads)
2173 {
2174 	return evsel__open_cpu(evsel, cpus, threads, 0, perf_cpu_map__nr(cpus));
2175 }
2176 
evsel__close(struct evsel * evsel)2177 void evsel__close(struct evsel *evsel)
2178 {
2179 	perf_evsel__close(&evsel->core);
2180 	perf_evsel__free_id(&evsel->core);
2181 }
2182 
evsel__open_per_cpu(struct evsel * evsel,struct perf_cpu_map * cpus,int cpu_map_idx)2183 int evsel__open_per_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int cpu_map_idx)
2184 {
2185 	if (cpu_map_idx == -1)
2186 		return evsel__open_cpu(evsel, cpus, NULL, 0, perf_cpu_map__nr(cpus));
2187 
2188 	return evsel__open_cpu(evsel, cpus, NULL, cpu_map_idx, cpu_map_idx + 1);
2189 }
2190 
evsel__open_per_thread(struct evsel * evsel,struct perf_thread_map * threads)2191 int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads)
2192 {
2193 	return evsel__open(evsel, NULL, threads);
2194 }
2195 
perf_evsel__parse_id_sample(const struct evsel * evsel,const union perf_event * event,struct perf_sample * sample)2196 static int perf_evsel__parse_id_sample(const struct evsel *evsel,
2197 				       const union perf_event *event,
2198 				       struct perf_sample *sample)
2199 {
2200 	u64 type = evsel->core.attr.sample_type;
2201 	const __u64 *array = event->sample.array;
2202 	bool swapped = evsel->needs_swap;
2203 	union u64_swap u;
2204 
2205 	array += ((event->header.size -
2206 		   sizeof(event->header)) / sizeof(u64)) - 1;
2207 
2208 	if (type & PERF_SAMPLE_IDENTIFIER) {
2209 		sample->id = *array;
2210 		array--;
2211 	}
2212 
2213 	if (type & PERF_SAMPLE_CPU) {
2214 		u.val64 = *array;
2215 		if (swapped) {
2216 			/* undo swap of u64, then swap on individual u32s */
2217 			u.val64 = bswap_64(u.val64);
2218 			u.val32[0] = bswap_32(u.val32[0]);
2219 		}
2220 
2221 		sample->cpu = u.val32[0];
2222 		array--;
2223 	}
2224 
2225 	if (type & PERF_SAMPLE_STREAM_ID) {
2226 		sample->stream_id = *array;
2227 		array--;
2228 	}
2229 
2230 	if (type & PERF_SAMPLE_ID) {
2231 		sample->id = *array;
2232 		array--;
2233 	}
2234 
2235 	if (type & PERF_SAMPLE_TIME) {
2236 		sample->time = *array;
2237 		array--;
2238 	}
2239 
2240 	if (type & PERF_SAMPLE_TID) {
2241 		u.val64 = *array;
2242 		if (swapped) {
2243 			/* undo swap of u64, then swap on individual u32s */
2244 			u.val64 = bswap_64(u.val64);
2245 			u.val32[0] = bswap_32(u.val32[0]);
2246 			u.val32[1] = bswap_32(u.val32[1]);
2247 		}
2248 
2249 		sample->pid = u.val32[0];
2250 		sample->tid = u.val32[1];
2251 		array--;
2252 	}
2253 
2254 	return 0;
2255 }
2256 
overflow(const void * endp,u16 max_size,const void * offset,u64 size)2257 static inline bool overflow(const void *endp, u16 max_size, const void *offset,
2258 			    u64 size)
2259 {
2260 	return size > max_size || offset + size > endp;
2261 }
2262 
2263 #define OVERFLOW_CHECK(offset, size, max_size)				\
2264 	do {								\
2265 		if (overflow(endp, (max_size), (offset), (size)))	\
2266 			return -EFAULT;					\
2267 	} while (0)
2268 
2269 #define OVERFLOW_CHECK_u64(offset) \
2270 	OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64))
2271 
2272 static int
perf_event__check_size(union perf_event * event,unsigned int sample_size)2273 perf_event__check_size(union perf_event *event, unsigned int sample_size)
2274 {
2275 	/*
2276 	 * The evsel's sample_size is based on PERF_SAMPLE_MASK which includes
2277 	 * up to PERF_SAMPLE_PERIOD.  After that overflow() must be used to
2278 	 * check the format does not go past the end of the event.
2279 	 */
2280 	if (sample_size + sizeof(event->header) > event->header.size)
2281 		return -EFAULT;
2282 
2283 	return 0;
2284 }
2285 
arch_perf_parse_sample_weight(struct perf_sample * data,const __u64 * array,u64 type __maybe_unused)2286 void __weak arch_perf_parse_sample_weight(struct perf_sample *data,
2287 					  const __u64 *array,
2288 					  u64 type __maybe_unused)
2289 {
2290 	data->weight = *array;
2291 }
2292 
evsel__bitfield_swap_branch_flags(u64 value)2293 u64 evsel__bitfield_swap_branch_flags(u64 value)
2294 {
2295 	u64 new_val = 0;
2296 
2297 	/*
2298 	 * branch_flags
2299 	 * union {
2300 	 * 	u64 values;
2301 	 * 	struct {
2302 	 * 		mispred:1	//target mispredicted
2303 	 * 		predicted:1	//target predicted
2304 	 * 		in_tx:1		//in transaction
2305 	 * 		abort:1		//transaction abort
2306 	 * 		cycles:16	//cycle count to last branch
2307 	 * 		type:4		//branch type
2308 	 * 		spec:2		//branch speculation info
2309 	 * 		new_type:4	//additional branch type
2310 	 * 		priv:3		//privilege level
2311 	 * 		reserved:31
2312 	 * 	}
2313 	 * }
2314 	 *
2315 	 * Avoid bswap64() the entire branch_flag.value,
2316 	 * as it has variable bit-field sizes. Instead the
2317 	 * macro takes the bit-field position/size,
2318 	 * swaps it based on the host endianness.
2319 	 */
2320 	if (host_is_bigendian()) {
2321 		new_val = bitfield_swap(value, 0, 1);
2322 		new_val |= bitfield_swap(value, 1, 1);
2323 		new_val |= bitfield_swap(value, 2, 1);
2324 		new_val |= bitfield_swap(value, 3, 1);
2325 		new_val |= bitfield_swap(value, 4, 16);
2326 		new_val |= bitfield_swap(value, 20, 4);
2327 		new_val |= bitfield_swap(value, 24, 2);
2328 		new_val |= bitfield_swap(value, 26, 4);
2329 		new_val |= bitfield_swap(value, 30, 3);
2330 		new_val |= bitfield_swap(value, 33, 31);
2331 	} else {
2332 		new_val = bitfield_swap(value, 63, 1);
2333 		new_val |= bitfield_swap(value, 62, 1);
2334 		new_val |= bitfield_swap(value, 61, 1);
2335 		new_val |= bitfield_swap(value, 60, 1);
2336 		new_val |= bitfield_swap(value, 44, 16);
2337 		new_val |= bitfield_swap(value, 40, 4);
2338 		new_val |= bitfield_swap(value, 38, 2);
2339 		new_val |= bitfield_swap(value, 34, 4);
2340 		new_val |= bitfield_swap(value, 31, 3);
2341 		new_val |= bitfield_swap(value, 0, 31);
2342 	}
2343 
2344 	return new_val;
2345 }
2346 
evsel__parse_sample(struct evsel * evsel,union perf_event * event,struct perf_sample * data)2347 int evsel__parse_sample(struct evsel *evsel, union perf_event *event,
2348 			struct perf_sample *data)
2349 {
2350 	u64 type = evsel->core.attr.sample_type;
2351 	bool swapped = evsel->needs_swap;
2352 	const __u64 *array;
2353 	u16 max_size = event->header.size;
2354 	const void *endp = (void *)event + max_size;
2355 	u64 sz;
2356 
2357 	/*
2358 	 * used for cross-endian analysis. See git commit 65014ab3
2359 	 * for why this goofiness is needed.
2360 	 */
2361 	union u64_swap u;
2362 
2363 	memset(data, 0, sizeof(*data));
2364 	data->cpu = data->pid = data->tid = -1;
2365 	data->stream_id = data->id = data->time = -1ULL;
2366 	data->period = evsel->core.attr.sample_period;
2367 	data->cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
2368 	data->misc    = event->header.misc;
2369 	data->id = -1ULL;
2370 	data->data_src = PERF_MEM_DATA_SRC_NONE;
2371 	data->vcpu = -1;
2372 
2373 	if (event->header.type != PERF_RECORD_SAMPLE) {
2374 		if (!evsel->core.attr.sample_id_all)
2375 			return 0;
2376 		return perf_evsel__parse_id_sample(evsel, event, data);
2377 	}
2378 
2379 	array = event->sample.array;
2380 
2381 	if (perf_event__check_size(event, evsel->sample_size))
2382 		return -EFAULT;
2383 
2384 	if (type & PERF_SAMPLE_IDENTIFIER) {
2385 		data->id = *array;
2386 		array++;
2387 	}
2388 
2389 	if (type & PERF_SAMPLE_IP) {
2390 		data->ip = *array;
2391 		array++;
2392 	}
2393 
2394 	if (type & PERF_SAMPLE_TID) {
2395 		u.val64 = *array;
2396 		if (swapped) {
2397 			/* undo swap of u64, then swap on individual u32s */
2398 			u.val64 = bswap_64(u.val64);
2399 			u.val32[0] = bswap_32(u.val32[0]);
2400 			u.val32[1] = bswap_32(u.val32[1]);
2401 		}
2402 
2403 		data->pid = u.val32[0];
2404 		data->tid = u.val32[1];
2405 		array++;
2406 	}
2407 
2408 	if (type & PERF_SAMPLE_TIME) {
2409 		data->time = *array;
2410 		array++;
2411 	}
2412 
2413 	if (type & PERF_SAMPLE_ADDR) {
2414 		data->addr = *array;
2415 		array++;
2416 	}
2417 
2418 	if (type & PERF_SAMPLE_ID) {
2419 		data->id = *array;
2420 		array++;
2421 	}
2422 
2423 	if (type & PERF_SAMPLE_STREAM_ID) {
2424 		data->stream_id = *array;
2425 		array++;
2426 	}
2427 
2428 	if (type & PERF_SAMPLE_CPU) {
2429 
2430 		u.val64 = *array;
2431 		if (swapped) {
2432 			/* undo swap of u64, then swap on individual u32s */
2433 			u.val64 = bswap_64(u.val64);
2434 			u.val32[0] = bswap_32(u.val32[0]);
2435 		}
2436 
2437 		data->cpu = u.val32[0];
2438 		array++;
2439 	}
2440 
2441 	if (type & PERF_SAMPLE_PERIOD) {
2442 		data->period = *array;
2443 		array++;
2444 	}
2445 
2446 	if (type & PERF_SAMPLE_READ) {
2447 		u64 read_format = evsel->core.attr.read_format;
2448 
2449 		OVERFLOW_CHECK_u64(array);
2450 		if (read_format & PERF_FORMAT_GROUP)
2451 			data->read.group.nr = *array;
2452 		else
2453 			data->read.one.value = *array;
2454 
2455 		array++;
2456 
2457 		if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2458 			OVERFLOW_CHECK_u64(array);
2459 			data->read.time_enabled = *array;
2460 			array++;
2461 		}
2462 
2463 		if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2464 			OVERFLOW_CHECK_u64(array);
2465 			data->read.time_running = *array;
2466 			array++;
2467 		}
2468 
2469 		/* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
2470 		if (read_format & PERF_FORMAT_GROUP) {
2471 			const u64 max_group_nr = UINT64_MAX /
2472 					sizeof(struct sample_read_value);
2473 
2474 			if (data->read.group.nr > max_group_nr)
2475 				return -EFAULT;
2476 
2477 			sz = data->read.group.nr * sample_read_value_size(read_format);
2478 			OVERFLOW_CHECK(array, sz, max_size);
2479 			data->read.group.values =
2480 					(struct sample_read_value *)array;
2481 			array = (void *)array + sz;
2482 		} else {
2483 			OVERFLOW_CHECK_u64(array);
2484 			data->read.one.id = *array;
2485 			array++;
2486 
2487 			if (read_format & PERF_FORMAT_LOST) {
2488 				OVERFLOW_CHECK_u64(array);
2489 				data->read.one.lost = *array;
2490 				array++;
2491 			}
2492 		}
2493 	}
2494 
2495 	if (type & PERF_SAMPLE_CALLCHAIN) {
2496 		const u64 max_callchain_nr = UINT64_MAX / sizeof(u64);
2497 
2498 		OVERFLOW_CHECK_u64(array);
2499 		data->callchain = (struct ip_callchain *)array++;
2500 		if (data->callchain->nr > max_callchain_nr)
2501 			return -EFAULT;
2502 		sz = data->callchain->nr * sizeof(u64);
2503 		OVERFLOW_CHECK(array, sz, max_size);
2504 		array = (void *)array + sz;
2505 	}
2506 
2507 	if (type & PERF_SAMPLE_RAW) {
2508 		OVERFLOW_CHECK_u64(array);
2509 		u.val64 = *array;
2510 
2511 		/*
2512 		 * Undo swap of u64, then swap on individual u32s,
2513 		 * get the size of the raw area and undo all of the
2514 		 * swap. The pevent interface handles endianness by
2515 		 * itself.
2516 		 */
2517 		if (swapped) {
2518 			u.val64 = bswap_64(u.val64);
2519 			u.val32[0] = bswap_32(u.val32[0]);
2520 			u.val32[1] = bswap_32(u.val32[1]);
2521 		}
2522 		data->raw_size = u.val32[0];
2523 
2524 		/*
2525 		 * The raw data is aligned on 64bits including the
2526 		 * u32 size, so it's safe to use mem_bswap_64.
2527 		 */
2528 		if (swapped)
2529 			mem_bswap_64((void *) array, data->raw_size);
2530 
2531 		array = (void *)array + sizeof(u32);
2532 
2533 		OVERFLOW_CHECK(array, data->raw_size, max_size);
2534 		data->raw_data = (void *)array;
2535 		array = (void *)array + data->raw_size;
2536 	}
2537 
2538 	if (type & PERF_SAMPLE_BRANCH_STACK) {
2539 		const u64 max_branch_nr = UINT64_MAX /
2540 					  sizeof(struct branch_entry);
2541 		struct branch_entry *e;
2542 		unsigned int i;
2543 
2544 		OVERFLOW_CHECK_u64(array);
2545 		data->branch_stack = (struct branch_stack *)array++;
2546 
2547 		if (data->branch_stack->nr > max_branch_nr)
2548 			return -EFAULT;
2549 
2550 		sz = data->branch_stack->nr * sizeof(struct branch_entry);
2551 		if (evsel__has_branch_hw_idx(evsel)) {
2552 			sz += sizeof(u64);
2553 			e = &data->branch_stack->entries[0];
2554 		} else {
2555 			data->no_hw_idx = true;
2556 			/*
2557 			 * if the PERF_SAMPLE_BRANCH_HW_INDEX is not applied,
2558 			 * only nr and entries[] will be output by kernel.
2559 			 */
2560 			e = (struct branch_entry *)&data->branch_stack->hw_idx;
2561 		}
2562 
2563 		if (swapped) {
2564 			/*
2565 			 * struct branch_flag does not have endian
2566 			 * specific bit field definition. And bswap
2567 			 * will not resolve the issue, since these
2568 			 * are bit fields.
2569 			 *
2570 			 * evsel__bitfield_swap_branch_flags() uses a
2571 			 * bitfield_swap macro to swap the bit position
2572 			 * based on the host endians.
2573 			 */
2574 			for (i = 0; i < data->branch_stack->nr; i++, e++)
2575 				e->flags.value = evsel__bitfield_swap_branch_flags(e->flags.value);
2576 		}
2577 
2578 		OVERFLOW_CHECK(array, sz, max_size);
2579 		array = (void *)array + sz;
2580 	}
2581 
2582 	if (type & PERF_SAMPLE_REGS_USER) {
2583 		OVERFLOW_CHECK_u64(array);
2584 		data->user_regs.abi = *array;
2585 		array++;
2586 
2587 		if (data->user_regs.abi) {
2588 			u64 mask = evsel->core.attr.sample_regs_user;
2589 
2590 			sz = hweight64(mask) * sizeof(u64);
2591 			OVERFLOW_CHECK(array, sz, max_size);
2592 			data->user_regs.mask = mask;
2593 			data->user_regs.regs = (u64 *)array;
2594 			array = (void *)array + sz;
2595 		}
2596 	}
2597 
2598 	if (type & PERF_SAMPLE_STACK_USER) {
2599 		OVERFLOW_CHECK_u64(array);
2600 		sz = *array++;
2601 
2602 		data->user_stack.offset = ((char *)(array - 1)
2603 					  - (char *) event);
2604 
2605 		if (!sz) {
2606 			data->user_stack.size = 0;
2607 		} else {
2608 			OVERFLOW_CHECK(array, sz, max_size);
2609 			data->user_stack.data = (char *)array;
2610 			array = (void *)array + sz;
2611 			OVERFLOW_CHECK_u64(array);
2612 			data->user_stack.size = *array++;
2613 			if (WARN_ONCE(data->user_stack.size > sz,
2614 				      "user stack dump failure\n"))
2615 				return -EFAULT;
2616 		}
2617 	}
2618 
2619 	if (type & PERF_SAMPLE_WEIGHT_TYPE) {
2620 		OVERFLOW_CHECK_u64(array);
2621 		arch_perf_parse_sample_weight(data, array, type);
2622 		array++;
2623 	}
2624 
2625 	if (type & PERF_SAMPLE_DATA_SRC) {
2626 		OVERFLOW_CHECK_u64(array);
2627 		data->data_src = *array;
2628 		array++;
2629 	}
2630 
2631 	if (type & PERF_SAMPLE_TRANSACTION) {
2632 		OVERFLOW_CHECK_u64(array);
2633 		data->transaction = *array;
2634 		array++;
2635 	}
2636 
2637 	data->intr_regs.abi = PERF_SAMPLE_REGS_ABI_NONE;
2638 	if (type & PERF_SAMPLE_REGS_INTR) {
2639 		OVERFLOW_CHECK_u64(array);
2640 		data->intr_regs.abi = *array;
2641 		array++;
2642 
2643 		if (data->intr_regs.abi != PERF_SAMPLE_REGS_ABI_NONE) {
2644 			u64 mask = evsel->core.attr.sample_regs_intr;
2645 
2646 			sz = hweight64(mask) * sizeof(u64);
2647 			OVERFLOW_CHECK(array, sz, max_size);
2648 			data->intr_regs.mask = mask;
2649 			data->intr_regs.regs = (u64 *)array;
2650 			array = (void *)array + sz;
2651 		}
2652 	}
2653 
2654 	data->phys_addr = 0;
2655 	if (type & PERF_SAMPLE_PHYS_ADDR) {
2656 		data->phys_addr = *array;
2657 		array++;
2658 	}
2659 
2660 	data->cgroup = 0;
2661 	if (type & PERF_SAMPLE_CGROUP) {
2662 		data->cgroup = *array;
2663 		array++;
2664 	}
2665 
2666 	data->data_page_size = 0;
2667 	if (type & PERF_SAMPLE_DATA_PAGE_SIZE) {
2668 		data->data_page_size = *array;
2669 		array++;
2670 	}
2671 
2672 	data->code_page_size = 0;
2673 	if (type & PERF_SAMPLE_CODE_PAGE_SIZE) {
2674 		data->code_page_size = *array;
2675 		array++;
2676 	}
2677 
2678 	if (type & PERF_SAMPLE_AUX) {
2679 		OVERFLOW_CHECK_u64(array);
2680 		sz = *array++;
2681 
2682 		OVERFLOW_CHECK(array, sz, max_size);
2683 		/* Undo swap of data */
2684 		if (swapped)
2685 			mem_bswap_64((char *)array, sz);
2686 		data->aux_sample.size = sz;
2687 		data->aux_sample.data = (char *)array;
2688 		array = (void *)array + sz;
2689 	}
2690 
2691 	return 0;
2692 }
2693 
evsel__parse_sample_timestamp(struct evsel * evsel,union perf_event * event,u64 * timestamp)2694 int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event,
2695 				  u64 *timestamp)
2696 {
2697 	u64 type = evsel->core.attr.sample_type;
2698 	const __u64 *array;
2699 
2700 	if (!(type & PERF_SAMPLE_TIME))
2701 		return -1;
2702 
2703 	if (event->header.type != PERF_RECORD_SAMPLE) {
2704 		struct perf_sample data = {
2705 			.time = -1ULL,
2706 		};
2707 
2708 		if (!evsel->core.attr.sample_id_all)
2709 			return -1;
2710 		if (perf_evsel__parse_id_sample(evsel, event, &data))
2711 			return -1;
2712 
2713 		*timestamp = data.time;
2714 		return 0;
2715 	}
2716 
2717 	array = event->sample.array;
2718 
2719 	if (perf_event__check_size(event, evsel->sample_size))
2720 		return -EFAULT;
2721 
2722 	if (type & PERF_SAMPLE_IDENTIFIER)
2723 		array++;
2724 
2725 	if (type & PERF_SAMPLE_IP)
2726 		array++;
2727 
2728 	if (type & PERF_SAMPLE_TID)
2729 		array++;
2730 
2731 	if (type & PERF_SAMPLE_TIME)
2732 		*timestamp = *array;
2733 
2734 	return 0;
2735 }
2736 
evsel__id_hdr_size(struct evsel * evsel)2737 u16 evsel__id_hdr_size(struct evsel *evsel)
2738 {
2739 	u64 sample_type = evsel->core.attr.sample_type;
2740 	u16 size = 0;
2741 
2742 	if (sample_type & PERF_SAMPLE_TID)
2743 		size += sizeof(u64);
2744 
2745 	if (sample_type & PERF_SAMPLE_TIME)
2746 		size += sizeof(u64);
2747 
2748 	if (sample_type & PERF_SAMPLE_ID)
2749 		size += sizeof(u64);
2750 
2751 	if (sample_type & PERF_SAMPLE_STREAM_ID)
2752 		size += sizeof(u64);
2753 
2754 	if (sample_type & PERF_SAMPLE_CPU)
2755 		size += sizeof(u64);
2756 
2757 	if (sample_type & PERF_SAMPLE_IDENTIFIER)
2758 		size += sizeof(u64);
2759 
2760 	return size;
2761 }
2762 
2763 #ifdef HAVE_LIBTRACEEVENT
evsel__field(struct evsel * evsel,const char * name)2764 struct tep_format_field *evsel__field(struct evsel *evsel, const char *name)
2765 {
2766 	return tep_find_field(evsel->tp_format, name);
2767 }
2768 
evsel__rawptr(struct evsel * evsel,struct perf_sample * sample,const char * name)2769 void *evsel__rawptr(struct evsel *evsel, struct perf_sample *sample, const char *name)
2770 {
2771 	struct tep_format_field *field = evsel__field(evsel, name);
2772 	int offset;
2773 
2774 	if (!field)
2775 		return NULL;
2776 
2777 	offset = field->offset;
2778 
2779 	if (field->flags & TEP_FIELD_IS_DYNAMIC) {
2780 		offset = *(int *)(sample->raw_data + field->offset);
2781 		offset &= 0xffff;
2782 		if (tep_field_is_relative(field->flags))
2783 			offset += field->offset + field->size;
2784 	}
2785 
2786 	return sample->raw_data + offset;
2787 }
2788 
format_field__intval(struct tep_format_field * field,struct perf_sample * sample,bool needs_swap)2789 u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample,
2790 			 bool needs_swap)
2791 {
2792 	u64 value;
2793 	void *ptr = sample->raw_data + field->offset;
2794 
2795 	switch (field->size) {
2796 	case 1:
2797 		return *(u8 *)ptr;
2798 	case 2:
2799 		value = *(u16 *)ptr;
2800 		break;
2801 	case 4:
2802 		value = *(u32 *)ptr;
2803 		break;
2804 	case 8:
2805 		memcpy(&value, ptr, sizeof(u64));
2806 		break;
2807 	default:
2808 		return 0;
2809 	}
2810 
2811 	if (!needs_swap)
2812 		return value;
2813 
2814 	switch (field->size) {
2815 	case 2:
2816 		return bswap_16(value);
2817 	case 4:
2818 		return bswap_32(value);
2819 	case 8:
2820 		return bswap_64(value);
2821 	default:
2822 		return 0;
2823 	}
2824 
2825 	return 0;
2826 }
2827 
evsel__intval(struct evsel * evsel,struct perf_sample * sample,const char * name)2828 u64 evsel__intval(struct evsel *evsel, struct perf_sample *sample, const char *name)
2829 {
2830 	struct tep_format_field *field = evsel__field(evsel, name);
2831 
2832 	return field ? format_field__intval(field, sample, evsel->needs_swap) : 0;
2833 }
2834 #endif
2835 
evsel__fallback(struct evsel * evsel,int err,char * msg,size_t msgsize)2836 bool evsel__fallback(struct evsel *evsel, int err, char *msg, size_t msgsize)
2837 {
2838 	int paranoid;
2839 
2840 	if ((err == ENOENT || err == ENXIO || err == ENODEV) &&
2841 	    evsel->core.attr.type   == PERF_TYPE_HARDWARE &&
2842 	    evsel->core.attr.config == PERF_COUNT_HW_CPU_CYCLES) {
2843 		/*
2844 		 * If it's cycles then fall back to hrtimer based
2845 		 * cpu-clock-tick sw counter, which is always available even if
2846 		 * no PMU support.
2847 		 *
2848 		 * PPC returns ENXIO until 2.6.37 (behavior changed with commit
2849 		 * b0a873e).
2850 		 */
2851 		scnprintf(msg, msgsize, "%s",
2852 "The cycles event is not supported, trying to fall back to cpu-clock-ticks");
2853 
2854 		evsel->core.attr.type   = PERF_TYPE_SOFTWARE;
2855 		evsel->core.attr.config = PERF_COUNT_SW_CPU_CLOCK;
2856 
2857 		zfree(&evsel->name);
2858 		return true;
2859 	} else if (err == EACCES && !evsel->core.attr.exclude_kernel &&
2860 		   (paranoid = perf_event_paranoid()) > 1) {
2861 		const char *name = evsel__name(evsel);
2862 		char *new_name;
2863 		const char *sep = ":";
2864 
2865 		/* If event has exclude user then don't exclude kernel. */
2866 		if (evsel->core.attr.exclude_user)
2867 			return false;
2868 
2869 		/* Is there already the separator in the name. */
2870 		if (strchr(name, '/') ||
2871 		    (strchr(name, ':') && !evsel->is_libpfm_event))
2872 			sep = "";
2873 
2874 		if (asprintf(&new_name, "%s%su", name, sep) < 0)
2875 			return false;
2876 
2877 		free(evsel->name);
2878 		evsel->name = new_name;
2879 		scnprintf(msg, msgsize, "kernel.perf_event_paranoid=%d, trying "
2880 			  "to fall back to excluding kernel and hypervisor "
2881 			  " samples", paranoid);
2882 		evsel->core.attr.exclude_kernel = 1;
2883 		evsel->core.attr.exclude_hv     = 1;
2884 
2885 		return true;
2886 	}
2887 
2888 	return false;
2889 }
2890 
find_process(const char * name)2891 static bool find_process(const char *name)
2892 {
2893 	size_t len = strlen(name);
2894 	DIR *dir;
2895 	struct dirent *d;
2896 	int ret = -1;
2897 
2898 	dir = opendir(procfs__mountpoint());
2899 	if (!dir)
2900 		return false;
2901 
2902 	/* Walk through the directory. */
2903 	while (ret && (d = readdir(dir)) != NULL) {
2904 		char path[PATH_MAX];
2905 		char *data;
2906 		size_t size;
2907 
2908 		if ((d->d_type != DT_DIR) ||
2909 		     !strcmp(".", d->d_name) ||
2910 		     !strcmp("..", d->d_name))
2911 			continue;
2912 
2913 		scnprintf(path, sizeof(path), "%s/%s/comm",
2914 			  procfs__mountpoint(), d->d_name);
2915 
2916 		if (filename__read_str(path, &data, &size))
2917 			continue;
2918 
2919 		ret = strncmp(name, data, len);
2920 		free(data);
2921 	}
2922 
2923 	closedir(dir);
2924 	return ret ? false : true;
2925 }
2926 
arch_evsel__open_strerror(struct evsel * evsel __maybe_unused,char * msg __maybe_unused,size_t size __maybe_unused)2927 int __weak arch_evsel__open_strerror(struct evsel *evsel __maybe_unused,
2928 				     char *msg __maybe_unused,
2929 				     size_t size __maybe_unused)
2930 {
2931 	return 0;
2932 }
2933 
evsel__open_strerror(struct evsel * evsel,struct target * target,int err,char * msg,size_t size)2934 int evsel__open_strerror(struct evsel *evsel, struct target *target,
2935 			 int err, char *msg, size_t size)
2936 {
2937 	char sbuf[STRERR_BUFSIZE];
2938 	int printed = 0, enforced = 0;
2939 	int ret;
2940 
2941 	switch (err) {
2942 	case EPERM:
2943 	case EACCES:
2944 		printed += scnprintf(msg + printed, size - printed,
2945 			"Access to performance monitoring and observability operations is limited.\n");
2946 
2947 		if (!sysfs__read_int("fs/selinux/enforce", &enforced)) {
2948 			if (enforced) {
2949 				printed += scnprintf(msg + printed, size - printed,
2950 					"Enforced MAC policy settings (SELinux) can limit access to performance\n"
2951 					"monitoring and observability operations. Inspect system audit records for\n"
2952 					"more perf_event access control information and adjusting the policy.\n");
2953 			}
2954 		}
2955 
2956 		if (err == EPERM)
2957 			printed += scnprintf(msg, size,
2958 				"No permission to enable %s event.\n\n", evsel__name(evsel));
2959 
2960 		return scnprintf(msg + printed, size - printed,
2961 		 "Consider adjusting /proc/sys/kernel/perf_event_paranoid setting to open\n"
2962 		 "access to performance monitoring and observability operations for processes\n"
2963 		 "without CAP_PERFMON, CAP_SYS_PTRACE or CAP_SYS_ADMIN Linux capability.\n"
2964 		 "More information can be found at 'Perf events and tool security' document:\n"
2965 		 "https://www.kernel.org/doc/html/latest/admin-guide/perf-security.html\n"
2966 		 "perf_event_paranoid setting is %d:\n"
2967 		 "  -1: Allow use of (almost) all events by all users\n"
2968 		 "      Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK\n"
2969 		 ">= 0: Disallow raw and ftrace function tracepoint access\n"
2970 		 ">= 1: Disallow CPU event access\n"
2971 		 ">= 2: Disallow kernel profiling\n"
2972 		 "To make the adjusted perf_event_paranoid setting permanent preserve it\n"
2973 		 "in /etc/sysctl.conf (e.g. kernel.perf_event_paranoid = <setting>)",
2974 		 perf_event_paranoid());
2975 	case ENOENT:
2976 		return scnprintf(msg, size, "The %s event is not supported.", evsel__name(evsel));
2977 	case EMFILE:
2978 		return scnprintf(msg, size, "%s",
2979 			 "Too many events are opened.\n"
2980 			 "Probably the maximum number of open file descriptors has been reached.\n"
2981 			 "Hint: Try again after reducing the number of events.\n"
2982 			 "Hint: Try increasing the limit with 'ulimit -n <limit>'");
2983 	case ENOMEM:
2984 		if (evsel__has_callchain(evsel) &&
2985 		    access("/proc/sys/kernel/perf_event_max_stack", F_OK) == 0)
2986 			return scnprintf(msg, size,
2987 					 "Not enough memory to setup event with callchain.\n"
2988 					 "Hint: Try tweaking /proc/sys/kernel/perf_event_max_stack\n"
2989 					 "Hint: Current value: %d", sysctl__max_stack());
2990 		break;
2991 	case ENODEV:
2992 		if (target->cpu_list)
2993 			return scnprintf(msg, size, "%s",
2994 	 "No such device - did you specify an out-of-range profile CPU?");
2995 		break;
2996 	case EOPNOTSUPP:
2997 		if (evsel->core.attr.sample_type & PERF_SAMPLE_BRANCH_STACK)
2998 			return scnprintf(msg, size,
2999 	"%s: PMU Hardware or event type doesn't support branch stack sampling.",
3000 					 evsel__name(evsel));
3001 		if (evsel->core.attr.aux_output)
3002 			return scnprintf(msg, size,
3003 	"%s: PMU Hardware doesn't support 'aux_output' feature",
3004 					 evsel__name(evsel));
3005 		if (evsel->core.attr.sample_period != 0)
3006 			return scnprintf(msg, size,
3007 	"%s: PMU Hardware doesn't support sampling/overflow-interrupts. Try 'perf stat'",
3008 					 evsel__name(evsel));
3009 		if (evsel->core.attr.precise_ip)
3010 			return scnprintf(msg, size, "%s",
3011 	"\'precise\' request may not be supported. Try removing 'p' modifier.");
3012 #if defined(__i386__) || defined(__x86_64__)
3013 		if (evsel->core.attr.type == PERF_TYPE_HARDWARE)
3014 			return scnprintf(msg, size, "%s",
3015 	"No hardware sampling interrupt available.\n");
3016 #endif
3017 		break;
3018 	case EBUSY:
3019 		if (find_process("oprofiled"))
3020 			return scnprintf(msg, size,
3021 	"The PMU counters are busy/taken by another profiler.\n"
3022 	"We found oprofile daemon running, please stop it and try again.");
3023 		break;
3024 	case EINVAL:
3025 		if (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE && perf_missing_features.code_page_size)
3026 			return scnprintf(msg, size, "Asking for the code page size isn't supported by this kernel.");
3027 		if (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE && perf_missing_features.data_page_size)
3028 			return scnprintf(msg, size, "Asking for the data page size isn't supported by this kernel.");
3029 		if (evsel->core.attr.write_backward && perf_missing_features.write_backward)
3030 			return scnprintf(msg, size, "Reading from overwrite event is not supported by this kernel.");
3031 		if (perf_missing_features.clockid)
3032 			return scnprintf(msg, size, "clockid feature not supported.");
3033 		if (perf_missing_features.clockid_wrong)
3034 			return scnprintf(msg, size, "wrong clockid (%d).", clockid);
3035 		if (perf_missing_features.aux_output)
3036 			return scnprintf(msg, size, "The 'aux_output' feature is not supported, update the kernel.");
3037 		if (!target__has_cpu(target))
3038 			return scnprintf(msg, size,
3039 	"Invalid event (%s) in per-thread mode, enable system wide with '-a'.",
3040 					evsel__name(evsel));
3041 
3042 		break;
3043 	case ENODATA:
3044 		return scnprintf(msg, size, "Cannot collect data source with the load latency event alone. "
3045 				 "Please add an auxiliary event in front of the load latency event.");
3046 	default:
3047 		break;
3048 	}
3049 
3050 	ret = arch_evsel__open_strerror(evsel, msg, size);
3051 	if (ret)
3052 		return ret;
3053 
3054 	return scnprintf(msg, size,
3055 	"The sys_perf_event_open() syscall returned with %d (%s) for event (%s).\n"
3056 	"/bin/dmesg | grep -i perf may provide additional information.\n",
3057 			 err, str_error_r(err, sbuf, sizeof(sbuf)), evsel__name(evsel));
3058 }
3059 
evsel__env(struct evsel * evsel)3060 struct perf_env *evsel__env(struct evsel *evsel)
3061 {
3062 	if (evsel && evsel->evlist && evsel->evlist->env)
3063 		return evsel->evlist->env;
3064 	return &perf_env;
3065 }
3066 
store_evsel_ids(struct evsel * evsel,struct evlist * evlist)3067 static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist)
3068 {
3069 	int cpu_map_idx, thread;
3070 
3071 	for (cpu_map_idx = 0; cpu_map_idx < xyarray__max_x(evsel->core.fd); cpu_map_idx++) {
3072 		for (thread = 0; thread < xyarray__max_y(evsel->core.fd);
3073 		     thread++) {
3074 			int fd = FD(evsel, cpu_map_idx, thread);
3075 
3076 			if (perf_evlist__id_add_fd(&evlist->core, &evsel->core,
3077 						   cpu_map_idx, thread, fd) < 0)
3078 				return -1;
3079 		}
3080 	}
3081 
3082 	return 0;
3083 }
3084 
evsel__store_ids(struct evsel * evsel,struct evlist * evlist)3085 int evsel__store_ids(struct evsel *evsel, struct evlist *evlist)
3086 {
3087 	struct perf_cpu_map *cpus = evsel->core.cpus;
3088 	struct perf_thread_map *threads = evsel->core.threads;
3089 
3090 	if (perf_evsel__alloc_id(&evsel->core, perf_cpu_map__nr(cpus), threads->nr))
3091 		return -ENOMEM;
3092 
3093 	return store_evsel_ids(evsel, evlist);
3094 }
3095 
evsel__zero_per_pkg(struct evsel * evsel)3096 void evsel__zero_per_pkg(struct evsel *evsel)
3097 {
3098 	struct hashmap_entry *cur;
3099 	size_t bkt;
3100 
3101 	if (evsel->per_pkg_mask) {
3102 		hashmap__for_each_entry(evsel->per_pkg_mask, cur, bkt)
3103 			zfree(&cur->pkey);
3104 
3105 		hashmap__clear(evsel->per_pkg_mask);
3106 	}
3107 }
3108 
3109 /**
3110  * evsel__is_hybrid - does the evsel have a known PMU that is hybrid. Note, this
3111  *                    will be false on hybrid systems for hardware and legacy
3112  *                    cache events.
3113  */
evsel__is_hybrid(const struct evsel * evsel)3114 bool evsel__is_hybrid(const struct evsel *evsel)
3115 {
3116 	if (perf_pmus__num_core_pmus() == 1)
3117 		return false;
3118 
3119 	return evsel->core.is_pmu_core;
3120 }
3121 
evsel__leader(const struct evsel * evsel)3122 struct evsel *evsel__leader(const struct evsel *evsel)
3123 {
3124 	return container_of(evsel->core.leader, struct evsel, core);
3125 }
3126 
evsel__has_leader(struct evsel * evsel,struct evsel * leader)3127 bool evsel__has_leader(struct evsel *evsel, struct evsel *leader)
3128 {
3129 	return evsel->core.leader == &leader->core;
3130 }
3131 
evsel__is_leader(struct evsel * evsel)3132 bool evsel__is_leader(struct evsel *evsel)
3133 {
3134 	return evsel__has_leader(evsel, evsel);
3135 }
3136 
evsel__set_leader(struct evsel * evsel,struct evsel * leader)3137 void evsel__set_leader(struct evsel *evsel, struct evsel *leader)
3138 {
3139 	evsel->core.leader = &leader->core;
3140 }
3141 
evsel__source_count(const struct evsel * evsel)3142 int evsel__source_count(const struct evsel *evsel)
3143 {
3144 	struct evsel *pos;
3145 	int count = 0;
3146 
3147 	evlist__for_each_entry(evsel->evlist, pos) {
3148 		if (pos->metric_leader == evsel)
3149 			count++;
3150 	}
3151 	return count;
3152 }
3153 
arch_evsel__must_be_in_group(const struct evsel * evsel __maybe_unused)3154 bool __weak arch_evsel__must_be_in_group(const struct evsel *evsel __maybe_unused)
3155 {
3156 	return false;
3157 }
3158 
3159 /*
3160  * Remove an event from a given group (leader).
3161  * Some events, e.g., perf metrics Topdown events,
3162  * must always be grouped. Ignore the events.
3163  */
evsel__remove_from_group(struct evsel * evsel,struct evsel * leader)3164 void evsel__remove_from_group(struct evsel *evsel, struct evsel *leader)
3165 {
3166 	if (!arch_evsel__must_be_in_group(evsel) && evsel != leader) {
3167 		evsel__set_leader(evsel, evsel);
3168 		evsel->core.nr_members = 0;
3169 		leader->core.nr_members--;
3170 	}
3171 }
3172