1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Arm Statistical Profiling Extensions (SPE) support
4  * Copyright (c) 2017-2018, Arm Ltd.
5  */
6 
7 #include <byteswap.h>
8 #include <endian.h>
9 #include <errno.h>
10 #include <inttypes.h>
11 #include <linux/bitops.h>
12 #include <linux/kernel.h>
13 #include <linux/log2.h>
14 #include <linux/types.h>
15 #include <linux/zalloc.h>
16 #include <stdlib.h>
17 #include <unistd.h>
18 
19 #include "auxtrace.h"
20 #include "color.h"
21 #include "debug.h"
22 #include "evlist.h"
23 #include "evsel.h"
24 #include "machine.h"
25 #include "session.h"
26 #include "symbol.h"
27 #include "thread.h"
28 #include "thread-stack.h"
29 #include "tsc.h"
30 #include "tool.h"
31 #include "util/synthetic-events.h"
32 
33 #include "arm-spe.h"
34 #include "arm-spe-decoder/arm-spe-decoder.h"
35 #include "arm-spe-decoder/arm-spe-pkt-decoder.h"
36 
37 #include "../../arch/arm64/include/asm/cputype.h"
38 #define MAX_TIMESTAMP (~0ULL)
39 
40 struct arm_spe {
41 	struct auxtrace			auxtrace;
42 	struct auxtrace_queues		queues;
43 	struct auxtrace_heap		heap;
44 	struct itrace_synth_opts        synth_opts;
45 	u32				auxtrace_type;
46 	struct perf_session		*session;
47 	struct machine			*machine;
48 	u32				pmu_type;
49 	u64				midr;
50 
51 	struct perf_tsc_conversion	tc;
52 
53 	u8				timeless_decoding;
54 	u8				data_queued;
55 
56 	u64				sample_type;
57 	u8				sample_flc;
58 	u8				sample_llc;
59 	u8				sample_tlb;
60 	u8				sample_branch;
61 	u8				sample_remote_access;
62 	u8				sample_memory;
63 	u8				sample_instructions;
64 	u64				instructions_sample_period;
65 
66 	u64				l1d_miss_id;
67 	u64				l1d_access_id;
68 	u64				llc_miss_id;
69 	u64				llc_access_id;
70 	u64				tlb_miss_id;
71 	u64				tlb_access_id;
72 	u64				branch_miss_id;
73 	u64				remote_access_id;
74 	u64				memory_id;
75 	u64				instructions_id;
76 
77 	u64				kernel_start;
78 
79 	unsigned long			num_events;
80 	u8				use_ctx_pkt_for_pid;
81 };
82 
83 struct arm_spe_queue {
84 	struct arm_spe			*spe;
85 	unsigned int			queue_nr;
86 	struct auxtrace_buffer		*buffer;
87 	struct auxtrace_buffer		*old_buffer;
88 	union perf_event		*event_buf;
89 	bool				on_heap;
90 	bool				done;
91 	pid_t				pid;
92 	pid_t				tid;
93 	int				cpu;
94 	struct arm_spe_decoder		*decoder;
95 	u64				time;
96 	u64				timestamp;
97 	struct thread			*thread;
98 	u64				period_instructions;
99 };
100 
arm_spe_dump(struct arm_spe * spe __maybe_unused,unsigned char * buf,size_t len)101 static void arm_spe_dump(struct arm_spe *spe __maybe_unused,
102 			 unsigned char *buf, size_t len)
103 {
104 	struct arm_spe_pkt packet;
105 	size_t pos = 0;
106 	int ret, pkt_len, i;
107 	char desc[ARM_SPE_PKT_DESC_MAX];
108 	const char *color = PERF_COLOR_BLUE;
109 
110 	color_fprintf(stdout, color,
111 		      ". ... ARM SPE data: size %#zx bytes\n",
112 		      len);
113 
114 	while (len) {
115 		ret = arm_spe_get_packet(buf, len, &packet);
116 		if (ret > 0)
117 			pkt_len = ret;
118 		else
119 			pkt_len = 1;
120 		printf(".");
121 		color_fprintf(stdout, color, "  %08x: ", pos);
122 		for (i = 0; i < pkt_len; i++)
123 			color_fprintf(stdout, color, " %02x", buf[i]);
124 		for (; i < 16; i++)
125 			color_fprintf(stdout, color, "   ");
126 		if (ret > 0) {
127 			ret = arm_spe_pkt_desc(&packet, desc,
128 					       ARM_SPE_PKT_DESC_MAX);
129 			if (!ret)
130 				color_fprintf(stdout, color, " %s\n", desc);
131 		} else {
132 			color_fprintf(stdout, color, " Bad packet!\n");
133 		}
134 		pos += pkt_len;
135 		buf += pkt_len;
136 		len -= pkt_len;
137 	}
138 }
139 
arm_spe_dump_event(struct arm_spe * spe,unsigned char * buf,size_t len)140 static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf,
141 			       size_t len)
142 {
143 	printf(".\n");
144 	arm_spe_dump(spe, buf, len);
145 }
146 
arm_spe_get_trace(struct arm_spe_buffer * b,void * data)147 static int arm_spe_get_trace(struct arm_spe_buffer *b, void *data)
148 {
149 	struct arm_spe_queue *speq = data;
150 	struct auxtrace_buffer *buffer = speq->buffer;
151 	struct auxtrace_buffer *old_buffer = speq->old_buffer;
152 	struct auxtrace_queue *queue;
153 
154 	queue = &speq->spe->queues.queue_array[speq->queue_nr];
155 
156 	buffer = auxtrace_buffer__next(queue, buffer);
157 	/* If no more data, drop the previous auxtrace_buffer and return */
158 	if (!buffer) {
159 		if (old_buffer)
160 			auxtrace_buffer__drop_data(old_buffer);
161 		b->len = 0;
162 		return 0;
163 	}
164 
165 	speq->buffer = buffer;
166 
167 	/* If the aux_buffer doesn't have data associated, try to load it */
168 	if (!buffer->data) {
169 		/* get the file desc associated with the perf data file */
170 		int fd = perf_data__fd(speq->spe->session->data);
171 
172 		buffer->data = auxtrace_buffer__get_data(buffer, fd);
173 		if (!buffer->data)
174 			return -ENOMEM;
175 	}
176 
177 	b->len = buffer->size;
178 	b->buf = buffer->data;
179 
180 	if (b->len) {
181 		if (old_buffer)
182 			auxtrace_buffer__drop_data(old_buffer);
183 		speq->old_buffer = buffer;
184 	} else {
185 		auxtrace_buffer__drop_data(buffer);
186 		return arm_spe_get_trace(b, data);
187 	}
188 
189 	return 0;
190 }
191 
arm_spe__alloc_queue(struct arm_spe * spe,unsigned int queue_nr)192 static struct arm_spe_queue *arm_spe__alloc_queue(struct arm_spe *spe,
193 		unsigned int queue_nr)
194 {
195 	struct arm_spe_params params = { .get_trace = 0, };
196 	struct arm_spe_queue *speq;
197 
198 	speq = zalloc(sizeof(*speq));
199 	if (!speq)
200 		return NULL;
201 
202 	speq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
203 	if (!speq->event_buf)
204 		goto out_free;
205 
206 	speq->spe = spe;
207 	speq->queue_nr = queue_nr;
208 	speq->pid = -1;
209 	speq->tid = -1;
210 	speq->cpu = -1;
211 	speq->period_instructions = 0;
212 
213 	/* params set */
214 	params.get_trace = arm_spe_get_trace;
215 	params.data = speq;
216 
217 	/* create new decoder */
218 	speq->decoder = arm_spe_decoder_new(&params);
219 	if (!speq->decoder)
220 		goto out_free;
221 
222 	return speq;
223 
224 out_free:
225 	zfree(&speq->event_buf);
226 	free(speq);
227 
228 	return NULL;
229 }
230 
arm_spe_cpumode(struct arm_spe * spe,u64 ip)231 static inline u8 arm_spe_cpumode(struct arm_spe *spe, u64 ip)
232 {
233 	return ip >= spe->kernel_start ?
234 		PERF_RECORD_MISC_KERNEL :
235 		PERF_RECORD_MISC_USER;
236 }
237 
arm_spe_set_pid_tid_cpu(struct arm_spe * spe,struct auxtrace_queue * queue)238 static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe,
239 				    struct auxtrace_queue *queue)
240 {
241 	struct arm_spe_queue *speq = queue->priv;
242 	pid_t tid;
243 
244 	tid = machine__get_current_tid(spe->machine, speq->cpu);
245 	if (tid != -1) {
246 		speq->tid = tid;
247 		thread__zput(speq->thread);
248 	} else
249 		speq->tid = queue->tid;
250 
251 	if ((!speq->thread) && (speq->tid != -1)) {
252 		speq->thread = machine__find_thread(spe->machine, -1,
253 						    speq->tid);
254 	}
255 
256 	if (speq->thread) {
257 		speq->pid = speq->thread->pid_;
258 		if (queue->cpu == -1)
259 			speq->cpu = speq->thread->cpu;
260 	}
261 }
262 
arm_spe_set_tid(struct arm_spe_queue * speq,pid_t tid)263 static int arm_spe_set_tid(struct arm_spe_queue *speq, pid_t tid)
264 {
265 	struct arm_spe *spe = speq->spe;
266 	int err = machine__set_current_tid(spe->machine, speq->cpu, -1, tid);
267 
268 	if (err)
269 		return err;
270 
271 	arm_spe_set_pid_tid_cpu(spe, &spe->queues.queue_array[speq->queue_nr]);
272 
273 	return 0;
274 }
275 
arm_spe_prep_sample(struct arm_spe * spe,struct arm_spe_queue * speq,union perf_event * event,struct perf_sample * sample)276 static void arm_spe_prep_sample(struct arm_spe *spe,
277 				struct arm_spe_queue *speq,
278 				union perf_event *event,
279 				struct perf_sample *sample)
280 {
281 	struct arm_spe_record *record = &speq->decoder->record;
282 
283 	if (!spe->timeless_decoding)
284 		sample->time = tsc_to_perf_time(record->timestamp, &spe->tc);
285 
286 	sample->ip = record->from_ip;
287 	sample->cpumode = arm_spe_cpumode(spe, sample->ip);
288 	sample->pid = speq->pid;
289 	sample->tid = speq->tid;
290 	sample->period = 1;
291 	sample->cpu = speq->cpu;
292 
293 	event->sample.header.type = PERF_RECORD_SAMPLE;
294 	event->sample.header.misc = sample->cpumode;
295 	event->sample.header.size = sizeof(struct perf_event_header);
296 }
297 
arm_spe__inject_event(union perf_event * event,struct perf_sample * sample,u64 type)298 static int arm_spe__inject_event(union perf_event *event, struct perf_sample *sample, u64 type)
299 {
300 	event->header.size = perf_event__sample_event_size(sample, type, 0);
301 	return perf_event__synthesize_sample(event, type, 0, sample);
302 }
303 
304 static inline int
arm_spe_deliver_synth_event(struct arm_spe * spe,struct arm_spe_queue * speq __maybe_unused,union perf_event * event,struct perf_sample * sample)305 arm_spe_deliver_synth_event(struct arm_spe *spe,
306 			    struct arm_spe_queue *speq __maybe_unused,
307 			    union perf_event *event,
308 			    struct perf_sample *sample)
309 {
310 	int ret;
311 
312 	if (spe->synth_opts.inject) {
313 		ret = arm_spe__inject_event(event, sample, spe->sample_type);
314 		if (ret)
315 			return ret;
316 	}
317 
318 	ret = perf_session__deliver_synth_event(spe->session, event, sample);
319 	if (ret)
320 		pr_err("ARM SPE: failed to deliver event, error %d\n", ret);
321 
322 	return ret;
323 }
324 
arm_spe__synth_mem_sample(struct arm_spe_queue * speq,u64 spe_events_id,u64 data_src)325 static int arm_spe__synth_mem_sample(struct arm_spe_queue *speq,
326 				     u64 spe_events_id, u64 data_src)
327 {
328 	struct arm_spe *spe = speq->spe;
329 	struct arm_spe_record *record = &speq->decoder->record;
330 	union perf_event *event = speq->event_buf;
331 	struct perf_sample sample = { .ip = 0, };
332 
333 	arm_spe_prep_sample(spe, speq, event, &sample);
334 
335 	sample.id = spe_events_id;
336 	sample.stream_id = spe_events_id;
337 	sample.addr = record->virt_addr;
338 	sample.phys_addr = record->phys_addr;
339 	sample.data_src = data_src;
340 	sample.weight = record->latency;
341 
342 	return arm_spe_deliver_synth_event(spe, speq, event, &sample);
343 }
344 
arm_spe__synth_branch_sample(struct arm_spe_queue * speq,u64 spe_events_id)345 static int arm_spe__synth_branch_sample(struct arm_spe_queue *speq,
346 					u64 spe_events_id)
347 {
348 	struct arm_spe *spe = speq->spe;
349 	struct arm_spe_record *record = &speq->decoder->record;
350 	union perf_event *event = speq->event_buf;
351 	struct perf_sample sample = { .ip = 0, };
352 
353 	arm_spe_prep_sample(spe, speq, event, &sample);
354 
355 	sample.id = spe_events_id;
356 	sample.stream_id = spe_events_id;
357 	sample.addr = record->to_ip;
358 	sample.weight = record->latency;
359 
360 	return arm_spe_deliver_synth_event(spe, speq, event, &sample);
361 }
362 
arm_spe__synth_instruction_sample(struct arm_spe_queue * speq,u64 spe_events_id,u64 data_src)363 static int arm_spe__synth_instruction_sample(struct arm_spe_queue *speq,
364 					     u64 spe_events_id, u64 data_src)
365 {
366 	struct arm_spe *spe = speq->spe;
367 	struct arm_spe_record *record = &speq->decoder->record;
368 	union perf_event *event = speq->event_buf;
369 	struct perf_sample sample = { .ip = 0, };
370 
371 	/*
372 	 * Handles perf instruction sampling period.
373 	 */
374 	speq->period_instructions++;
375 	if (speq->period_instructions < spe->instructions_sample_period)
376 		return 0;
377 	speq->period_instructions = 0;
378 
379 	arm_spe_prep_sample(spe, speq, event, &sample);
380 
381 	sample.id = spe_events_id;
382 	sample.stream_id = spe_events_id;
383 	sample.addr = record->virt_addr;
384 	sample.phys_addr = record->phys_addr;
385 	sample.data_src = data_src;
386 	sample.period = spe->instructions_sample_period;
387 	sample.weight = record->latency;
388 
389 	return arm_spe_deliver_synth_event(spe, speq, event, &sample);
390 }
391 
392 static const struct midr_range neoverse_spe[] = {
393 	MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
394 	MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
395 	MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
396 	{},
397 };
398 
arm_spe__synth_data_source_neoverse(const struct arm_spe_record * record,union perf_mem_data_src * data_src)399 static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *record,
400 						union perf_mem_data_src *data_src)
401 {
402 	/*
403 	 * Even though four levels of cache hierarchy are possible, no known
404 	 * production Neoverse systems currently include more than three levels
405 	 * so for the time being we assume three exist. If a production system
406 	 * is built with four the this function would have to be changed to
407 	 * detect the number of levels for reporting.
408 	 */
409 
410 	/*
411 	 * We have no data on the hit level or data source for stores in the
412 	 * Neoverse SPE records.
413 	 */
414 	if (record->op & ARM_SPE_ST) {
415 		data_src->mem_lvl = PERF_MEM_LVL_NA;
416 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_NA;
417 		data_src->mem_snoop = PERF_MEM_SNOOP_NA;
418 		return;
419 	}
420 
421 	switch (record->source) {
422 	case ARM_SPE_NV_L1D:
423 		data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT;
424 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1;
425 		data_src->mem_snoop = PERF_MEM_SNOOP_NONE;
426 		break;
427 	case ARM_SPE_NV_L2:
428 		data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT;
429 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2;
430 		data_src->mem_snoop = PERF_MEM_SNOOP_NONE;
431 		break;
432 	case ARM_SPE_NV_PEER_CORE:
433 		data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT;
434 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2;
435 		data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER;
436 		break;
437 	/*
438 	 * We don't know if this is L1, L2 but we do know it was a cache-2-cache
439 	 * transfer, so set SNOOPX_PEER
440 	 */
441 	case ARM_SPE_NV_LOCAL_CLUSTER:
442 	case ARM_SPE_NV_PEER_CLUSTER:
443 		data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT;
444 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3;
445 		data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER;
446 		break;
447 	/*
448 	 * System cache is assumed to be L3
449 	 */
450 	case ARM_SPE_NV_SYS_CACHE:
451 		data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT;
452 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3;
453 		data_src->mem_snoop = PERF_MEM_SNOOP_HIT;
454 		break;
455 	/*
456 	 * We don't know what level it hit in, except it came from the other
457 	 * socket
458 	 */
459 	case ARM_SPE_NV_REMOTE:
460 		data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1;
461 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_ANY_CACHE;
462 		data_src->mem_remote = PERF_MEM_REMOTE_REMOTE;
463 		data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER;
464 		break;
465 	case ARM_SPE_NV_DRAM:
466 		data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT;
467 		data_src->mem_lvl_num = PERF_MEM_LVLNUM_RAM;
468 		data_src->mem_snoop = PERF_MEM_SNOOP_NONE;
469 		break;
470 	default:
471 		break;
472 	}
473 }
474 
arm_spe__synth_data_source_generic(const struct arm_spe_record * record,union perf_mem_data_src * data_src)475 static void arm_spe__synth_data_source_generic(const struct arm_spe_record *record,
476 					       union perf_mem_data_src *data_src)
477 {
478 	if (record->type & (ARM_SPE_LLC_ACCESS | ARM_SPE_LLC_MISS)) {
479 		data_src->mem_lvl = PERF_MEM_LVL_L3;
480 
481 		if (record->type & ARM_SPE_LLC_MISS)
482 			data_src->mem_lvl |= PERF_MEM_LVL_MISS;
483 		else
484 			data_src->mem_lvl |= PERF_MEM_LVL_HIT;
485 	} else if (record->type & (ARM_SPE_L1D_ACCESS | ARM_SPE_L1D_MISS)) {
486 		data_src->mem_lvl = PERF_MEM_LVL_L1;
487 
488 		if (record->type & ARM_SPE_L1D_MISS)
489 			data_src->mem_lvl |= PERF_MEM_LVL_MISS;
490 		else
491 			data_src->mem_lvl |= PERF_MEM_LVL_HIT;
492 	}
493 
494 	if (record->type & ARM_SPE_REMOTE_ACCESS)
495 		data_src->mem_lvl |= PERF_MEM_LVL_REM_CCE1;
496 }
497 
arm_spe__synth_data_source(const struct arm_spe_record * record,u64 midr)498 static u64 arm_spe__synth_data_source(const struct arm_spe_record *record, u64 midr)
499 {
500 	union perf_mem_data_src	data_src = { 0 };
501 	bool is_neoverse = is_midr_in_range_list(midr, neoverse_spe);
502 
503 	if (record->op == ARM_SPE_LD)
504 		data_src.mem_op = PERF_MEM_OP_LOAD;
505 	else if (record->op == ARM_SPE_ST)
506 		data_src.mem_op = PERF_MEM_OP_STORE;
507 	else
508 		return 0;
509 
510 	if (is_neoverse)
511 		arm_spe__synth_data_source_neoverse(record, &data_src);
512 	else
513 		arm_spe__synth_data_source_generic(record, &data_src);
514 
515 	if (record->type & (ARM_SPE_TLB_ACCESS | ARM_SPE_TLB_MISS)) {
516 		data_src.mem_dtlb = PERF_MEM_TLB_WK;
517 
518 		if (record->type & ARM_SPE_TLB_MISS)
519 			data_src.mem_dtlb |= PERF_MEM_TLB_MISS;
520 		else
521 			data_src.mem_dtlb |= PERF_MEM_TLB_HIT;
522 	}
523 
524 	return data_src.val;
525 }
526 
arm_spe_sample(struct arm_spe_queue * speq)527 static int arm_spe_sample(struct arm_spe_queue *speq)
528 {
529 	const struct arm_spe_record *record = &speq->decoder->record;
530 	struct arm_spe *spe = speq->spe;
531 	u64 data_src;
532 	int err;
533 
534 	data_src = arm_spe__synth_data_source(record, spe->midr);
535 
536 	if (spe->sample_flc) {
537 		if (record->type & ARM_SPE_L1D_MISS) {
538 			err = arm_spe__synth_mem_sample(speq, spe->l1d_miss_id,
539 							data_src);
540 			if (err)
541 				return err;
542 		}
543 
544 		if (record->type & ARM_SPE_L1D_ACCESS) {
545 			err = arm_spe__synth_mem_sample(speq, spe->l1d_access_id,
546 							data_src);
547 			if (err)
548 				return err;
549 		}
550 	}
551 
552 	if (spe->sample_llc) {
553 		if (record->type & ARM_SPE_LLC_MISS) {
554 			err = arm_spe__synth_mem_sample(speq, spe->llc_miss_id,
555 							data_src);
556 			if (err)
557 				return err;
558 		}
559 
560 		if (record->type & ARM_SPE_LLC_ACCESS) {
561 			err = arm_spe__synth_mem_sample(speq, spe->llc_access_id,
562 							data_src);
563 			if (err)
564 				return err;
565 		}
566 	}
567 
568 	if (spe->sample_tlb) {
569 		if (record->type & ARM_SPE_TLB_MISS) {
570 			err = arm_spe__synth_mem_sample(speq, spe->tlb_miss_id,
571 							data_src);
572 			if (err)
573 				return err;
574 		}
575 
576 		if (record->type & ARM_SPE_TLB_ACCESS) {
577 			err = arm_spe__synth_mem_sample(speq, spe->tlb_access_id,
578 							data_src);
579 			if (err)
580 				return err;
581 		}
582 	}
583 
584 	if (spe->sample_branch && (record->type & ARM_SPE_BRANCH_MISS)) {
585 		err = arm_spe__synth_branch_sample(speq, spe->branch_miss_id);
586 		if (err)
587 			return err;
588 	}
589 
590 	if (spe->sample_remote_access &&
591 	    (record->type & ARM_SPE_REMOTE_ACCESS)) {
592 		err = arm_spe__synth_mem_sample(speq, spe->remote_access_id,
593 						data_src);
594 		if (err)
595 			return err;
596 	}
597 
598 	/*
599 	 * When data_src is zero it means the record is not a memory operation,
600 	 * skip to synthesize memory sample for this case.
601 	 */
602 	if (spe->sample_memory && data_src) {
603 		err = arm_spe__synth_mem_sample(speq, spe->memory_id, data_src);
604 		if (err)
605 			return err;
606 	}
607 
608 	if (spe->sample_instructions) {
609 		err = arm_spe__synth_instruction_sample(speq, spe->instructions_id, data_src);
610 		if (err)
611 			return err;
612 	}
613 
614 	return 0;
615 }
616 
arm_spe_run_decoder(struct arm_spe_queue * speq,u64 * timestamp)617 static int arm_spe_run_decoder(struct arm_spe_queue *speq, u64 *timestamp)
618 {
619 	struct arm_spe *spe = speq->spe;
620 	struct arm_spe_record *record;
621 	int ret;
622 
623 	if (!spe->kernel_start)
624 		spe->kernel_start = machine__kernel_start(spe->machine);
625 
626 	while (1) {
627 		/*
628 		 * The usual logic is firstly to decode the packets, and then
629 		 * based the record to synthesize sample; but here the flow is
630 		 * reversed: it calls arm_spe_sample() for synthesizing samples
631 		 * prior to arm_spe_decode().
632 		 *
633 		 * Two reasons for this code logic:
634 		 * 1. Firstly, when setup queue in arm_spe__setup_queue(), it
635 		 * has decoded trace data and generated a record, but the record
636 		 * is left to generate sample until run to here, so it's correct
637 		 * to synthesize sample for the left record.
638 		 * 2. After decoding trace data, it needs to compare the record
639 		 * timestamp with the coming perf event, if the record timestamp
640 		 * is later than the perf event, it needs bail out and pushs the
641 		 * record into auxtrace heap, thus the record can be deferred to
642 		 * synthesize sample until run to here at the next time; so this
643 		 * can correlate samples between Arm SPE trace data and other
644 		 * perf events with correct time ordering.
645 		 */
646 
647 		/*
648 		 * Update pid/tid info.
649 		 */
650 		record = &speq->decoder->record;
651 		if (!spe->timeless_decoding && record->context_id != (u64)-1) {
652 			ret = arm_spe_set_tid(speq, record->context_id);
653 			if (ret)
654 				return ret;
655 
656 			spe->use_ctx_pkt_for_pid = true;
657 		}
658 
659 		ret = arm_spe_sample(speq);
660 		if (ret)
661 			return ret;
662 
663 		ret = arm_spe_decode(speq->decoder);
664 		if (!ret) {
665 			pr_debug("No data or all data has been processed.\n");
666 			return 1;
667 		}
668 
669 		/*
670 		 * Error is detected when decode SPE trace data, continue to
671 		 * the next trace data and find out more records.
672 		 */
673 		if (ret < 0)
674 			continue;
675 
676 		record = &speq->decoder->record;
677 
678 		/* Update timestamp for the last record */
679 		if (record->timestamp > speq->timestamp)
680 			speq->timestamp = record->timestamp;
681 
682 		/*
683 		 * If the timestamp of the queue is later than timestamp of the
684 		 * coming perf event, bail out so can allow the perf event to
685 		 * be processed ahead.
686 		 */
687 		if (!spe->timeless_decoding && speq->timestamp >= *timestamp) {
688 			*timestamp = speq->timestamp;
689 			return 0;
690 		}
691 	}
692 
693 	return 0;
694 }
695 
arm_spe__setup_queue(struct arm_spe * spe,struct auxtrace_queue * queue,unsigned int queue_nr)696 static int arm_spe__setup_queue(struct arm_spe *spe,
697 			       struct auxtrace_queue *queue,
698 			       unsigned int queue_nr)
699 {
700 	struct arm_spe_queue *speq = queue->priv;
701 	struct arm_spe_record *record;
702 
703 	if (list_empty(&queue->head) || speq)
704 		return 0;
705 
706 	speq = arm_spe__alloc_queue(spe, queue_nr);
707 
708 	if (!speq)
709 		return -ENOMEM;
710 
711 	queue->priv = speq;
712 
713 	if (queue->cpu != -1)
714 		speq->cpu = queue->cpu;
715 
716 	if (!speq->on_heap) {
717 		int ret;
718 
719 		if (spe->timeless_decoding)
720 			return 0;
721 
722 retry:
723 		ret = arm_spe_decode(speq->decoder);
724 
725 		if (!ret)
726 			return 0;
727 
728 		if (ret < 0)
729 			goto retry;
730 
731 		record = &speq->decoder->record;
732 
733 		speq->timestamp = record->timestamp;
734 		ret = auxtrace_heap__add(&spe->heap, queue_nr, speq->timestamp);
735 		if (ret)
736 			return ret;
737 		speq->on_heap = true;
738 	}
739 
740 	return 0;
741 }
742 
arm_spe__setup_queues(struct arm_spe * spe)743 static int arm_spe__setup_queues(struct arm_spe *spe)
744 {
745 	unsigned int i;
746 	int ret;
747 
748 	for (i = 0; i < spe->queues.nr_queues; i++) {
749 		ret = arm_spe__setup_queue(spe, &spe->queues.queue_array[i], i);
750 		if (ret)
751 			return ret;
752 	}
753 
754 	return 0;
755 }
756 
arm_spe__update_queues(struct arm_spe * spe)757 static int arm_spe__update_queues(struct arm_spe *spe)
758 {
759 	if (spe->queues.new_data) {
760 		spe->queues.new_data = false;
761 		return arm_spe__setup_queues(spe);
762 	}
763 
764 	return 0;
765 }
766 
arm_spe__is_timeless_decoding(struct arm_spe * spe)767 static bool arm_spe__is_timeless_decoding(struct arm_spe *spe)
768 {
769 	struct evsel *evsel;
770 	struct evlist *evlist = spe->session->evlist;
771 	bool timeless_decoding = true;
772 
773 	/*
774 	 * Circle through the list of event and complain if we find one
775 	 * with the time bit set.
776 	 */
777 	evlist__for_each_entry(evlist, evsel) {
778 		if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
779 			timeless_decoding = false;
780 	}
781 
782 	return timeless_decoding;
783 }
784 
arm_spe_process_queues(struct arm_spe * spe,u64 timestamp)785 static int arm_spe_process_queues(struct arm_spe *spe, u64 timestamp)
786 {
787 	unsigned int queue_nr;
788 	u64 ts;
789 	int ret;
790 
791 	while (1) {
792 		struct auxtrace_queue *queue;
793 		struct arm_spe_queue *speq;
794 
795 		if (!spe->heap.heap_cnt)
796 			return 0;
797 
798 		if (spe->heap.heap_array[0].ordinal >= timestamp)
799 			return 0;
800 
801 		queue_nr = spe->heap.heap_array[0].queue_nr;
802 		queue = &spe->queues.queue_array[queue_nr];
803 		speq = queue->priv;
804 
805 		auxtrace_heap__pop(&spe->heap);
806 
807 		if (spe->heap.heap_cnt) {
808 			ts = spe->heap.heap_array[0].ordinal + 1;
809 			if (ts > timestamp)
810 				ts = timestamp;
811 		} else {
812 			ts = timestamp;
813 		}
814 
815 		/*
816 		 * A previous context-switch event has set pid/tid in the machine's context, so
817 		 * here we need to update the pid/tid in the thread and SPE queue.
818 		 */
819 		if (!spe->use_ctx_pkt_for_pid)
820 			arm_spe_set_pid_tid_cpu(spe, queue);
821 
822 		ret = arm_spe_run_decoder(speq, &ts);
823 		if (ret < 0) {
824 			auxtrace_heap__add(&spe->heap, queue_nr, ts);
825 			return ret;
826 		}
827 
828 		if (!ret) {
829 			ret = auxtrace_heap__add(&spe->heap, queue_nr, ts);
830 			if (ret < 0)
831 				return ret;
832 		} else {
833 			speq->on_heap = false;
834 		}
835 	}
836 
837 	return 0;
838 }
839 
arm_spe_process_timeless_queues(struct arm_spe * spe,pid_t tid,u64 time_)840 static int arm_spe_process_timeless_queues(struct arm_spe *spe, pid_t tid,
841 					    u64 time_)
842 {
843 	struct auxtrace_queues *queues = &spe->queues;
844 	unsigned int i;
845 	u64 ts = 0;
846 
847 	for (i = 0; i < queues->nr_queues; i++) {
848 		struct auxtrace_queue *queue = &spe->queues.queue_array[i];
849 		struct arm_spe_queue *speq = queue->priv;
850 
851 		if (speq && (tid == -1 || speq->tid == tid)) {
852 			speq->time = time_;
853 			arm_spe_set_pid_tid_cpu(spe, queue);
854 			arm_spe_run_decoder(speq, &ts);
855 		}
856 	}
857 	return 0;
858 }
859 
arm_spe_context_switch(struct arm_spe * spe,union perf_event * event,struct perf_sample * sample)860 static int arm_spe_context_switch(struct arm_spe *spe, union perf_event *event,
861 				  struct perf_sample *sample)
862 {
863 	pid_t pid, tid;
864 	int cpu;
865 
866 	if (!(event->header.misc & PERF_RECORD_MISC_SWITCH_OUT))
867 		return 0;
868 
869 	pid = event->context_switch.next_prev_pid;
870 	tid = event->context_switch.next_prev_tid;
871 	cpu = sample->cpu;
872 
873 	if (tid == -1)
874 		pr_warning("context_switch event has no tid\n");
875 
876 	return machine__set_current_tid(spe->machine, cpu, pid, tid);
877 }
878 
arm_spe_process_event(struct perf_session * session,union perf_event * event,struct perf_sample * sample,struct perf_tool * tool)879 static int arm_spe_process_event(struct perf_session *session,
880 				 union perf_event *event,
881 				 struct perf_sample *sample,
882 				 struct perf_tool *tool)
883 {
884 	int err = 0;
885 	u64 timestamp;
886 	struct arm_spe *spe = container_of(session->auxtrace,
887 			struct arm_spe, auxtrace);
888 
889 	if (dump_trace)
890 		return 0;
891 
892 	if (!tool->ordered_events) {
893 		pr_err("SPE trace requires ordered events\n");
894 		return -EINVAL;
895 	}
896 
897 	if (sample->time && (sample->time != (u64) -1))
898 		timestamp = perf_time_to_tsc(sample->time, &spe->tc);
899 	else
900 		timestamp = 0;
901 
902 	if (timestamp || spe->timeless_decoding) {
903 		err = arm_spe__update_queues(spe);
904 		if (err)
905 			return err;
906 	}
907 
908 	if (spe->timeless_decoding) {
909 		if (event->header.type == PERF_RECORD_EXIT) {
910 			err = arm_spe_process_timeless_queues(spe,
911 					event->fork.tid,
912 					sample->time);
913 		}
914 	} else if (timestamp) {
915 		err = arm_spe_process_queues(spe, timestamp);
916 		if (err)
917 			return err;
918 
919 		if (!spe->use_ctx_pkt_for_pid &&
920 		    (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE ||
921 		    event->header.type == PERF_RECORD_SWITCH))
922 			err = arm_spe_context_switch(spe, event, sample);
923 	}
924 
925 	return err;
926 }
927 
arm_spe_process_auxtrace_event(struct perf_session * session,union perf_event * event,struct perf_tool * tool __maybe_unused)928 static int arm_spe_process_auxtrace_event(struct perf_session *session,
929 					  union perf_event *event,
930 					  struct perf_tool *tool __maybe_unused)
931 {
932 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
933 					     auxtrace);
934 
935 	if (!spe->data_queued) {
936 		struct auxtrace_buffer *buffer;
937 		off_t data_offset;
938 		int fd = perf_data__fd(session->data);
939 		int err;
940 
941 		if (perf_data__is_pipe(session->data)) {
942 			data_offset = 0;
943 		} else {
944 			data_offset = lseek(fd, 0, SEEK_CUR);
945 			if (data_offset == -1)
946 				return -errno;
947 		}
948 
949 		err = auxtrace_queues__add_event(&spe->queues, session, event,
950 				data_offset, &buffer);
951 		if (err)
952 			return err;
953 
954 		/* Dump here now we have copied a piped trace out of the pipe */
955 		if (dump_trace) {
956 			if (auxtrace_buffer__get_data(buffer, fd)) {
957 				arm_spe_dump_event(spe, buffer->data,
958 						buffer->size);
959 				auxtrace_buffer__put_data(buffer);
960 			}
961 		}
962 	}
963 
964 	return 0;
965 }
966 
arm_spe_flush(struct perf_session * session __maybe_unused,struct perf_tool * tool __maybe_unused)967 static int arm_spe_flush(struct perf_session *session __maybe_unused,
968 			 struct perf_tool *tool __maybe_unused)
969 {
970 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
971 			auxtrace);
972 	int ret;
973 
974 	if (dump_trace)
975 		return 0;
976 
977 	if (!tool->ordered_events)
978 		return -EINVAL;
979 
980 	ret = arm_spe__update_queues(spe);
981 	if (ret < 0)
982 		return ret;
983 
984 	if (spe->timeless_decoding)
985 		return arm_spe_process_timeless_queues(spe, -1,
986 				MAX_TIMESTAMP - 1);
987 
988 	ret = arm_spe_process_queues(spe, MAX_TIMESTAMP);
989 	if (ret)
990 		return ret;
991 
992 	if (!spe->use_ctx_pkt_for_pid)
993 		ui__warning("Arm SPE CONTEXT packets not found in the traces.\n"
994 			    "Matching of TIDs to SPE events could be inaccurate.\n");
995 
996 	return 0;
997 }
998 
arm_spe_free_queue(void * priv)999 static void arm_spe_free_queue(void *priv)
1000 {
1001 	struct arm_spe_queue *speq = priv;
1002 
1003 	if (!speq)
1004 		return;
1005 	thread__zput(speq->thread);
1006 	arm_spe_decoder_free(speq->decoder);
1007 	zfree(&speq->event_buf);
1008 	free(speq);
1009 }
1010 
arm_spe_free_events(struct perf_session * session)1011 static void arm_spe_free_events(struct perf_session *session)
1012 {
1013 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
1014 					     auxtrace);
1015 	struct auxtrace_queues *queues = &spe->queues;
1016 	unsigned int i;
1017 
1018 	for (i = 0; i < queues->nr_queues; i++) {
1019 		arm_spe_free_queue(queues->queue_array[i].priv);
1020 		queues->queue_array[i].priv = NULL;
1021 	}
1022 	auxtrace_queues__free(queues);
1023 }
1024 
arm_spe_free(struct perf_session * session)1025 static void arm_spe_free(struct perf_session *session)
1026 {
1027 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
1028 					     auxtrace);
1029 
1030 	auxtrace_heap__free(&spe->heap);
1031 	arm_spe_free_events(session);
1032 	session->auxtrace = NULL;
1033 	free(spe);
1034 }
1035 
arm_spe_evsel_is_auxtrace(struct perf_session * session,struct evsel * evsel)1036 static bool arm_spe_evsel_is_auxtrace(struct perf_session *session,
1037 				      struct evsel *evsel)
1038 {
1039 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, auxtrace);
1040 
1041 	return evsel->core.attr.type == spe->pmu_type;
1042 }
1043 
1044 static const char * const arm_spe_info_fmts[] = {
1045 	[ARM_SPE_PMU_TYPE]		= "  PMU Type           %"PRId64"\n",
1046 };
1047 
arm_spe_print_info(__u64 * arr)1048 static void arm_spe_print_info(__u64 *arr)
1049 {
1050 	if (!dump_trace)
1051 		return;
1052 
1053 	fprintf(stdout, arm_spe_info_fmts[ARM_SPE_PMU_TYPE], arr[ARM_SPE_PMU_TYPE]);
1054 }
1055 
1056 struct arm_spe_synth {
1057 	struct perf_tool dummy_tool;
1058 	struct perf_session *session;
1059 };
1060 
arm_spe_event_synth(struct perf_tool * tool,union perf_event * event,struct perf_sample * sample __maybe_unused,struct machine * machine __maybe_unused)1061 static int arm_spe_event_synth(struct perf_tool *tool,
1062 			       union perf_event *event,
1063 			       struct perf_sample *sample __maybe_unused,
1064 			       struct machine *machine __maybe_unused)
1065 {
1066 	struct arm_spe_synth *arm_spe_synth =
1067 		      container_of(tool, struct arm_spe_synth, dummy_tool);
1068 
1069 	return perf_session__deliver_synth_event(arm_spe_synth->session,
1070 						 event, NULL);
1071 }
1072 
arm_spe_synth_event(struct perf_session * session,struct perf_event_attr * attr,u64 id)1073 static int arm_spe_synth_event(struct perf_session *session,
1074 			       struct perf_event_attr *attr, u64 id)
1075 {
1076 	struct arm_spe_synth arm_spe_synth;
1077 
1078 	memset(&arm_spe_synth, 0, sizeof(struct arm_spe_synth));
1079 	arm_spe_synth.session = session;
1080 
1081 	return perf_event__synthesize_attr(&arm_spe_synth.dummy_tool, attr, 1,
1082 					   &id, arm_spe_event_synth);
1083 }
1084 
arm_spe_set_event_name(struct evlist * evlist,u64 id,const char * name)1085 static void arm_spe_set_event_name(struct evlist *evlist, u64 id,
1086 				    const char *name)
1087 {
1088 	struct evsel *evsel;
1089 
1090 	evlist__for_each_entry(evlist, evsel) {
1091 		if (evsel->core.id && evsel->core.id[0] == id) {
1092 			if (evsel->name)
1093 				zfree(&evsel->name);
1094 			evsel->name = strdup(name);
1095 			break;
1096 		}
1097 	}
1098 }
1099 
1100 static int
arm_spe_synth_events(struct arm_spe * spe,struct perf_session * session)1101 arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session)
1102 {
1103 	struct evlist *evlist = session->evlist;
1104 	struct evsel *evsel;
1105 	struct perf_event_attr attr;
1106 	bool found = false;
1107 	u64 id;
1108 	int err;
1109 
1110 	evlist__for_each_entry(evlist, evsel) {
1111 		if (evsel->core.attr.type == spe->pmu_type) {
1112 			found = true;
1113 			break;
1114 		}
1115 	}
1116 
1117 	if (!found) {
1118 		pr_debug("No selected events with SPE trace data\n");
1119 		return 0;
1120 	}
1121 
1122 	memset(&attr, 0, sizeof(struct perf_event_attr));
1123 	attr.size = sizeof(struct perf_event_attr);
1124 	attr.type = PERF_TYPE_HARDWARE;
1125 	attr.sample_type = evsel->core.attr.sample_type &
1126 				(PERF_SAMPLE_MASK | PERF_SAMPLE_PHYS_ADDR);
1127 	attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1128 			    PERF_SAMPLE_PERIOD | PERF_SAMPLE_DATA_SRC |
1129 			    PERF_SAMPLE_WEIGHT | PERF_SAMPLE_ADDR;
1130 	if (spe->timeless_decoding)
1131 		attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1132 	else
1133 		attr.sample_type |= PERF_SAMPLE_TIME;
1134 
1135 	spe->sample_type = attr.sample_type;
1136 
1137 	attr.exclude_user = evsel->core.attr.exclude_user;
1138 	attr.exclude_kernel = evsel->core.attr.exclude_kernel;
1139 	attr.exclude_hv = evsel->core.attr.exclude_hv;
1140 	attr.exclude_host = evsel->core.attr.exclude_host;
1141 	attr.exclude_guest = evsel->core.attr.exclude_guest;
1142 	attr.sample_id_all = evsel->core.attr.sample_id_all;
1143 	attr.read_format = evsel->core.attr.read_format;
1144 
1145 	/* create new id val to be a fixed offset from evsel id */
1146 	id = evsel->core.id[0] + 1000000000;
1147 
1148 	if (!id)
1149 		id = 1;
1150 
1151 	if (spe->synth_opts.flc) {
1152 		spe->sample_flc = true;
1153 
1154 		/* Level 1 data cache miss */
1155 		err = arm_spe_synth_event(session, &attr, id);
1156 		if (err)
1157 			return err;
1158 		spe->l1d_miss_id = id;
1159 		arm_spe_set_event_name(evlist, id, "l1d-miss");
1160 		id += 1;
1161 
1162 		/* Level 1 data cache access */
1163 		err = arm_spe_synth_event(session, &attr, id);
1164 		if (err)
1165 			return err;
1166 		spe->l1d_access_id = id;
1167 		arm_spe_set_event_name(evlist, id, "l1d-access");
1168 		id += 1;
1169 	}
1170 
1171 	if (spe->synth_opts.llc) {
1172 		spe->sample_llc = true;
1173 
1174 		/* Last level cache miss */
1175 		err = arm_spe_synth_event(session, &attr, id);
1176 		if (err)
1177 			return err;
1178 		spe->llc_miss_id = id;
1179 		arm_spe_set_event_name(evlist, id, "llc-miss");
1180 		id += 1;
1181 
1182 		/* Last level cache access */
1183 		err = arm_spe_synth_event(session, &attr, id);
1184 		if (err)
1185 			return err;
1186 		spe->llc_access_id = id;
1187 		arm_spe_set_event_name(evlist, id, "llc-access");
1188 		id += 1;
1189 	}
1190 
1191 	if (spe->synth_opts.tlb) {
1192 		spe->sample_tlb = true;
1193 
1194 		/* TLB miss */
1195 		err = arm_spe_synth_event(session, &attr, id);
1196 		if (err)
1197 			return err;
1198 		spe->tlb_miss_id = id;
1199 		arm_spe_set_event_name(evlist, id, "tlb-miss");
1200 		id += 1;
1201 
1202 		/* TLB access */
1203 		err = arm_spe_synth_event(session, &attr, id);
1204 		if (err)
1205 			return err;
1206 		spe->tlb_access_id = id;
1207 		arm_spe_set_event_name(evlist, id, "tlb-access");
1208 		id += 1;
1209 	}
1210 
1211 	if (spe->synth_opts.branches) {
1212 		spe->sample_branch = true;
1213 
1214 		/* Branch miss */
1215 		err = arm_spe_synth_event(session, &attr, id);
1216 		if (err)
1217 			return err;
1218 		spe->branch_miss_id = id;
1219 		arm_spe_set_event_name(evlist, id, "branch-miss");
1220 		id += 1;
1221 	}
1222 
1223 	if (spe->synth_opts.remote_access) {
1224 		spe->sample_remote_access = true;
1225 
1226 		/* Remote access */
1227 		err = arm_spe_synth_event(session, &attr, id);
1228 		if (err)
1229 			return err;
1230 		spe->remote_access_id = id;
1231 		arm_spe_set_event_name(evlist, id, "remote-access");
1232 		id += 1;
1233 	}
1234 
1235 	if (spe->synth_opts.mem) {
1236 		spe->sample_memory = true;
1237 
1238 		err = arm_spe_synth_event(session, &attr, id);
1239 		if (err)
1240 			return err;
1241 		spe->memory_id = id;
1242 		arm_spe_set_event_name(evlist, id, "memory");
1243 		id += 1;
1244 	}
1245 
1246 	if (spe->synth_opts.instructions) {
1247 		if (spe->synth_opts.period_type != PERF_ITRACE_PERIOD_INSTRUCTIONS) {
1248 			pr_warning("Only instruction-based sampling period is currently supported by Arm SPE.\n");
1249 			goto synth_instructions_out;
1250 		}
1251 		if (spe->synth_opts.period > 1)
1252 			pr_warning("Arm SPE has a hardware-based sample period.\n"
1253 				   "Additional instruction events will be discarded by --itrace\n");
1254 
1255 		spe->sample_instructions = true;
1256 		attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1257 		attr.sample_period = spe->synth_opts.period;
1258 		spe->instructions_sample_period = attr.sample_period;
1259 		err = arm_spe_synth_event(session, &attr, id);
1260 		if (err)
1261 			return err;
1262 		spe->instructions_id = id;
1263 		arm_spe_set_event_name(evlist, id, "instructions");
1264 	}
1265 synth_instructions_out:
1266 
1267 	return 0;
1268 }
1269 
arm_spe_process_auxtrace_info(union perf_event * event,struct perf_session * session)1270 int arm_spe_process_auxtrace_info(union perf_event *event,
1271 				  struct perf_session *session)
1272 {
1273 	struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
1274 	size_t min_sz = sizeof(u64) * ARM_SPE_AUXTRACE_PRIV_MAX;
1275 	struct perf_record_time_conv *tc = &session->time_conv;
1276 	const char *cpuid = perf_env__cpuid(session->evlist->env);
1277 	u64 midr = strtol(cpuid, NULL, 16);
1278 	struct arm_spe *spe;
1279 	int err;
1280 
1281 	if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) +
1282 					min_sz)
1283 		return -EINVAL;
1284 
1285 	spe = zalloc(sizeof(struct arm_spe));
1286 	if (!spe)
1287 		return -ENOMEM;
1288 
1289 	err = auxtrace_queues__init(&spe->queues);
1290 	if (err)
1291 		goto err_free;
1292 
1293 	spe->session = session;
1294 	spe->machine = &session->machines.host; /* No kvm support */
1295 	spe->auxtrace_type = auxtrace_info->type;
1296 	spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE];
1297 	spe->midr = midr;
1298 
1299 	spe->timeless_decoding = arm_spe__is_timeless_decoding(spe);
1300 
1301 	/*
1302 	 * The synthesized event PERF_RECORD_TIME_CONV has been handled ahead
1303 	 * and the parameters for hardware clock are stored in the session
1304 	 * context.  Passes these parameters to the struct perf_tsc_conversion
1305 	 * in "spe->tc", which is used for later conversion between clock
1306 	 * counter and timestamp.
1307 	 *
1308 	 * For backward compatibility, copies the fields starting from
1309 	 * "time_cycles" only if they are contained in the event.
1310 	 */
1311 	spe->tc.time_shift = tc->time_shift;
1312 	spe->tc.time_mult = tc->time_mult;
1313 	spe->tc.time_zero = tc->time_zero;
1314 
1315 	if (event_contains(*tc, time_cycles)) {
1316 		spe->tc.time_cycles = tc->time_cycles;
1317 		spe->tc.time_mask = tc->time_mask;
1318 		spe->tc.cap_user_time_zero = tc->cap_user_time_zero;
1319 		spe->tc.cap_user_time_short = tc->cap_user_time_short;
1320 	}
1321 
1322 	spe->auxtrace.process_event = arm_spe_process_event;
1323 	spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event;
1324 	spe->auxtrace.flush_events = arm_spe_flush;
1325 	spe->auxtrace.free_events = arm_spe_free_events;
1326 	spe->auxtrace.free = arm_spe_free;
1327 	spe->auxtrace.evsel_is_auxtrace = arm_spe_evsel_is_auxtrace;
1328 	session->auxtrace = &spe->auxtrace;
1329 
1330 	arm_spe_print_info(&auxtrace_info->priv[0]);
1331 
1332 	if (dump_trace)
1333 		return 0;
1334 
1335 	if (session->itrace_synth_opts && session->itrace_synth_opts->set)
1336 		spe->synth_opts = *session->itrace_synth_opts;
1337 	else
1338 		itrace_synth_opts__set_default(&spe->synth_opts, false);
1339 
1340 	err = arm_spe_synth_events(spe, session);
1341 	if (err)
1342 		goto err_free_queues;
1343 
1344 	err = auxtrace_queues__process_index(&spe->queues, session);
1345 	if (err)
1346 		goto err_free_queues;
1347 
1348 	if (spe->queues.populated)
1349 		spe->data_queued = true;
1350 
1351 	return 0;
1352 
1353 err_free_queues:
1354 	auxtrace_queues__free(&spe->queues);
1355 	session->auxtrace = NULL;
1356 err_free:
1357 	free(spe);
1358 	return err;
1359 }
1360