1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright(C) 2015-2018 Linaro Limited.
4  *
5  * Author: Tor Jeremiassen <tor@ti.com>
6  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
7  */
8 
9 #include <linux/kernel.h>
10 #include <linux/bitfield.h>
11 #include <linux/bitops.h>
12 #include <linux/coresight-pmu.h>
13 #include <linux/err.h>
14 #include <linux/log2.h>
15 #include <linux/types.h>
16 #include <linux/zalloc.h>
17 
18 #include <stdlib.h>
19 
20 #include "auxtrace.h"
21 #include "color.h"
22 #include "cs-etm.h"
23 #include "cs-etm-decoder/cs-etm-decoder.h"
24 #include "debug.h"
25 #include "dso.h"
26 #include "evlist.h"
27 #include "intlist.h"
28 #include "machine.h"
29 #include "map.h"
30 #include "perf.h"
31 #include "session.h"
32 #include "map_symbol.h"
33 #include "branch.h"
34 #include "symbol.h"
35 #include "tool.h"
36 #include "thread.h"
37 #include "thread-stack.h"
38 #include "tsc.h"
39 #include <tools/libc_compat.h>
40 #include "util/synthetic-events.h"
41 #include "util/util.h"
42 
43 struct cs_etm_auxtrace {
44 	struct auxtrace auxtrace;
45 	struct auxtrace_queues queues;
46 	struct auxtrace_heap heap;
47 	struct itrace_synth_opts synth_opts;
48 	struct perf_session *session;
49 	struct perf_tsc_conversion tc;
50 
51 	/*
52 	 * Timeless has no timestamps in the trace so overlapping mmap lookups
53 	 * are less accurate but produces smaller trace data. We use context IDs
54 	 * in the trace instead of matching timestamps with fork records so
55 	 * they're not really needed in the general case. Overlapping mmaps
56 	 * happen in cases like between a fork and an exec.
57 	 */
58 	bool timeless_decoding;
59 
60 	/*
61 	 * Per-thread ignores the trace channel ID and instead assumes that
62 	 * everything in a buffer comes from the same process regardless of
63 	 * which CPU it ran on. It also implies no context IDs so the TID is
64 	 * taken from the auxtrace buffer.
65 	 */
66 	bool per_thread_decoding;
67 	bool snapshot_mode;
68 	bool data_queued;
69 	bool has_virtual_ts; /* Virtual/Kernel timestamps in the trace. */
70 
71 	int num_cpu;
72 	u64 latest_kernel_timestamp;
73 	u32 auxtrace_type;
74 	u64 branches_sample_type;
75 	u64 branches_id;
76 	u64 instructions_sample_type;
77 	u64 instructions_sample_period;
78 	u64 instructions_id;
79 	u64 **metadata;
80 	unsigned int pmu_type;
81 	enum cs_etm_pid_fmt pid_fmt;
82 };
83 
84 struct cs_etm_traceid_queue {
85 	u8 trace_chan_id;
86 	u64 period_instructions;
87 	size_t last_branch_pos;
88 	union perf_event *event_buf;
89 	struct thread *thread;
90 	struct thread *prev_packet_thread;
91 	ocsd_ex_level prev_packet_el;
92 	ocsd_ex_level el;
93 	struct branch_stack *last_branch;
94 	struct branch_stack *last_branch_rb;
95 	struct cs_etm_packet *prev_packet;
96 	struct cs_etm_packet *packet;
97 	struct cs_etm_packet_queue packet_queue;
98 };
99 
100 struct cs_etm_queue {
101 	struct cs_etm_auxtrace *etm;
102 	struct cs_etm_decoder *decoder;
103 	struct auxtrace_buffer *buffer;
104 	unsigned int queue_nr;
105 	u8 pending_timestamp_chan_id;
106 	u64 offset;
107 	const unsigned char *buf;
108 	size_t buf_len, buf_used;
109 	/* Conversion between traceID and index in traceid_queues array */
110 	struct intlist *traceid_queues_list;
111 	struct cs_etm_traceid_queue **traceid_queues;
112 };
113 
114 /* RB tree for quick conversion between traceID and metadata pointers */
115 static struct intlist *traceid_list;
116 
117 static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm);
118 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
119 					   pid_t tid);
120 static int cs_etm__get_data_block(struct cs_etm_queue *etmq);
121 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq);
122 
123 /* PTMs ETMIDR [11:8] set to b0011 */
124 #define ETMIDR_PTM_VERSION 0x00000300
125 
126 /*
127  * A struct auxtrace_heap_item only has a queue_nr and a timestamp to
128  * work with.  One option is to modify to auxtrace_heap_XYZ() API or simply
129  * encode the etm queue number as the upper 16 bit and the channel as
130  * the lower 16 bit.
131  */
132 #define TO_CS_QUEUE_NR(queue_nr, trace_chan_id)	\
133 		      (queue_nr << 16 | trace_chan_id)
134 #define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16)
135 #define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff)
136 
cs_etm__get_v7_protocol_version(u32 etmidr)137 static u32 cs_etm__get_v7_protocol_version(u32 etmidr)
138 {
139 	etmidr &= ETMIDR_PTM_VERSION;
140 
141 	if (etmidr == ETMIDR_PTM_VERSION)
142 		return CS_ETM_PROTO_PTM;
143 
144 	return CS_ETM_PROTO_ETMV3;
145 }
146 
cs_etm__get_magic(u8 trace_chan_id,u64 * magic)147 static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic)
148 {
149 	struct int_node *inode;
150 	u64 *metadata;
151 
152 	inode = intlist__find(traceid_list, trace_chan_id);
153 	if (!inode)
154 		return -EINVAL;
155 
156 	metadata = inode->priv;
157 	*magic = metadata[CS_ETM_MAGIC];
158 	return 0;
159 }
160 
cs_etm__get_cpu(u8 trace_chan_id,int * cpu)161 int cs_etm__get_cpu(u8 trace_chan_id, int *cpu)
162 {
163 	struct int_node *inode;
164 	u64 *metadata;
165 
166 	inode = intlist__find(traceid_list, trace_chan_id);
167 	if (!inode)
168 		return -EINVAL;
169 
170 	metadata = inode->priv;
171 	*cpu = (int)metadata[CS_ETM_CPU];
172 	return 0;
173 }
174 
175 /*
176  * The returned PID format is presented as an enum:
177  *
178  *   CS_ETM_PIDFMT_CTXTID: CONTEXTIDR or CONTEXTIDR_EL1 is traced.
179  *   CS_ETM_PIDFMT_CTXTID2: CONTEXTIDR_EL2 is traced.
180  *   CS_ETM_PIDFMT_NONE: No context IDs
181  *
182  * It's possible that the two bits ETM_OPT_CTXTID and ETM_OPT_CTXTID2
183  * are enabled at the same time when the session runs on an EL2 kernel.
184  * This means the CONTEXTIDR_EL1 and CONTEXTIDR_EL2 both will be
185  * recorded in the trace data, the tool will selectively use
186  * CONTEXTIDR_EL2 as PID.
187  *
188  * The result is cached in etm->pid_fmt so this function only needs to be called
189  * when processing the aux info.
190  */
cs_etm__init_pid_fmt(u64 * metadata)191 static enum cs_etm_pid_fmt cs_etm__init_pid_fmt(u64 *metadata)
192 {
193 	u64 val;
194 
195 	if (metadata[CS_ETM_MAGIC] == __perf_cs_etmv3_magic) {
196 		val = metadata[CS_ETM_ETMCR];
197 		/* CONTEXTIDR is traced */
198 		if (val & BIT(ETM_OPT_CTXTID))
199 			return CS_ETM_PIDFMT_CTXTID;
200 	} else {
201 		val = metadata[CS_ETMV4_TRCCONFIGR];
202 		/* CONTEXTIDR_EL2 is traced */
203 		if (val & (BIT(ETM4_CFG_BIT_VMID) | BIT(ETM4_CFG_BIT_VMID_OPT)))
204 			return CS_ETM_PIDFMT_CTXTID2;
205 		/* CONTEXTIDR_EL1 is traced */
206 		else if (val & BIT(ETM4_CFG_BIT_CTXTID))
207 			return CS_ETM_PIDFMT_CTXTID;
208 	}
209 
210 	return CS_ETM_PIDFMT_NONE;
211 }
212 
cs_etm__get_pid_fmt(struct cs_etm_queue * etmq)213 enum cs_etm_pid_fmt cs_etm__get_pid_fmt(struct cs_etm_queue *etmq)
214 {
215 	return etmq->etm->pid_fmt;
216 }
217 
cs_etm__map_trace_id(u8 trace_chan_id,u64 * cpu_metadata)218 static int cs_etm__map_trace_id(u8 trace_chan_id, u64 *cpu_metadata)
219 {
220 	struct int_node *inode;
221 
222 	/* Get an RB node for this CPU */
223 	inode = intlist__findnew(traceid_list, trace_chan_id);
224 
225 	/* Something went wrong, no need to continue */
226 	if (!inode)
227 		return -ENOMEM;
228 
229 	/*
230 	 * The node for that CPU should not be taken.
231 	 * Back out if that's the case.
232 	 */
233 	if (inode->priv)
234 		return -EINVAL;
235 
236 	/* All good, associate the traceID with the metadata pointer */
237 	inode->priv = cpu_metadata;
238 
239 	return 0;
240 }
241 
cs_etm__metadata_get_trace_id(u8 * trace_chan_id,u64 * cpu_metadata)242 static int cs_etm__metadata_get_trace_id(u8 *trace_chan_id, u64 *cpu_metadata)
243 {
244 	u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC];
245 
246 	switch (cs_etm_magic) {
247 	case __perf_cs_etmv3_magic:
248 		*trace_chan_id = (u8)(cpu_metadata[CS_ETM_ETMTRACEIDR] &
249 				      CORESIGHT_TRACE_ID_VAL_MASK);
250 		break;
251 	case __perf_cs_etmv4_magic:
252 	case __perf_cs_ete_magic:
253 		*trace_chan_id = (u8)(cpu_metadata[CS_ETMV4_TRCTRACEIDR] &
254 				      CORESIGHT_TRACE_ID_VAL_MASK);
255 		break;
256 	default:
257 		return -EINVAL;
258 	}
259 	return 0;
260 }
261 
262 /*
263  * update metadata trace ID from the value found in the AUX_HW_INFO packet.
264  * This will also clear the CORESIGHT_TRACE_ID_UNUSED_FLAG flag if present.
265  */
cs_etm__metadata_set_trace_id(u8 trace_chan_id,u64 * cpu_metadata)266 static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata)
267 {
268 	u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC];
269 
270 	switch (cs_etm_magic) {
271 	case __perf_cs_etmv3_magic:
272 		 cpu_metadata[CS_ETM_ETMTRACEIDR] = trace_chan_id;
273 		break;
274 	case __perf_cs_etmv4_magic:
275 	case __perf_cs_ete_magic:
276 		cpu_metadata[CS_ETMV4_TRCTRACEIDR] = trace_chan_id;
277 		break;
278 
279 	default:
280 		return -EINVAL;
281 	}
282 	return 0;
283 }
284 
285 /*
286  * Get a metadata for a specific cpu from an array.
287  *
288  */
get_cpu_data(struct cs_etm_auxtrace * etm,int cpu)289 static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu)
290 {
291 	int i;
292 	u64 *metadata = NULL;
293 
294 	for (i = 0; i < etm->num_cpu; i++) {
295 		if (etm->metadata[i][CS_ETM_CPU] == (u64)cpu) {
296 			metadata = etm->metadata[i];
297 			break;
298 		}
299 	}
300 
301 	return metadata;
302 }
303 
304 /*
305  * Handle the PERF_RECORD_AUX_OUTPUT_HW_ID event.
306  *
307  * The payload associates the Trace ID and the CPU.
308  * The routine is tolerant of seeing multiple packets with the same association,
309  * but a CPU / Trace ID association changing during a session is an error.
310  */
cs_etm__process_aux_output_hw_id(struct perf_session * session,union perf_event * event)311 static int cs_etm__process_aux_output_hw_id(struct perf_session *session,
312 					    union perf_event *event)
313 {
314 	struct cs_etm_auxtrace *etm;
315 	struct perf_sample sample;
316 	struct int_node *inode;
317 	struct evsel *evsel;
318 	u64 *cpu_data;
319 	u64 hw_id;
320 	int cpu, version, err;
321 	u8 trace_chan_id, curr_chan_id;
322 
323 	/* extract and parse the HW ID */
324 	hw_id = event->aux_output_hw_id.hw_id;
325 	version = FIELD_GET(CS_AUX_HW_ID_VERSION_MASK, hw_id);
326 	trace_chan_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id);
327 
328 	/* check that we can handle this version */
329 	if (version > CS_AUX_HW_ID_CURR_VERSION)
330 		return -EINVAL;
331 
332 	/* get access to the etm metadata */
333 	etm = container_of(session->auxtrace, struct cs_etm_auxtrace, auxtrace);
334 	if (!etm || !etm->metadata)
335 		return -EINVAL;
336 
337 	/* parse the sample to get the CPU */
338 	evsel = evlist__event2evsel(session->evlist, event);
339 	if (!evsel)
340 		return -EINVAL;
341 	err = evsel__parse_sample(evsel, event, &sample);
342 	if (err)
343 		return err;
344 	cpu = sample.cpu;
345 	if (cpu == -1) {
346 		/* no CPU in the sample - possibly recorded with an old version of perf */
347 		pr_err("CS_ETM: no CPU AUX_OUTPUT_HW_ID sample. Use compatible perf to record.");
348 		return -EINVAL;
349 	}
350 
351 	/* See if the ID is mapped to a CPU, and it matches the current CPU */
352 	inode = intlist__find(traceid_list, trace_chan_id);
353 	if (inode) {
354 		cpu_data = inode->priv;
355 		if ((int)cpu_data[CS_ETM_CPU] != cpu) {
356 			pr_err("CS_ETM: map mismatch between HW_ID packet CPU and Trace ID\n");
357 			return -EINVAL;
358 		}
359 
360 		/* check that the mapped ID matches */
361 		err = cs_etm__metadata_get_trace_id(&curr_chan_id, cpu_data);
362 		if (err)
363 			return err;
364 		if (curr_chan_id != trace_chan_id) {
365 			pr_err("CS_ETM: mismatch between CPU trace ID and HW_ID packet ID\n");
366 			return -EINVAL;
367 		}
368 
369 		/* mapped and matched - return OK */
370 		return 0;
371 	}
372 
373 	cpu_data = get_cpu_data(etm, cpu);
374 	if (cpu_data == NULL)
375 		return err;
376 
377 	/* not one we've seen before - lets map it */
378 	err = cs_etm__map_trace_id(trace_chan_id, cpu_data);
379 	if (err)
380 		return err;
381 
382 	/*
383 	 * if we are picking up the association from the packet, need to plug
384 	 * the correct trace ID into the metadata for setting up decoders later.
385 	 */
386 	err = cs_etm__metadata_set_trace_id(trace_chan_id, cpu_data);
387 	return err;
388 }
389 
cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue * etmq,u8 trace_chan_id)390 void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq,
391 					      u8 trace_chan_id)
392 {
393 	/*
394 	 * When a timestamp packet is encountered the backend code
395 	 * is stopped so that the front end has time to process packets
396 	 * that were accumulated in the traceID queue.  Since there can
397 	 * be more than one channel per cs_etm_queue, we need to specify
398 	 * what traceID queue needs servicing.
399 	 */
400 	etmq->pending_timestamp_chan_id = trace_chan_id;
401 }
402 
cs_etm__etmq_get_timestamp(struct cs_etm_queue * etmq,u8 * trace_chan_id)403 static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq,
404 				      u8 *trace_chan_id)
405 {
406 	struct cs_etm_packet_queue *packet_queue;
407 
408 	if (!etmq->pending_timestamp_chan_id)
409 		return 0;
410 
411 	if (trace_chan_id)
412 		*trace_chan_id = etmq->pending_timestamp_chan_id;
413 
414 	packet_queue = cs_etm__etmq_get_packet_queue(etmq,
415 						     etmq->pending_timestamp_chan_id);
416 	if (!packet_queue)
417 		return 0;
418 
419 	/* Acknowledge pending status */
420 	etmq->pending_timestamp_chan_id = 0;
421 
422 	/* See function cs_etm_decoder__do_{hard|soft}_timestamp() */
423 	return packet_queue->cs_timestamp;
424 }
425 
cs_etm__clear_packet_queue(struct cs_etm_packet_queue * queue)426 static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue)
427 {
428 	int i;
429 
430 	queue->head = 0;
431 	queue->tail = 0;
432 	queue->packet_count = 0;
433 	for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) {
434 		queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN;
435 		queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR;
436 		queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR;
437 		queue->packet_buffer[i].instr_count = 0;
438 		queue->packet_buffer[i].last_instr_taken_branch = false;
439 		queue->packet_buffer[i].last_instr_size = 0;
440 		queue->packet_buffer[i].last_instr_type = 0;
441 		queue->packet_buffer[i].last_instr_subtype = 0;
442 		queue->packet_buffer[i].last_instr_cond = 0;
443 		queue->packet_buffer[i].flags = 0;
444 		queue->packet_buffer[i].exception_number = UINT32_MAX;
445 		queue->packet_buffer[i].trace_chan_id = UINT8_MAX;
446 		queue->packet_buffer[i].cpu = INT_MIN;
447 	}
448 }
449 
cs_etm__clear_all_packet_queues(struct cs_etm_queue * etmq)450 static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq)
451 {
452 	int idx;
453 	struct int_node *inode;
454 	struct cs_etm_traceid_queue *tidq;
455 	struct intlist *traceid_queues_list = etmq->traceid_queues_list;
456 
457 	intlist__for_each_entry(inode, traceid_queues_list) {
458 		idx = (int)(intptr_t)inode->priv;
459 		tidq = etmq->traceid_queues[idx];
460 		cs_etm__clear_packet_queue(&tidq->packet_queue);
461 	}
462 }
463 
cs_etm__init_traceid_queue(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq,u8 trace_chan_id)464 static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
465 				      struct cs_etm_traceid_queue *tidq,
466 				      u8 trace_chan_id)
467 {
468 	int rc = -ENOMEM;
469 	struct auxtrace_queue *queue;
470 	struct cs_etm_auxtrace *etm = etmq->etm;
471 
472 	cs_etm__clear_packet_queue(&tidq->packet_queue);
473 
474 	queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
475 	tidq->trace_chan_id = trace_chan_id;
476 	tidq->el = tidq->prev_packet_el = ocsd_EL_unknown;
477 	tidq->thread = machine__findnew_thread(&etm->session->machines.host, -1,
478 					       queue->tid);
479 	tidq->prev_packet_thread = machine__idle_thread(&etm->session->machines.host);
480 
481 	tidq->packet = zalloc(sizeof(struct cs_etm_packet));
482 	if (!tidq->packet)
483 		goto out;
484 
485 	tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));
486 	if (!tidq->prev_packet)
487 		goto out_free;
488 
489 	if (etm->synth_opts.last_branch) {
490 		size_t sz = sizeof(struct branch_stack);
491 
492 		sz += etm->synth_opts.last_branch_sz *
493 		      sizeof(struct branch_entry);
494 		tidq->last_branch = zalloc(sz);
495 		if (!tidq->last_branch)
496 			goto out_free;
497 		tidq->last_branch_rb = zalloc(sz);
498 		if (!tidq->last_branch_rb)
499 			goto out_free;
500 	}
501 
502 	tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
503 	if (!tidq->event_buf)
504 		goto out_free;
505 
506 	return 0;
507 
508 out_free:
509 	zfree(&tidq->last_branch_rb);
510 	zfree(&tidq->last_branch);
511 	zfree(&tidq->prev_packet);
512 	zfree(&tidq->packet);
513 out:
514 	return rc;
515 }
516 
517 static struct cs_etm_traceid_queue
cs_etm__etmq_get_traceid_queue(struct cs_etm_queue * etmq,u8 trace_chan_id)518 *cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
519 {
520 	int idx;
521 	struct int_node *inode;
522 	struct intlist *traceid_queues_list;
523 	struct cs_etm_traceid_queue *tidq, **traceid_queues;
524 	struct cs_etm_auxtrace *etm = etmq->etm;
525 
526 	if (etm->per_thread_decoding)
527 		trace_chan_id = CS_ETM_PER_THREAD_TRACEID;
528 
529 	traceid_queues_list = etmq->traceid_queues_list;
530 
531 	/*
532 	 * Check if the traceid_queue exist for this traceID by looking
533 	 * in the queue list.
534 	 */
535 	inode = intlist__find(traceid_queues_list, trace_chan_id);
536 	if (inode) {
537 		idx = (int)(intptr_t)inode->priv;
538 		return etmq->traceid_queues[idx];
539 	}
540 
541 	/* We couldn't find a traceid_queue for this traceID, allocate one */
542 	tidq = malloc(sizeof(*tidq));
543 	if (!tidq)
544 		return NULL;
545 
546 	memset(tidq, 0, sizeof(*tidq));
547 
548 	/* Get a valid index for the new traceid_queue */
549 	idx = intlist__nr_entries(traceid_queues_list);
550 	/* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */
551 	inode = intlist__findnew(traceid_queues_list, trace_chan_id);
552 	if (!inode)
553 		goto out_free;
554 
555 	/* Associate this traceID with this index */
556 	inode->priv = (void *)(intptr_t)idx;
557 
558 	if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))
559 		goto out_free;
560 
561 	/* Grow the traceid_queues array by one unit */
562 	traceid_queues = etmq->traceid_queues;
563 	traceid_queues = reallocarray(traceid_queues,
564 				      idx + 1,
565 				      sizeof(*traceid_queues));
566 
567 	/*
568 	 * On failure reallocarray() returns NULL and the original block of
569 	 * memory is left untouched.
570 	 */
571 	if (!traceid_queues)
572 		goto out_free;
573 
574 	traceid_queues[idx] = tidq;
575 	etmq->traceid_queues = traceid_queues;
576 
577 	return etmq->traceid_queues[idx];
578 
579 out_free:
580 	/*
581 	 * Function intlist__remove() removes the inode from the list
582 	 * and delete the memory associated to it.
583 	 */
584 	intlist__remove(traceid_queues_list, inode);
585 	free(tidq);
586 
587 	return NULL;
588 }
589 
590 struct cs_etm_packet_queue
cs_etm__etmq_get_packet_queue(struct cs_etm_queue * etmq,u8 trace_chan_id)591 *cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
592 {
593 	struct cs_etm_traceid_queue *tidq;
594 
595 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
596 	if (tidq)
597 		return &tidq->packet_queue;
598 
599 	return NULL;
600 }
601 
cs_etm__packet_swap(struct cs_etm_auxtrace * etm,struct cs_etm_traceid_queue * tidq)602 static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm,
603 				struct cs_etm_traceid_queue *tidq)
604 {
605 	struct cs_etm_packet *tmp;
606 
607 	if (etm->synth_opts.branches || etm->synth_opts.last_branch ||
608 	    etm->synth_opts.instructions) {
609 		/*
610 		 * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
611 		 * the next incoming packet.
612 		 *
613 		 * Threads and exception levels are also tracked for both the
614 		 * previous and current packets. This is because the previous
615 		 * packet is used for the 'from' IP for branch samples, so the
616 		 * thread at that time must also be assigned to that sample.
617 		 * Across discontinuity packets the thread can change, so by
618 		 * tracking the thread for the previous packet the branch sample
619 		 * will have the correct info.
620 		 */
621 		tmp = tidq->packet;
622 		tidq->packet = tidq->prev_packet;
623 		tidq->prev_packet = tmp;
624 		tidq->prev_packet_el = tidq->el;
625 		thread__put(tidq->prev_packet_thread);
626 		tidq->prev_packet_thread = thread__get(tidq->thread);
627 	}
628 }
629 
cs_etm__packet_dump(const char * pkt_string)630 static void cs_etm__packet_dump(const char *pkt_string)
631 {
632 	const char *color = PERF_COLOR_BLUE;
633 	int len = strlen(pkt_string);
634 
635 	if (len && (pkt_string[len-1] == '\n'))
636 		color_fprintf(stdout, color, "	%s", pkt_string);
637 	else
638 		color_fprintf(stdout, color, "	%s\n", pkt_string);
639 
640 	fflush(stdout);
641 }
642 
cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params * t_params,struct cs_etm_auxtrace * etm,int idx,u32 etmidr)643 static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params,
644 					  struct cs_etm_auxtrace *etm, int idx,
645 					  u32 etmidr)
646 {
647 	u64 **metadata = etm->metadata;
648 
649 	t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr);
650 	t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR];
651 	t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR];
652 }
653 
cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params * t_params,struct cs_etm_auxtrace * etm,int idx)654 static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params,
655 					  struct cs_etm_auxtrace *etm, int idx)
656 {
657 	u64 **metadata = etm->metadata;
658 
659 	t_params[idx].protocol = CS_ETM_PROTO_ETMV4i;
660 	t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0];
661 	t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1];
662 	t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2];
663 	t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8];
664 	t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR];
665 	t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR];
666 }
667 
cs_etm__set_trace_param_ete(struct cs_etm_trace_params * t_params,struct cs_etm_auxtrace * etm,int idx)668 static void cs_etm__set_trace_param_ete(struct cs_etm_trace_params *t_params,
669 					  struct cs_etm_auxtrace *etm, int idx)
670 {
671 	u64 **metadata = etm->metadata;
672 
673 	t_params[idx].protocol = CS_ETM_PROTO_ETE;
674 	t_params[idx].ete.reg_idr0 = metadata[idx][CS_ETE_TRCIDR0];
675 	t_params[idx].ete.reg_idr1 = metadata[idx][CS_ETE_TRCIDR1];
676 	t_params[idx].ete.reg_idr2 = metadata[idx][CS_ETE_TRCIDR2];
677 	t_params[idx].ete.reg_idr8 = metadata[idx][CS_ETE_TRCIDR8];
678 	t_params[idx].ete.reg_configr = metadata[idx][CS_ETE_TRCCONFIGR];
679 	t_params[idx].ete.reg_traceidr = metadata[idx][CS_ETE_TRCTRACEIDR];
680 	t_params[idx].ete.reg_devarch = metadata[idx][CS_ETE_TRCDEVARCH];
681 }
682 
cs_etm__init_trace_params(struct cs_etm_trace_params * t_params,struct cs_etm_auxtrace * etm,int decoders)683 static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params,
684 				     struct cs_etm_auxtrace *etm,
685 				     int decoders)
686 {
687 	int i;
688 	u32 etmidr;
689 	u64 architecture;
690 
691 	for (i = 0; i < decoders; i++) {
692 		architecture = etm->metadata[i][CS_ETM_MAGIC];
693 
694 		switch (architecture) {
695 		case __perf_cs_etmv3_magic:
696 			etmidr = etm->metadata[i][CS_ETM_ETMIDR];
697 			cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr);
698 			break;
699 		case __perf_cs_etmv4_magic:
700 			cs_etm__set_trace_param_etmv4(t_params, etm, i);
701 			break;
702 		case __perf_cs_ete_magic:
703 			cs_etm__set_trace_param_ete(t_params, etm, i);
704 			break;
705 		default:
706 			return -EINVAL;
707 		}
708 	}
709 
710 	return 0;
711 }
712 
cs_etm__init_decoder_params(struct cs_etm_decoder_params * d_params,struct cs_etm_queue * etmq,enum cs_etm_decoder_operation mode,bool formatted)713 static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params,
714 				       struct cs_etm_queue *etmq,
715 				       enum cs_etm_decoder_operation mode,
716 				       bool formatted)
717 {
718 	int ret = -EINVAL;
719 
720 	if (!(mode < CS_ETM_OPERATION_MAX))
721 		goto out;
722 
723 	d_params->packet_printer = cs_etm__packet_dump;
724 	d_params->operation = mode;
725 	d_params->data = etmq;
726 	d_params->formatted = formatted;
727 	d_params->fsyncs = false;
728 	d_params->hsyncs = false;
729 	d_params->frame_aligned = true;
730 
731 	ret = 0;
732 out:
733 	return ret;
734 }
735 
cs_etm__dump_event(struct cs_etm_queue * etmq,struct auxtrace_buffer * buffer)736 static void cs_etm__dump_event(struct cs_etm_queue *etmq,
737 			       struct auxtrace_buffer *buffer)
738 {
739 	int ret;
740 	const char *color = PERF_COLOR_BLUE;
741 	size_t buffer_used = 0;
742 
743 	fprintf(stdout, "\n");
744 	color_fprintf(stdout, color,
745 		     ". ... CoreSight %s Trace data: size %#zx bytes\n",
746 		     cs_etm_decoder__get_name(etmq->decoder), buffer->size);
747 
748 	do {
749 		size_t consumed;
750 
751 		ret = cs_etm_decoder__process_data_block(
752 				etmq->decoder, buffer->offset,
753 				&((u8 *)buffer->data)[buffer_used],
754 				buffer->size - buffer_used, &consumed);
755 		if (ret)
756 			break;
757 
758 		buffer_used += consumed;
759 	} while (buffer_used < buffer->size);
760 
761 	cs_etm_decoder__reset(etmq->decoder);
762 }
763 
cs_etm__flush_events(struct perf_session * session,struct perf_tool * tool)764 static int cs_etm__flush_events(struct perf_session *session,
765 				struct perf_tool *tool)
766 {
767 	struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
768 						   struct cs_etm_auxtrace,
769 						   auxtrace);
770 	if (dump_trace)
771 		return 0;
772 
773 	if (!tool->ordered_events)
774 		return -EINVAL;
775 
776 	if (etm->timeless_decoding) {
777 		/*
778 		 * Pass tid = -1 to process all queues. But likely they will have
779 		 * already been processed on PERF_RECORD_EXIT anyway.
780 		 */
781 		return cs_etm__process_timeless_queues(etm, -1);
782 	}
783 
784 	return cs_etm__process_timestamped_queues(etm);
785 }
786 
cs_etm__free_traceid_queues(struct cs_etm_queue * etmq)787 static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq)
788 {
789 	int idx;
790 	uintptr_t priv;
791 	struct int_node *inode, *tmp;
792 	struct cs_etm_traceid_queue *tidq;
793 	struct intlist *traceid_queues_list = etmq->traceid_queues_list;
794 
795 	intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) {
796 		priv = (uintptr_t)inode->priv;
797 		idx = priv;
798 
799 		/* Free this traceid_queue from the array */
800 		tidq = etmq->traceid_queues[idx];
801 		thread__zput(tidq->thread);
802 		thread__zput(tidq->prev_packet_thread);
803 		zfree(&tidq->event_buf);
804 		zfree(&tidq->last_branch);
805 		zfree(&tidq->last_branch_rb);
806 		zfree(&tidq->prev_packet);
807 		zfree(&tidq->packet);
808 		zfree(&tidq);
809 
810 		/*
811 		 * Function intlist__remove() removes the inode from the list
812 		 * and delete the memory associated to it.
813 		 */
814 		intlist__remove(traceid_queues_list, inode);
815 	}
816 
817 	/* Then the RB tree itself */
818 	intlist__delete(traceid_queues_list);
819 	etmq->traceid_queues_list = NULL;
820 
821 	/* finally free the traceid_queues array */
822 	zfree(&etmq->traceid_queues);
823 }
824 
cs_etm__free_queue(void * priv)825 static void cs_etm__free_queue(void *priv)
826 {
827 	struct cs_etm_queue *etmq = priv;
828 
829 	if (!etmq)
830 		return;
831 
832 	cs_etm_decoder__free(etmq->decoder);
833 	cs_etm__free_traceid_queues(etmq);
834 	free(etmq);
835 }
836 
cs_etm__free_events(struct perf_session * session)837 static void cs_etm__free_events(struct perf_session *session)
838 {
839 	unsigned int i;
840 	struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
841 						   struct cs_etm_auxtrace,
842 						   auxtrace);
843 	struct auxtrace_queues *queues = &aux->queues;
844 
845 	for (i = 0; i < queues->nr_queues; i++) {
846 		cs_etm__free_queue(queues->queue_array[i].priv);
847 		queues->queue_array[i].priv = NULL;
848 	}
849 
850 	auxtrace_queues__free(queues);
851 }
852 
cs_etm__free(struct perf_session * session)853 static void cs_etm__free(struct perf_session *session)
854 {
855 	int i;
856 	struct int_node *inode, *tmp;
857 	struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
858 						   struct cs_etm_auxtrace,
859 						   auxtrace);
860 	cs_etm__free_events(session);
861 	session->auxtrace = NULL;
862 
863 	/* First remove all traceID/metadata nodes for the RB tree */
864 	intlist__for_each_entry_safe(inode, tmp, traceid_list)
865 		intlist__remove(traceid_list, inode);
866 	/* Then the RB tree itself */
867 	intlist__delete(traceid_list);
868 
869 	for (i = 0; i < aux->num_cpu; i++)
870 		zfree(&aux->metadata[i]);
871 
872 	zfree(&aux->metadata);
873 	zfree(&aux);
874 }
875 
cs_etm__evsel_is_auxtrace(struct perf_session * session,struct evsel * evsel)876 static bool cs_etm__evsel_is_auxtrace(struct perf_session *session,
877 				      struct evsel *evsel)
878 {
879 	struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
880 						   struct cs_etm_auxtrace,
881 						   auxtrace);
882 
883 	return evsel->core.attr.type == aux->pmu_type;
884 }
885 
cs_etm__get_machine(struct cs_etm_queue * etmq,ocsd_ex_level el)886 static struct machine *cs_etm__get_machine(struct cs_etm_queue *etmq,
887 					   ocsd_ex_level el)
888 {
889 	enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq);
890 
891 	/*
892 	 * For any virtualisation based on nVHE (e.g. pKVM), or host kernels
893 	 * running at EL1 assume everything is the host.
894 	 */
895 	if (pid_fmt == CS_ETM_PIDFMT_CTXTID)
896 		return &etmq->etm->session->machines.host;
897 
898 	/*
899 	 * Not perfect, but otherwise assume anything in EL1 is the default
900 	 * guest, and everything else is the host. Distinguishing between guest
901 	 * and host userspaces isn't currently supported either. Neither is
902 	 * multiple guest support. All this does is reduce the likeliness of
903 	 * decode errors where we look into the host kernel maps when it should
904 	 * have been the guest maps.
905 	 */
906 	switch (el) {
907 	case ocsd_EL1:
908 		return machines__find_guest(&etmq->etm->session->machines,
909 					    DEFAULT_GUEST_KERNEL_ID);
910 	case ocsd_EL3:
911 	case ocsd_EL2:
912 	case ocsd_EL0:
913 	case ocsd_EL_unknown:
914 	default:
915 		return &etmq->etm->session->machines.host;
916 	}
917 }
918 
cs_etm__cpu_mode(struct cs_etm_queue * etmq,u64 address,ocsd_ex_level el)919 static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address,
920 			   ocsd_ex_level el)
921 {
922 	struct machine *machine = cs_etm__get_machine(etmq, el);
923 
924 	if (address >= machine__kernel_start(machine)) {
925 		if (machine__is_host(machine))
926 			return PERF_RECORD_MISC_KERNEL;
927 		else
928 			return PERF_RECORD_MISC_GUEST_KERNEL;
929 	} else {
930 		if (machine__is_host(machine))
931 			return PERF_RECORD_MISC_USER;
932 		else {
933 			/*
934 			 * Can't really happen at the moment because
935 			 * cs_etm__get_machine() will always return
936 			 * machines.host for any non EL1 trace.
937 			 */
938 			return PERF_RECORD_MISC_GUEST_USER;
939 		}
940 	}
941 }
942 
cs_etm__mem_access(struct cs_etm_queue * etmq,u8 trace_chan_id,u64 address,size_t size,u8 * buffer,const ocsd_mem_space_acc_t mem_space)943 static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
944 			      u64 address, size_t size, u8 *buffer,
945 			      const ocsd_mem_space_acc_t mem_space)
946 {
947 	u8  cpumode;
948 	u64 offset;
949 	int len;
950 	struct addr_location al;
951 	struct dso *dso;
952 	struct cs_etm_traceid_queue *tidq;
953 	int ret = 0;
954 
955 	if (!etmq)
956 		return 0;
957 
958 	addr_location__init(&al);
959 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
960 	if (!tidq)
961 		goto out;
962 
963 	/*
964 	 * We've already tracked EL along side the PID in cs_etm__set_thread()
965 	 * so double check that it matches what OpenCSD thinks as well. It
966 	 * doesn't distinguish between EL0 and EL1 for this mem access callback
967 	 * so we had to do the extra tracking. Skip validation if it's any of
968 	 * the 'any' values.
969 	 */
970 	if (!(mem_space == OCSD_MEM_SPACE_ANY ||
971 	      mem_space == OCSD_MEM_SPACE_N || mem_space == OCSD_MEM_SPACE_S)) {
972 		if (mem_space & OCSD_MEM_SPACE_EL1N) {
973 			/* Includes both non secure EL1 and EL0 */
974 			assert(tidq->el == ocsd_EL1 || tidq->el == ocsd_EL0);
975 		} else if (mem_space & OCSD_MEM_SPACE_EL2)
976 			assert(tidq->el == ocsd_EL2);
977 		else if (mem_space & OCSD_MEM_SPACE_EL3)
978 			assert(tidq->el == ocsd_EL3);
979 	}
980 
981 	cpumode = cs_etm__cpu_mode(etmq, address, tidq->el);
982 
983 	if (!thread__find_map(tidq->thread, cpumode, address, &al))
984 		goto out;
985 
986 	dso = map__dso(al.map);
987 	if (!dso)
988 		goto out;
989 
990 	if (dso->data.status == DSO_DATA_STATUS_ERROR &&
991 	    dso__data_status_seen(dso, DSO_DATA_STATUS_SEEN_ITRACE))
992 		goto out;
993 
994 	offset = map__map_ip(al.map, address);
995 
996 	map__load(al.map);
997 
998 	len = dso__data_read_offset(dso, maps__machine(thread__maps(tidq->thread)),
999 				    offset, buffer, size);
1000 
1001 	if (len <= 0) {
1002 		ui__warning_once("CS ETM Trace: Missing DSO. Use 'perf archive' or debuginfod to export data from the traced system.\n"
1003 				 "              Enable CONFIG_PROC_KCORE or use option '-k /path/to/vmlinux' for kernel symbols.\n");
1004 		if (!dso->auxtrace_warned) {
1005 			pr_err("CS ETM Trace: Debug data not found for address %#"PRIx64" in %s\n",
1006 				    address,
1007 				    dso->long_name ? dso->long_name : "Unknown");
1008 			dso->auxtrace_warned = true;
1009 		}
1010 		goto out;
1011 	}
1012 	ret = len;
1013 out:
1014 	addr_location__exit(&al);
1015 	return ret;
1016 }
1017 
cs_etm__alloc_queue(struct cs_etm_auxtrace * etm,bool formatted)1018 static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm,
1019 						bool formatted)
1020 {
1021 	struct cs_etm_decoder_params d_params;
1022 	struct cs_etm_trace_params  *t_params = NULL;
1023 	struct cs_etm_queue *etmq;
1024 	/*
1025 	 * Each queue can only contain data from one CPU when unformatted, so only one decoder is
1026 	 * needed.
1027 	 */
1028 	int decoders = formatted ? etm->num_cpu : 1;
1029 
1030 	etmq = zalloc(sizeof(*etmq));
1031 	if (!etmq)
1032 		return NULL;
1033 
1034 	etmq->traceid_queues_list = intlist__new(NULL);
1035 	if (!etmq->traceid_queues_list)
1036 		goto out_free;
1037 
1038 	/* Use metadata to fill in trace parameters for trace decoder */
1039 	t_params = zalloc(sizeof(*t_params) * decoders);
1040 
1041 	if (!t_params)
1042 		goto out_free;
1043 
1044 	if (cs_etm__init_trace_params(t_params, etm, decoders))
1045 		goto out_free;
1046 
1047 	/* Set decoder parameters to decode trace packets */
1048 	if (cs_etm__init_decoder_params(&d_params, etmq,
1049 					dump_trace ? CS_ETM_OPERATION_PRINT :
1050 						     CS_ETM_OPERATION_DECODE,
1051 					formatted))
1052 		goto out_free;
1053 
1054 	etmq->decoder = cs_etm_decoder__new(decoders, &d_params,
1055 					    t_params);
1056 
1057 	if (!etmq->decoder)
1058 		goto out_free;
1059 
1060 	/*
1061 	 * Register a function to handle all memory accesses required by
1062 	 * the trace decoder library.
1063 	 */
1064 	if (cs_etm_decoder__add_mem_access_cb(etmq->decoder,
1065 					      0x0L, ((u64) -1L),
1066 					      cs_etm__mem_access))
1067 		goto out_free_decoder;
1068 
1069 	zfree(&t_params);
1070 	return etmq;
1071 
1072 out_free_decoder:
1073 	cs_etm_decoder__free(etmq->decoder);
1074 out_free:
1075 	intlist__delete(etmq->traceid_queues_list);
1076 	free(etmq);
1077 
1078 	return NULL;
1079 }
1080 
cs_etm__setup_queue(struct cs_etm_auxtrace * etm,struct auxtrace_queue * queue,unsigned int queue_nr,bool formatted)1081 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,
1082 			       struct auxtrace_queue *queue,
1083 			       unsigned int queue_nr,
1084 			       bool formatted)
1085 {
1086 	struct cs_etm_queue *etmq = queue->priv;
1087 
1088 	if (list_empty(&queue->head) || etmq)
1089 		return 0;
1090 
1091 	etmq = cs_etm__alloc_queue(etm, formatted);
1092 
1093 	if (!etmq)
1094 		return -ENOMEM;
1095 
1096 	queue->priv = etmq;
1097 	etmq->etm = etm;
1098 	etmq->queue_nr = queue_nr;
1099 	etmq->offset = 0;
1100 
1101 	return 0;
1102 }
1103 
cs_etm__queue_first_cs_timestamp(struct cs_etm_auxtrace * etm,struct cs_etm_queue * etmq,unsigned int queue_nr)1104 static int cs_etm__queue_first_cs_timestamp(struct cs_etm_auxtrace *etm,
1105 					    struct cs_etm_queue *etmq,
1106 					    unsigned int queue_nr)
1107 {
1108 	int ret = 0;
1109 	unsigned int cs_queue_nr;
1110 	u8 trace_chan_id;
1111 	u64 cs_timestamp;
1112 
1113 	/*
1114 	 * We are under a CPU-wide trace scenario.  As such we need to know
1115 	 * when the code that generated the traces started to execute so that
1116 	 * it can be correlated with execution on other CPUs.  So we get a
1117 	 * handle on the beginning of traces and decode until we find a
1118 	 * timestamp.  The timestamp is then added to the auxtrace min heap
1119 	 * in order to know what nibble (of all the etmqs) to decode first.
1120 	 */
1121 	while (1) {
1122 		/*
1123 		 * Fetch an aux_buffer from this etmq.  Bail if no more
1124 		 * blocks or an error has been encountered.
1125 		 */
1126 		ret = cs_etm__get_data_block(etmq);
1127 		if (ret <= 0)
1128 			goto out;
1129 
1130 		/*
1131 		 * Run decoder on the trace block.  The decoder will stop when
1132 		 * encountering a CS timestamp, a full packet queue or the end of
1133 		 * trace for that block.
1134 		 */
1135 		ret = cs_etm__decode_data_block(etmq);
1136 		if (ret)
1137 			goto out;
1138 
1139 		/*
1140 		 * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all
1141 		 * the timestamp calculation for us.
1142 		 */
1143 		cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
1144 
1145 		/* We found a timestamp, no need to continue. */
1146 		if (cs_timestamp)
1147 			break;
1148 
1149 		/*
1150 		 * We didn't find a timestamp so empty all the traceid packet
1151 		 * queues before looking for another timestamp packet, either
1152 		 * in the current data block or a new one.  Packets that were
1153 		 * just decoded are useless since no timestamp has been
1154 		 * associated with them.  As such simply discard them.
1155 		 */
1156 		cs_etm__clear_all_packet_queues(etmq);
1157 	}
1158 
1159 	/*
1160 	 * We have a timestamp.  Add it to the min heap to reflect when
1161 	 * instructions conveyed by the range packets of this traceID queue
1162 	 * started to execute.  Once the same has been done for all the traceID
1163 	 * queues of each etmq, redenring and decoding can start in
1164 	 * chronological order.
1165 	 *
1166 	 * Note that packets decoded above are still in the traceID's packet
1167 	 * queue and will be processed in cs_etm__process_timestamped_queues().
1168 	 */
1169 	cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
1170 	ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);
1171 out:
1172 	return ret;
1173 }
1174 
1175 static inline
cs_etm__copy_last_branch_rb(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq)1176 void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
1177 				 struct cs_etm_traceid_queue *tidq)
1178 {
1179 	struct branch_stack *bs_src = tidq->last_branch_rb;
1180 	struct branch_stack *bs_dst = tidq->last_branch;
1181 	size_t nr = 0;
1182 
1183 	/*
1184 	 * Set the number of records before early exit: ->nr is used to
1185 	 * determine how many branches to copy from ->entries.
1186 	 */
1187 	bs_dst->nr = bs_src->nr;
1188 
1189 	/*
1190 	 * Early exit when there is nothing to copy.
1191 	 */
1192 	if (!bs_src->nr)
1193 		return;
1194 
1195 	/*
1196 	 * As bs_src->entries is a circular buffer, we need to copy from it in
1197 	 * two steps.  First, copy the branches from the most recently inserted
1198 	 * branch ->last_branch_pos until the end of bs_src->entries buffer.
1199 	 */
1200 	nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
1201 	memcpy(&bs_dst->entries[0],
1202 	       &bs_src->entries[tidq->last_branch_pos],
1203 	       sizeof(struct branch_entry) * nr);
1204 
1205 	/*
1206 	 * If we wrapped around at least once, the branches from the beginning
1207 	 * of the bs_src->entries buffer and until the ->last_branch_pos element
1208 	 * are older valid branches: copy them over.  The total number of
1209 	 * branches copied over will be equal to the number of branches asked by
1210 	 * the user in last_branch_sz.
1211 	 */
1212 	if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
1213 		memcpy(&bs_dst->entries[nr],
1214 		       &bs_src->entries[0],
1215 		       sizeof(struct branch_entry) * tidq->last_branch_pos);
1216 	}
1217 }
1218 
1219 static inline
cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue * tidq)1220 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
1221 {
1222 	tidq->last_branch_pos = 0;
1223 	tidq->last_branch_rb->nr = 0;
1224 }
1225 
cs_etm__t32_instr_size(struct cs_etm_queue * etmq,u8 trace_chan_id,u64 addr)1226 static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
1227 					 u8 trace_chan_id, u64 addr)
1228 {
1229 	u8 instrBytes[2];
1230 
1231 	cs_etm__mem_access(etmq, trace_chan_id, addr, ARRAY_SIZE(instrBytes),
1232 			   instrBytes, 0);
1233 	/*
1234 	 * T32 instruction size is indicated by bits[15:11] of the first
1235 	 * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
1236 	 * denote a 32-bit instruction.
1237 	 */
1238 	return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
1239 }
1240 
cs_etm__first_executed_instr(struct cs_etm_packet * packet)1241 static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
1242 {
1243 	/* Returns 0 for the CS_ETM_DISCONTINUITY packet */
1244 	if (packet->sample_type == CS_ETM_DISCONTINUITY)
1245 		return 0;
1246 
1247 	return packet->start_addr;
1248 }
1249 
1250 static inline
cs_etm__last_executed_instr(const struct cs_etm_packet * packet)1251 u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
1252 {
1253 	/* Returns 0 for the CS_ETM_DISCONTINUITY packet */
1254 	if (packet->sample_type == CS_ETM_DISCONTINUITY)
1255 		return 0;
1256 
1257 	return packet->end_addr - packet->last_instr_size;
1258 }
1259 
cs_etm__instr_addr(struct cs_etm_queue * etmq,u64 trace_chan_id,const struct cs_etm_packet * packet,u64 offset)1260 static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
1261 				     u64 trace_chan_id,
1262 				     const struct cs_etm_packet *packet,
1263 				     u64 offset)
1264 {
1265 	if (packet->isa == CS_ETM_ISA_T32) {
1266 		u64 addr = packet->start_addr;
1267 
1268 		while (offset) {
1269 			addr += cs_etm__t32_instr_size(etmq,
1270 						       trace_chan_id, addr);
1271 			offset--;
1272 		}
1273 		return addr;
1274 	}
1275 
1276 	/* Assume a 4 byte instruction size (A32/A64) */
1277 	return packet->start_addr + offset * 4;
1278 }
1279 
cs_etm__update_last_branch_rb(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq)1280 static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
1281 					  struct cs_etm_traceid_queue *tidq)
1282 {
1283 	struct branch_stack *bs = tidq->last_branch_rb;
1284 	struct branch_entry *be;
1285 
1286 	/*
1287 	 * The branches are recorded in a circular buffer in reverse
1288 	 * chronological order: we start recording from the last element of the
1289 	 * buffer down.  After writing the first element of the stack, move the
1290 	 * insert position back to the end of the buffer.
1291 	 */
1292 	if (!tidq->last_branch_pos)
1293 		tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
1294 
1295 	tidq->last_branch_pos -= 1;
1296 
1297 	be       = &bs->entries[tidq->last_branch_pos];
1298 	be->from = cs_etm__last_executed_instr(tidq->prev_packet);
1299 	be->to	 = cs_etm__first_executed_instr(tidq->packet);
1300 	/* No support for mispredict */
1301 	be->flags.mispred = 0;
1302 	be->flags.predicted = 1;
1303 
1304 	/*
1305 	 * Increment bs->nr until reaching the number of last branches asked by
1306 	 * the user on the command line.
1307 	 */
1308 	if (bs->nr < etmq->etm->synth_opts.last_branch_sz)
1309 		bs->nr += 1;
1310 }
1311 
cs_etm__inject_event(union perf_event * event,struct perf_sample * sample,u64 type)1312 static int cs_etm__inject_event(union perf_event *event,
1313 			       struct perf_sample *sample, u64 type)
1314 {
1315 	event->header.size = perf_event__sample_event_size(sample, type, 0);
1316 	return perf_event__synthesize_sample(event, type, 0, sample);
1317 }
1318 
1319 
1320 static int
cs_etm__get_trace(struct cs_etm_queue * etmq)1321 cs_etm__get_trace(struct cs_etm_queue *etmq)
1322 {
1323 	struct auxtrace_buffer *aux_buffer = etmq->buffer;
1324 	struct auxtrace_buffer *old_buffer = aux_buffer;
1325 	struct auxtrace_queue *queue;
1326 
1327 	queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
1328 
1329 	aux_buffer = auxtrace_buffer__next(queue, aux_buffer);
1330 
1331 	/* If no more data, drop the previous auxtrace_buffer and return */
1332 	if (!aux_buffer) {
1333 		if (old_buffer)
1334 			auxtrace_buffer__drop_data(old_buffer);
1335 		etmq->buf_len = 0;
1336 		return 0;
1337 	}
1338 
1339 	etmq->buffer = aux_buffer;
1340 
1341 	/* If the aux_buffer doesn't have data associated, try to load it */
1342 	if (!aux_buffer->data) {
1343 		/* get the file desc associated with the perf data file */
1344 		int fd = perf_data__fd(etmq->etm->session->data);
1345 
1346 		aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd);
1347 		if (!aux_buffer->data)
1348 			return -ENOMEM;
1349 	}
1350 
1351 	/* If valid, drop the previous buffer */
1352 	if (old_buffer)
1353 		auxtrace_buffer__drop_data(old_buffer);
1354 
1355 	etmq->buf_used = 0;
1356 	etmq->buf_len = aux_buffer->size;
1357 	etmq->buf = aux_buffer->data;
1358 
1359 	return etmq->buf_len;
1360 }
1361 
cs_etm__set_thread(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq,pid_t tid,ocsd_ex_level el)1362 static void cs_etm__set_thread(struct cs_etm_queue *etmq,
1363 			       struct cs_etm_traceid_queue *tidq, pid_t tid,
1364 			       ocsd_ex_level el)
1365 {
1366 	struct machine *machine = cs_etm__get_machine(etmq, el);
1367 
1368 	if (tid != -1) {
1369 		thread__zput(tidq->thread);
1370 		tidq->thread = machine__find_thread(machine, -1, tid);
1371 	}
1372 
1373 	/* Couldn't find a known thread */
1374 	if (!tidq->thread)
1375 		tidq->thread = machine__idle_thread(machine);
1376 
1377 	tidq->el = el;
1378 }
1379 
cs_etm__etmq_set_tid_el(struct cs_etm_queue * etmq,pid_t tid,u8 trace_chan_id,ocsd_ex_level el)1380 int cs_etm__etmq_set_tid_el(struct cs_etm_queue *etmq, pid_t tid,
1381 			    u8 trace_chan_id, ocsd_ex_level el)
1382 {
1383 	struct cs_etm_traceid_queue *tidq;
1384 
1385 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
1386 	if (!tidq)
1387 		return -EINVAL;
1388 
1389 	cs_etm__set_thread(etmq, tidq, tid, el);
1390 	return 0;
1391 }
1392 
cs_etm__etmq_is_timeless(struct cs_etm_queue * etmq)1393 bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq)
1394 {
1395 	return !!etmq->etm->timeless_decoding;
1396 }
1397 
cs_etm__copy_insn(struct cs_etm_queue * etmq,u64 trace_chan_id,const struct cs_etm_packet * packet,struct perf_sample * sample)1398 static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
1399 			      u64 trace_chan_id,
1400 			      const struct cs_etm_packet *packet,
1401 			      struct perf_sample *sample)
1402 {
1403 	/*
1404 	 * It's pointless to read instructions for the CS_ETM_DISCONTINUITY
1405 	 * packet, so directly bail out with 'insn_len' = 0.
1406 	 */
1407 	if (packet->sample_type == CS_ETM_DISCONTINUITY) {
1408 		sample->insn_len = 0;
1409 		return;
1410 	}
1411 
1412 	/*
1413 	 * T32 instruction size might be 32-bit or 16-bit, decide by calling
1414 	 * cs_etm__t32_instr_size().
1415 	 */
1416 	if (packet->isa == CS_ETM_ISA_T32)
1417 		sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id,
1418 							  sample->ip);
1419 	/* Otherwise, A64 and A32 instruction size are always 32-bit. */
1420 	else
1421 		sample->insn_len = 4;
1422 
1423 	cs_etm__mem_access(etmq, trace_chan_id, sample->ip, sample->insn_len,
1424 			   (void *)sample->insn, 0);
1425 }
1426 
cs_etm__convert_sample_time(struct cs_etm_queue * etmq,u64 cs_timestamp)1427 u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp)
1428 {
1429 	struct cs_etm_auxtrace *etm = etmq->etm;
1430 
1431 	if (etm->has_virtual_ts)
1432 		return tsc_to_perf_time(cs_timestamp, &etm->tc);
1433 	else
1434 		return cs_timestamp;
1435 }
1436 
cs_etm__resolve_sample_time(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq)1437 static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq,
1438 					       struct cs_etm_traceid_queue *tidq)
1439 {
1440 	struct cs_etm_auxtrace *etm = etmq->etm;
1441 	struct cs_etm_packet_queue *packet_queue = &tidq->packet_queue;
1442 
1443 	if (!etm->timeless_decoding && etm->has_virtual_ts)
1444 		return packet_queue->cs_timestamp;
1445 	else
1446 		return etm->latest_kernel_timestamp;
1447 }
1448 
cs_etm__synth_instruction_sample(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq,u64 addr,u64 period)1449 static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
1450 					    struct cs_etm_traceid_queue *tidq,
1451 					    u64 addr, u64 period)
1452 {
1453 	int ret = 0;
1454 	struct cs_etm_auxtrace *etm = etmq->etm;
1455 	union perf_event *event = tidq->event_buf;
1456 	struct perf_sample sample = {.ip = 0,};
1457 
1458 	event->sample.header.type = PERF_RECORD_SAMPLE;
1459 	event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, tidq->el);
1460 	event->sample.header.size = sizeof(struct perf_event_header);
1461 
1462 	/* Set time field based on etm auxtrace config. */
1463 	sample.time = cs_etm__resolve_sample_time(etmq, tidq);
1464 
1465 	sample.ip = addr;
1466 	sample.pid = thread__pid(tidq->thread);
1467 	sample.tid = thread__tid(tidq->thread);
1468 	sample.id = etmq->etm->instructions_id;
1469 	sample.stream_id = etmq->etm->instructions_id;
1470 	sample.period = period;
1471 	sample.cpu = tidq->packet->cpu;
1472 	sample.flags = tidq->prev_packet->flags;
1473 	sample.cpumode = event->sample.header.misc;
1474 
1475 	cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);
1476 
1477 	if (etm->synth_opts.last_branch)
1478 		sample.branch_stack = tidq->last_branch;
1479 
1480 	if (etm->synth_opts.inject) {
1481 		ret = cs_etm__inject_event(event, &sample,
1482 					   etm->instructions_sample_type);
1483 		if (ret)
1484 			return ret;
1485 	}
1486 
1487 	ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1488 
1489 	if (ret)
1490 		pr_err(
1491 			"CS ETM Trace: failed to deliver instruction event, error %d\n",
1492 			ret);
1493 
1494 	return ret;
1495 }
1496 
1497 /*
1498  * The cs etm packet encodes an instruction range between a branch target
1499  * and the next taken branch. Generate sample accordingly.
1500  */
cs_etm__synth_branch_sample(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq)1501 static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
1502 				       struct cs_etm_traceid_queue *tidq)
1503 {
1504 	int ret = 0;
1505 	struct cs_etm_auxtrace *etm = etmq->etm;
1506 	struct perf_sample sample = {.ip = 0,};
1507 	union perf_event *event = tidq->event_buf;
1508 	struct dummy_branch_stack {
1509 		u64			nr;
1510 		u64			hw_idx;
1511 		struct branch_entry	entries;
1512 	} dummy_bs;
1513 	u64 ip;
1514 
1515 	ip = cs_etm__last_executed_instr(tidq->prev_packet);
1516 
1517 	event->sample.header.type = PERF_RECORD_SAMPLE;
1518 	event->sample.header.misc = cs_etm__cpu_mode(etmq, ip,
1519 						     tidq->prev_packet_el);
1520 	event->sample.header.size = sizeof(struct perf_event_header);
1521 
1522 	/* Set time field based on etm auxtrace config. */
1523 	sample.time = cs_etm__resolve_sample_time(etmq, tidq);
1524 
1525 	sample.ip = ip;
1526 	sample.pid = thread__pid(tidq->prev_packet_thread);
1527 	sample.tid = thread__tid(tidq->prev_packet_thread);
1528 	sample.addr = cs_etm__first_executed_instr(tidq->packet);
1529 	sample.id = etmq->etm->branches_id;
1530 	sample.stream_id = etmq->etm->branches_id;
1531 	sample.period = 1;
1532 	sample.cpu = tidq->packet->cpu;
1533 	sample.flags = tidq->prev_packet->flags;
1534 	sample.cpumode = event->sample.header.misc;
1535 
1536 	cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,
1537 			  &sample);
1538 
1539 	/*
1540 	 * perf report cannot handle events without a branch stack
1541 	 */
1542 	if (etm->synth_opts.last_branch) {
1543 		dummy_bs = (struct dummy_branch_stack){
1544 			.nr = 1,
1545 			.hw_idx = -1ULL,
1546 			.entries = {
1547 				.from = sample.ip,
1548 				.to = sample.addr,
1549 			},
1550 		};
1551 		sample.branch_stack = (struct branch_stack *)&dummy_bs;
1552 	}
1553 
1554 	if (etm->synth_opts.inject) {
1555 		ret = cs_etm__inject_event(event, &sample,
1556 					   etm->branches_sample_type);
1557 		if (ret)
1558 			return ret;
1559 	}
1560 
1561 	ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1562 
1563 	if (ret)
1564 		pr_err(
1565 		"CS ETM Trace: failed to deliver instruction event, error %d\n",
1566 		ret);
1567 
1568 	return ret;
1569 }
1570 
1571 struct cs_etm_synth {
1572 	struct perf_tool dummy_tool;
1573 	struct perf_session *session;
1574 };
1575 
cs_etm__event_synth(struct perf_tool * tool,union perf_event * event,struct perf_sample * sample __maybe_unused,struct machine * machine __maybe_unused)1576 static int cs_etm__event_synth(struct perf_tool *tool,
1577 			       union perf_event *event,
1578 			       struct perf_sample *sample __maybe_unused,
1579 			       struct machine *machine __maybe_unused)
1580 {
1581 	struct cs_etm_synth *cs_etm_synth =
1582 		      container_of(tool, struct cs_etm_synth, dummy_tool);
1583 
1584 	return perf_session__deliver_synth_event(cs_etm_synth->session,
1585 						 event, NULL);
1586 }
1587 
cs_etm__synth_event(struct perf_session * session,struct perf_event_attr * attr,u64 id)1588 static int cs_etm__synth_event(struct perf_session *session,
1589 			       struct perf_event_attr *attr, u64 id)
1590 {
1591 	struct cs_etm_synth cs_etm_synth;
1592 
1593 	memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth));
1594 	cs_etm_synth.session = session;
1595 
1596 	return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1,
1597 					   &id, cs_etm__event_synth);
1598 }
1599 
cs_etm__synth_events(struct cs_etm_auxtrace * etm,struct perf_session * session)1600 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
1601 				struct perf_session *session)
1602 {
1603 	struct evlist *evlist = session->evlist;
1604 	struct evsel *evsel;
1605 	struct perf_event_attr attr;
1606 	bool found = false;
1607 	u64 id;
1608 	int err;
1609 
1610 	evlist__for_each_entry(evlist, evsel) {
1611 		if (evsel->core.attr.type == etm->pmu_type) {
1612 			found = true;
1613 			break;
1614 		}
1615 	}
1616 
1617 	if (!found) {
1618 		pr_debug("No selected events with CoreSight Trace data\n");
1619 		return 0;
1620 	}
1621 
1622 	memset(&attr, 0, sizeof(struct perf_event_attr));
1623 	attr.size = sizeof(struct perf_event_attr);
1624 	attr.type = PERF_TYPE_HARDWARE;
1625 	attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
1626 	attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1627 			    PERF_SAMPLE_PERIOD;
1628 	if (etm->timeless_decoding)
1629 		attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1630 	else
1631 		attr.sample_type |= PERF_SAMPLE_TIME;
1632 
1633 	attr.exclude_user = evsel->core.attr.exclude_user;
1634 	attr.exclude_kernel = evsel->core.attr.exclude_kernel;
1635 	attr.exclude_hv = evsel->core.attr.exclude_hv;
1636 	attr.exclude_host = evsel->core.attr.exclude_host;
1637 	attr.exclude_guest = evsel->core.attr.exclude_guest;
1638 	attr.sample_id_all = evsel->core.attr.sample_id_all;
1639 	attr.read_format = evsel->core.attr.read_format;
1640 
1641 	/* create new id val to be a fixed offset from evsel id */
1642 	id = evsel->core.id[0] + 1000000000;
1643 
1644 	if (!id)
1645 		id = 1;
1646 
1647 	if (etm->synth_opts.branches) {
1648 		attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1649 		attr.sample_period = 1;
1650 		attr.sample_type |= PERF_SAMPLE_ADDR;
1651 		err = cs_etm__synth_event(session, &attr, id);
1652 		if (err)
1653 			return err;
1654 		etm->branches_sample_type = attr.sample_type;
1655 		etm->branches_id = id;
1656 		id += 1;
1657 		attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
1658 	}
1659 
1660 	if (etm->synth_opts.last_branch) {
1661 		attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1662 		/*
1663 		 * We don't use the hardware index, but the sample generation
1664 		 * code uses the new format branch_stack with this field,
1665 		 * so the event attributes must indicate that it's present.
1666 		 */
1667 		attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX;
1668 	}
1669 
1670 	if (etm->synth_opts.instructions) {
1671 		attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1672 		attr.sample_period = etm->synth_opts.period;
1673 		etm->instructions_sample_period = attr.sample_period;
1674 		err = cs_etm__synth_event(session, &attr, id);
1675 		if (err)
1676 			return err;
1677 		etm->instructions_sample_type = attr.sample_type;
1678 		etm->instructions_id = id;
1679 		id += 1;
1680 	}
1681 
1682 	return 0;
1683 }
1684 
cs_etm__sample(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq)1685 static int cs_etm__sample(struct cs_etm_queue *etmq,
1686 			  struct cs_etm_traceid_queue *tidq)
1687 {
1688 	struct cs_etm_auxtrace *etm = etmq->etm;
1689 	int ret;
1690 	u8 trace_chan_id = tidq->trace_chan_id;
1691 	u64 instrs_prev;
1692 
1693 	/* Get instructions remainder from previous packet */
1694 	instrs_prev = tidq->period_instructions;
1695 
1696 	tidq->period_instructions += tidq->packet->instr_count;
1697 
1698 	/*
1699 	 * Record a branch when the last instruction in
1700 	 * PREV_PACKET is a branch.
1701 	 */
1702 	if (etm->synth_opts.last_branch &&
1703 	    tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1704 	    tidq->prev_packet->last_instr_taken_branch)
1705 		cs_etm__update_last_branch_rb(etmq, tidq);
1706 
1707 	if (etm->synth_opts.instructions &&
1708 	    tidq->period_instructions >= etm->instructions_sample_period) {
1709 		/*
1710 		 * Emit instruction sample periodically
1711 		 * TODO: allow period to be defined in cycles and clock time
1712 		 */
1713 
1714 		/*
1715 		 * Below diagram demonstrates the instruction samples
1716 		 * generation flows:
1717 		 *
1718 		 *    Instrs     Instrs       Instrs       Instrs
1719 		 *   Sample(n)  Sample(n+1)  Sample(n+2)  Sample(n+3)
1720 		 *    |            |            |            |
1721 		 *    V            V            V            V
1722 		 *   --------------------------------------------------
1723 		 *            ^                                  ^
1724 		 *            |                                  |
1725 		 *         Period                             Period
1726 		 *    instructions(Pi)                   instructions(Pi')
1727 		 *
1728 		 *            |                                  |
1729 		 *            \---------------- -----------------/
1730 		 *                             V
1731 		 *                 tidq->packet->instr_count
1732 		 *
1733 		 * Instrs Sample(n...) are the synthesised samples occurring
1734 		 * every etm->instructions_sample_period instructions - as
1735 		 * defined on the perf command line.  Sample(n) is being the
1736 		 * last sample before the current etm packet, n+1 to n+3
1737 		 * samples are generated from the current etm packet.
1738 		 *
1739 		 * tidq->packet->instr_count represents the number of
1740 		 * instructions in the current etm packet.
1741 		 *
1742 		 * Period instructions (Pi) contains the number of
1743 		 * instructions executed after the sample point(n) from the
1744 		 * previous etm packet.  This will always be less than
1745 		 * etm->instructions_sample_period.
1746 		 *
1747 		 * When generate new samples, it combines with two parts
1748 		 * instructions, one is the tail of the old packet and another
1749 		 * is the head of the new coming packet, to generate
1750 		 * sample(n+1); sample(n+2) and sample(n+3) consume the
1751 		 * instructions with sample period.  After sample(n+3), the rest
1752 		 * instructions will be used by later packet and it is assigned
1753 		 * to tidq->period_instructions for next round calculation.
1754 		 */
1755 
1756 		/*
1757 		 * Get the initial offset into the current packet instructions;
1758 		 * entry conditions ensure that instrs_prev is less than
1759 		 * etm->instructions_sample_period.
1760 		 */
1761 		u64 offset = etm->instructions_sample_period - instrs_prev;
1762 		u64 addr;
1763 
1764 		/* Prepare last branches for instruction sample */
1765 		if (etm->synth_opts.last_branch)
1766 			cs_etm__copy_last_branch_rb(etmq, tidq);
1767 
1768 		while (tidq->period_instructions >=
1769 				etm->instructions_sample_period) {
1770 			/*
1771 			 * Calculate the address of the sampled instruction (-1
1772 			 * as sample is reported as though instruction has just
1773 			 * been executed, but PC has not advanced to next
1774 			 * instruction)
1775 			 */
1776 			addr = cs_etm__instr_addr(etmq, trace_chan_id,
1777 						  tidq->packet, offset - 1);
1778 			ret = cs_etm__synth_instruction_sample(
1779 				etmq, tidq, addr,
1780 				etm->instructions_sample_period);
1781 			if (ret)
1782 				return ret;
1783 
1784 			offset += etm->instructions_sample_period;
1785 			tidq->period_instructions -=
1786 				etm->instructions_sample_period;
1787 		}
1788 	}
1789 
1790 	if (etm->synth_opts.branches) {
1791 		bool generate_sample = false;
1792 
1793 		/* Generate sample for tracing on packet */
1794 		if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1795 			generate_sample = true;
1796 
1797 		/* Generate sample for branch taken packet */
1798 		if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1799 		    tidq->prev_packet->last_instr_taken_branch)
1800 			generate_sample = true;
1801 
1802 		if (generate_sample) {
1803 			ret = cs_etm__synth_branch_sample(etmq, tidq);
1804 			if (ret)
1805 				return ret;
1806 		}
1807 	}
1808 
1809 	cs_etm__packet_swap(etm, tidq);
1810 
1811 	return 0;
1812 }
1813 
cs_etm__exception(struct cs_etm_traceid_queue * tidq)1814 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
1815 {
1816 	/*
1817 	 * When the exception packet is inserted, whether the last instruction
1818 	 * in previous range packet is taken branch or not, we need to force
1819 	 * to set 'prev_packet->last_instr_taken_branch' to true.  This ensures
1820 	 * to generate branch sample for the instruction range before the
1821 	 * exception is trapped to kernel or before the exception returning.
1822 	 *
1823 	 * The exception packet includes the dummy address values, so don't
1824 	 * swap PACKET with PREV_PACKET.  This keeps PREV_PACKET to be useful
1825 	 * for generating instruction and branch samples.
1826 	 */
1827 	if (tidq->prev_packet->sample_type == CS_ETM_RANGE)
1828 		tidq->prev_packet->last_instr_taken_branch = true;
1829 
1830 	return 0;
1831 }
1832 
cs_etm__flush(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq)1833 static int cs_etm__flush(struct cs_etm_queue *etmq,
1834 			 struct cs_etm_traceid_queue *tidq)
1835 {
1836 	int err = 0;
1837 	struct cs_etm_auxtrace *etm = etmq->etm;
1838 
1839 	/* Handle start tracing packet */
1840 	if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)
1841 		goto swap_packet;
1842 
1843 	if (etmq->etm->synth_opts.last_branch &&
1844 	    etmq->etm->synth_opts.instructions &&
1845 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1846 		u64 addr;
1847 
1848 		/* Prepare last branches for instruction sample */
1849 		cs_etm__copy_last_branch_rb(etmq, tidq);
1850 
1851 		/*
1852 		 * Generate a last branch event for the branches left in the
1853 		 * circular buffer at the end of the trace.
1854 		 *
1855 		 * Use the address of the end of the last reported execution
1856 		 * range
1857 		 */
1858 		addr = cs_etm__last_executed_instr(tidq->prev_packet);
1859 
1860 		err = cs_etm__synth_instruction_sample(
1861 			etmq, tidq, addr,
1862 			tidq->period_instructions);
1863 		if (err)
1864 			return err;
1865 
1866 		tidq->period_instructions = 0;
1867 
1868 	}
1869 
1870 	if (etm->synth_opts.branches &&
1871 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1872 		err = cs_etm__synth_branch_sample(etmq, tidq);
1873 		if (err)
1874 			return err;
1875 	}
1876 
1877 swap_packet:
1878 	cs_etm__packet_swap(etm, tidq);
1879 
1880 	/* Reset last branches after flush the trace */
1881 	if (etm->synth_opts.last_branch)
1882 		cs_etm__reset_last_branch_rb(tidq);
1883 
1884 	return err;
1885 }
1886 
cs_etm__end_block(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq)1887 static int cs_etm__end_block(struct cs_etm_queue *etmq,
1888 			     struct cs_etm_traceid_queue *tidq)
1889 {
1890 	int err;
1891 
1892 	/*
1893 	 * It has no new packet coming and 'etmq->packet' contains the stale
1894 	 * packet which was set at the previous time with packets swapping;
1895 	 * so skip to generate branch sample to avoid stale packet.
1896 	 *
1897 	 * For this case only flush branch stack and generate a last branch
1898 	 * event for the branches left in the circular buffer at the end of
1899 	 * the trace.
1900 	 */
1901 	if (etmq->etm->synth_opts.last_branch &&
1902 	    etmq->etm->synth_opts.instructions &&
1903 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1904 		u64 addr;
1905 
1906 		/* Prepare last branches for instruction sample */
1907 		cs_etm__copy_last_branch_rb(etmq, tidq);
1908 
1909 		/*
1910 		 * Use the address of the end of the last reported execution
1911 		 * range.
1912 		 */
1913 		addr = cs_etm__last_executed_instr(tidq->prev_packet);
1914 
1915 		err = cs_etm__synth_instruction_sample(
1916 			etmq, tidq, addr,
1917 			tidq->period_instructions);
1918 		if (err)
1919 			return err;
1920 
1921 		tidq->period_instructions = 0;
1922 	}
1923 
1924 	return 0;
1925 }
1926 /*
1927  * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
1928  *			   if need be.
1929  * Returns:	< 0	if error
1930  *		= 0	if no more auxtrace_buffer to read
1931  *		> 0	if the current buffer isn't empty yet
1932  */
cs_etm__get_data_block(struct cs_etm_queue * etmq)1933 static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
1934 {
1935 	int ret;
1936 
1937 	if (!etmq->buf_len) {
1938 		ret = cs_etm__get_trace(etmq);
1939 		if (ret <= 0)
1940 			return ret;
1941 		/*
1942 		 * We cannot assume consecutive blocks in the data file
1943 		 * are contiguous, reset the decoder to force re-sync.
1944 		 */
1945 		ret = cs_etm_decoder__reset(etmq->decoder);
1946 		if (ret)
1947 			return ret;
1948 	}
1949 
1950 	return etmq->buf_len;
1951 }
1952 
cs_etm__is_svc_instr(struct cs_etm_queue * etmq,u8 trace_chan_id,struct cs_etm_packet * packet,u64 end_addr)1953 static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
1954 				 struct cs_etm_packet *packet,
1955 				 u64 end_addr)
1956 {
1957 	/* Initialise to keep compiler happy */
1958 	u16 instr16 = 0;
1959 	u32 instr32 = 0;
1960 	u64 addr;
1961 
1962 	switch (packet->isa) {
1963 	case CS_ETM_ISA_T32:
1964 		/*
1965 		 * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
1966 		 *
1967 		 *  b'15         b'8
1968 		 * +-----------------+--------+
1969 		 * | 1 1 0 1 1 1 1 1 |  imm8  |
1970 		 * +-----------------+--------+
1971 		 *
1972 		 * According to the specification, it only defines SVC for T32
1973 		 * with 16 bits instruction and has no definition for 32bits;
1974 		 * so below only read 2 bytes as instruction size for T32.
1975 		 */
1976 		addr = end_addr - 2;
1977 		cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr16),
1978 				   (u8 *)&instr16, 0);
1979 		if ((instr16 & 0xFF00) == 0xDF00)
1980 			return true;
1981 
1982 		break;
1983 	case CS_ETM_ISA_A32:
1984 		/*
1985 		 * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
1986 		 *
1987 		 *  b'31 b'28 b'27 b'24
1988 		 * +---------+---------+-------------------------+
1989 		 * |  !1111  | 1 1 1 1 |        imm24            |
1990 		 * +---------+---------+-------------------------+
1991 		 */
1992 		addr = end_addr - 4;
1993 		cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32),
1994 				   (u8 *)&instr32, 0);
1995 		if ((instr32 & 0x0F000000) == 0x0F000000 &&
1996 		    (instr32 & 0xF0000000) != 0xF0000000)
1997 			return true;
1998 
1999 		break;
2000 	case CS_ETM_ISA_A64:
2001 		/*
2002 		 * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
2003 		 *
2004 		 *  b'31               b'21           b'4     b'0
2005 		 * +-----------------------+---------+-----------+
2006 		 * | 1 1 0 1 0 1 0 0 0 0 0 |  imm16  | 0 0 0 0 1 |
2007 		 * +-----------------------+---------+-----------+
2008 		 */
2009 		addr = end_addr - 4;
2010 		cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32),
2011 				   (u8 *)&instr32, 0);
2012 		if ((instr32 & 0xFFE0001F) == 0xd4000001)
2013 			return true;
2014 
2015 		break;
2016 	case CS_ETM_ISA_UNKNOWN:
2017 	default:
2018 		break;
2019 	}
2020 
2021 	return false;
2022 }
2023 
cs_etm__is_syscall(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq,u64 magic)2024 static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
2025 			       struct cs_etm_traceid_queue *tidq, u64 magic)
2026 {
2027 	u8 trace_chan_id = tidq->trace_chan_id;
2028 	struct cs_etm_packet *packet = tidq->packet;
2029 	struct cs_etm_packet *prev_packet = tidq->prev_packet;
2030 
2031 	if (magic == __perf_cs_etmv3_magic)
2032 		if (packet->exception_number == CS_ETMV3_EXC_SVC)
2033 			return true;
2034 
2035 	/*
2036 	 * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
2037 	 * HVC cases; need to check if it's SVC instruction based on
2038 	 * packet address.
2039 	 */
2040 	if (magic == __perf_cs_etmv4_magic) {
2041 		if (packet->exception_number == CS_ETMV4_EXC_CALL &&
2042 		    cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
2043 					 prev_packet->end_addr))
2044 			return true;
2045 	}
2046 
2047 	return false;
2048 }
2049 
cs_etm__is_async_exception(struct cs_etm_traceid_queue * tidq,u64 magic)2050 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
2051 				       u64 magic)
2052 {
2053 	struct cs_etm_packet *packet = tidq->packet;
2054 
2055 	if (magic == __perf_cs_etmv3_magic)
2056 		if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT ||
2057 		    packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT ||
2058 		    packet->exception_number == CS_ETMV3_EXC_PE_RESET ||
2059 		    packet->exception_number == CS_ETMV3_EXC_IRQ ||
2060 		    packet->exception_number == CS_ETMV3_EXC_FIQ)
2061 			return true;
2062 
2063 	if (magic == __perf_cs_etmv4_magic)
2064 		if (packet->exception_number == CS_ETMV4_EXC_RESET ||
2065 		    packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT ||
2066 		    packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR ||
2067 		    packet->exception_number == CS_ETMV4_EXC_INST_DEBUG ||
2068 		    packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG ||
2069 		    packet->exception_number == CS_ETMV4_EXC_IRQ ||
2070 		    packet->exception_number == CS_ETMV4_EXC_FIQ)
2071 			return true;
2072 
2073 	return false;
2074 }
2075 
cs_etm__is_sync_exception(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq,u64 magic)2076 static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
2077 				      struct cs_etm_traceid_queue *tidq,
2078 				      u64 magic)
2079 {
2080 	u8 trace_chan_id = tidq->trace_chan_id;
2081 	struct cs_etm_packet *packet = tidq->packet;
2082 	struct cs_etm_packet *prev_packet = tidq->prev_packet;
2083 
2084 	if (magic == __perf_cs_etmv3_magic)
2085 		if (packet->exception_number == CS_ETMV3_EXC_SMC ||
2086 		    packet->exception_number == CS_ETMV3_EXC_HYP ||
2087 		    packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE ||
2088 		    packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR ||
2089 		    packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT ||
2090 		    packet->exception_number == CS_ETMV3_EXC_DATA_FAULT ||
2091 		    packet->exception_number == CS_ETMV3_EXC_GENERIC)
2092 			return true;
2093 
2094 	if (magic == __perf_cs_etmv4_magic) {
2095 		if (packet->exception_number == CS_ETMV4_EXC_TRAP ||
2096 		    packet->exception_number == CS_ETMV4_EXC_ALIGNMENT ||
2097 		    packet->exception_number == CS_ETMV4_EXC_INST_FAULT ||
2098 		    packet->exception_number == CS_ETMV4_EXC_DATA_FAULT)
2099 			return true;
2100 
2101 		/*
2102 		 * For CS_ETMV4_EXC_CALL, except SVC other instructions
2103 		 * (SMC, HVC) are taken as sync exceptions.
2104 		 */
2105 		if (packet->exception_number == CS_ETMV4_EXC_CALL &&
2106 		    !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
2107 					  prev_packet->end_addr))
2108 			return true;
2109 
2110 		/*
2111 		 * ETMv4 has 5 bits for exception number; if the numbers
2112 		 * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
2113 		 * they are implementation defined exceptions.
2114 		 *
2115 		 * For this case, simply take it as sync exception.
2116 		 */
2117 		if (packet->exception_number > CS_ETMV4_EXC_FIQ &&
2118 		    packet->exception_number <= CS_ETMV4_EXC_END)
2119 			return true;
2120 	}
2121 
2122 	return false;
2123 }
2124 
cs_etm__set_sample_flags(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq)2125 static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
2126 				    struct cs_etm_traceid_queue *tidq)
2127 {
2128 	struct cs_etm_packet *packet = tidq->packet;
2129 	struct cs_etm_packet *prev_packet = tidq->prev_packet;
2130 	u8 trace_chan_id = tidq->trace_chan_id;
2131 	u64 magic;
2132 	int ret;
2133 
2134 	switch (packet->sample_type) {
2135 	case CS_ETM_RANGE:
2136 		/*
2137 		 * Immediate branch instruction without neither link nor
2138 		 * return flag, it's normal branch instruction within
2139 		 * the function.
2140 		 */
2141 		if (packet->last_instr_type == OCSD_INSTR_BR &&
2142 		    packet->last_instr_subtype == OCSD_S_INSTR_NONE) {
2143 			packet->flags = PERF_IP_FLAG_BRANCH;
2144 
2145 			if (packet->last_instr_cond)
2146 				packet->flags |= PERF_IP_FLAG_CONDITIONAL;
2147 		}
2148 
2149 		/*
2150 		 * Immediate branch instruction with link (e.g. BL), this is
2151 		 * branch instruction for function call.
2152 		 */
2153 		if (packet->last_instr_type == OCSD_INSTR_BR &&
2154 		    packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
2155 			packet->flags = PERF_IP_FLAG_BRANCH |
2156 					PERF_IP_FLAG_CALL;
2157 
2158 		/*
2159 		 * Indirect branch instruction with link (e.g. BLR), this is
2160 		 * branch instruction for function call.
2161 		 */
2162 		if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2163 		    packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
2164 			packet->flags = PERF_IP_FLAG_BRANCH |
2165 					PERF_IP_FLAG_CALL;
2166 
2167 		/*
2168 		 * Indirect branch instruction with subtype of
2169 		 * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
2170 		 * function return for A32/T32.
2171 		 */
2172 		if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2173 		    packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET)
2174 			packet->flags = PERF_IP_FLAG_BRANCH |
2175 					PERF_IP_FLAG_RETURN;
2176 
2177 		/*
2178 		 * Indirect branch instruction without link (e.g. BR), usually
2179 		 * this is used for function return, especially for functions
2180 		 * within dynamic link lib.
2181 		 */
2182 		if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2183 		    packet->last_instr_subtype == OCSD_S_INSTR_NONE)
2184 			packet->flags = PERF_IP_FLAG_BRANCH |
2185 					PERF_IP_FLAG_RETURN;
2186 
2187 		/* Return instruction for function return. */
2188 		if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2189 		    packet->last_instr_subtype == OCSD_S_INSTR_V8_RET)
2190 			packet->flags = PERF_IP_FLAG_BRANCH |
2191 					PERF_IP_FLAG_RETURN;
2192 
2193 		/*
2194 		 * Decoder might insert a discontinuity in the middle of
2195 		 * instruction packets, fixup prev_packet with flag
2196 		 * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
2197 		 */
2198 		if (prev_packet->sample_type == CS_ETM_DISCONTINUITY)
2199 			prev_packet->flags |= PERF_IP_FLAG_BRANCH |
2200 					      PERF_IP_FLAG_TRACE_BEGIN;
2201 
2202 		/*
2203 		 * If the previous packet is an exception return packet
2204 		 * and the return address just follows SVC instruction,
2205 		 * it needs to calibrate the previous packet sample flags
2206 		 * as PERF_IP_FLAG_SYSCALLRET.
2207 		 */
2208 		if (prev_packet->flags == (PERF_IP_FLAG_BRANCH |
2209 					   PERF_IP_FLAG_RETURN |
2210 					   PERF_IP_FLAG_INTERRUPT) &&
2211 		    cs_etm__is_svc_instr(etmq, trace_chan_id,
2212 					 packet, packet->start_addr))
2213 			prev_packet->flags = PERF_IP_FLAG_BRANCH |
2214 					     PERF_IP_FLAG_RETURN |
2215 					     PERF_IP_FLAG_SYSCALLRET;
2216 		break;
2217 	case CS_ETM_DISCONTINUITY:
2218 		/*
2219 		 * The trace is discontinuous, if the previous packet is
2220 		 * instruction packet, set flag PERF_IP_FLAG_TRACE_END
2221 		 * for previous packet.
2222 		 */
2223 		if (prev_packet->sample_type == CS_ETM_RANGE)
2224 			prev_packet->flags |= PERF_IP_FLAG_BRANCH |
2225 					      PERF_IP_FLAG_TRACE_END;
2226 		break;
2227 	case CS_ETM_EXCEPTION:
2228 		ret = cs_etm__get_magic(packet->trace_chan_id, &magic);
2229 		if (ret)
2230 			return ret;
2231 
2232 		/* The exception is for system call. */
2233 		if (cs_etm__is_syscall(etmq, tidq, magic))
2234 			packet->flags = PERF_IP_FLAG_BRANCH |
2235 					PERF_IP_FLAG_CALL |
2236 					PERF_IP_FLAG_SYSCALLRET;
2237 		/*
2238 		 * The exceptions are triggered by external signals from bus,
2239 		 * interrupt controller, debug module, PE reset or halt.
2240 		 */
2241 		else if (cs_etm__is_async_exception(tidq, magic))
2242 			packet->flags = PERF_IP_FLAG_BRANCH |
2243 					PERF_IP_FLAG_CALL |
2244 					PERF_IP_FLAG_ASYNC |
2245 					PERF_IP_FLAG_INTERRUPT;
2246 		/*
2247 		 * Otherwise, exception is caused by trap, instruction &
2248 		 * data fault, or alignment errors.
2249 		 */
2250 		else if (cs_etm__is_sync_exception(etmq, tidq, magic))
2251 			packet->flags = PERF_IP_FLAG_BRANCH |
2252 					PERF_IP_FLAG_CALL |
2253 					PERF_IP_FLAG_INTERRUPT;
2254 
2255 		/*
2256 		 * When the exception packet is inserted, since exception
2257 		 * packet is not used standalone for generating samples
2258 		 * and it's affiliation to the previous instruction range
2259 		 * packet; so set previous range packet flags to tell perf
2260 		 * it is an exception taken branch.
2261 		 */
2262 		if (prev_packet->sample_type == CS_ETM_RANGE)
2263 			prev_packet->flags = packet->flags;
2264 		break;
2265 	case CS_ETM_EXCEPTION_RET:
2266 		/*
2267 		 * When the exception return packet is inserted, since
2268 		 * exception return packet is not used standalone for
2269 		 * generating samples and it's affiliation to the previous
2270 		 * instruction range packet; so set previous range packet
2271 		 * flags to tell perf it is an exception return branch.
2272 		 *
2273 		 * The exception return can be for either system call or
2274 		 * other exception types; unfortunately the packet doesn't
2275 		 * contain exception type related info so we cannot decide
2276 		 * the exception type purely based on exception return packet.
2277 		 * If we record the exception number from exception packet and
2278 		 * reuse it for exception return packet, this is not reliable
2279 		 * due the trace can be discontinuity or the interrupt can
2280 		 * be nested, thus the recorded exception number cannot be
2281 		 * used for exception return packet for these two cases.
2282 		 *
2283 		 * For exception return packet, we only need to distinguish the
2284 		 * packet is for system call or for other types.  Thus the
2285 		 * decision can be deferred when receive the next packet which
2286 		 * contains the return address, based on the return address we
2287 		 * can read out the previous instruction and check if it's a
2288 		 * system call instruction and then calibrate the sample flag
2289 		 * as needed.
2290 		 */
2291 		if (prev_packet->sample_type == CS_ETM_RANGE)
2292 			prev_packet->flags = PERF_IP_FLAG_BRANCH |
2293 					     PERF_IP_FLAG_RETURN |
2294 					     PERF_IP_FLAG_INTERRUPT;
2295 		break;
2296 	case CS_ETM_EMPTY:
2297 	default:
2298 		break;
2299 	}
2300 
2301 	return 0;
2302 }
2303 
cs_etm__decode_data_block(struct cs_etm_queue * etmq)2304 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq)
2305 {
2306 	int ret = 0;
2307 	size_t processed = 0;
2308 
2309 	/*
2310 	 * Packets are decoded and added to the decoder's packet queue
2311 	 * until the decoder packet processing callback has requested that
2312 	 * processing stops or there is nothing left in the buffer.  Normal
2313 	 * operations that stop processing are a timestamp packet or a full
2314 	 * decoder buffer queue.
2315 	 */
2316 	ret = cs_etm_decoder__process_data_block(etmq->decoder,
2317 						 etmq->offset,
2318 						 &etmq->buf[etmq->buf_used],
2319 						 etmq->buf_len,
2320 						 &processed);
2321 	if (ret)
2322 		goto out;
2323 
2324 	etmq->offset += processed;
2325 	etmq->buf_used += processed;
2326 	etmq->buf_len -= processed;
2327 
2328 out:
2329 	return ret;
2330 }
2331 
cs_etm__process_traceid_queue(struct cs_etm_queue * etmq,struct cs_etm_traceid_queue * tidq)2332 static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
2333 					 struct cs_etm_traceid_queue *tidq)
2334 {
2335 	int ret;
2336 	struct cs_etm_packet_queue *packet_queue;
2337 
2338 	packet_queue = &tidq->packet_queue;
2339 
2340 	/* Process each packet in this chunk */
2341 	while (1) {
2342 		ret = cs_etm_decoder__get_packet(packet_queue,
2343 						 tidq->packet);
2344 		if (ret <= 0)
2345 			/*
2346 			 * Stop processing this chunk on
2347 			 * end of data or error
2348 			 */
2349 			break;
2350 
2351 		/*
2352 		 * Since packet addresses are swapped in packet
2353 		 * handling within below switch() statements,
2354 		 * thus setting sample flags must be called
2355 		 * prior to switch() statement to use address
2356 		 * information before packets swapping.
2357 		 */
2358 		ret = cs_etm__set_sample_flags(etmq, tidq);
2359 		if (ret < 0)
2360 			break;
2361 
2362 		switch (tidq->packet->sample_type) {
2363 		case CS_ETM_RANGE:
2364 			/*
2365 			 * If the packet contains an instruction
2366 			 * range, generate instruction sequence
2367 			 * events.
2368 			 */
2369 			cs_etm__sample(etmq, tidq);
2370 			break;
2371 		case CS_ETM_EXCEPTION:
2372 		case CS_ETM_EXCEPTION_RET:
2373 			/*
2374 			 * If the exception packet is coming,
2375 			 * make sure the previous instruction
2376 			 * range packet to be handled properly.
2377 			 */
2378 			cs_etm__exception(tidq);
2379 			break;
2380 		case CS_ETM_DISCONTINUITY:
2381 			/*
2382 			 * Discontinuity in trace, flush
2383 			 * previous branch stack
2384 			 */
2385 			cs_etm__flush(etmq, tidq);
2386 			break;
2387 		case CS_ETM_EMPTY:
2388 			/*
2389 			 * Should not receive empty packet,
2390 			 * report error.
2391 			 */
2392 			pr_err("CS ETM Trace: empty packet\n");
2393 			return -EINVAL;
2394 		default:
2395 			break;
2396 		}
2397 	}
2398 
2399 	return ret;
2400 }
2401 
cs_etm__clear_all_traceid_queues(struct cs_etm_queue * etmq)2402 static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq)
2403 {
2404 	int idx;
2405 	struct int_node *inode;
2406 	struct cs_etm_traceid_queue *tidq;
2407 	struct intlist *traceid_queues_list = etmq->traceid_queues_list;
2408 
2409 	intlist__for_each_entry(inode, traceid_queues_list) {
2410 		idx = (int)(intptr_t)inode->priv;
2411 		tidq = etmq->traceid_queues[idx];
2412 
2413 		/* Ignore return value */
2414 		cs_etm__process_traceid_queue(etmq, tidq);
2415 
2416 		/*
2417 		 * Generate an instruction sample with the remaining
2418 		 * branchstack entries.
2419 		 */
2420 		cs_etm__flush(etmq, tidq);
2421 	}
2422 }
2423 
cs_etm__run_per_thread_timeless_decoder(struct cs_etm_queue * etmq)2424 static int cs_etm__run_per_thread_timeless_decoder(struct cs_etm_queue *etmq)
2425 {
2426 	int err = 0;
2427 	struct cs_etm_traceid_queue *tidq;
2428 
2429 	tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
2430 	if (!tidq)
2431 		return -EINVAL;
2432 
2433 	/* Go through each buffer in the queue and decode them one by one */
2434 	while (1) {
2435 		err = cs_etm__get_data_block(etmq);
2436 		if (err <= 0)
2437 			return err;
2438 
2439 		/* Run trace decoder until buffer consumed or end of trace */
2440 		do {
2441 			err = cs_etm__decode_data_block(etmq);
2442 			if (err)
2443 				return err;
2444 
2445 			/*
2446 			 * Process each packet in this chunk, nothing to do if
2447 			 * an error occurs other than hoping the next one will
2448 			 * be better.
2449 			 */
2450 			err = cs_etm__process_traceid_queue(etmq, tidq);
2451 
2452 		} while (etmq->buf_len);
2453 
2454 		if (err == 0)
2455 			/* Flush any remaining branch stack entries */
2456 			err = cs_etm__end_block(etmq, tidq);
2457 	}
2458 
2459 	return err;
2460 }
2461 
cs_etm__run_per_cpu_timeless_decoder(struct cs_etm_queue * etmq)2462 static int cs_etm__run_per_cpu_timeless_decoder(struct cs_etm_queue *etmq)
2463 {
2464 	int idx, err = 0;
2465 	struct cs_etm_traceid_queue *tidq;
2466 	struct int_node *inode;
2467 
2468 	/* Go through each buffer in the queue and decode them one by one */
2469 	while (1) {
2470 		err = cs_etm__get_data_block(etmq);
2471 		if (err <= 0)
2472 			return err;
2473 
2474 		/* Run trace decoder until buffer consumed or end of trace */
2475 		do {
2476 			err = cs_etm__decode_data_block(etmq);
2477 			if (err)
2478 				return err;
2479 
2480 			/*
2481 			 * cs_etm__run_per_thread_timeless_decoder() runs on a
2482 			 * single traceID queue because each TID has a separate
2483 			 * buffer. But here in per-cpu mode we need to iterate
2484 			 * over each channel instead.
2485 			 */
2486 			intlist__for_each_entry(inode,
2487 						etmq->traceid_queues_list) {
2488 				idx = (int)(intptr_t)inode->priv;
2489 				tidq = etmq->traceid_queues[idx];
2490 				cs_etm__process_traceid_queue(etmq, tidq);
2491 			}
2492 		} while (etmq->buf_len);
2493 
2494 		intlist__for_each_entry(inode, etmq->traceid_queues_list) {
2495 			idx = (int)(intptr_t)inode->priv;
2496 			tidq = etmq->traceid_queues[idx];
2497 			/* Flush any remaining branch stack entries */
2498 			err = cs_etm__end_block(etmq, tidq);
2499 			if (err)
2500 				return err;
2501 		}
2502 	}
2503 
2504 	return err;
2505 }
2506 
cs_etm__process_timeless_queues(struct cs_etm_auxtrace * etm,pid_t tid)2507 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
2508 					   pid_t tid)
2509 {
2510 	unsigned int i;
2511 	struct auxtrace_queues *queues = &etm->queues;
2512 
2513 	for (i = 0; i < queues->nr_queues; i++) {
2514 		struct auxtrace_queue *queue = &etm->queues.queue_array[i];
2515 		struct cs_etm_queue *etmq = queue->priv;
2516 		struct cs_etm_traceid_queue *tidq;
2517 
2518 		if (!etmq)
2519 			continue;
2520 
2521 		if (etm->per_thread_decoding) {
2522 			tidq = cs_etm__etmq_get_traceid_queue(
2523 				etmq, CS_ETM_PER_THREAD_TRACEID);
2524 
2525 			if (!tidq)
2526 				continue;
2527 
2528 			if (tid == -1 || thread__tid(tidq->thread) == tid)
2529 				cs_etm__run_per_thread_timeless_decoder(etmq);
2530 		} else
2531 			cs_etm__run_per_cpu_timeless_decoder(etmq);
2532 	}
2533 
2534 	return 0;
2535 }
2536 
cs_etm__process_timestamped_queues(struct cs_etm_auxtrace * etm)2537 static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm)
2538 {
2539 	int ret = 0;
2540 	unsigned int cs_queue_nr, queue_nr, i;
2541 	u8 trace_chan_id;
2542 	u64 cs_timestamp;
2543 	struct auxtrace_queue *queue;
2544 	struct cs_etm_queue *etmq;
2545 	struct cs_etm_traceid_queue *tidq;
2546 
2547 	/*
2548 	 * Pre-populate the heap with one entry from each queue so that we can
2549 	 * start processing in time order across all queues.
2550 	 */
2551 	for (i = 0; i < etm->queues.nr_queues; i++) {
2552 		etmq = etm->queues.queue_array[i].priv;
2553 		if (!etmq)
2554 			continue;
2555 
2556 		ret = cs_etm__queue_first_cs_timestamp(etm, etmq, i);
2557 		if (ret)
2558 			return ret;
2559 	}
2560 
2561 	while (1) {
2562 		if (!etm->heap.heap_cnt)
2563 			goto out;
2564 
2565 		/* Take the entry at the top of the min heap */
2566 		cs_queue_nr = etm->heap.heap_array[0].queue_nr;
2567 		queue_nr = TO_QUEUE_NR(cs_queue_nr);
2568 		trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr);
2569 		queue = &etm->queues.queue_array[queue_nr];
2570 		etmq = queue->priv;
2571 
2572 		/*
2573 		 * Remove the top entry from the heap since we are about
2574 		 * to process it.
2575 		 */
2576 		auxtrace_heap__pop(&etm->heap);
2577 
2578 		tidq  = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
2579 		if (!tidq) {
2580 			/*
2581 			 * No traceID queue has been allocated for this traceID,
2582 			 * which means something somewhere went very wrong.  No
2583 			 * other choice than simply exit.
2584 			 */
2585 			ret = -EINVAL;
2586 			goto out;
2587 		}
2588 
2589 		/*
2590 		 * Packets associated with this timestamp are already in
2591 		 * the etmq's traceID queue, so process them.
2592 		 */
2593 		ret = cs_etm__process_traceid_queue(etmq, tidq);
2594 		if (ret < 0)
2595 			goto out;
2596 
2597 		/*
2598 		 * Packets for this timestamp have been processed, time to
2599 		 * move on to the next timestamp, fetching a new auxtrace_buffer
2600 		 * if need be.
2601 		 */
2602 refetch:
2603 		ret = cs_etm__get_data_block(etmq);
2604 		if (ret < 0)
2605 			goto out;
2606 
2607 		/*
2608 		 * No more auxtrace_buffers to process in this etmq, simply
2609 		 * move on to another entry in the auxtrace_heap.
2610 		 */
2611 		if (!ret)
2612 			continue;
2613 
2614 		ret = cs_etm__decode_data_block(etmq);
2615 		if (ret)
2616 			goto out;
2617 
2618 		cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
2619 
2620 		if (!cs_timestamp) {
2621 			/*
2622 			 * Function cs_etm__decode_data_block() returns when
2623 			 * there is no more traces to decode in the current
2624 			 * auxtrace_buffer OR when a timestamp has been
2625 			 * encountered on any of the traceID queues.  Since we
2626 			 * did not get a timestamp, there is no more traces to
2627 			 * process in this auxtrace_buffer.  As such empty and
2628 			 * flush all traceID queues.
2629 			 */
2630 			cs_etm__clear_all_traceid_queues(etmq);
2631 
2632 			/* Fetch another auxtrace_buffer for this etmq */
2633 			goto refetch;
2634 		}
2635 
2636 		/*
2637 		 * Add to the min heap the timestamp for packets that have
2638 		 * just been decoded.  They will be processed and synthesized
2639 		 * during the next call to cs_etm__process_traceid_queue() for
2640 		 * this queue/traceID.
2641 		 */
2642 		cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
2643 		ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);
2644 	}
2645 
2646 out:
2647 	return ret;
2648 }
2649 
cs_etm__process_itrace_start(struct cs_etm_auxtrace * etm,union perf_event * event)2650 static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,
2651 					union perf_event *event)
2652 {
2653 	struct thread *th;
2654 
2655 	if (etm->timeless_decoding)
2656 		return 0;
2657 
2658 	/*
2659 	 * Add the tid/pid to the log so that we can get a match when we get a
2660 	 * contextID from the decoder. Only track for the host: only kernel
2661 	 * trace is supported for guests which wouldn't need pids so this should
2662 	 * be fine.
2663 	 */
2664 	th = machine__findnew_thread(&etm->session->machines.host,
2665 				     event->itrace_start.pid,
2666 				     event->itrace_start.tid);
2667 	if (!th)
2668 		return -ENOMEM;
2669 
2670 	thread__put(th);
2671 
2672 	return 0;
2673 }
2674 
cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace * etm,union perf_event * event)2675 static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,
2676 					   union perf_event *event)
2677 {
2678 	struct thread *th;
2679 	bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
2680 
2681 	/*
2682 	 * Context switch in per-thread mode are irrelevant since perf
2683 	 * will start/stop tracing as the process is scheduled.
2684 	 */
2685 	if (etm->timeless_decoding)
2686 		return 0;
2687 
2688 	/*
2689 	 * SWITCH_IN events carry the next process to be switched out while
2690 	 * SWITCH_OUT events carry the process to be switched in.  As such
2691 	 * we don't care about IN events.
2692 	 */
2693 	if (!out)
2694 		return 0;
2695 
2696 	/*
2697 	 * Add the tid/pid to the log so that we can get a match when we get a
2698 	 * contextID from the decoder. Only track for the host: only kernel
2699 	 * trace is supported for guests which wouldn't need pids so this should
2700 	 * be fine.
2701 	 */
2702 	th = machine__findnew_thread(&etm->session->machines.host,
2703 				     event->context_switch.next_prev_pid,
2704 				     event->context_switch.next_prev_tid);
2705 	if (!th)
2706 		return -ENOMEM;
2707 
2708 	thread__put(th);
2709 
2710 	return 0;
2711 }
2712 
cs_etm__process_event(struct perf_session * session,union perf_event * event,struct perf_sample * sample,struct perf_tool * tool)2713 static int cs_etm__process_event(struct perf_session *session,
2714 				 union perf_event *event,
2715 				 struct perf_sample *sample,
2716 				 struct perf_tool *tool)
2717 {
2718 	struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2719 						   struct cs_etm_auxtrace,
2720 						   auxtrace);
2721 
2722 	if (dump_trace)
2723 		return 0;
2724 
2725 	if (!tool->ordered_events) {
2726 		pr_err("CoreSight ETM Trace requires ordered events\n");
2727 		return -EINVAL;
2728 	}
2729 
2730 	switch (event->header.type) {
2731 	case PERF_RECORD_EXIT:
2732 		/*
2733 		 * Don't need to wait for cs_etm__flush_events() in per-thread mode to
2734 		 * start the decode because we know there will be no more trace from
2735 		 * this thread. All this does is emit samples earlier than waiting for
2736 		 * the flush in other modes, but with timestamps it makes sense to wait
2737 		 * for flush so that events from different threads are interleaved
2738 		 * properly.
2739 		 */
2740 		if (etm->per_thread_decoding && etm->timeless_decoding)
2741 			return cs_etm__process_timeless_queues(etm,
2742 							       event->fork.tid);
2743 		break;
2744 
2745 	case PERF_RECORD_ITRACE_START:
2746 		return cs_etm__process_itrace_start(etm, event);
2747 
2748 	case PERF_RECORD_SWITCH_CPU_WIDE:
2749 		return cs_etm__process_switch_cpu_wide(etm, event);
2750 
2751 	case PERF_RECORD_AUX:
2752 		/*
2753 		 * Record the latest kernel timestamp available in the header
2754 		 * for samples so that synthesised samples occur from this point
2755 		 * onwards.
2756 		 */
2757 		if (sample->time && (sample->time != (u64)-1))
2758 			etm->latest_kernel_timestamp = sample->time;
2759 		break;
2760 
2761 	default:
2762 		break;
2763 	}
2764 
2765 	return 0;
2766 }
2767 
dump_queued_data(struct cs_etm_auxtrace * etm,struct perf_record_auxtrace * event)2768 static void dump_queued_data(struct cs_etm_auxtrace *etm,
2769 			     struct perf_record_auxtrace *event)
2770 {
2771 	struct auxtrace_buffer *buf;
2772 	unsigned int i;
2773 	/*
2774 	 * Find all buffers with same reference in the queues and dump them.
2775 	 * This is because the queues can contain multiple entries of the same
2776 	 * buffer that were split on aux records.
2777 	 */
2778 	for (i = 0; i < etm->queues.nr_queues; ++i)
2779 		list_for_each_entry(buf, &etm->queues.queue_array[i].head, list)
2780 			if (buf->reference == event->reference)
2781 				cs_etm__dump_event(etm->queues.queue_array[i].priv, buf);
2782 }
2783 
cs_etm__process_auxtrace_event(struct perf_session * session,union perf_event * event,struct perf_tool * tool __maybe_unused)2784 static int cs_etm__process_auxtrace_event(struct perf_session *session,
2785 					  union perf_event *event,
2786 					  struct perf_tool *tool __maybe_unused)
2787 {
2788 	struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2789 						   struct cs_etm_auxtrace,
2790 						   auxtrace);
2791 	if (!etm->data_queued) {
2792 		struct auxtrace_buffer *buffer;
2793 		off_t  data_offset;
2794 		int fd = perf_data__fd(session->data);
2795 		bool is_pipe = perf_data__is_pipe(session->data);
2796 		int err;
2797 		int idx = event->auxtrace.idx;
2798 
2799 		if (is_pipe)
2800 			data_offset = 0;
2801 		else {
2802 			data_offset = lseek(fd, 0, SEEK_CUR);
2803 			if (data_offset == -1)
2804 				return -errno;
2805 		}
2806 
2807 		err = auxtrace_queues__add_event(&etm->queues, session,
2808 						 event, data_offset, &buffer);
2809 		if (err)
2810 			return err;
2811 
2812 		/*
2813 		 * Knowing if the trace is formatted or not requires a lookup of
2814 		 * the aux record so only works in non-piped mode where data is
2815 		 * queued in cs_etm__queue_aux_records(). Always assume
2816 		 * formatted in piped mode (true).
2817 		 */
2818 		err = cs_etm__setup_queue(etm, &etm->queues.queue_array[idx],
2819 					  idx, true);
2820 		if (err)
2821 			return err;
2822 
2823 		if (dump_trace)
2824 			if (auxtrace_buffer__get_data(buffer, fd)) {
2825 				cs_etm__dump_event(etm->queues.queue_array[idx].priv, buffer);
2826 				auxtrace_buffer__put_data(buffer);
2827 			}
2828 	} else if (dump_trace)
2829 		dump_queued_data(etm, &event->auxtrace);
2830 
2831 	return 0;
2832 }
2833 
cs_etm__setup_timeless_decoding(struct cs_etm_auxtrace * etm)2834 static int cs_etm__setup_timeless_decoding(struct cs_etm_auxtrace *etm)
2835 {
2836 	struct evsel *evsel;
2837 	struct evlist *evlist = etm->session->evlist;
2838 
2839 	/* Override timeless mode with user input from --itrace=Z */
2840 	if (etm->synth_opts.timeless_decoding) {
2841 		etm->timeless_decoding = true;
2842 		return 0;
2843 	}
2844 
2845 	/*
2846 	 * Find the cs_etm evsel and look at what its timestamp setting was
2847 	 */
2848 	evlist__for_each_entry(evlist, evsel)
2849 		if (cs_etm__evsel_is_auxtrace(etm->session, evsel)) {
2850 			etm->timeless_decoding =
2851 				!(evsel->core.attr.config & BIT(ETM_OPT_TS));
2852 			return 0;
2853 		}
2854 
2855 	pr_err("CS ETM: Couldn't find ETM evsel\n");
2856 	return -EINVAL;
2857 }
2858 
2859 /*
2860  * Read a single cpu parameter block from the auxtrace_info priv block.
2861  *
2862  * For version 1 there is a per cpu nr_params entry. If we are handling
2863  * version 1 file, then there may be less, the same, or more params
2864  * indicated by this value than the compile time number we understand.
2865  *
2866  * For a version 0 info block, there are a fixed number, and we need to
2867  * fill out the nr_param value in the metadata we create.
2868  */
cs_etm__create_meta_blk(u64 * buff_in,int * buff_in_offset,int out_blk_size,int nr_params_v0)2869 static u64 *cs_etm__create_meta_blk(u64 *buff_in, int *buff_in_offset,
2870 				    int out_blk_size, int nr_params_v0)
2871 {
2872 	u64 *metadata = NULL;
2873 	int hdr_version;
2874 	int nr_in_params, nr_out_params, nr_cmn_params;
2875 	int i, k;
2876 
2877 	metadata = zalloc(sizeof(*metadata) * out_blk_size);
2878 	if (!metadata)
2879 		return NULL;
2880 
2881 	/* read block current index & version */
2882 	i = *buff_in_offset;
2883 	hdr_version = buff_in[CS_HEADER_VERSION];
2884 
2885 	if (!hdr_version) {
2886 	/* read version 0 info block into a version 1 metadata block  */
2887 		nr_in_params = nr_params_v0;
2888 		metadata[CS_ETM_MAGIC] = buff_in[i + CS_ETM_MAGIC];
2889 		metadata[CS_ETM_CPU] = buff_in[i + CS_ETM_CPU];
2890 		metadata[CS_ETM_NR_TRC_PARAMS] = nr_in_params;
2891 		/* remaining block params at offset +1 from source */
2892 		for (k = CS_ETM_COMMON_BLK_MAX_V1 - 1; k < nr_in_params; k++)
2893 			metadata[k + 1] = buff_in[i + k];
2894 		/* version 0 has 2 common params */
2895 		nr_cmn_params = 2;
2896 	} else {
2897 	/* read version 1 info block - input and output nr_params may differ */
2898 		/* version 1 has 3 common params */
2899 		nr_cmn_params = 3;
2900 		nr_in_params = buff_in[i + CS_ETM_NR_TRC_PARAMS];
2901 
2902 		/* if input has more params than output - skip excess */
2903 		nr_out_params = nr_in_params + nr_cmn_params;
2904 		if (nr_out_params > out_blk_size)
2905 			nr_out_params = out_blk_size;
2906 
2907 		for (k = CS_ETM_MAGIC; k < nr_out_params; k++)
2908 			metadata[k] = buff_in[i + k];
2909 
2910 		/* record the actual nr params we copied */
2911 		metadata[CS_ETM_NR_TRC_PARAMS] = nr_out_params - nr_cmn_params;
2912 	}
2913 
2914 	/* adjust in offset by number of in params used */
2915 	i += nr_in_params + nr_cmn_params;
2916 	*buff_in_offset = i;
2917 	return metadata;
2918 }
2919 
2920 /**
2921  * Puts a fragment of an auxtrace buffer into the auxtrace queues based
2922  * on the bounds of aux_event, if it matches with the buffer that's at
2923  * file_offset.
2924  *
2925  * Normally, whole auxtrace buffers would be added to the queue. But we
2926  * want to reset the decoder for every PERF_RECORD_AUX event, and the decoder
2927  * is reset across each buffer, so splitting the buffers up in advance has
2928  * the same effect.
2929  */
cs_etm__queue_aux_fragment(struct perf_session * session,off_t file_offset,size_t sz,struct perf_record_aux * aux_event,struct perf_sample * sample)2930 static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_offset, size_t sz,
2931 				      struct perf_record_aux *aux_event, struct perf_sample *sample)
2932 {
2933 	int err;
2934 	char buf[PERF_SAMPLE_MAX_SIZE];
2935 	union perf_event *auxtrace_event_union;
2936 	struct perf_record_auxtrace *auxtrace_event;
2937 	union perf_event auxtrace_fragment;
2938 	__u64 aux_offset, aux_size;
2939 	__u32 idx;
2940 	bool formatted;
2941 
2942 	struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2943 						   struct cs_etm_auxtrace,
2944 						   auxtrace);
2945 
2946 	/*
2947 	 * There should be a PERF_RECORD_AUXTRACE event at the file_offset that we got
2948 	 * from looping through the auxtrace index.
2949 	 */
2950 	err = perf_session__peek_event(session, file_offset, buf,
2951 				       PERF_SAMPLE_MAX_SIZE, &auxtrace_event_union, NULL);
2952 	if (err)
2953 		return err;
2954 	auxtrace_event = &auxtrace_event_union->auxtrace;
2955 	if (auxtrace_event->header.type != PERF_RECORD_AUXTRACE)
2956 		return -EINVAL;
2957 
2958 	if (auxtrace_event->header.size < sizeof(struct perf_record_auxtrace) ||
2959 		auxtrace_event->header.size != sz) {
2960 		return -EINVAL;
2961 	}
2962 
2963 	/*
2964 	 * In per-thread mode, auxtrace CPU is set to -1, but TID will be set instead. See
2965 	 * auxtrace_mmap_params__set_idx(). However, the sample AUX event will contain a
2966 	 * CPU as we set this always for the AUX_OUTPUT_HW_ID event.
2967 	 * So now compare only TIDs if auxtrace CPU is -1, and CPUs if auxtrace CPU is not -1.
2968 	 * Return 'not found' if mismatch.
2969 	 */
2970 	if (auxtrace_event->cpu == (__u32) -1) {
2971 		etm->per_thread_decoding = true;
2972 		if (auxtrace_event->tid != sample->tid)
2973 			return 1;
2974 	} else if (auxtrace_event->cpu != sample->cpu) {
2975 		if (etm->per_thread_decoding) {
2976 			/*
2977 			 * Found a per-cpu buffer after a per-thread one was
2978 			 * already found
2979 			 */
2980 			pr_err("CS ETM: Inconsistent per-thread/per-cpu mode.\n");
2981 			return -EINVAL;
2982 		}
2983 		return 1;
2984 	}
2985 
2986 	if (aux_event->flags & PERF_AUX_FLAG_OVERWRITE) {
2987 		/*
2988 		 * Clamp size in snapshot mode. The buffer size is clamped in
2989 		 * __auxtrace_mmap__read() for snapshots, so the aux record size doesn't reflect
2990 		 * the buffer size.
2991 		 */
2992 		aux_size = min(aux_event->aux_size, auxtrace_event->size);
2993 
2994 		/*
2995 		 * In this mode, the head also points to the end of the buffer so aux_offset
2996 		 * needs to have the size subtracted so it points to the beginning as in normal mode
2997 		 */
2998 		aux_offset = aux_event->aux_offset - aux_size;
2999 	} else {
3000 		aux_size = aux_event->aux_size;
3001 		aux_offset = aux_event->aux_offset;
3002 	}
3003 
3004 	if (aux_offset >= auxtrace_event->offset &&
3005 	    aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) {
3006 		/*
3007 		 * If this AUX event was inside this buffer somewhere, create a new auxtrace event
3008 		 * based on the sizes of the aux event, and queue that fragment.
3009 		 */
3010 		auxtrace_fragment.auxtrace = *auxtrace_event;
3011 		auxtrace_fragment.auxtrace.size = aux_size;
3012 		auxtrace_fragment.auxtrace.offset = aux_offset;
3013 		file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size;
3014 
3015 		pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64
3016 			  " tid: %d cpu: %d\n", aux_size, aux_offset, sample->tid, sample->cpu);
3017 		err = auxtrace_queues__add_event(&etm->queues, session, &auxtrace_fragment,
3018 						 file_offset, NULL);
3019 		if (err)
3020 			return err;
3021 
3022 		idx = auxtrace_event->idx;
3023 		formatted = !(aux_event->flags & PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
3024 		return cs_etm__setup_queue(etm, &etm->queues.queue_array[idx],
3025 					   idx, formatted);
3026 	}
3027 
3028 	/* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */
3029 	return 1;
3030 }
3031 
cs_etm__process_aux_hw_id_cb(struct perf_session * session,union perf_event * event,u64 offset __maybe_unused,void * data __maybe_unused)3032 static int cs_etm__process_aux_hw_id_cb(struct perf_session *session, union perf_event *event,
3033 					u64 offset __maybe_unused, void *data __maybe_unused)
3034 {
3035 	/* look to handle PERF_RECORD_AUX_OUTPUT_HW_ID early to ensure decoders can be set up */
3036 	if (event->header.type == PERF_RECORD_AUX_OUTPUT_HW_ID) {
3037 		(*(int *)data)++; /* increment found count */
3038 		return cs_etm__process_aux_output_hw_id(session, event);
3039 	}
3040 	return 0;
3041 }
3042 
cs_etm__queue_aux_records_cb(struct perf_session * session,union perf_event * event,u64 offset __maybe_unused,void * data __maybe_unused)3043 static int cs_etm__queue_aux_records_cb(struct perf_session *session, union perf_event *event,
3044 					u64 offset __maybe_unused, void *data __maybe_unused)
3045 {
3046 	struct perf_sample sample;
3047 	int ret;
3048 	struct auxtrace_index_entry *ent;
3049 	struct auxtrace_index *auxtrace_index;
3050 	struct evsel *evsel;
3051 	size_t i;
3052 
3053 	/* Don't care about any other events, we're only queuing buffers for AUX events */
3054 	if (event->header.type != PERF_RECORD_AUX)
3055 		return 0;
3056 
3057 	if (event->header.size < sizeof(struct perf_record_aux))
3058 		return -EINVAL;
3059 
3060 	/* Truncated Aux records can have 0 size and shouldn't result in anything being queued. */
3061 	if (!event->aux.aux_size)
3062 		return 0;
3063 
3064 	/*
3065 	 * Parse the sample, we need the sample_id_all data that comes after the event so that the
3066 	 * CPU or PID can be matched to an AUXTRACE buffer's CPU or PID.
3067 	 */
3068 	evsel = evlist__event2evsel(session->evlist, event);
3069 	if (!evsel)
3070 		return -EINVAL;
3071 	ret = evsel__parse_sample(evsel, event, &sample);
3072 	if (ret)
3073 		return ret;
3074 
3075 	/*
3076 	 * Loop through the auxtrace index to find the buffer that matches up with this aux event.
3077 	 */
3078 	list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) {
3079 		for (i = 0; i < auxtrace_index->nr; i++) {
3080 			ent = &auxtrace_index->entries[i];
3081 			ret = cs_etm__queue_aux_fragment(session, ent->file_offset,
3082 							 ent->sz, &event->aux, &sample);
3083 			/*
3084 			 * Stop search on error or successful values. Continue search on
3085 			 * 1 ('not found')
3086 			 */
3087 			if (ret != 1)
3088 				return ret;
3089 		}
3090 	}
3091 
3092 	/*
3093 	 * Couldn't find the buffer corresponding to this aux record, something went wrong. Warn but
3094 	 * don't exit with an error because it will still be possible to decode other aux records.
3095 	 */
3096 	pr_err("CS ETM: Couldn't find auxtrace buffer for aux_offset: %#"PRI_lx64
3097 	       " tid: %d cpu: %d\n", event->aux.aux_offset, sample.tid, sample.cpu);
3098 	return 0;
3099 }
3100 
cs_etm__queue_aux_records(struct perf_session * session)3101 static int cs_etm__queue_aux_records(struct perf_session *session)
3102 {
3103 	struct auxtrace_index *index = list_first_entry_or_null(&session->auxtrace_index,
3104 								struct auxtrace_index, list);
3105 	if (index && index->nr > 0)
3106 		return perf_session__peek_events(session, session->header.data_offset,
3107 						 session->header.data_size,
3108 						 cs_etm__queue_aux_records_cb, NULL);
3109 
3110 	/*
3111 	 * We would get here if there are no entries in the index (either no auxtrace
3112 	 * buffers or no index at all). Fail silently as there is the possibility of
3113 	 * queueing them in cs_etm__process_auxtrace_event() if etm->data_queued is still
3114 	 * false.
3115 	 *
3116 	 * In that scenario, buffers will not be split by AUX records.
3117 	 */
3118 	return 0;
3119 }
3120 
3121 #define HAS_PARAM(j, type, param) (metadata[(j)][CS_ETM_NR_TRC_PARAMS] <= \
3122 				  (CS_##type##_##param - CS_ETM_COMMON_BLK_MAX_V1))
3123 
3124 /*
3125  * Loop through the ETMs and complain if we find at least one where ts_source != 1 (virtual
3126  * timestamps).
3127  */
cs_etm__has_virtual_ts(u64 ** metadata,int num_cpu)3128 static bool cs_etm__has_virtual_ts(u64 **metadata, int num_cpu)
3129 {
3130 	int j;
3131 
3132 	for (j = 0; j < num_cpu; j++) {
3133 		switch (metadata[j][CS_ETM_MAGIC]) {
3134 		case __perf_cs_etmv4_magic:
3135 			if (HAS_PARAM(j, ETMV4, TS_SOURCE) || metadata[j][CS_ETMV4_TS_SOURCE] != 1)
3136 				return false;
3137 			break;
3138 		case __perf_cs_ete_magic:
3139 			if (HAS_PARAM(j, ETE, TS_SOURCE) || metadata[j][CS_ETE_TS_SOURCE] != 1)
3140 				return false;
3141 			break;
3142 		default:
3143 			/* Unknown / unsupported magic number. */
3144 			return false;
3145 		}
3146 	}
3147 	return true;
3148 }
3149 
3150 /* map trace ids to correct metadata block, from information in metadata */
cs_etm__map_trace_ids_metadata(int num_cpu,u64 ** metadata)3151 static int cs_etm__map_trace_ids_metadata(int num_cpu, u64 **metadata)
3152 {
3153 	u64 cs_etm_magic;
3154 	u8 trace_chan_id;
3155 	int i, err;
3156 
3157 	for (i = 0; i < num_cpu; i++) {
3158 		cs_etm_magic = metadata[i][CS_ETM_MAGIC];
3159 		switch (cs_etm_magic) {
3160 		case __perf_cs_etmv3_magic:
3161 			metadata[i][CS_ETM_ETMTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK;
3162 			trace_chan_id = (u8)(metadata[i][CS_ETM_ETMTRACEIDR]);
3163 			break;
3164 		case __perf_cs_etmv4_magic:
3165 		case __perf_cs_ete_magic:
3166 			metadata[i][CS_ETMV4_TRCTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK;
3167 			trace_chan_id = (u8)(metadata[i][CS_ETMV4_TRCTRACEIDR]);
3168 			break;
3169 		default:
3170 			/* unknown magic number */
3171 			return -EINVAL;
3172 		}
3173 		err = cs_etm__map_trace_id(trace_chan_id, metadata[i]);
3174 		if (err)
3175 			return err;
3176 	}
3177 	return 0;
3178 }
3179 
3180 /*
3181  * If we found AUX_HW_ID packets, then set any metadata marked as unused to the
3182  * unused value to reduce the number of unneeded decoders created.
3183  */
cs_etm__clear_unused_trace_ids_metadata(int num_cpu,u64 ** metadata)3184 static int cs_etm__clear_unused_trace_ids_metadata(int num_cpu, u64 **metadata)
3185 {
3186 	u64 cs_etm_magic;
3187 	int i;
3188 
3189 	for (i = 0; i < num_cpu; i++) {
3190 		cs_etm_magic = metadata[i][CS_ETM_MAGIC];
3191 		switch (cs_etm_magic) {
3192 		case __perf_cs_etmv3_magic:
3193 			if (metadata[i][CS_ETM_ETMTRACEIDR] & CORESIGHT_TRACE_ID_UNUSED_FLAG)
3194 				metadata[i][CS_ETM_ETMTRACEIDR] = CORESIGHT_TRACE_ID_UNUSED_VAL;
3195 			break;
3196 		case __perf_cs_etmv4_magic:
3197 		case __perf_cs_ete_magic:
3198 			if (metadata[i][CS_ETMV4_TRCTRACEIDR] & CORESIGHT_TRACE_ID_UNUSED_FLAG)
3199 				metadata[i][CS_ETMV4_TRCTRACEIDR] = CORESIGHT_TRACE_ID_UNUSED_VAL;
3200 			break;
3201 		default:
3202 			/* unknown magic number */
3203 			return -EINVAL;
3204 		}
3205 	}
3206 	return 0;
3207 }
3208 
cs_etm__process_auxtrace_info_full(union perf_event * event,struct perf_session * session)3209 int cs_etm__process_auxtrace_info_full(union perf_event *event,
3210 				       struct perf_session *session)
3211 {
3212 	struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
3213 	struct cs_etm_auxtrace *etm = NULL;
3214 	struct perf_record_time_conv *tc = &session->time_conv;
3215 	int event_header_size = sizeof(struct perf_event_header);
3216 	int total_size = auxtrace_info->header.size;
3217 	int priv_size = 0;
3218 	int num_cpu;
3219 	int err = 0;
3220 	int aux_hw_id_found;
3221 	int i, j;
3222 	u64 *ptr = NULL;
3223 	u64 **metadata = NULL;
3224 
3225 	/*
3226 	 * Create an RB tree for traceID-metadata tuple.  Since the conversion
3227 	 * has to be made for each packet that gets decoded, optimizing access
3228 	 * in anything other than a sequential array is worth doing.
3229 	 */
3230 	traceid_list = intlist__new(NULL);
3231 	if (!traceid_list)
3232 		return -ENOMEM;
3233 
3234 	/* First the global part */
3235 	ptr = (u64 *) auxtrace_info->priv;
3236 	num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff;
3237 	metadata = zalloc(sizeof(*metadata) * num_cpu);
3238 	if (!metadata) {
3239 		err = -ENOMEM;
3240 		goto err_free_traceid_list;
3241 	}
3242 
3243 	/* Start parsing after the common part of the header */
3244 	i = CS_HEADER_VERSION_MAX;
3245 
3246 	/*
3247 	 * The metadata is stored in the auxtrace_info section and encodes
3248 	 * the configuration of the ARM embedded trace macrocell which is
3249 	 * required by the trace decoder to properly decode the trace due
3250 	 * to its highly compressed nature.
3251 	 */
3252 	for (j = 0; j < num_cpu; j++) {
3253 		if (ptr[i] == __perf_cs_etmv3_magic) {
3254 			metadata[j] =
3255 				cs_etm__create_meta_blk(ptr, &i,
3256 							CS_ETM_PRIV_MAX,
3257 							CS_ETM_NR_TRC_PARAMS_V0);
3258 		} else if (ptr[i] == __perf_cs_etmv4_magic) {
3259 			metadata[j] =
3260 				cs_etm__create_meta_blk(ptr, &i,
3261 							CS_ETMV4_PRIV_MAX,
3262 							CS_ETMV4_NR_TRC_PARAMS_V0);
3263 		} else if (ptr[i] == __perf_cs_ete_magic) {
3264 			metadata[j] = cs_etm__create_meta_blk(ptr, &i, CS_ETE_PRIV_MAX, -1);
3265 		} else {
3266 			ui__error("CS ETM Trace: Unrecognised magic number %#"PRIx64". File could be from a newer version of perf.\n",
3267 				  ptr[i]);
3268 			err = -EINVAL;
3269 			goto err_free_metadata;
3270 		}
3271 
3272 		if (!metadata[j]) {
3273 			err = -ENOMEM;
3274 			goto err_free_metadata;
3275 		}
3276 	}
3277 
3278 	/*
3279 	 * Each of CS_HEADER_VERSION_MAX, CS_ETM_PRIV_MAX and
3280 	 * CS_ETMV4_PRIV_MAX mark how many double words are in the
3281 	 * global metadata, and each cpu's metadata respectively.
3282 	 * The following tests if the correct number of double words was
3283 	 * present in the auxtrace info section.
3284 	 */
3285 	priv_size = total_size - event_header_size - INFO_HEADER_SIZE;
3286 	if (i * 8 != priv_size) {
3287 		err = -EINVAL;
3288 		goto err_free_metadata;
3289 	}
3290 
3291 	etm = zalloc(sizeof(*etm));
3292 
3293 	if (!etm) {
3294 		err = -ENOMEM;
3295 		goto err_free_metadata;
3296 	}
3297 
3298 	/*
3299 	 * As all the ETMs run at the same exception level, the system should
3300 	 * have the same PID format crossing CPUs.  So cache the PID format
3301 	 * and reuse it for sequential decoding.
3302 	 */
3303 	etm->pid_fmt = cs_etm__init_pid_fmt(metadata[0]);
3304 
3305 	err = auxtrace_queues__init(&etm->queues);
3306 	if (err)
3307 		goto err_free_etm;
3308 
3309 	if (session->itrace_synth_opts->set) {
3310 		etm->synth_opts = *session->itrace_synth_opts;
3311 	} else {
3312 		itrace_synth_opts__set_default(&etm->synth_opts,
3313 				session->itrace_synth_opts->default_no_sample);
3314 		etm->synth_opts.callchain = false;
3315 	}
3316 
3317 	etm->session = session;
3318 
3319 	etm->num_cpu = num_cpu;
3320 	etm->pmu_type = (unsigned int) ((ptr[CS_PMU_TYPE_CPUS] >> 32) & 0xffffffff);
3321 	etm->snapshot_mode = (ptr[CS_ETM_SNAPSHOT] != 0);
3322 	etm->metadata = metadata;
3323 	etm->auxtrace_type = auxtrace_info->type;
3324 
3325 	/* Use virtual timestamps if all ETMs report ts_source = 1 */
3326 	etm->has_virtual_ts = cs_etm__has_virtual_ts(metadata, num_cpu);
3327 
3328 	if (!etm->has_virtual_ts)
3329 		ui__warning("Virtual timestamps are not enabled, or not supported by the traced system.\n"
3330 			    "The time field of the samples will not be set accurately.\n\n");
3331 
3332 	etm->auxtrace.process_event = cs_etm__process_event;
3333 	etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;
3334 	etm->auxtrace.flush_events = cs_etm__flush_events;
3335 	etm->auxtrace.free_events = cs_etm__free_events;
3336 	etm->auxtrace.free = cs_etm__free;
3337 	etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace;
3338 	session->auxtrace = &etm->auxtrace;
3339 
3340 	err = cs_etm__setup_timeless_decoding(etm);
3341 	if (err)
3342 		return err;
3343 
3344 	etm->tc.time_shift = tc->time_shift;
3345 	etm->tc.time_mult = tc->time_mult;
3346 	etm->tc.time_zero = tc->time_zero;
3347 	if (event_contains(*tc, time_cycles)) {
3348 		etm->tc.time_cycles = tc->time_cycles;
3349 		etm->tc.time_mask = tc->time_mask;
3350 		etm->tc.cap_user_time_zero = tc->cap_user_time_zero;
3351 		etm->tc.cap_user_time_short = tc->cap_user_time_short;
3352 	}
3353 	err = cs_etm__synth_events(etm, session);
3354 	if (err)
3355 		goto err_free_queues;
3356 
3357 	/*
3358 	 * Map Trace ID values to CPU metadata.
3359 	 *
3360 	 * Trace metadata will always contain Trace ID values from the legacy algorithm. If the
3361 	 * files has been recorded by a "new" perf updated to handle AUX_HW_ID then the metadata
3362 	 * ID value will also have the CORESIGHT_TRACE_ID_UNUSED_FLAG set.
3363 	 *
3364 	 * The updated kernel drivers that use AUX_HW_ID to sent Trace IDs will attempt to use
3365 	 * the same IDs as the old algorithm as far as is possible, unless there are clashes
3366 	 * in which case a different value will be used. This means an older perf may still
3367 	 * be able to record and read files generate on a newer system.
3368 	 *
3369 	 * For a perf able to interpret AUX_HW_ID packets we first check for the presence of
3370 	 * those packets. If they are there then the values will be mapped and plugged into
3371 	 * the metadata. We then set any remaining metadata values with the used flag to a
3372 	 * value CORESIGHT_TRACE_ID_UNUSED_VAL - which indicates no decoder is required.
3373 	 *
3374 	 * If no AUX_HW_ID packets are present - which means a file recorded on an old kernel
3375 	 * then we map Trace ID values to CPU directly from the metadata - clearing any unused
3376 	 * flags if present.
3377 	 */
3378 
3379 	/* first scan for AUX_OUTPUT_HW_ID records to map trace ID values to CPU metadata */
3380 	aux_hw_id_found = 0;
3381 	err = perf_session__peek_events(session, session->header.data_offset,
3382 					session->header.data_size,
3383 					cs_etm__process_aux_hw_id_cb, &aux_hw_id_found);
3384 	if (err)
3385 		goto err_free_queues;
3386 
3387 	/* if HW ID found then clear any unused metadata ID values */
3388 	if (aux_hw_id_found)
3389 		err = cs_etm__clear_unused_trace_ids_metadata(num_cpu, metadata);
3390 	/* otherwise, this is a file with metadata values only, map from metadata */
3391 	else
3392 		err = cs_etm__map_trace_ids_metadata(num_cpu, metadata);
3393 
3394 	if (err)
3395 		goto err_free_queues;
3396 
3397 	err = cs_etm__queue_aux_records(session);
3398 	if (err)
3399 		goto err_free_queues;
3400 
3401 	etm->data_queued = etm->queues.populated;
3402 	return 0;
3403 
3404 err_free_queues:
3405 	auxtrace_queues__free(&etm->queues);
3406 	session->auxtrace = NULL;
3407 err_free_etm:
3408 	zfree(&etm);
3409 err_free_metadata:
3410 	/* No need to check @metadata[j], free(NULL) is supported */
3411 	for (j = 0; j < num_cpu; j++)
3412 		zfree(&metadata[j]);
3413 	zfree(&metadata);
3414 err_free_traceid_list:
3415 	intlist__delete(traceid_list);
3416 	return err;
3417 }
3418