260 	struct cs_etm_traceid_queue *tidq;  in cs_etm__clear_all_packet_queues()  local
265 		tidq = etmq->traceid_queues[idx];  in cs_etm__clear_all_packet_queues()
266 		cs_etm__clear_packet_queue(&tidq->packet_queue);  in cs_etm__clear_all_packet_queues()
271 				      struct cs_etm_traceid_queue *tidq,  in cs_etm__init_traceid_queue()  argument
278 	cs_etm__clear_packet_queue(&tidq->packet_queue);  in cs_etm__init_traceid_queue()
281 	tidq->tid = queue->tid;  in cs_etm__init_traceid_queue()
282 	tidq->pid = -1;  in cs_etm__init_traceid_queue()
283 	tidq->trace_chan_id = trace_chan_id;  in cs_etm__init_traceid_queue()
285 	tidq->packet = zalloc(sizeof(struct cs_etm_packet));  in cs_etm__init_traceid_queue()
286 	if (!tidq->packet)  in cs_etm__init_traceid_queue()
289 	tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));  in cs_etm__init_traceid_queue()
290 	if (!tidq->prev_packet)  in cs_etm__init_traceid_queue()
298 		tidq->last_branch = zalloc(sz);  in cs_etm__init_traceid_queue()
299 		if (!tidq->last_branch)  in cs_etm__init_traceid_queue()
301 		tidq->last_branch_rb = zalloc(sz);  in cs_etm__init_traceid_queue()
302 		if (!tidq->last_branch_rb)  in cs_etm__init_traceid_queue()
306 	tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);  in cs_etm__init_traceid_queue()
307 	if (!tidq->event_buf)  in cs_etm__init_traceid_queue()
313 	zfree(&tidq->last_branch_rb);  in cs_etm__init_traceid_queue()
314 	zfree(&tidq->last_branch);  in cs_etm__init_traceid_queue()
315 	zfree(&tidq->prev_packet);  in cs_etm__init_traceid_queue()
316 	zfree(&tidq->packet);  in cs_etm__init_traceid_queue()
327 	struct cs_etm_traceid_queue *tidq, **traceid_queues;  in cs_etm__etmq_get_traceid_queue()  local
346 	tidq = malloc(sizeof(*tidq));  in cs_etm__etmq_get_traceid_queue()
347 	if (!tidq)  in cs_etm__etmq_get_traceid_queue()
350 	memset(tidq, 0, sizeof(*tidq));  in cs_etm__etmq_get_traceid_queue()
362 	if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))  in cs_etm__etmq_get_traceid_queue()
378 	traceid_queues[idx] = tidq;  in cs_etm__etmq_get_traceid_queue()
389 	free(tidq);  in cs_etm__etmq_get_traceid_queue()
397 	struct cs_etm_traceid_queue *tidq;  in cs_etm__etmq_get_packet_queue()  local
399 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__etmq_get_packet_queue()
400 	if (tidq)  in cs_etm__etmq_get_packet_queue()
401 		return &tidq->packet_queue;  in cs_etm__etmq_get_packet_queue()
407 				struct cs_etm_traceid_queue *tidq)  in cs_etm__packet_swap()  argument
417 		tmp = tidq->packet;  in cs_etm__packet_swap()
418 		tidq->packet = tidq->prev_packet;  in cs_etm__packet_swap()
419 		tidq->prev_packet = tmp;  in cs_etm__packet_swap()
580 	struct cs_etm_traceid_queue *tidq;  in cs_etm__free_traceid_queues()  local
588 		tidq = etmq->traceid_queues[idx];  in cs_etm__free_traceid_queues()
589 		thread__zput(tidq->thread);  in cs_etm__free_traceid_queues()
590 		zfree(&tidq->event_buf);  in cs_etm__free_traceid_queues()
591 		zfree(&tidq->last_branch);  in cs_etm__free_traceid_queues()
592 		zfree(&tidq->last_branch_rb);  in cs_etm__free_traceid_queues()
593 		zfree(&tidq->prev_packet);  in cs_etm__free_traceid_queues()
594 		zfree(&tidq->packet);  in cs_etm__free_traceid_queues()
595 		zfree(&tidq);  in cs_etm__free_traceid_queues()
704 	struct cs_etm_traceid_queue *tidq;  in cs_etm__mem_access()  local
711 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__mem_access()
712 	if (!tidq)  in cs_etm__mem_access()
715 	thread = tidq->thread;  in cs_etm__mem_access()
909 				 struct cs_etm_traceid_queue *tidq)  in cs_etm__copy_last_branch_rb()  argument
911 	struct branch_stack *bs_src = tidq->last_branch_rb;  in cs_etm__copy_last_branch_rb()
912 	struct branch_stack *bs_dst = tidq->last_branch;  in cs_etm__copy_last_branch_rb()
932 	nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;  in cs_etm__copy_last_branch_rb()
934 	       &bs_src->entries[tidq->last_branch_pos],  in cs_etm__copy_last_branch_rb()
947 		       sizeof(struct branch_entry) * tidq->last_branch_pos);  in cs_etm__copy_last_branch_rb()
952 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)  in cs_etm__reset_last_branch_rb()  argument
954 	tidq->last_branch_pos = 0;  in cs_etm__reset_last_branch_rb()
955 	tidq->last_branch_rb->nr = 0;  in cs_etm__reset_last_branch_rb()
1013 					  struct cs_etm_traceid_queue *tidq)  in cs_etm__update_last_branch_rb()  argument
1015 	struct branch_stack *bs = tidq->last_branch_rb;  in cs_etm__update_last_branch_rb()
1024 	if (!tidq->last_branch_pos)  in cs_etm__update_last_branch_rb()
1025 		tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;  in cs_etm__update_last_branch_rb()
1027 	tidq->last_branch_pos -= 1;  in cs_etm__update_last_branch_rb()
1029 	be       = &bs->entries[tidq->last_branch_pos];  in cs_etm__update_last_branch_rb()
1030 	be->from = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__update_last_branch_rb()
1031 	be->to	 = cs_etm__first_executed_instr(tidq->packet);  in cs_etm__update_last_branch_rb()
1095 				    struct cs_etm_traceid_queue *tidq)  in cs_etm__set_pid_tid_cpu()  argument
1097 	if ((!tidq->thread) && (tidq->tid != -1))  in cs_etm__set_pid_tid_cpu()
1098 		tidq->thread = machine__find_thread(etm->machine, -1,  in cs_etm__set_pid_tid_cpu()
1099 						    tidq->tid);  in cs_etm__set_pid_tid_cpu()
1101 	if (tidq->thread)  in cs_etm__set_pid_tid_cpu()
1102 		tidq->pid = tidq->thread->pid_;  in cs_etm__set_pid_tid_cpu()
1110 	struct cs_etm_traceid_queue *tidq;  in cs_etm__etmq_set_tid()  local
1112 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__etmq_set_tid()
1113 	if (!tidq)  in cs_etm__etmq_set_tid()
1123 	tidq->tid = tid;  in cs_etm__etmq_set_tid()
1124 	thread__zput(tidq->thread);  in cs_etm__etmq_set_tid()
1126 	cs_etm__set_pid_tid_cpu(etm, tidq);  in cs_etm__etmq_set_tid()
1165 					    struct cs_etm_traceid_queue *tidq,  in cs_etm__synth_instruction_sample()  argument
1170 	union perf_event *event = tidq->event_buf;  in cs_etm__synth_instruction_sample()
1180 	sample.pid = tidq->pid;  in cs_etm__synth_instruction_sample()
1181 	sample.tid = tidq->tid;  in cs_etm__synth_instruction_sample()
1185 	sample.cpu = tidq->packet->cpu;  in cs_etm__synth_instruction_sample()
1186 	sample.flags = tidq->prev_packet->flags;  in cs_etm__synth_instruction_sample()
1189 	cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);  in cs_etm__synth_instruction_sample()
1192 		sample.branch_stack = tidq->last_branch;  in cs_etm__synth_instruction_sample()
1216 				       struct cs_etm_traceid_queue *tidq)  in cs_etm__synth_branch_sample()  argument
1221 	union perf_event *event = tidq->event_buf;  in cs_etm__synth_branch_sample()
1229 	ip = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__synth_branch_sample()
1238 	sample.pid = tidq->pid;  in cs_etm__synth_branch_sample()
1239 	sample.tid = tidq->tid;  in cs_etm__synth_branch_sample()
1240 	sample.addr = cs_etm__first_executed_instr(tidq->packet);  in cs_etm__synth_branch_sample()
1244 	sample.cpu = tidq->packet->cpu;  in cs_etm__synth_branch_sample()
1245 	sample.flags = tidq->prev_packet->flags;  in cs_etm__synth_branch_sample()
1248 	cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,  in cs_etm__synth_branch_sample()
1398 			  struct cs_etm_traceid_queue *tidq)  in cs_etm__sample()  argument
1402 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__sample()
1406 	instrs_prev = tidq->period_instructions;  in cs_etm__sample()
1408 	tidq->period_instructions += tidq->packet->instr_count;  in cs_etm__sample()
1415 	    tidq->prev_packet->sample_type == CS_ETM_RANGE &&  in cs_etm__sample()
1416 	    tidq->prev_packet->last_instr_taken_branch)  in cs_etm__sample()
1417 		cs_etm__update_last_branch_rb(etmq, tidq);  in cs_etm__sample()
1420 	    tidq->period_instructions >= etm->instructions_sample_period) {  in cs_etm__sample()
1478 			cs_etm__copy_last_branch_rb(etmq, tidq);  in cs_etm__sample()
1480 		while (tidq->period_instructions >=  in cs_etm__sample()
1489 						  tidq->packet, offset - 1);  in cs_etm__sample()
1491 				etmq, tidq, addr,  in cs_etm__sample()
1497 			tidq->period_instructions -=  in cs_etm__sample()
1506 		if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)  in cs_etm__sample()
1510 		if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&  in cs_etm__sample()
1511 		    tidq->prev_packet->last_instr_taken_branch)  in cs_etm__sample()
1515 			ret = cs_etm__synth_branch_sample(etmq, tidq);  in cs_etm__sample()
1521 	cs_etm__packet_swap(etm, tidq);  in cs_etm__sample()
1526 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)  in cs_etm__exception()  argument
1539 	if (tidq->prev_packet->sample_type == CS_ETM_RANGE)  in cs_etm__exception()
1540 		tidq->prev_packet->last_instr_taken_branch = true;  in cs_etm__exception()
1546 			 struct cs_etm_traceid_queue *tidq)  in cs_etm__flush()  argument
1552 	if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)  in cs_etm__flush()
1557 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {  in cs_etm__flush()
1561 		cs_etm__copy_last_branch_rb(etmq, tidq);  in cs_etm__flush()
1570 		addr = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__flush()
1573 			etmq, tidq, addr,  in cs_etm__flush()
1574 			tidq->period_instructions);  in cs_etm__flush()
1578 		tidq->period_instructions = 0;  in cs_etm__flush()
1583 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {  in cs_etm__flush()
1584 		err = cs_etm__synth_branch_sample(etmq, tidq);  in cs_etm__flush()
1590 	cs_etm__packet_swap(etm, tidq);  in cs_etm__flush()
1594 		cs_etm__reset_last_branch_rb(tidq);  in cs_etm__flush()
1600 			     struct cs_etm_traceid_queue *tidq)  in cs_etm__end_block()  argument
1615 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {  in cs_etm__end_block()
1619 		cs_etm__copy_last_branch_rb(etmq, tidq);  in cs_etm__end_block()
1625 		addr = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__end_block()
1628 			etmq, tidq, addr,  in cs_etm__end_block()
1629 			tidq->period_instructions);  in cs_etm__end_block()
1633 		tidq->period_instructions = 0;  in cs_etm__end_block()
1737 			       struct cs_etm_traceid_queue *tidq, u64 magic)  in cs_etm__is_syscall()  argument
1739 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__is_syscall()
1740 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__is_syscall()
1741 	struct cs_etm_packet *prev_packet = tidq->prev_packet;  in cs_etm__is_syscall()
1762 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,  in cs_etm__is_async_exception()  argument
1765 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__is_async_exception()
1789 				      struct cs_etm_traceid_queue *tidq,  in cs_etm__is_sync_exception()  argument
1792 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__is_sync_exception()
1793 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__is_sync_exception()
1794 	struct cs_etm_packet *prev_packet = tidq->prev_packet;  in cs_etm__is_sync_exception()
1838 				    struct cs_etm_traceid_queue *tidq)  in cs_etm__set_sample_flags()  argument
1840 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__set_sample_flags()
1841 	struct cs_etm_packet *prev_packet = tidq->prev_packet;  in cs_etm__set_sample_flags()
1842 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__set_sample_flags()
1945 		if (cs_etm__is_syscall(etmq, tidq, magic))  in cs_etm__set_sample_flags()
1953 		else if (cs_etm__is_async_exception(tidq, magic))  in cs_etm__set_sample_flags()
1962 		else if (cs_etm__is_sync_exception(etmq, tidq, magic))  in cs_etm__set_sample_flags()
2045 					 struct cs_etm_traceid_queue *tidq)  in cs_etm__process_traceid_queue()  argument
2050 	packet_queue = &tidq->packet_queue;  in cs_etm__process_traceid_queue()
2055 						 tidq->packet);  in cs_etm__process_traceid_queue()
2070 		ret = cs_etm__set_sample_flags(etmq, tidq);  in cs_etm__process_traceid_queue()
2074 		switch (tidq->packet->sample_type) {  in cs_etm__process_traceid_queue()
2081 			cs_etm__sample(etmq, tidq);  in cs_etm__process_traceid_queue()
2090 			cs_etm__exception(tidq);  in cs_etm__process_traceid_queue()
2097 			cs_etm__flush(etmq, tidq);  in cs_etm__process_traceid_queue()
2118 	struct cs_etm_traceid_queue *tidq;  in cs_etm__clear_all_traceid_queues()  local
2123 		tidq = etmq->traceid_queues[idx];  in cs_etm__clear_all_traceid_queues()
2126 		cs_etm__process_traceid_queue(etmq, tidq);  in cs_etm__clear_all_traceid_queues()
2132 		cs_etm__flush(etmq, tidq);  in cs_etm__clear_all_traceid_queues()
2139 	struct cs_etm_traceid_queue *tidq;  in cs_etm__run_decoder()  local
2141 	tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);  in cs_etm__run_decoder()
2142 	if (!tidq)  in cs_etm__run_decoder()
2162 			err = cs_etm__process_traceid_queue(etmq, tidq);  in cs_etm__run_decoder()
2168 			err = cs_etm__end_block(etmq, tidq);  in cs_etm__run_decoder()
2183 		struct cs_etm_traceid_queue *tidq;  in cs_etm__process_timeless_queues()  local
2188 		tidq = cs_etm__etmq_get_traceid_queue(etmq,  in cs_etm__process_timeless_queues()
2191 		if (!tidq)  in cs_etm__process_timeless_queues()
2194 		if ((tid == -1) || (tidq->tid == tid)) {  in cs_etm__process_timeless_queues()
2195 			cs_etm__set_pid_tid_cpu(etm, tidq);  in cs_etm__process_timeless_queues()
2211 	struct cs_etm_traceid_queue *tidq;  in cs_etm__process_queues()  local
2244 		tidq  = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__process_queues()
2245 		if (!tidq) {  in cs_etm__process_queues()
2259 		ret = cs_etm__process_traceid_queue(etmq, tidq);  in cs_etm__process_queues()