345 	struct intel_pt_queue *ptq = data;  in intel_pt_findnew_vmcs_info()  local
346 	struct intel_pt *pt = ptq->pt;  in intel_pt_findnew_vmcs_info()
393 static int intel_pt_get_buffer(struct intel_pt_queue *ptq,  in intel_pt_get_buffer()  argument
401 		int fd = perf_data__fd(ptq->pt->session->data);  in intel_pt_get_buffer()
408 	might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode;  in intel_pt_get_buffer()
410 	    intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))  in intel_pt_get_buffer()
433 static void intel_pt_lookahead_drop_buffer(struct intel_pt_queue *ptq,  in intel_pt_lookahead_drop_buffer()  argument
436 	if (!buffer || buffer == ptq->buffer || buffer == ptq->old_buffer)  in intel_pt_lookahead_drop_buffer()
446 	struct intel_pt_queue *ptq = data;  in intel_pt_lookahead()  local
447 	struct auxtrace_buffer *buffer = ptq->buffer;  in intel_pt_lookahead()
448 	struct auxtrace_buffer *old_buffer = ptq->old_buffer;  in intel_pt_lookahead()
452 	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];  in intel_pt_lookahead()
461 		err = intel_pt_get_buffer(ptq, buffer, old_buffer, &b);  in intel_pt_lookahead()
466 			intel_pt_lookahead_drop_buffer(ptq, old_buffer);  in intel_pt_lookahead()
469 			intel_pt_lookahead_drop_buffer(ptq, buffer);  in intel_pt_lookahead()
479 		intel_pt_lookahead_drop_buffer(ptq, buffer);  in intel_pt_lookahead()
480 	intel_pt_lookahead_drop_buffer(ptq, old_buffer);  in intel_pt_lookahead()
491 	struct intel_pt_queue *ptq = data;  in intel_pt_get_trace()  local
492 	struct auxtrace_buffer *buffer = ptq->buffer;  in intel_pt_get_trace()
493 	struct auxtrace_buffer *old_buffer = ptq->old_buffer;  in intel_pt_get_trace()
497 	if (ptq->stop) {  in intel_pt_get_trace()
502 	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];  in intel_pt_get_trace()
512 	ptq->buffer = buffer;  in intel_pt_get_trace()
514 	err = intel_pt_get_buffer(ptq, buffer, old_buffer, b);  in intel_pt_get_trace()
518 	if (ptq->step_through_buffers)  in intel_pt_get_trace()
519 		ptq->stop = true;  in intel_pt_get_trace()
524 		ptq->old_buffer = buffer;  in intel_pt_get_trace()
666 static inline u8 intel_pt_nr_cpumode(struct intel_pt_queue *ptq, uint64_t ip, bool nr)  in intel_pt_nr_cpumode()  argument
674 	return ip >= ptq->pt->kernel_start ?  in intel_pt_nr_cpumode()
679 static inline u8 intel_pt_cpumode(struct intel_pt_queue *ptq, uint64_t from_ip, uint64_t to_ip)  in intel_pt_cpumode()  argument
683 		return intel_pt_nr_cpumode(ptq, from_ip, ptq->state->from_nr);  in intel_pt_cpumode()
684 	return intel_pt_nr_cpumode(ptq, to_ip, ptq->state->to_nr);  in intel_pt_cpumode()
687 static int intel_pt_get_guest(struct intel_pt_queue *ptq)  in intel_pt_get_guest()  argument
689 	struct machines *machines = &ptq->pt->session->machines;  in intel_pt_get_guest()
691 	pid_t pid = ptq->pid <= 0 ? DEFAULT_GUEST_KERNEL_ID : ptq->pid;  in intel_pt_get_guest()
693 	if (ptq->guest_machine && pid == ptq->guest_machine->pid)  in intel_pt_get_guest()
696 	ptq->guest_machine = NULL;  in intel_pt_get_guest()
697 	thread__zput(ptq->unknown_guest_thread);  in intel_pt_get_guest()
700 		thread__zput(ptq->guest_thread);  in intel_pt_get_guest()
701 		ptq->guest_thread = machines__findnew_guest_code(machines, pid);  in intel_pt_get_guest()
708 	ptq->unknown_guest_thread = machine__idle_thread(machine);  in intel_pt_get_guest()
709 	if (!ptq->unknown_guest_thread)  in intel_pt_get_guest()
712 	ptq->guest_machine = machine;  in intel_pt_get_guest()
744 	struct intel_pt_queue *ptq = data;  in intel_pt_walk_next_insn()  local
745 	struct machine *machine = ptq->pt->machine;  in intel_pt_walk_next_insn()
762 	nr = ptq->state->to_nr;  in intel_pt_walk_next_insn()
763 	cpumode = intel_pt_nr_cpumode(ptq, *ip, nr);  in intel_pt_walk_next_insn()
766 		if (ptq->pt->have_guest_sideband) {  in intel_pt_walk_next_insn()
767 			if (!ptq->guest_machine || ptq->guest_machine_pid != ptq->pid) {  in intel_pt_walk_next_insn()
772 			   intel_pt_get_guest(ptq)) {  in intel_pt_walk_next_insn()
776 		machine = ptq->guest_machine;  in intel_pt_walk_next_insn()
777 		thread = ptq->guest_thread;  in intel_pt_walk_next_insn()
783 			thread = ptq->unknown_guest_thread;  in intel_pt_walk_next_insn()
786 		thread = ptq->thread;  in intel_pt_walk_next_insn()
792 			thread = ptq->pt->unknown_thread;  in intel_pt_walk_next_insn()
958 	struct intel_pt_queue *ptq = data;  in __intel_pt_pgd_ip()  local
964 	if (ptq->state->to_nr) {  in __intel_pt_pgd_ip()
966 			return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);  in __intel_pt_pgd_ip()
969 	} else if (ip >= ptq->pt->kernel_start) {  in __intel_pt_pgd_ip()
970 		return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);  in __intel_pt_pgd_ip()
975 	thread = ptq->thread;  in __intel_pt_pgd_ip()
984 	return intel_pt_match_pgd_ip(ptq->pt, ip, offset,  in __intel_pt_pgd_ip()
1265 	struct intel_pt_queue *ptq;  in intel_pt_alloc_queue()  local
1267 	ptq = zalloc(sizeof(struct intel_pt_queue));  in intel_pt_alloc_queue()
1268 	if (!ptq)  in intel_pt_alloc_queue()
1272 		ptq->chain = intel_pt_alloc_chain(pt);  in intel_pt_alloc_queue()
1273 		if (!ptq->chain)  in intel_pt_alloc_queue()
1280 		ptq->last_branch = intel_pt_alloc_br_stack(entry_cnt);  in intel_pt_alloc_queue()
1281 		if (!ptq->last_branch)  in intel_pt_alloc_queue()
1285 	ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);  in intel_pt_alloc_queue()
1286 	if (!ptq->event_buf)  in intel_pt_alloc_queue()
1289 	ptq->pt = pt;  in intel_pt_alloc_queue()
1290 	ptq->queue_nr = queue_nr;  in intel_pt_alloc_queue()
1291 	ptq->exclude_kernel = intel_pt_exclude_kernel(pt);  in intel_pt_alloc_queue()
1292 	ptq->pid = -1;  in intel_pt_alloc_queue()
1293 	ptq->tid = -1;  in intel_pt_alloc_queue()
1294 	ptq->cpu = -1;  in intel_pt_alloc_queue()
1295 	ptq->next_tid = -1;  in intel_pt_alloc_queue()
1301 	params.data = ptq;  in intel_pt_alloc_queue()
1353 	ptq->decoder = intel_pt_decoder_new(&params);  in intel_pt_alloc_queue()
1354 	if (!ptq->decoder)  in intel_pt_alloc_queue()
1357 	return ptq;  in intel_pt_alloc_queue()
1360 	zfree(&ptq->event_buf);  in intel_pt_alloc_queue()
1361 	zfree(&ptq->last_branch);  in intel_pt_alloc_queue()
1362 	zfree(&ptq->chain);  in intel_pt_alloc_queue()
1363 	free(ptq);  in intel_pt_alloc_queue()
1369 	struct intel_pt_queue *ptq = priv;  in intel_pt_free_queue()  local
1371 	if (!ptq)  in intel_pt_free_queue()
1373 	thread__zput(ptq->thread);  in intel_pt_free_queue()
1374 	thread__zput(ptq->guest_thread);  in intel_pt_free_queue()
1375 	thread__zput(ptq->unknown_guest_thread);  in intel_pt_free_queue()
1376 	intel_pt_decoder_free(ptq->decoder);  in intel_pt_free_queue()
1377 	zfree(&ptq->event_buf);  in intel_pt_free_queue()
1378 	zfree(&ptq->last_branch);  in intel_pt_free_queue()
1379 	zfree(&ptq->chain);  in intel_pt_free_queue()
1380 	free(ptq);  in intel_pt_free_queue()
1391 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_first_timestamp()  local
1393 		if (ptq && ptq->decoder)  in intel_pt_first_timestamp()
1394 			intel_pt_set_first_timestamp(ptq->decoder, timestamp);  in intel_pt_first_timestamp()
1398 static int intel_pt_get_guest_from_sideband(struct intel_pt_queue *ptq)  in intel_pt_get_guest_from_sideband()  argument
1400 	struct machines *machines = &ptq->pt->session->machines;  in intel_pt_get_guest_from_sideband()
1402 	pid_t machine_pid = ptq->pid;  in intel_pt_get_guest_from_sideband()
1413 	if (ptq->guest_machine != machine) {  in intel_pt_get_guest_from_sideband()
1414 		ptq->guest_machine = NULL;  in intel_pt_get_guest_from_sideband()
1415 		thread__zput(ptq->guest_thread);  in intel_pt_get_guest_from_sideband()
1416 		thread__zput(ptq->unknown_guest_thread);  in intel_pt_get_guest_from_sideband()
1418 		ptq->unknown_guest_thread = machine__find_thread(machine, 0, 0);  in intel_pt_get_guest_from_sideband()
1419 		if (!ptq->unknown_guest_thread)  in intel_pt_get_guest_from_sideband()
1421 		ptq->guest_machine = machine;  in intel_pt_get_guest_from_sideband()
1424 	vcpu = ptq->thread ? ptq->thread->guest_cpu : -1;  in intel_pt_get_guest_from_sideband()
1430 	if (ptq->guest_thread && ptq->guest_thread->tid != tid)  in intel_pt_get_guest_from_sideband()
1431 		thread__zput(ptq->guest_thread);  in intel_pt_get_guest_from_sideband()
1433 	if (!ptq->guest_thread) {  in intel_pt_get_guest_from_sideband()
1434 		ptq->guest_thread = machine__find_thread(machine, -1, tid);  in intel_pt_get_guest_from_sideband()
1435 		if (!ptq->guest_thread)  in intel_pt_get_guest_from_sideband()
1439 	ptq->guest_machine_pid = machine_pid;  in intel_pt_get_guest_from_sideband()
1440 	ptq->guest_pid = ptq->guest_thread->pid_;  in intel_pt_get_guest_from_sideband()
1441 	ptq->guest_tid = tid;  in intel_pt_get_guest_from_sideband()
1442 	ptq->vcpu = vcpu;  in intel_pt_get_guest_from_sideband()
1450 	struct intel_pt_queue *ptq = queue->priv;  in intel_pt_set_pid_tid_cpu()  local
1453 		ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);  in intel_pt_set_pid_tid_cpu()
1454 		if (ptq->tid == -1)  in intel_pt_set_pid_tid_cpu()
1455 			ptq->pid = -1;  in intel_pt_set_pid_tid_cpu()
1456 		thread__zput(ptq->thread);  in intel_pt_set_pid_tid_cpu()
1459 	if (!ptq->thread && ptq->tid != -1)  in intel_pt_set_pid_tid_cpu()
1460 		ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);  in intel_pt_set_pid_tid_cpu()
1462 	if (ptq->thread) {  in intel_pt_set_pid_tid_cpu()
1463 		ptq->pid = ptq->thread->pid_;  in intel_pt_set_pid_tid_cpu()
1465 			ptq->cpu = ptq->thread->cpu;  in intel_pt_set_pid_tid_cpu()
1468 	if (pt->have_guest_sideband && intel_pt_get_guest_from_sideband(ptq)) {  in intel_pt_set_pid_tid_cpu()
1469 		ptq->guest_machine_pid = 0;  in intel_pt_set_pid_tid_cpu()
1470 		ptq->guest_pid = -1;  in intel_pt_set_pid_tid_cpu()
1471 		ptq->guest_tid = -1;  in intel_pt_set_pid_tid_cpu()
1472 		ptq->vcpu = -1;  in intel_pt_set_pid_tid_cpu()
1476 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)  in intel_pt_sample_flags()  argument
1478 	struct intel_pt *pt = ptq->pt;  in intel_pt_sample_flags()
1480 	ptq->insn_len = 0;  in intel_pt_sample_flags()
1481 	if (ptq->state->flags & INTEL_PT_ABORT_TX) {  in intel_pt_sample_flags()
1482 		ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;  in intel_pt_sample_flags()
1483 	} else if (ptq->state->flags & INTEL_PT_ASYNC) {  in intel_pt_sample_flags()
1484 		if (!ptq->state->to_ip)  in intel_pt_sample_flags()
1485 			ptq->flags = PERF_IP_FLAG_BRANCH |  in intel_pt_sample_flags()
1487 		else if (ptq->state->from_nr && !ptq->state->to_nr)  in intel_pt_sample_flags()
1488 			ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |  in intel_pt_sample_flags()
1491 			ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |  in intel_pt_sample_flags()
1495 		if (ptq->state->from_ip)  in intel_pt_sample_flags()
1496 			ptq->flags = intel_pt_insn_type(ptq->state->insn_op);  in intel_pt_sample_flags()
1498 			ptq->flags = PERF_IP_FLAG_BRANCH |  in intel_pt_sample_flags()
1500 		if (ptq->state->flags & INTEL_PT_IN_TX)  in intel_pt_sample_flags()
1501 			ptq->flags |= PERF_IP_FLAG_IN_TX;  in intel_pt_sample_flags()
1502 		ptq->insn_len = ptq->state->insn_len;  in intel_pt_sample_flags()
1503 		memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_sample_flags()
1506 	if (ptq->state->type & INTEL_PT_TRACE_BEGIN)  in intel_pt_sample_flags()
1507 		ptq->flags |= PERF_IP_FLAG_TRACE_BEGIN;  in intel_pt_sample_flags()
1508 	if (ptq->state->type & INTEL_PT_TRACE_END)  in intel_pt_sample_flags()
1509 		ptq->flags |= PERF_IP_FLAG_TRACE_END;  in intel_pt_sample_flags()
1512 		if (ptq->state->type & INTEL_PT_IFLAG_CHG) {  in intel_pt_sample_flags()
1513 			if (!ptq->state->from_iflag)  in intel_pt_sample_flags()
1514 				ptq->flags |= PERF_IP_FLAG_INTR_DISABLE;  in intel_pt_sample_flags()
1515 			if (ptq->state->from_iflag != ptq->state->to_iflag)  in intel_pt_sample_flags()
1516 				ptq->flags |= PERF_IP_FLAG_INTR_TOGGLE;  in intel_pt_sample_flags()
1517 		} else if (!ptq->state->to_iflag) {  in intel_pt_sample_flags()
1518 			ptq->flags |= PERF_IP_FLAG_INTR_DISABLE;  in intel_pt_sample_flags()
1524 				      struct intel_pt_queue *ptq)  in intel_pt_setup_time_range()  argument
1529 	ptq->sel_timestamp = pt->time_ranges[0].start;  in intel_pt_setup_time_range()
1530 	ptq->sel_idx = 0;  in intel_pt_setup_time_range()
1532 	if (ptq->sel_timestamp) {  in intel_pt_setup_time_range()
1533 		ptq->sel_start = true;  in intel_pt_setup_time_range()
1535 		ptq->sel_timestamp = pt->time_ranges[0].end;  in intel_pt_setup_time_range()
1536 		ptq->sel_start = false;  in intel_pt_setup_time_range()
1544 	struct intel_pt_queue *ptq = queue->priv;  in intel_pt_setup_queue()  local
1549 	if (!ptq) {  in intel_pt_setup_queue()
1550 		ptq = intel_pt_alloc_queue(pt, queue_nr);  in intel_pt_setup_queue()
1551 		if (!ptq)  in intel_pt_setup_queue()
1553 		queue->priv = ptq;  in intel_pt_setup_queue()
1556 			ptq->cpu = queue->cpu;  in intel_pt_setup_queue()
1557 		ptq->tid = queue->tid;  in intel_pt_setup_queue()
1559 		ptq->cbr_seen = UINT_MAX;  in intel_pt_setup_queue()
1563 			ptq->step_through_buffers = true;  in intel_pt_setup_queue()
1565 		ptq->sync_switch = pt->sync_switch;  in intel_pt_setup_queue()
1567 		intel_pt_setup_time_range(pt, ptq);  in intel_pt_setup_queue()
1570 	if (!ptq->on_heap &&  in intel_pt_setup_queue()
1571 	    (!ptq->sync_switch ||  in intel_pt_setup_queue()
1572 	     ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {  in intel_pt_setup_queue()
1581 			     queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_setup_queue()
1583 		if (ptq->sel_start && ptq->sel_timestamp) {  in intel_pt_setup_queue()
1584 			ret = intel_pt_fast_forward(ptq->decoder,  in intel_pt_setup_queue()
1585 						    ptq->sel_timestamp);  in intel_pt_setup_queue()
1591 			state = intel_pt_decode(ptq->decoder);  in intel_pt_setup_queue()
1604 		ptq->timestamp = state->timestamp;  in intel_pt_setup_queue()
1606 			     queue_nr, ptq->timestamp);  in intel_pt_setup_queue()
1607 		ptq->state = state;  in intel_pt_setup_queue()
1608 		ptq->have_sample = true;  in intel_pt_setup_queue()
1609 		if (ptq->sel_start && ptq->sel_timestamp &&  in intel_pt_setup_queue()
1610 		    ptq->timestamp < ptq->sel_timestamp)  in intel_pt_setup_queue()
1611 			ptq->have_sample = false;  in intel_pt_setup_queue()
1612 		intel_pt_sample_flags(ptq);  in intel_pt_setup_queue()
1613 		ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);  in intel_pt_setup_queue()
1616 		ptq->on_heap = true;  in intel_pt_setup_queue()
1652 static void intel_pt_prep_a_sample(struct intel_pt_queue *ptq,  in intel_pt_prep_a_sample()  argument
1659 	sample->pid = ptq->pid;  in intel_pt_prep_a_sample()
1660 	sample->tid = ptq->tid;  in intel_pt_prep_a_sample()
1662 	if (ptq->pt->have_guest_sideband) {  in intel_pt_prep_a_sample()
1663 		if ((ptq->state->from_ip && ptq->state->from_nr) ||  in intel_pt_prep_a_sample()
1664 		    (ptq->state->to_ip && ptq->state->to_nr)) {  in intel_pt_prep_a_sample()
1665 			sample->pid = ptq->guest_pid;  in intel_pt_prep_a_sample()
1666 			sample->tid = ptq->guest_tid;  in intel_pt_prep_a_sample()
1667 			sample->machine_pid = ptq->guest_machine_pid;  in intel_pt_prep_a_sample()
1668 			sample->vcpu = ptq->vcpu;  in intel_pt_prep_a_sample()
1672 	sample->cpu = ptq->cpu;  in intel_pt_prep_a_sample()
1673 	sample->insn_len = ptq->insn_len;  in intel_pt_prep_a_sample()
1674 	memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_prep_a_sample()
1678 				   struct intel_pt_queue *ptq,  in intel_pt_prep_b_sample()  argument
1682 	intel_pt_prep_a_sample(ptq, event, sample);  in intel_pt_prep_b_sample()
1685 		sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);  in intel_pt_prep_b_sample()
1687 	sample->ip = ptq->state->from_ip;  in intel_pt_prep_b_sample()
1688 	sample->addr = ptq->state->to_ip;  in intel_pt_prep_b_sample()
1689 	sample->cpumode = intel_pt_cpumode(ptq, sample->ip, sample->addr);  in intel_pt_prep_b_sample()
1691 	sample->flags = ptq->flags;  in intel_pt_prep_b_sample()
1730 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_branch_sample()  argument
1732 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_branch_sample()
1733 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_branch_sample()
1741 	if (pt->branches_filter && !(pt->branches_filter & ptq->flags))  in intel_pt_synth_branch_sample()
1747 	intel_pt_prep_b_sample(pt, ptq, event, &sample);  in intel_pt_synth_branch_sample()
1749 	sample.id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1750 	sample.stream_id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1768 	if (ptq->sample_ipc)  in intel_pt_synth_branch_sample()
1769 		sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt;  in intel_pt_synth_branch_sample()
1771 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt;  in intel_pt_synth_branch_sample()
1772 		ptq->last_br_insn_cnt = ptq->ipc_insn_cnt;  in intel_pt_synth_branch_sample()
1773 		ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt;  in intel_pt_synth_branch_sample()
1781 				 struct intel_pt_queue *ptq,  in intel_pt_prep_sample()  argument
1785 	intel_pt_prep_b_sample(pt, ptq, event, sample);  in intel_pt_prep_sample()
1788 		thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,  in intel_pt_prep_sample()
1791 		sample->callchain = ptq->chain;  in intel_pt_prep_sample()
1795 		thread_stack__br_sample(ptq->thread, ptq->cpu, ptq->last_branch,  in intel_pt_prep_sample()
1797 		sample->branch_stack = ptq->last_branch;  in intel_pt_prep_sample()
1801 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_instruction_sample()  argument
1803 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_instruction_sample()
1804 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_instruction_sample()
1810 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_instruction_sample()
1812 	sample.id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1813 	sample.stream_id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1817 		sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;  in intel_pt_synth_instruction_sample()
1819 	if (ptq->sample_ipc)  in intel_pt_synth_instruction_sample()
1820 		sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt;  in intel_pt_synth_instruction_sample()
1822 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt;  in intel_pt_synth_instruction_sample()
1823 		ptq->last_in_insn_cnt = ptq->ipc_insn_cnt;  in intel_pt_synth_instruction_sample()
1824 		ptq->last_in_cyc_cnt = ptq->ipc_cyc_cnt;  in intel_pt_synth_instruction_sample()
1827 	ptq->last_insn_cnt = ptq->state->tot_insn_cnt;  in intel_pt_synth_instruction_sample()
1833 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_transaction_sample()  argument
1835 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_transaction_sample()
1836 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_transaction_sample()
1842 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_transaction_sample()
1844 	sample.id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1845 	sample.stream_id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1852 				   struct intel_pt_queue *ptq,  in intel_pt_prep_p_sample()  argument
1856 	intel_pt_prep_sample(pt, ptq, event, sample);  in intel_pt_prep_p_sample()
1866 static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_ptwrite_sample()  argument
1868 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_ptwrite_sample()
1869 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_ptwrite_sample()
1876 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_ptwrite_sample()
1878 	sample.id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1879 	sample.stream_id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1882 	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);  in intel_pt_synth_ptwrite_sample()
1883 	raw.payload = cpu_to_le64(ptq->state->ptw_payload);  in intel_pt_synth_ptwrite_sample()
1892 static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_cbr_sample()  argument
1894 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_cbr_sample()
1895 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_cbr_sample()
1903 	ptq->cbr_seen = ptq->state->cbr;  in intel_pt_synth_cbr_sample()
1905 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_cbr_sample()
1907 	sample.id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1908 	sample.stream_id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1910 	flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);  in intel_pt_synth_cbr_sample()
1922 static int intel_pt_synth_psb_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_psb_sample()  argument
1924 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_psb_sample()
1925 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_psb_sample()
1932 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_psb_sample()
1934 	sample.id = ptq->pt->psb_id;  in intel_pt_synth_psb_sample()
1935 	sample.stream_id = ptq->pt->psb_id;  in intel_pt_synth_psb_sample()
1939 	raw.offset = ptq->state->psb_offset;  in intel_pt_synth_psb_sample()
1948 static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_mwait_sample()  argument
1950 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_mwait_sample()
1951 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_mwait_sample()
1958 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_mwait_sample()
1960 	sample.id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1961 	sample.stream_id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1964 	raw.payload = cpu_to_le64(ptq->state->mwait_payload);  in intel_pt_synth_mwait_sample()
1973 static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_pwre_sample()  argument
1975 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pwre_sample()
1976 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_pwre_sample()
1983 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwre_sample()
1985 	sample.id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1986 	sample.stream_id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1989 	raw.payload = cpu_to_le64(ptq->state->pwre_payload);  in intel_pt_synth_pwre_sample()
1998 static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_exstop_sample()  argument
2000 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_exstop_sample()
2001 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_exstop_sample()
2008 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_exstop_sample()
2010 	sample.id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
2011 	sample.stream_id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
2014 	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);  in intel_pt_synth_exstop_sample()
2023 static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_pwrx_sample()  argument
2025 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pwrx_sample()
2026 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_pwrx_sample()
2033 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwrx_sample()
2035 	sample.id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
2036 	sample.stream_id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
2039 	raw.payload = cpu_to_le64(ptq->state->pwrx_payload);  in intel_pt_synth_pwrx_sample()
2175 static int intel_pt_do_synth_pebs_sample(struct intel_pt_queue *ptq, struct evsel *evsel, u64 id)  in intel_pt_do_synth_pebs_sample()  argument
2177 	const struct intel_pt_blk_items *items = &ptq->state->items;  in intel_pt_do_synth_pebs_sample()
2179 	union perf_event *event = ptq->event_buf;  in intel_pt_do_synth_pebs_sample()
2180 	struct intel_pt *pt = ptq->pt;  in intel_pt_do_synth_pebs_sample()
2188 	intel_pt_prep_a_sample(ptq, event, &sample);  in intel_pt_do_synth_pebs_sample()
2202 		sample.ip = ptq->state->from_ip;  in intel_pt_do_synth_pebs_sample()
2204 	cpumode = intel_pt_cpumode(ptq, sample.ip, 0);  in intel_pt_do_synth_pebs_sample()
2216 			timestamp = ptq->timestamp;  in intel_pt_do_synth_pebs_sample()
2223 		thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,  in intel_pt_do_synth_pebs_sample()
2226 		sample.callchain = ptq->chain;  in intel_pt_do_synth_pebs_sample()
2249 			intel_pt_add_lbrs(ptq->last_branch, items);  in intel_pt_do_synth_pebs_sample()
2251 			thread_stack__br_sample(ptq->thread, ptq->cpu,  in intel_pt_do_synth_pebs_sample()
2252 						ptq->last_branch,  in intel_pt_do_synth_pebs_sample()
2255 			ptq->last_branch->nr = 0;  in intel_pt_do_synth_pebs_sample()
2257 		sample.branch_stack = ptq->last_branch;  in intel_pt_do_synth_pebs_sample()
2307 static int intel_pt_synth_single_pebs_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_single_pebs_sample()  argument
2309 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_single_pebs_sample()
2313 	return intel_pt_do_synth_pebs_sample(ptq, evsel, id);  in intel_pt_synth_single_pebs_sample()
2316 static int intel_pt_synth_pebs_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_pebs_sample()  argument
2318 	const struct intel_pt_blk_items *items = &ptq->state->items;  in intel_pt_synth_pebs_sample()
2320 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pebs_sample()
2327 		return intel_pt_synth_single_pebs_sample(ptq);  in intel_pt_synth_pebs_sample()
2331 		pe = &ptq->pebs[hw_id];  in intel_pt_synth_pebs_sample()
2336 			return intel_pt_synth_single_pebs_sample(ptq);  in intel_pt_synth_pebs_sample()
2338 		err = intel_pt_do_synth_pebs_sample(ptq, pe->evsel, pe->id);  in intel_pt_synth_pebs_sample()
2346 static int intel_pt_synth_events_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_events_sample()  argument
2348 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_events_sample()
2349 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_events_sample()
2360 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_events_sample()
2362 	sample.id        = ptq->pt->evt_id;  in intel_pt_synth_events_sample()
2363 	sample.stream_id = ptq->pt->evt_id;  in intel_pt_synth_events_sample()
2365 	raw.cfe.type     = ptq->state->cfe_type;  in intel_pt_synth_events_sample()
2367 	raw.cfe.ip       = !!(ptq->state->flags & INTEL_PT_FUP_IP);  in intel_pt_synth_events_sample()
2368 	raw.cfe.vector   = ptq->state->cfe_vector;  in intel_pt_synth_events_sample()
2369 	raw.cfe.evd_cnt  = ptq->state->evd_cnt;  in intel_pt_synth_events_sample()
2371 	for (i = 0; i < ptq->state->evd_cnt; i++) {  in intel_pt_synth_events_sample()
2373 		raw.evd[i].evd_type = ptq->state->evd[i].type;  in intel_pt_synth_events_sample()
2374 		raw.evd[i].payload  = ptq->state->evd[i].payload;  in intel_pt_synth_events_sample()
2378 			  ptq->state->evd_cnt * sizeof(struct perf_synth_intel_evd);  in intel_pt_synth_events_sample()
2385 static int intel_pt_synth_iflag_chg_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_iflag_chg_sample()  argument
2387 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_iflag_chg_sample()
2388 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_iflag_chg_sample()
2395 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_iflag_chg_sample()
2397 	sample.id = ptq->pt->iflag_chg_id;  in intel_pt_synth_iflag_chg_sample()
2398 	sample.stream_id = ptq->pt->iflag_chg_id;  in intel_pt_synth_iflag_chg_sample()
2401 	raw.iflag = ptq->state->to_iflag;  in intel_pt_synth_iflag_chg_sample()
2403 	if (ptq->state->type & INTEL_PT_BRANCH) {  in intel_pt_synth_iflag_chg_sample()
2405 		raw.branch_ip = ptq->state->to_ip;  in intel_pt_synth_iflag_chg_sample()
2409 	sample.flags = ptq->flags;  in intel_pt_synth_iflag_chg_sample()
2461 static int intel_ptq_synth_error(struct intel_pt_queue *ptq,  in intel_ptq_synth_error()  argument
2464 	struct intel_pt *pt = ptq->pt;  in intel_ptq_synth_error()
2465 	u64 tm = ptq->timestamp;  in intel_ptq_synth_error()
2467 	pid_t pid = ptq->pid;  in intel_ptq_synth_error()
2468 	pid_t tid = ptq->tid;  in intel_ptq_synth_error()
2474 		machine_pid = ptq->guest_machine_pid;  in intel_ptq_synth_error()
2475 		vcpu = ptq->vcpu;  in intel_ptq_synth_error()
2476 		pid = ptq->guest_pid;  in intel_ptq_synth_error()
2477 		tid = ptq->guest_tid;  in intel_ptq_synth_error()
2480 	return intel_pt_synth_error(pt, state->err, ptq->cpu, pid, tid,  in intel_ptq_synth_error()
2484 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)  in intel_pt_next_tid()  argument
2487 	pid_t tid = ptq->next_tid;  in intel_pt_next_tid()
2493 	intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);  in intel_pt_next_tid()
2495 	err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);  in intel_pt_next_tid()
2497 	queue = &pt->queues.queue_array[ptq->queue_nr];  in intel_pt_next_tid()
2500 	ptq->next_tid = -1;  in intel_pt_next_tid()
2505 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)  in intel_pt_is_switch_ip()  argument
2507 	struct intel_pt *pt = ptq->pt;  in intel_pt_is_switch_ip()
2510 	       (ptq->flags & PERF_IP_FLAG_BRANCH) &&  in intel_pt_is_switch_ip()
2511 	       !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |  in intel_pt_is_switch_ip()
2518 static int intel_pt_sample(struct intel_pt_queue *ptq)  in intel_pt_sample()  argument
2520 	const struct intel_pt_state *state = ptq->state;  in intel_pt_sample()
2521 	struct intel_pt *pt = ptq->pt;  in intel_pt_sample()
2524 	if (!ptq->have_sample)  in intel_pt_sample()
2527 	ptq->have_sample = false;  in intel_pt_sample()
2530 		ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt;  in intel_pt_sample()
2531 		ptq->ipc_cyc_cnt = ptq->state->cycles;  in intel_pt_sample()
2532 		ptq->sample_ipc = true;  in intel_pt_sample()
2534 		ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt;  in intel_pt_sample()
2535 		ptq->ipc_cyc_cnt = ptq->state->tot_cyc_cnt;  in intel_pt_sample()
2536 		ptq->sample_ipc = ptq->state->flags & INTEL_PT_SAMPLE_IPC;  in intel_pt_sample()
2541 		intel_pt_get_guest(ptq);  in intel_pt_sample()
2548 		err = intel_pt_synth_pebs_sample(ptq);  in intel_pt_sample()
2555 			err = intel_pt_synth_events_sample(ptq);  in intel_pt_sample()
2560 			err = intel_pt_synth_iflag_chg_sample(ptq);  in intel_pt_sample()
2568 			err = intel_pt_synth_psb_sample(ptq);  in intel_pt_sample()
2572 		if (ptq->state->cbr != ptq->cbr_seen) {  in intel_pt_sample()
2573 			err = intel_pt_synth_cbr_sample(ptq);  in intel_pt_sample()
2579 				err = intel_pt_synth_mwait_sample(ptq);  in intel_pt_sample()
2584 				err = intel_pt_synth_pwre_sample(ptq);  in intel_pt_sample()
2589 				err = intel_pt_synth_exstop_sample(ptq);  in intel_pt_sample()
2594 				err = intel_pt_synth_pwrx_sample(ptq);  in intel_pt_sample()
2602 		err = intel_pt_synth_instruction_sample(ptq);  in intel_pt_sample()
2608 		err = intel_pt_synth_transaction_sample(ptq);  in intel_pt_sample()
2614 		err = intel_pt_synth_ptwrite_sample(ptq);  in intel_pt_sample()
2623 		thread_stack__event(ptq->thread, ptq->cpu, ptq->flags,  in intel_pt_sample()
2624 				    state->from_ip, state->to_ip, ptq->insn_len,  in intel_pt_sample()
2629 		thread_stack__set_trace_nr(ptq->thread, ptq->cpu, state->trace_nr);  in intel_pt_sample()
2644 			err = intel_pt_synth_branch_sample(ptq);  in intel_pt_sample()
2649 			err = intel_pt_synth_branch_sample(ptq);  in intel_pt_sample()
2652 			err = intel_pt_synth_branch_sample(ptq);  in intel_pt_sample()
2658 	if (!ptq->sync_switch)  in intel_pt_sample()
2661 	if (intel_pt_is_switch_ip(ptq, state->to_ip)) {  in intel_pt_sample()
2662 		switch (ptq->switch_state) {  in intel_pt_sample()
2666 			err = intel_pt_next_tid(pt, ptq);  in intel_pt_sample()
2669 			ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sample()
2672 			ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;  in intel_pt_sample()
2676 		ptq->switch_state = INTEL_PT_SS_NOT_TRACING;  in intel_pt_sample()
2677 	} else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {  in intel_pt_sample()
2678 		ptq->switch_state = INTEL_PT_SS_UNKNOWN;  in intel_pt_sample()
2679 	} else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&  in intel_pt_sample()
2681 		   (ptq->flags & PERF_IP_FLAG_CALL)) {  in intel_pt_sample()
2682 		ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sample()
2751 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_enable_sync_switch()  local
2753 		if (ptq)  in intel_pt_enable_sync_switch()
2754 			ptq->sync_switch = true;  in intel_pt_enable_sync_switch()
2766 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_disable_sync_switch()  local
2768 		if (ptq) {  in intel_pt_disable_sync_switch()
2769 			ptq->sync_switch = false;  in intel_pt_disable_sync_switch()
2770 			intel_pt_next_tid(pt, ptq);  in intel_pt_disable_sync_switch()
2779 static bool intel_pt_next_time(struct intel_pt_queue *ptq)  in intel_pt_next_time()  argument
2781 	struct intel_pt *pt = ptq->pt;  in intel_pt_next_time()
2783 	if (ptq->sel_start) {  in intel_pt_next_time()
2785 		ptq->sel_start = false;  in intel_pt_next_time()
2786 		ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end;  in intel_pt_next_time()
2788 	} else if (ptq->sel_idx + 1 < pt->range_cnt) {  in intel_pt_next_time()
2790 		ptq->sel_start = true;  in intel_pt_next_time()
2791 		ptq->sel_idx += 1;  in intel_pt_next_time()
2792 		ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start;  in intel_pt_next_time()
2800 static int intel_pt_time_filter(struct intel_pt_queue *ptq, u64 *ff_timestamp)  in intel_pt_time_filter()  argument
2805 		if (ptq->sel_start) {  in intel_pt_time_filter()
2806 			if (ptq->timestamp >= ptq->sel_timestamp) {  in intel_pt_time_filter()
2808 				intel_pt_next_time(ptq);  in intel_pt_time_filter()
2809 				if (!ptq->sel_timestamp) {  in intel_pt_time_filter()
2817 			ptq->have_sample = false;  in intel_pt_time_filter()
2818 			if (ptq->sel_timestamp > *ff_timestamp) {  in intel_pt_time_filter()
2819 				if (ptq->sync_switch) {  in intel_pt_time_filter()
2820 					intel_pt_next_tid(ptq->pt, ptq);  in intel_pt_time_filter()
2821 					ptq->switch_state = INTEL_PT_SS_UNKNOWN;  in intel_pt_time_filter()
2823 				*ff_timestamp = ptq->sel_timestamp;  in intel_pt_time_filter()
2824 				err = intel_pt_fast_forward(ptq->decoder,  in intel_pt_time_filter()
2825 							    ptq->sel_timestamp);  in intel_pt_time_filter()
2830 		} else if (ptq->timestamp > ptq->sel_timestamp) {  in intel_pt_time_filter()
2832 			if (!intel_pt_next_time(ptq)) {  in intel_pt_time_filter()
2834 				ptq->have_sample = false;  in intel_pt_time_filter()
2835 				ptq->switch_state = INTEL_PT_SS_NOT_TRACING;  in intel_pt_time_filter()
2847 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)  in intel_pt_run_decoder()  argument
2849 	const struct intel_pt_state *state = ptq->state;  in intel_pt_run_decoder()
2850 	struct intel_pt *pt = ptq->pt;  in intel_pt_run_decoder()
2870 		     ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_run_decoder()
2872 		err = intel_pt_sample(ptq);  in intel_pt_run_decoder()
2876 		state = intel_pt_decode(ptq->decoder);  in intel_pt_run_decoder()
2880 			if (ptq->sync_switch &&  in intel_pt_run_decoder()
2882 				ptq->sync_switch = false;  in intel_pt_run_decoder()
2883 				intel_pt_next_tid(pt, ptq);  in intel_pt_run_decoder()
2885 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
2887 				err = intel_ptq_synth_error(ptq, state);  in intel_pt_run_decoder()
2894 		ptq->state = state;  in intel_pt_run_decoder()
2895 		ptq->have_sample = true;  in intel_pt_run_decoder()
2896 		intel_pt_sample_flags(ptq);  in intel_pt_run_decoder()
2904 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
2906 		} else if (ptq->sync_switch &&  in intel_pt_run_decoder()
2907 			   ptq->switch_state == INTEL_PT_SS_UNKNOWN &&  in intel_pt_run_decoder()
2908 			   intel_pt_is_switch_ip(ptq, state->to_ip) &&  in intel_pt_run_decoder()
2909 			   ptq->next_tid == -1) {  in intel_pt_run_decoder()
2912 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
2913 		} else if (state->timestamp > ptq->timestamp) {  in intel_pt_run_decoder()
2914 			ptq->timestamp = state->timestamp;  in intel_pt_run_decoder()
2917 		if (ptq->sel_timestamp) {  in intel_pt_run_decoder()
2918 			err = intel_pt_time_filter(ptq, &ff_timestamp);  in intel_pt_run_decoder()
2923 		if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {  in intel_pt_run_decoder()
2924 			*timestamp = ptq->timestamp;  in intel_pt_run_decoder()
2948 		struct intel_pt_queue *ptq;  in intel_pt_process_queues()  local
2958 		ptq = queue->priv;  in intel_pt_process_queues()
2976 		ret = intel_pt_run_decoder(ptq, &ts);  in intel_pt_process_queues()
2988 			ptq->on_heap = false;  in intel_pt_process_queues()
3004 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_process_timeless_queues()  local
3006 		if (ptq && (tid == -1 || ptq->tid == tid)) {  in intel_pt_process_timeless_queues()
3007 			ptq->time = time_;  in intel_pt_process_timeless_queues()
3009 			intel_pt_run_decoder(ptq, &ts);  in intel_pt_process_timeless_queues()
3015 static void intel_pt_sample_set_pid_tid_cpu(struct intel_pt_queue *ptq,  in intel_pt_sample_set_pid_tid_cpu()  argument
3019 	struct machine *m = ptq->pt->machine;  in intel_pt_sample_set_pid_tid_cpu()
3021 	ptq->pid = sample->pid;  in intel_pt_sample_set_pid_tid_cpu()
3022 	ptq->tid = sample->tid;  in intel_pt_sample_set_pid_tid_cpu()
3023 	ptq->cpu = queue->cpu;  in intel_pt_sample_set_pid_tid_cpu()
3026 		     ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_sample_set_pid_tid_cpu()
3028 	thread__zput(ptq->thread);  in intel_pt_sample_set_pid_tid_cpu()
3030 	if (ptq->tid == -1)  in intel_pt_sample_set_pid_tid_cpu()
3033 	if (ptq->pid == -1) {  in intel_pt_sample_set_pid_tid_cpu()
3034 		ptq->thread = machine__find_thread(m, -1, ptq->tid);  in intel_pt_sample_set_pid_tid_cpu()
3035 		if (ptq->thread)  in intel_pt_sample_set_pid_tid_cpu()
3036 			ptq->pid = ptq->thread->pid_;  in intel_pt_sample_set_pid_tid_cpu()
3040 	ptq->thread = machine__findnew_thread(m, ptq->pid, ptq->tid);  in intel_pt_sample_set_pid_tid_cpu()
3047 	struct intel_pt_queue *ptq;  in intel_pt_process_timeless_sample()  local
3054 	ptq = queue->priv;  in intel_pt_process_timeless_sample()
3055 	if (!ptq)  in intel_pt_process_timeless_sample()
3058 	ptq->stop = false;  in intel_pt_process_timeless_sample()
3059 	ptq->time = sample->time;  in intel_pt_process_timeless_sample()
3060 	intel_pt_sample_set_pid_tid_cpu(ptq, queue, sample);  in intel_pt_process_timeless_sample()
3061 	intel_pt_run_decoder(ptq, &ts);  in intel_pt_process_timeless_sample()
3103 	struct intel_pt_queue *ptq;  in intel_pt_sync_switch()  local
3109 	ptq = intel_pt_cpu_to_ptq(pt, cpu);  in intel_pt_sync_switch()
3110 	if (!ptq || !ptq->sync_switch)  in intel_pt_sync_switch()
3113 	switch (ptq->switch_state) {  in intel_pt_sync_switch()
3118 		ptq->next_tid = tid;  in intel_pt_sync_switch()
3119 		ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;  in intel_pt_sync_switch()
3122 		if (!ptq->on_heap) {  in intel_pt_sync_switch()
3123 			ptq->timestamp = perf_time_to_tsc(timestamp,  in intel_pt_sync_switch()
3125 			err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,  in intel_pt_sync_switch()
3126 						 ptq->timestamp);  in intel_pt_sync_switch()
3129 			ptq->on_heap = true;  in intel_pt_sync_switch()
3131 		ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sync_switch()
3140 	ptq->next_tid = -1;  in intel_pt_sync_switch()
3177 		struct intel_pt_queue *ptq;  in intel_pt_context_switch_in()  local
3179 		ptq = intel_pt_cpu_to_ptq(pt, cpu);  in intel_pt_context_switch_in()
3180 		if (ptq && ptq->sync_switch) {  in intel_pt_context_switch_in()
3181 			ptq->next_tid = -1;  in intel_pt_context_switch_in()
3182 			switch (ptq->switch_state) {  in intel_pt_context_switch_in()
3189 				ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_context_switch_in()
3295 	struct intel_pt_queue *ptq;  in intel_pt_process_aux_output_hw_id()  local
3305 	ptq = queue->priv;  in intel_pt_process_aux_output_hw_id()
3307 	ptq->pebs[hw_id].evsel = evsel;  in intel_pt_process_aux_output_hw_id()
3308 	ptq->pebs[hw_id].id = sample->id;  in intel_pt_process_aux_output_hw_id()