30 static DEFINE_PER_CPU(struct pt, pt_ctx);
106 PMU_FORMAT_ATTR(pt,		"config:0"	);
418 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config_start()  local
422 	if (READ_ONCE(pt->vmx_on))  in pt_config_start()
423 		perf_aux_output_flag(&pt->handle, PERF_AUX_FLAG_PARTIAL);  in pt_config_start()
461 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config_filters()  local
483 		if (pt->filters.filter[range].msr_a != filter->msr_a) {  in pt_config_filters()
485 			pt->filters.filter[range].msr_a = filter->msr_a;  in pt_config_filters()
488 		if (pt->filters.filter[range].msr_b != filter->msr_b) {  in pt_config_filters()
490 			pt->filters.filter[range].msr_b = filter->msr_b;  in pt_config_filters()
501 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config()  local
502 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_config()
542 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config_stop()  local
550 	if (!READ_ONCE(pt->vmx_on))  in pt_config_stop()
625 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config_buffer()  local
638 	if (pt->output_base != reg) {  in pt_config_buffer()
639 		pt->output_base = reg;  in pt_config_buffer()
644 	if (pt->output_mask != reg) {  in pt_config_buffer()
645 		pt->output_mask = reg;  in pt_config_buffer()
845 static void pt_update_head(struct pt *pt)  in pt_update_head()  argument
847 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_update_head()
896 static void pt_handle_status(struct pt *pt)  in pt_handle_status()  argument
898 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_handle_status()
921 			perf_aux_output_flag(&pt->handle,  in pt_handle_status()
957 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_read_offset()  local
961 		rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, pt->output_base);  in pt_read_offset()
962 		tp = phys_to_virt(pt->output_base);  in pt_read_offset()
966 	rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, pt->output_mask);  in pt_read_offset()
968 	buf->output_off = pt->output_mask >> 32;  in pt_read_offset()
971 		buf->cur_idx = (pt->output_mask & 0xffffff80) >> 7;  in pt_read_offset()
1475 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in intel_pt_interrupt()  local
1477 	struct perf_event *event = pt->handle.event;  in intel_pt_interrupt()
1484 	if (!READ_ONCE(pt->handle_nmi))  in intel_pt_interrupt()
1492 	buf = perf_get_aux(&pt->handle);  in intel_pt_interrupt()
1498 	pt_handle_status(pt);  in intel_pt_interrupt()
1500 	pt_update_head(pt);  in intel_pt_interrupt()
1502 	perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0));  in intel_pt_interrupt()
1507 		buf = perf_aux_output_begin(&pt->handle, event);  in intel_pt_interrupt()
1513 		pt_buffer_reset_offsets(buf, pt->handle.head);  in intel_pt_interrupt()
1515 		ret = pt_buffer_reset_markers(buf, &pt->handle);  in intel_pt_interrupt()
1517 			perf_aux_output_end(&pt->handle, 0);  in intel_pt_interrupt()
1528 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in intel_pt_handle_vmx()  local
1543 	WRITE_ONCE(pt->vmx_on, on);  in intel_pt_handle_vmx()
1549 	event = pt->handle.event;  in intel_pt_handle_vmx()
1551 		perf_aux_output_flag(&pt->handle,  in intel_pt_handle_vmx()
1569 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_start()  local
1572 	buf = perf_aux_output_begin(&pt->handle, event);  in pt_event_start()
1576 	pt_buffer_reset_offsets(buf, pt->handle.head);  in pt_event_start()
1578 		if (pt_buffer_reset_markers(buf, &pt->handle))  in pt_event_start()
1582 	WRITE_ONCE(pt->handle_nmi, 1);  in pt_event_start()
1591 	perf_aux_output_end(&pt->handle, 0);  in pt_event_start()
1598 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_stop()  local
1604 	WRITE_ONCE(pt->handle_nmi, 0);  in pt_event_stop()
1614 		struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_event_stop()
1619 		if (WARN_ON_ONCE(pt->handle.event != event))  in pt_event_stop()
1624 		pt_handle_status(pt);  in pt_event_stop()
1626 		pt_update_head(pt);  in pt_event_stop()
1629 			pt->handle.head =  in pt_event_stop()
1632 		perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0));  in pt_event_stop()
1640 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_snapshot_aux()  local
1641 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_event_snapshot_aux()
1658 	if (READ_ONCE(pt->handle_nmi))  in pt_event_snapshot_aux()
1662 	pt_update_head(pt);  in pt_event_snapshot_aux()
1669 	ret = perf_output_copy_aux(&pt->handle, handle, from, to);  in pt_event_snapshot_aux()
1676 	if (pt->handle_nmi)  in pt_event_snapshot_aux()
1689 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_add()  local
1693 	if (pt->handle.event)  in pt_event_add()
1744 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in cpu_emergency_stop_pt()  local
1746 	if (pt->handle.event)  in cpu_emergency_stop_pt()
1747 		pt_event_stop(pt->handle.event, PERF_EF_UPDATE);  in cpu_emergency_stop_pt()