78 static inline void cond_local_irq_enable(struct pt_regs *regs)  in cond_local_irq_enable()  argument
80 	if (regs->flags & X86_EFLAGS_IF)  in cond_local_irq_enable()
84 static inline void cond_local_irq_disable(struct pt_regs *regs)  in cond_local_irq_disable()  argument
86 	if (regs->flags & X86_EFLAGS_IF)  in cond_local_irq_disable()
104 		  struct pt_regs *regs,	long error_code)  in do_trap_no_signal()  argument
106 	if (v8086_mode(regs)) {  in do_trap_no_signal()
112 			if (!handle_vm86_trap((struct kernel_vm86_regs *) regs,  in do_trap_no_signal()
116 	} else if (!user_mode(regs)) {  in do_trap_no_signal()
117 		if (fixup_exception(regs, trapnr, error_code, 0))  in do_trap_no_signal()
122 		die(str, regs, error_code);  in do_trap_no_signal()
124 		if (fixup_vdso_exception(regs, trapnr, error_code, 0))  in do_trap_no_signal()
145 			struct pt_regs *regs, long error_code)  in show_signal()  argument
151 			regs->ip, regs->sp, error_code);  in show_signal()
152 		print_vma_addr(KERN_CONT " in ", regs->ip);  in show_signal()
158 do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,  in do_trap()  argument
163 	if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code))  in do_trap()
166 	show_signal(tsk, signr, "trap ", str, regs, error_code);  in do_trap()
175 static void do_error_trap(struct pt_regs *regs, long error_code, char *str,  in do_error_trap()  argument
180 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=  in do_error_trap()
182 		cond_local_irq_enable(regs);  in do_error_trap()
183 		do_trap(trapnr, signr, str, regs, error_code, sicode, addr);  in do_error_trap()
184 		cond_local_irq_disable(regs);  in do_error_trap()
198 static __always_inline void __user *error_get_trap_addr(struct pt_regs *regs)  in error_get_trap_addr()  argument
200 	return (void __user *)uprobe_get_trap_addr(regs);  in error_get_trap_addr()
205 	do_error_trap(regs, 0, "divide error", X86_TRAP_DE, SIGFPE,  in DEFINE_IDTENTRY()
206 		      FPE_INTDIV, error_get_trap_addr(regs));  in DEFINE_IDTENTRY()
211 	do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL);  in DEFINE_IDTENTRY()
239 	if (WARN_ON_ONCE(user_mode(regs) || (error_code & CP_EC) != CP_ENDBR))  in DEFINE_IDTENTRY_ERRORCODE()
242 	if (unlikely(regs->ip == (unsigned long)&ibt_selftest_ip)) {  in DEFINE_IDTENTRY_ERRORCODE()
243 		regs->ax = 0;  in DEFINE_IDTENTRY_ERRORCODE()
247 	pr_err("Missing ENDBR: %pS\n", (void *)instruction_pointer(regs));  in DEFINE_IDTENTRY_ERRORCODE()
250 		__warn(__FILE__, __LINE__, (void *)regs->ip, TAINT_WARN, regs, NULL);  in DEFINE_IDTENTRY_ERRORCODE()
290 void handle_invalid_op(struct pt_regs *regs)  in handle_invalid_op()  argument
292 static inline void handle_invalid_op(struct pt_regs *regs)  in handle_invalid_op()
295 	do_error_trap(regs, 0, "invalid opcode", X86_TRAP_UD, SIGILL,  in handle_invalid_op()
296 		      ILL_ILLOPN, error_get_trap_addr(regs));  in handle_invalid_op()
299 static noinstr bool handle_bug(struct pt_regs *regs)  in handle_bug()  argument
303 	if (!is_valid_bugaddr(regs->ip))  in handle_bug()
314 	if (regs->flags & X86_EFLAGS_IF)  in handle_bug()
316 	if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) {  in handle_bug()
317 		regs->ip += LEN_UD2;  in handle_bug()
320 	if (regs->flags & X86_EFLAGS_IF)  in handle_bug()
336 	if (!user_mode(regs) && handle_bug(regs))  in DEFINE_IDTENTRY_RAW()
339 	state = irqentry_enter(regs);  in DEFINE_IDTENTRY_RAW()
341 	handle_invalid_op(regs);  in DEFINE_IDTENTRY_RAW()
343 	irqentry_exit(regs, state);  in DEFINE_IDTENTRY_RAW()
348 	do_error_trap(regs, 0, "coprocessor segment overrun",  in DEFINE_IDTENTRY()
354 	do_error_trap(regs, error_code, "invalid TSS", X86_TRAP_TS, SIGSEGV,  in DEFINE_IDTENTRY_ERRORCODE()
360 	do_error_trap(regs, error_code, "segment not present", X86_TRAP_NP,  in DEFINE_IDTENTRY_ERRORCODE()
366 	do_error_trap(regs, error_code, "stack segment", X86_TRAP_SS, SIGBUS,  in DEFINE_IDTENTRY_ERRORCODE()
374 	if (notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_AC, SIGBUS) == NOTIFY_STOP)  in DEFINE_IDTENTRY_ERRORCODE()
377 	if (!user_mode(regs))  in DEFINE_IDTENTRY_ERRORCODE()
378 		die("Split lock detected\n", regs, error_code);  in DEFINE_IDTENTRY_ERRORCODE()
382 	if (handle_user_split_lock(regs, error_code))  in DEFINE_IDTENTRY_ERRORCODE()
385 	do_trap(X86_TRAP_AC, SIGBUS, "alignment check", regs,  in DEFINE_IDTENTRY_ERRORCODE()
393 __visible void __noreturn handle_stack_overflow(struct pt_regs *regs,  in handle_stack_overflow()  argument
402 	die("stack guard page", regs, 0);  in handle_stack_overflow()
455 	if (((long)regs->sp >> P4D_SHIFT) == ESPFIX_PGD_ENTRY &&  in DEFINE_IDTENTRY_DF()
456 		regs->cs == __KERNEL_CS &&  in DEFINE_IDTENTRY_DF()
457 		regs->ip == (unsigned long)native_irq_return_iret)  in DEFINE_IDTENTRY_DF()
460 		unsigned long *p = (unsigned long *)regs->sp;  in DEFINE_IDTENTRY_DF()
486 		regs->ip = (unsigned long)asm_exc_general_protection;  in DEFINE_IDTENTRY_DF()
487 		regs->sp = (unsigned long)&gpregs->orig_ax;  in DEFINE_IDTENTRY_DF()
493 	irqentry_nmi_enter(regs);  in DEFINE_IDTENTRY_DF()
495 	notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);  in DEFINE_IDTENTRY_DF()
539 		handle_stack_overflow(regs, address, &info);  in DEFINE_IDTENTRY_DF()
543 	die("double fault", regs, error_code);  in DEFINE_IDTENTRY_DF()
550 	if (notify_die(DIE_TRAP, "bounds", regs, 0,  in DEFINE_IDTENTRY()
553 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
555 	if (!user_mode(regs))  in DEFINE_IDTENTRY()
556 		die("bounds", regs, 0);  in DEFINE_IDTENTRY()
558 	do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, 0, 0, NULL);  in DEFINE_IDTENTRY()
560 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
574 static enum kernel_gp_hint get_kernel_gp_address(struct pt_regs *regs,  in get_kernel_gp_address()  argument
581 	if (copy_from_kernel_nofault(insn_buf, (void *)regs->ip,  in get_kernel_gp_address()
589 	*addr = (unsigned long)insn_get_addr_ref(&insn, regs);  in get_kernel_gp_address()
609 static bool fixup_iopl_exception(struct pt_regs *regs)  in fixup_iopl_exception()  argument
618 	if (insn_get_effective_ip(regs, &ip))  in fixup_iopl_exception()
635 	regs->ip += 1;  in fixup_iopl_exception()
690 static bool gp_try_fixup_and_notify(struct pt_regs *regs, int trapnr,  in gp_try_fixup_and_notify()  argument
693 	if (fixup_exception(regs, trapnr, error_code, 0))  in gp_try_fixup_and_notify()
704 	    kprobe_fault_handler(regs, trapnr))  in gp_try_fixup_and_notify()
707 	return notify_die(DIE_GPF, str, regs, error_code, trapnr, SIGSEGV) == NOTIFY_STOP;  in gp_try_fixup_and_notify()
710 static void gp_user_force_sig_segv(struct pt_regs *regs, int trapnr,  in gp_user_force_sig_segv()  argument
715 	show_signal(current, SIGSEGV, "", str, regs, error_code);  in gp_user_force_sig_segv()
725 	if (user_mode(regs) && try_fixup_enqcmd_gp())  in DEFINE_IDTENTRY_ERRORCODE()
728 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY_ERRORCODE()
731 		if (user_mode(regs) && fixup_umip_exception(regs))  in DEFINE_IDTENTRY_ERRORCODE()
735 	if (v8086_mode(regs)) {  in DEFINE_IDTENTRY_ERRORCODE()
737 		handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);  in DEFINE_IDTENTRY_ERRORCODE()
742 	if (user_mode(regs)) {  in DEFINE_IDTENTRY_ERRORCODE()
743 		if (fixup_iopl_exception(regs))  in DEFINE_IDTENTRY_ERRORCODE()
746 		if (fixup_vdso_exception(regs, X86_TRAP_GP, error_code, 0))  in DEFINE_IDTENTRY_ERRORCODE()
749 		gp_user_force_sig_segv(regs, X86_TRAP_GP, error_code, desc);  in DEFINE_IDTENTRY_ERRORCODE()
753 	if (gp_try_fixup_and_notify(regs, X86_TRAP_GP, error_code, desc))  in DEFINE_IDTENTRY_ERRORCODE()
759 		hint = get_kernel_gp_address(regs, &gp_addr);  in DEFINE_IDTENTRY_ERRORCODE()
774 	die_addr(desc, regs, error_code, gp_addr);  in DEFINE_IDTENTRY_ERRORCODE()
777 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY_ERRORCODE()
780 static bool do_int3(struct pt_regs *regs)  in do_int3()  argument
785 	if (kgdb_ll_trap(DIE_INT3, "int3", regs, 0, X86_TRAP_BP,  in do_int3()
791 	if (kprobe_int3_handler(regs))  in do_int3()
794 	res = notify_die(DIE_INT3, "int3", regs, 0, X86_TRAP_BP, SIGTRAP);  in do_int3()
800 static void do_int3_user(struct pt_regs *regs)  in do_int3_user()  argument
802 	if (do_int3(regs))  in do_int3_user()
805 	cond_local_irq_enable(regs);  in do_int3_user()
806 	do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, 0, 0, NULL);  in do_int3_user()
807 	cond_local_irq_disable(regs);  in do_int3_user()
817 	if (poke_int3_handler(regs))  in DEFINE_IDTENTRY_RAW()
827 	if (user_mode(regs)) {  in DEFINE_IDTENTRY_RAW()
828 		irqentry_enter_from_user_mode(regs);  in DEFINE_IDTENTRY_RAW()
830 		do_int3_user(regs);  in DEFINE_IDTENTRY_RAW()
832 		irqentry_exit_to_user_mode(regs);  in DEFINE_IDTENTRY_RAW()
834 		irqentry_state_t irq_state = irqentry_nmi_enter(regs);  in DEFINE_IDTENTRY_RAW()
837 		if (!do_int3(regs))  in DEFINE_IDTENTRY_RAW()
838 			die("int3", regs, 0);  in DEFINE_IDTENTRY_RAW()
840 		irqentry_nmi_exit(regs, irq_state);  in DEFINE_IDTENTRY_RAW()
852 	struct pt_regs *regs = (struct pt_regs *)this_cpu_read(cpu_current_top_of_stack) - 1;  in sync_regs()  local
853 	if (regs != eregs)  in sync_regs()
854 		*regs = *eregs;  in sync_regs()
855 	return regs;  in sync_regs()
859 asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *regs)  in vc_switch_off_ist()  argument
869 	if (ip_within_syscall_gap(regs)) {  in vc_switch_off_ist()
879 	sp    = regs->sp;  in vc_switch_off_ist()
895 	*regs_ret = *regs;  in vc_switch_off_ist()
929 static bool is_sysenter_singlestep(struct pt_regs *regs)  in is_sysenter_singlestep()  argument
940 	return (regs->ip - (unsigned long)__begin_SYSENTER_singlestep_region) <  in is_sysenter_singlestep()
944 	return (regs->ip - (unsigned long)entry_SYSENTER_compat) <  in is_sysenter_singlestep()
999 static bool notify_debug(struct pt_regs *regs, unsigned long *dr6)  in notify_debug()  argument
1008 	if (notify_die(DIE_DEBUG, "debug", regs, (long)dr6, 0, SIGTRAP) == NOTIFY_STOP)  in notify_debug()
1014 static __always_inline void exc_debug_kernel(struct pt_regs *regs,  in exc_debug_kernel()  argument
1029 	irqentry_state_t irq_state = irqentry_nmi_enter(regs);  in exc_debug_kernel()
1036 	WARN_ON_ONCE(user_mode(regs));  in exc_debug_kernel()
1056 	if ((dr6 & DR_STEP) && is_sysenter_singlestep(regs))  in exc_debug_kernel()
1065 	if (notify_debug(regs, &dr6))  in exc_debug_kernel()
1080 		regs->flags &= ~X86_EFLAGS_TF;  in exc_debug_kernel()
1083 	irqentry_nmi_exit(regs, irq_state);  in exc_debug_kernel()
1088 static __always_inline void exc_debug_user(struct pt_regs *regs,  in exc_debug_user()  argument
1097 	WARN_ON_ONCE(!user_mode(regs));  in exc_debug_user()
1108 	irqentry_enter_from_user_mode(regs);  in exc_debug_user()
1134 	if (notify_debug(regs, &dr6))  in exc_debug_user()
1140 	if (v8086_mode(regs)) {  in exc_debug_user()
1141 		handle_vm86_trap((struct kernel_vm86_regs *)regs, 0, X86_TRAP_DB);  in exc_debug_user()
1147 		handle_bus_lock(regs);  in exc_debug_user()
1152 		send_sigtrap(regs, 0, get_si_code(dr6));  in exc_debug_user()
1158 	irqentry_exit_to_user_mode(regs);  in exc_debug_user()
1165 	exc_debug_kernel(regs, debug_read_clear_dr6());  in DEFINE_IDTENTRY_DEBUG()
1171 	exc_debug_user(regs, debug_read_clear_dr6());  in DEFINE_IDTENTRY_DEBUG_USER()
1179 	if (user_mode(regs))  in DEFINE_IDTENTRY_RAW()
1180 		exc_debug_user(regs, dr6);  in DEFINE_IDTENTRY_RAW()
1182 		exc_debug_kernel(regs, dr6);  in DEFINE_IDTENTRY_RAW()
1191 static void math_error(struct pt_regs *regs, int trapnr)  in math_error()  argument
1199 	cond_local_irq_enable(regs);  in math_error()
1201 	if (!user_mode(regs)) {  in math_error()
1202 		if (fixup_exception(regs, trapnr, 0, 0))  in math_error()
1208 		if (notify_die(DIE_TRAP, str, regs, 0, trapnr,  in math_error()
1210 			die(str, regs, 0);  in math_error()
1228 	if (fixup_vdso_exception(regs, trapnr, 0, 0))  in math_error()
1232 			(void __user *)uprobe_get_trap_addr(regs));  in math_error()
1234 	cond_local_irq_disable(regs);  in math_error()
1239 	math_error(regs, X86_TRAP_MF);  in DEFINE_IDTENTRY()
1247 			__exc_general_protection(regs, 0);  in DEFINE_IDTENTRY()
1251 	math_error(regs, X86_TRAP_XF);  in DEFINE_IDTENTRY()
1277 static bool handle_xfd_event(struct pt_regs *regs)  in handle_xfd_event()  argument
1292 	if (WARN_ON(!user_mode(regs)))  in handle_xfd_event()
1301 		force_sig_fault(SIGILL, ILL_ILLOPC, error_get_trap_addr(regs));  in handle_xfd_event()
1316 	if (handle_xfd_event(regs))  in DEFINE_IDTENTRY()
1323 		cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
1325 		info.regs = regs;  in DEFINE_IDTENTRY()
1328 		cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
1343 		die("unexpected #NM exception", regs, 0);  in DEFINE_IDTENTRY()
1351 static void ve_raise_fault(struct pt_regs *regs, long error_code)  in ve_raise_fault()  argument
1353 	if (user_mode(regs)) {  in ve_raise_fault()
1354 		gp_user_force_sig_segv(regs, X86_TRAP_VE, error_code, VE_FAULT_STR);  in ve_raise_fault()
1358 	if (gp_try_fixup_and_notify(regs, X86_TRAP_VE, error_code, VE_FAULT_STR))  in ve_raise_fault()
1361 	die_addr(VE_FAULT_STR, regs, error_code, 0);  in ve_raise_fault()
1418 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
1424 	if (!tdx_handle_virt_exception(regs, &ve))  in DEFINE_IDTENTRY()
1425 		ve_raise_fault(regs, 0);  in DEFINE_IDTENTRY()
1427 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
1436 	if (notify_die(DIE_TRAP, "iret exception", regs, 0,  in DEFINE_IDTENTRY_SW()
1438 		do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, 0,  in DEFINE_IDTENTRY_SW()