94 		  struct pt_regs *regs,	long error_code)  in do_trap_no_signal()  argument
96 	if (v8086_mode(regs)) {  in do_trap_no_signal()
102 			if (!handle_vm86_trap((struct kernel_vm86_regs *) regs,  in do_trap_no_signal()
106 	} else if (!user_mode(regs)) {  in do_trap_no_signal()
107 		if (fixup_exception(regs, trapnr, error_code, 0))  in do_trap_no_signal()
112 		die(str, regs, error_code);  in do_trap_no_signal()
114 		if (fixup_vdso_exception(regs, trapnr, error_code, 0))  in do_trap_no_signal()
135 			struct pt_regs *regs, long error_code)  in show_signal()  argument
141 			regs->ip, regs->sp, error_code);  in show_signal()
142 		print_vma_addr(KERN_CONT " in ", regs->ip);  in show_signal()
148 do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,  in do_trap()  argument
153 	if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code))  in do_trap()
156 	show_signal(tsk, signr, "trap ", str, regs, error_code);  in do_trap()
165 static void do_error_trap(struct pt_regs *regs, long error_code, char *str,  in do_error_trap()  argument
170 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=  in do_error_trap()
172 		cond_local_irq_enable(regs);  in do_error_trap()
173 		do_trap(trapnr, signr, str, regs, error_code, sicode, addr);  in do_error_trap()
174 		cond_local_irq_disable(regs);  in do_error_trap()
188 static __always_inline void __user *error_get_trap_addr(struct pt_regs *regs)  in error_get_trap_addr()  argument
190 	return (void __user *)uprobe_get_trap_addr(regs);  in error_get_trap_addr()
195 	do_error_trap(regs, 0, "divide error", X86_TRAP_DE, SIGFPE,  in DEFINE_IDTENTRY()
196 		      FPE_INTDIV, error_get_trap_addr(regs));  in DEFINE_IDTENTRY()
201 	do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL);  in DEFINE_IDTENTRY()
205 void handle_invalid_op(struct pt_regs *regs)  in handle_invalid_op()  argument
207 static inline void handle_invalid_op(struct pt_regs *regs)  in handle_invalid_op()
210 	do_error_trap(regs, 0, "invalid opcode", X86_TRAP_UD, SIGILL,  in handle_invalid_op()
211 		      ILL_ILLOPN, error_get_trap_addr(regs));  in handle_invalid_op()
214 static noinstr bool handle_bug(struct pt_regs *regs)  in handle_bug()  argument
223 	kmsan_unpoison_entry_regs(regs);  in handle_bug()
224 	if (!is_valid_bugaddr(regs->ip))  in handle_bug()
235 	if (regs->flags & X86_EFLAGS_IF)  in handle_bug()
237 	if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN ||  in handle_bug()
238 	    handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) {  in handle_bug()
239 		regs->ip += LEN_UD2;  in handle_bug()
242 	if (regs->flags & X86_EFLAGS_IF)  in handle_bug()
258 	if (!user_mode(regs) && handle_bug(regs))  in DEFINE_IDTENTRY_RAW()
261 	state = irqentry_enter(regs);  in DEFINE_IDTENTRY_RAW()
263 	handle_invalid_op(regs);  in DEFINE_IDTENTRY_RAW()
265 	irqentry_exit(regs, state);  in DEFINE_IDTENTRY_RAW()
270 	do_error_trap(regs, 0, "coprocessor segment overrun",  in DEFINE_IDTENTRY()
276 	do_error_trap(regs, error_code, "invalid TSS", X86_TRAP_TS, SIGSEGV,  in DEFINE_IDTENTRY_ERRORCODE()
282 	do_error_trap(regs, error_code, "segment not present", X86_TRAP_NP,  in DEFINE_IDTENTRY_ERRORCODE()
288 	do_error_trap(regs, error_code, "stack segment", X86_TRAP_SS, SIGBUS,  in DEFINE_IDTENTRY_ERRORCODE()
296 	if (notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_AC, SIGBUS) == NOTIFY_STOP)  in DEFINE_IDTENTRY_ERRORCODE()
299 	if (!user_mode(regs))  in DEFINE_IDTENTRY_ERRORCODE()
300 		die("Split lock detected\n", regs, error_code);  in DEFINE_IDTENTRY_ERRORCODE()
304 	if (handle_user_split_lock(regs, error_code))  in DEFINE_IDTENTRY_ERRORCODE()
307 	do_trap(X86_TRAP_AC, SIGBUS, "alignment check", regs,  in DEFINE_IDTENTRY_ERRORCODE()
315 __visible void __noreturn handle_stack_overflow(struct pt_regs *regs,  in handle_stack_overflow()  argument
324 	die("stack guard page", regs, 0);  in handle_stack_overflow()
377 	if (((long)regs->sp >> P4D_SHIFT) == ESPFIX_PGD_ENTRY &&  in DEFINE_IDTENTRY_DF()
378 		regs->cs == __KERNEL_CS &&  in DEFINE_IDTENTRY_DF()
379 		regs->ip == (unsigned long)native_irq_return_iret)  in DEFINE_IDTENTRY_DF()
382 		unsigned long *p = (unsigned long *)regs->sp;  in DEFINE_IDTENTRY_DF()
408 		regs->ip = (unsigned long)asm_exc_general_protection;  in DEFINE_IDTENTRY_DF()
409 		regs->sp = (unsigned long)&gpregs->orig_ax;  in DEFINE_IDTENTRY_DF()
415 	irqentry_nmi_enter(regs);  in DEFINE_IDTENTRY_DF()
417 	notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);  in DEFINE_IDTENTRY_DF()
461 		handle_stack_overflow(regs, address, &info);  in DEFINE_IDTENTRY_DF()
465 	die("double fault", regs, error_code);  in DEFINE_IDTENTRY_DF()
472 	if (notify_die(DIE_TRAP, "bounds", regs, 0,  in DEFINE_IDTENTRY()
475 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
477 	if (!user_mode(regs))  in DEFINE_IDTENTRY()
478 		die("bounds", regs, 0);  in DEFINE_IDTENTRY()
480 	do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, 0, 0, NULL);  in DEFINE_IDTENTRY()
482 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
496 static enum kernel_gp_hint get_kernel_gp_address(struct pt_regs *regs,  in get_kernel_gp_address()  argument
503 	if (copy_from_kernel_nofault(insn_buf, (void *)regs->ip,  in get_kernel_gp_address()
511 	*addr = (unsigned long)insn_get_addr_ref(&insn, regs);  in get_kernel_gp_address()
531 static bool fixup_iopl_exception(struct pt_regs *regs)  in fixup_iopl_exception()  argument
540 	if (insn_get_effective_ip(regs, &ip))  in fixup_iopl_exception()
557 	regs->ip += 1;  in fixup_iopl_exception()
612 static bool gp_try_fixup_and_notify(struct pt_regs *regs, int trapnr,  in gp_try_fixup_and_notify()  argument
616 	if (fixup_exception(regs, trapnr, error_code, address))  in gp_try_fixup_and_notify()
627 	    kprobe_fault_handler(regs, trapnr))  in gp_try_fixup_and_notify()
630 	return notify_die(DIE_GPF, str, regs, error_code, trapnr, SIGSEGV) == NOTIFY_STOP;  in gp_try_fixup_and_notify()
633 static void gp_user_force_sig_segv(struct pt_regs *regs, int trapnr,  in gp_user_force_sig_segv()  argument
638 	show_signal(current, SIGSEGV, "", str, regs, error_code);  in gp_user_force_sig_segv()
648 	if (user_mode(regs) && try_fixup_enqcmd_gp())  in DEFINE_IDTENTRY_ERRORCODE()
651 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY_ERRORCODE()
654 		if (user_mode(regs) && fixup_umip_exception(regs))  in DEFINE_IDTENTRY_ERRORCODE()
658 	if (v8086_mode(regs)) {  in DEFINE_IDTENTRY_ERRORCODE()
660 		handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);  in DEFINE_IDTENTRY_ERRORCODE()
665 	if (user_mode(regs)) {  in DEFINE_IDTENTRY_ERRORCODE()
666 		if (fixup_iopl_exception(regs))  in DEFINE_IDTENTRY_ERRORCODE()
669 		if (fixup_vdso_exception(regs, X86_TRAP_GP, error_code, 0))  in DEFINE_IDTENTRY_ERRORCODE()
672 		gp_user_force_sig_segv(regs, X86_TRAP_GP, error_code, desc);  in DEFINE_IDTENTRY_ERRORCODE()
676 	if (gp_try_fixup_and_notify(regs, X86_TRAP_GP, error_code, desc, 0))  in DEFINE_IDTENTRY_ERRORCODE()
682 		hint = get_kernel_gp_address(regs, &gp_addr);  in DEFINE_IDTENTRY_ERRORCODE()
697 	die_addr(desc, regs, error_code, gp_addr);  in DEFINE_IDTENTRY_ERRORCODE()
700 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY_ERRORCODE()
703 static bool do_int3(struct pt_regs *regs)  in do_int3()  argument
708 	if (kgdb_ll_trap(DIE_INT3, "int3", regs, 0, X86_TRAP_BP,  in do_int3()
714 	if (kprobe_int3_handler(regs))  in do_int3()
717 	res = notify_die(DIE_INT3, "int3", regs, 0, X86_TRAP_BP, SIGTRAP);  in do_int3()
723 static void do_int3_user(struct pt_regs *regs)  in do_int3_user()  argument
725 	if (do_int3(regs))  in do_int3_user()
728 	cond_local_irq_enable(regs);  in do_int3_user()
729 	do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, 0, 0, NULL);  in do_int3_user()
730 	cond_local_irq_disable(regs);  in do_int3_user()
740 	if (poke_int3_handler(regs))  in DEFINE_IDTENTRY_RAW()
750 	if (user_mode(regs)) {  in DEFINE_IDTENTRY_RAW()
751 		irqentry_enter_from_user_mode(regs);  in DEFINE_IDTENTRY_RAW()
753 		do_int3_user(regs);  in DEFINE_IDTENTRY_RAW()
755 		irqentry_exit_to_user_mode(regs);  in DEFINE_IDTENTRY_RAW()
757 		irqentry_state_t irq_state = irqentry_nmi_enter(regs);  in DEFINE_IDTENTRY_RAW()
760 		if (!do_int3(regs))  in DEFINE_IDTENTRY_RAW()
761 			die("int3", regs, 0);  in DEFINE_IDTENTRY_RAW()
763 		irqentry_nmi_exit(regs, irq_state);  in DEFINE_IDTENTRY_RAW()
775 	struct pt_regs *regs = (struct pt_regs *)this_cpu_read(pcpu_hot.top_of_stack) - 1;  in sync_regs()  local
776 	if (regs != eregs)  in sync_regs()
777 		*regs = *eregs;  in sync_regs()
778 	return regs;  in sync_regs()
782 asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *regs)  in vc_switch_off_ist()  argument
792 	if (ip_within_syscall_gap(regs)) {  in vc_switch_off_ist()
802 	sp    = regs->sp;  in vc_switch_off_ist()
818 	*regs_ret = *regs;  in vc_switch_off_ist()
852 static bool is_sysenter_singlestep(struct pt_regs *regs)  in is_sysenter_singlestep()  argument
863 	return (regs->ip - (unsigned long)__begin_SYSENTER_singlestep_region) <  in is_sysenter_singlestep()
867 	return (regs->ip - (unsigned long)entry_SYSENTER_compat) <  in is_sysenter_singlestep()
922 static bool notify_debug(struct pt_regs *regs, unsigned long *dr6)  in notify_debug()  argument
931 	if (notify_die(DIE_DEBUG, "debug", regs, (long)dr6, 0, SIGTRAP) == NOTIFY_STOP)  in notify_debug()
937 static __always_inline void exc_debug_kernel(struct pt_regs *regs,  in exc_debug_kernel()  argument
952 	irqentry_state_t irq_state = irqentry_nmi_enter(regs);  in exc_debug_kernel()
959 	WARN_ON_ONCE(user_mode(regs));  in exc_debug_kernel()
979 	if ((dr6 & DR_STEP) && is_sysenter_singlestep(regs))  in exc_debug_kernel()
988 	if (notify_debug(regs, &dr6))  in exc_debug_kernel()
1003 		regs->flags &= ~X86_EFLAGS_TF;  in exc_debug_kernel()
1006 	irqentry_nmi_exit(regs, irq_state);  in exc_debug_kernel()
1011 static __always_inline void exc_debug_user(struct pt_regs *regs,  in exc_debug_user()  argument
1020 	WARN_ON_ONCE(!user_mode(regs));  in exc_debug_user()
1031 	irqentry_enter_from_user_mode(regs);  in exc_debug_user()
1057 	if (notify_debug(regs, &dr6))  in exc_debug_user()
1063 	if (v8086_mode(regs)) {  in exc_debug_user()
1064 		handle_vm86_trap((struct kernel_vm86_regs *)regs, 0, X86_TRAP_DB);  in exc_debug_user()
1070 		handle_bus_lock(regs);  in exc_debug_user()
1075 		send_sigtrap(regs, 0, get_si_code(dr6));  in exc_debug_user()
1081 	irqentry_exit_to_user_mode(regs);  in exc_debug_user()
1088 	exc_debug_kernel(regs, debug_read_clear_dr6());  in DEFINE_IDTENTRY_DEBUG()
1094 	exc_debug_user(regs, debug_read_clear_dr6());  in DEFINE_IDTENTRY_DEBUG_USER()
1102 	if (user_mode(regs))  in DEFINE_IDTENTRY_RAW()
1103 		exc_debug_user(regs, dr6);  in DEFINE_IDTENTRY_RAW()
1105 		exc_debug_kernel(regs, dr6);  in DEFINE_IDTENTRY_RAW()
1114 static void math_error(struct pt_regs *regs, int trapnr)  in math_error()  argument
1122 	cond_local_irq_enable(regs);  in math_error()
1124 	if (!user_mode(regs)) {  in math_error()
1125 		if (fixup_exception(regs, trapnr, 0, 0))  in math_error()
1131 		if (notify_die(DIE_TRAP, str, regs, 0, trapnr,  in math_error()
1133 			die(str, regs, 0);  in math_error()
1151 	if (fixup_vdso_exception(regs, trapnr, 0, 0))  in math_error()
1155 			(void __user *)uprobe_get_trap_addr(regs));  in math_error()
1157 	cond_local_irq_disable(regs);  in math_error()
1162 	math_error(regs, X86_TRAP_MF);  in DEFINE_IDTENTRY()
1170 			__exc_general_protection(regs, 0);  in DEFINE_IDTENTRY()
1174 	math_error(regs, X86_TRAP_XF);  in DEFINE_IDTENTRY()
1200 static bool handle_xfd_event(struct pt_regs *regs)  in handle_xfd_event()  argument
1215 	if (WARN_ON(!user_mode(regs)))  in handle_xfd_event()
1224 		force_sig_fault(SIGILL, ILL_ILLOPC, error_get_trap_addr(regs));  in handle_xfd_event()
1239 	if (handle_xfd_event(regs))  in DEFINE_IDTENTRY()
1246 		cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
1248 		info.regs = regs;  in DEFINE_IDTENTRY()
1251 		cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
1266 		die("unexpected #NM exception", regs, 0);  in DEFINE_IDTENTRY()
1274 static void ve_raise_fault(struct pt_regs *regs, long error_code,  in ve_raise_fault()  argument
1277 	if (user_mode(regs)) {  in ve_raise_fault()
1278 		gp_user_force_sig_segv(regs, X86_TRAP_VE, error_code, VE_FAULT_STR);  in ve_raise_fault()
1282 	if (gp_try_fixup_and_notify(regs, X86_TRAP_VE, error_code,  in ve_raise_fault()
1287 	die_addr(VE_FAULT_STR, regs, error_code, address);  in ve_raise_fault()
1344 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
1350 	if (!tdx_handle_virt_exception(regs, &ve))  in DEFINE_IDTENTRY()
1351 		ve_raise_fault(regs, 0, ve.gla);  in DEFINE_IDTENTRY()
1353 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
1362 	if (notify_die(DIE_TRAP, "iret exception", regs, 0,  in DEFINE_IDTENTRY_SW()
1364 		do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, 0,  in DEFINE_IDTENTRY_SW()