80 static inline void cond_local_irq_enable(struct pt_regs *regs)  in cond_local_irq_enable()  argument
82 	if (regs->flags & X86_EFLAGS_IF)  in cond_local_irq_enable()
86 static inline void cond_local_irq_disable(struct pt_regs *regs)  in cond_local_irq_disable()  argument
88 	if (regs->flags & X86_EFLAGS_IF)  in cond_local_irq_disable()
106 		  struct pt_regs *regs,	long error_code)  in do_trap_no_signal()  argument
108 	if (v8086_mode(regs)) {  in do_trap_no_signal()
114 			if (!handle_vm86_trap((struct kernel_vm86_regs *) regs,  in do_trap_no_signal()
118 	} else if (!user_mode(regs)) {  in do_trap_no_signal()
119 		if (fixup_exception(regs, trapnr, error_code, 0))  in do_trap_no_signal()
124 		die(str, regs, error_code);  in do_trap_no_signal()
126 		if (fixup_vdso_exception(regs, trapnr, error_code, 0))  in do_trap_no_signal()
147 			struct pt_regs *regs, long error_code)  in show_signal()  argument
153 			regs->ip, regs->sp, error_code);  in show_signal()
154 		print_vma_addr(KERN_CONT " in ", regs->ip);  in show_signal()
160 do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,  in do_trap()  argument
165 	if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code))  in do_trap()
168 	show_signal(tsk, signr, "trap ", str, regs, error_code);  in do_trap()
177 static void do_error_trap(struct pt_regs *regs, long error_code, char *str,  in do_error_trap()  argument
182 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=  in do_error_trap()
184 		cond_local_irq_enable(regs);  in do_error_trap()
185 		do_trap(trapnr, signr, str, regs, error_code, sicode, addr);  in do_error_trap()
186 		cond_local_irq_disable(regs);  in do_error_trap()
200 static __always_inline void __user *error_get_trap_addr(struct pt_regs *regs)  in error_get_trap_addr()  argument
202 	return (void __user *)uprobe_get_trap_addr(regs);  in error_get_trap_addr()
207 	do_error_trap(regs, 0, "divide error", X86_TRAP_DE, SIGFPE,  in DEFINE_IDTENTRY()
208 		      FPE_INTDIV, error_get_trap_addr(regs));  in DEFINE_IDTENTRY()
213 	do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL);  in DEFINE_IDTENTRY()
241 	if (WARN_ON_ONCE(user_mode(regs) || (error_code & CP_EC) != CP_ENDBR))  in DEFINE_IDTENTRY_ERRORCODE()
244 	if (unlikely(regs->ip == (unsigned long)&ibt_selftest_ip)) {  in DEFINE_IDTENTRY_ERRORCODE()
245 		regs->ax = 0;  in DEFINE_IDTENTRY_ERRORCODE()
249 	pr_err("Missing ENDBR: %pS\n", (void *)instruction_pointer(regs));  in DEFINE_IDTENTRY_ERRORCODE()
252 		__warn(__FILE__, __LINE__, (void *)regs->ip, TAINT_WARN, regs, NULL);  in DEFINE_IDTENTRY_ERRORCODE()
292 void handle_invalid_op(struct pt_regs *regs)  in handle_invalid_op()  argument
294 static inline void handle_invalid_op(struct pt_regs *regs)  in handle_invalid_op()
297 	do_error_trap(regs, 0, "invalid opcode", X86_TRAP_UD, SIGILL,  in handle_invalid_op()
298 		      ILL_ILLOPN, error_get_trap_addr(regs));  in handle_invalid_op()
301 static noinstr bool handle_bug(struct pt_regs *regs)  in handle_bug()  argument
310 	kmsan_unpoison_entry_regs(regs);  in handle_bug()
311 	if (!is_valid_bugaddr(regs->ip))  in handle_bug()
322 	if (regs->flags & X86_EFLAGS_IF)  in handle_bug()
324 	if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN ||  in handle_bug()
325 	    handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) {  in handle_bug()
326 		regs->ip += LEN_UD2;  in handle_bug()
329 	if (regs->flags & X86_EFLAGS_IF)  in handle_bug()
345 	if (!user_mode(regs) && handle_bug(regs))  in DEFINE_IDTENTRY_RAW()
348 	state = irqentry_enter(regs);  in DEFINE_IDTENTRY_RAW()
350 	handle_invalid_op(regs);  in DEFINE_IDTENTRY_RAW()
352 	irqentry_exit(regs, state);  in DEFINE_IDTENTRY_RAW()
357 	do_error_trap(regs, 0, "coprocessor segment overrun",  in DEFINE_IDTENTRY()
363 	do_error_trap(regs, error_code, "invalid TSS", X86_TRAP_TS, SIGSEGV,  in DEFINE_IDTENTRY_ERRORCODE()
369 	do_error_trap(regs, error_code, "segment not present", X86_TRAP_NP,  in DEFINE_IDTENTRY_ERRORCODE()
375 	do_error_trap(regs, error_code, "stack segment", X86_TRAP_SS, SIGBUS,  in DEFINE_IDTENTRY_ERRORCODE()
383 	if (notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_AC, SIGBUS) == NOTIFY_STOP)  in DEFINE_IDTENTRY_ERRORCODE()
386 	if (!user_mode(regs))  in DEFINE_IDTENTRY_ERRORCODE()
387 		die("Split lock detected\n", regs, error_code);  in DEFINE_IDTENTRY_ERRORCODE()
391 	if (handle_user_split_lock(regs, error_code))  in DEFINE_IDTENTRY_ERRORCODE()
394 	do_trap(X86_TRAP_AC, SIGBUS, "alignment check", regs,  in DEFINE_IDTENTRY_ERRORCODE()
402 __visible void __noreturn handle_stack_overflow(struct pt_regs *regs,  in handle_stack_overflow()  argument
411 	die("stack guard page", regs, 0);  in handle_stack_overflow()
464 	if (((long)regs->sp >> P4D_SHIFT) == ESPFIX_PGD_ENTRY &&  in DEFINE_IDTENTRY_DF()
465 		regs->cs == __KERNEL_CS &&  in DEFINE_IDTENTRY_DF()
466 		regs->ip == (unsigned long)native_irq_return_iret)  in DEFINE_IDTENTRY_DF()
469 		unsigned long *p = (unsigned long *)regs->sp;  in DEFINE_IDTENTRY_DF()
495 		regs->ip = (unsigned long)asm_exc_general_protection;  in DEFINE_IDTENTRY_DF()
496 		regs->sp = (unsigned long)&gpregs->orig_ax;  in DEFINE_IDTENTRY_DF()
502 	irqentry_nmi_enter(regs);  in DEFINE_IDTENTRY_DF()
504 	notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);  in DEFINE_IDTENTRY_DF()
548 		handle_stack_overflow(regs, address, &info);  in DEFINE_IDTENTRY_DF()
552 	die("double fault", regs, error_code);  in DEFINE_IDTENTRY_DF()
559 	if (notify_die(DIE_TRAP, "bounds", regs, 0,  in DEFINE_IDTENTRY()
562 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
564 	if (!user_mode(regs))  in DEFINE_IDTENTRY()
565 		die("bounds", regs, 0);  in DEFINE_IDTENTRY()
567 	do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, 0, 0, NULL);  in DEFINE_IDTENTRY()
569 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
583 static enum kernel_gp_hint get_kernel_gp_address(struct pt_regs *regs,  in get_kernel_gp_address()  argument
590 	if (copy_from_kernel_nofault(insn_buf, (void *)regs->ip,  in get_kernel_gp_address()
598 	*addr = (unsigned long)insn_get_addr_ref(&insn, regs);  in get_kernel_gp_address()
618 static bool fixup_iopl_exception(struct pt_regs *regs)  in fixup_iopl_exception()  argument
627 	if (insn_get_effective_ip(regs, &ip))  in fixup_iopl_exception()
644 	regs->ip += 1;  in fixup_iopl_exception()
699 static bool gp_try_fixup_and_notify(struct pt_regs *regs, int trapnr,  in gp_try_fixup_and_notify()  argument
702 	if (fixup_exception(regs, trapnr, error_code, 0))  in gp_try_fixup_and_notify()
713 	    kprobe_fault_handler(regs, trapnr))  in gp_try_fixup_and_notify()
716 	return notify_die(DIE_GPF, str, regs, error_code, trapnr, SIGSEGV) == NOTIFY_STOP;  in gp_try_fixup_and_notify()
719 static void gp_user_force_sig_segv(struct pt_regs *regs, int trapnr,  in gp_user_force_sig_segv()  argument
724 	show_signal(current, SIGSEGV, "", str, regs, error_code);  in gp_user_force_sig_segv()
734 	if (user_mode(regs) && try_fixup_enqcmd_gp())  in DEFINE_IDTENTRY_ERRORCODE()
737 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY_ERRORCODE()
740 		if (user_mode(regs) && fixup_umip_exception(regs))  in DEFINE_IDTENTRY_ERRORCODE()
744 	if (v8086_mode(regs)) {  in DEFINE_IDTENTRY_ERRORCODE()
746 		handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);  in DEFINE_IDTENTRY_ERRORCODE()
751 	if (user_mode(regs)) {  in DEFINE_IDTENTRY_ERRORCODE()
752 		if (fixup_iopl_exception(regs))  in DEFINE_IDTENTRY_ERRORCODE()
755 		if (fixup_vdso_exception(regs, X86_TRAP_GP, error_code, 0))  in DEFINE_IDTENTRY_ERRORCODE()
758 		gp_user_force_sig_segv(regs, X86_TRAP_GP, error_code, desc);  in DEFINE_IDTENTRY_ERRORCODE()
762 	if (gp_try_fixup_and_notify(regs, X86_TRAP_GP, error_code, desc))  in DEFINE_IDTENTRY_ERRORCODE()
768 		hint = get_kernel_gp_address(regs, &gp_addr);  in DEFINE_IDTENTRY_ERRORCODE()
783 	die_addr(desc, regs, error_code, gp_addr);  in DEFINE_IDTENTRY_ERRORCODE()
786 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY_ERRORCODE()
789 static bool do_int3(struct pt_regs *regs)  in do_int3()  argument
794 	if (kgdb_ll_trap(DIE_INT3, "int3", regs, 0, X86_TRAP_BP,  in do_int3()
800 	if (kprobe_int3_handler(regs))  in do_int3()
803 	res = notify_die(DIE_INT3, "int3", regs, 0, X86_TRAP_BP, SIGTRAP);  in do_int3()
809 static void do_int3_user(struct pt_regs *regs)  in do_int3_user()  argument
811 	if (do_int3(regs))  in do_int3_user()
814 	cond_local_irq_enable(regs);  in do_int3_user()
815 	do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, 0, 0, NULL);  in do_int3_user()
816 	cond_local_irq_disable(regs);  in do_int3_user()
826 	if (poke_int3_handler(regs))  in DEFINE_IDTENTRY_RAW()
836 	if (user_mode(regs)) {  in DEFINE_IDTENTRY_RAW()
837 		irqentry_enter_from_user_mode(regs);  in DEFINE_IDTENTRY_RAW()
839 		do_int3_user(regs);  in DEFINE_IDTENTRY_RAW()
841 		irqentry_exit_to_user_mode(regs);  in DEFINE_IDTENTRY_RAW()
843 		irqentry_state_t irq_state = irqentry_nmi_enter(regs);  in DEFINE_IDTENTRY_RAW()
846 		if (!do_int3(regs))  in DEFINE_IDTENTRY_RAW()
847 			die("int3", regs, 0);  in DEFINE_IDTENTRY_RAW()
849 		irqentry_nmi_exit(regs, irq_state);  in DEFINE_IDTENTRY_RAW()
861 	struct pt_regs *regs = (struct pt_regs *)this_cpu_read(cpu_current_top_of_stack) - 1;  in sync_regs()  local
862 	if (regs != eregs)  in sync_regs()
863 		*regs = *eregs;  in sync_regs()
864 	return regs;  in sync_regs()
868 asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *regs)  in vc_switch_off_ist()  argument
878 	if (ip_within_syscall_gap(regs)) {  in vc_switch_off_ist()
888 	sp    = regs->sp;  in vc_switch_off_ist()
904 	*regs_ret = *regs;  in vc_switch_off_ist()
938 static bool is_sysenter_singlestep(struct pt_regs *regs)  in is_sysenter_singlestep()  argument
949 	return (regs->ip - (unsigned long)__begin_SYSENTER_singlestep_region) <  in is_sysenter_singlestep()
953 	return (regs->ip - (unsigned long)entry_SYSENTER_compat) <  in is_sysenter_singlestep()
1008 static bool notify_debug(struct pt_regs *regs, unsigned long *dr6)  in notify_debug()  argument
1017 	if (notify_die(DIE_DEBUG, "debug", regs, (long)dr6, 0, SIGTRAP) == NOTIFY_STOP)  in notify_debug()
1023 static __always_inline void exc_debug_kernel(struct pt_regs *regs,  in exc_debug_kernel()  argument
1038 	irqentry_state_t irq_state = irqentry_nmi_enter(regs);  in exc_debug_kernel()
1045 	WARN_ON_ONCE(user_mode(regs));  in exc_debug_kernel()
1065 	if ((dr6 & DR_STEP) && is_sysenter_singlestep(regs))  in exc_debug_kernel()
1074 	if (notify_debug(regs, &dr6))  in exc_debug_kernel()
1089 		regs->flags &= ~X86_EFLAGS_TF;  in exc_debug_kernel()
1092 	irqentry_nmi_exit(regs, irq_state);  in exc_debug_kernel()
1097 static __always_inline void exc_debug_user(struct pt_regs *regs,  in exc_debug_user()  argument
1106 	WARN_ON_ONCE(!user_mode(regs));  in exc_debug_user()
1117 	irqentry_enter_from_user_mode(regs);  in exc_debug_user()
1143 	if (notify_debug(regs, &dr6))  in exc_debug_user()
1149 	if (v8086_mode(regs)) {  in exc_debug_user()
1150 		handle_vm86_trap((struct kernel_vm86_regs *)regs, 0, X86_TRAP_DB);  in exc_debug_user()
1156 		handle_bus_lock(regs);  in exc_debug_user()
1161 		send_sigtrap(regs, 0, get_si_code(dr6));  in exc_debug_user()
1167 	irqentry_exit_to_user_mode(regs);  in exc_debug_user()
1174 	exc_debug_kernel(regs, debug_read_clear_dr6());  in DEFINE_IDTENTRY_DEBUG()
1180 	exc_debug_user(regs, debug_read_clear_dr6());  in DEFINE_IDTENTRY_DEBUG_USER()
1188 	if (user_mode(regs))  in DEFINE_IDTENTRY_RAW()
1189 		exc_debug_user(regs, dr6);  in DEFINE_IDTENTRY_RAW()
1191 		exc_debug_kernel(regs, dr6);  in DEFINE_IDTENTRY_RAW()
1200 static void math_error(struct pt_regs *regs, int trapnr)  in math_error()  argument
1208 	cond_local_irq_enable(regs);  in math_error()
1210 	if (!user_mode(regs)) {  in math_error()
1211 		if (fixup_exception(regs, trapnr, 0, 0))  in math_error()
1217 		if (notify_die(DIE_TRAP, str, regs, 0, trapnr,  in math_error()
1219 			die(str, regs, 0);  in math_error()
1237 	if (fixup_vdso_exception(regs, trapnr, 0, 0))  in math_error()
1241 			(void __user *)uprobe_get_trap_addr(regs));  in math_error()
1243 	cond_local_irq_disable(regs);  in math_error()
1248 	math_error(regs, X86_TRAP_MF);  in DEFINE_IDTENTRY()
1256 			__exc_general_protection(regs, 0);  in DEFINE_IDTENTRY()
1260 	math_error(regs, X86_TRAP_XF);  in DEFINE_IDTENTRY()
1286 static bool handle_xfd_event(struct pt_regs *regs)  in handle_xfd_event()  argument
1301 	if (WARN_ON(!user_mode(regs)))  in handle_xfd_event()
1310 		force_sig_fault(SIGILL, ILL_ILLOPC, error_get_trap_addr(regs));  in handle_xfd_event()
1325 	if (handle_xfd_event(regs))  in DEFINE_IDTENTRY()
1332 		cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
1334 		info.regs = regs;  in DEFINE_IDTENTRY()
1337 		cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
1352 		die("unexpected #NM exception", regs, 0);  in DEFINE_IDTENTRY()
1360 static void ve_raise_fault(struct pt_regs *regs, long error_code)  in ve_raise_fault()  argument
1362 	if (user_mode(regs)) {  in ve_raise_fault()
1363 		gp_user_force_sig_segv(regs, X86_TRAP_VE, error_code, VE_FAULT_STR);  in ve_raise_fault()
1367 	if (gp_try_fixup_and_notify(regs, X86_TRAP_VE, error_code, VE_FAULT_STR))  in ve_raise_fault()
1370 	die_addr(VE_FAULT_STR, regs, error_code, 0);  in ve_raise_fault()
1427 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
1433 	if (!tdx_handle_virt_exception(regs, &ve))  in DEFINE_IDTENTRY()
1434 		ve_raise_fault(regs, 0);  in DEFINE_IDTENTRY()
1436 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
1445 	if (notify_die(DIE_TRAP, "iret exception", regs, 0,  in DEFINE_IDTENTRY_SW()
1447 		do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, 0,  in DEFINE_IDTENTRY_SW()