75 static inline void cond_local_irq_enable(struct pt_regs *regs)  in cond_local_irq_enable()  argument
77 	if (regs->flags & X86_EFLAGS_IF)  in cond_local_irq_enable()
81 static inline void cond_local_irq_disable(struct pt_regs *regs)  in cond_local_irq_disable()  argument
83 	if (regs->flags & X86_EFLAGS_IF)  in cond_local_irq_disable()
101 		  struct pt_regs *regs,	long error_code)  in do_trap_no_signal()  argument
103 	if (v8086_mode(regs)) {  in do_trap_no_signal()
109 			if (!handle_vm86_trap((struct kernel_vm86_regs *) regs,  in do_trap_no_signal()
113 	} else if (!user_mode(regs)) {  in do_trap_no_signal()
114 		if (fixup_exception(regs, trapnr, error_code, 0))  in do_trap_no_signal()
119 		die(str, regs, error_code);  in do_trap_no_signal()
139 			struct pt_regs *regs, long error_code)  in show_signal()  argument
145 			regs->ip, regs->sp, error_code);  in show_signal()
146 		print_vma_addr(KERN_CONT " in ", regs->ip);  in show_signal()
152 do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,  in do_trap()  argument
157 	if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code))  in do_trap()
160 	show_signal(tsk, signr, "trap ", str, regs, error_code);  in do_trap()
169 static void do_error_trap(struct pt_regs *regs, long error_code, char *str,  in do_error_trap()  argument
174 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=  in do_error_trap()
176 		cond_local_irq_enable(regs);  in do_error_trap()
177 		do_trap(trapnr, signr, str, regs, error_code, sicode, addr);  in do_error_trap()
178 		cond_local_irq_disable(regs);  in do_error_trap()
192 static __always_inline void __user *error_get_trap_addr(struct pt_regs *regs)  in error_get_trap_addr()  argument
194 	return (void __user *)uprobe_get_trap_addr(regs);  in error_get_trap_addr()
199 	do_error_trap(regs, 0, "divide error", X86_TRAP_DE, SIGFPE,  in DEFINE_IDTENTRY()
200 		      FPE_INTDIV, error_get_trap_addr(regs));  in DEFINE_IDTENTRY()
205 	do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL);  in DEFINE_IDTENTRY()
209 void handle_invalid_op(struct pt_regs *regs)  in handle_invalid_op()  argument
211 static inline void handle_invalid_op(struct pt_regs *regs)  in handle_invalid_op()
214 	do_error_trap(regs, 0, "invalid opcode", X86_TRAP_UD, SIGILL,  in handle_invalid_op()
215 		      ILL_ILLOPN, error_get_trap_addr(regs));  in handle_invalid_op()
218 static noinstr bool handle_bug(struct pt_regs *regs)  in handle_bug()  argument
222 	if (!is_valid_bugaddr(regs->ip))  in handle_bug()
233 	if (regs->flags & X86_EFLAGS_IF)  in handle_bug()
235 	if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) {  in handle_bug()
236 		regs->ip += LEN_UD2;  in handle_bug()
239 	if (regs->flags & X86_EFLAGS_IF)  in handle_bug()
255 	if (!user_mode(regs) && handle_bug(regs))  in DEFINE_IDTENTRY_RAW()
258 	state = irqentry_enter(regs);  in DEFINE_IDTENTRY_RAW()
260 	handle_invalid_op(regs);  in DEFINE_IDTENTRY_RAW()
262 	irqentry_exit(regs, state);  in DEFINE_IDTENTRY_RAW()
267 	do_error_trap(regs, 0, "coprocessor segment overrun",  in DEFINE_IDTENTRY()
273 	do_error_trap(regs, error_code, "invalid TSS", X86_TRAP_TS, SIGSEGV,  in DEFINE_IDTENTRY_ERRORCODE()
279 	do_error_trap(regs, error_code, "segment not present", X86_TRAP_NP,  in DEFINE_IDTENTRY_ERRORCODE()
285 	do_error_trap(regs, error_code, "stack segment", X86_TRAP_SS, SIGBUS,  in DEFINE_IDTENTRY_ERRORCODE()
293 	if (notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_AC, SIGBUS) == NOTIFY_STOP)  in DEFINE_IDTENTRY_ERRORCODE()
296 	if (!user_mode(regs))  in DEFINE_IDTENTRY_ERRORCODE()
297 		die("Split lock detected\n", regs, error_code);  in DEFINE_IDTENTRY_ERRORCODE()
301 	if (handle_user_split_lock(regs, error_code))  in DEFINE_IDTENTRY_ERRORCODE()
304 	do_trap(X86_TRAP_AC, SIGBUS, "alignment check", regs,  in DEFINE_IDTENTRY_ERRORCODE()
312 						struct pt_regs *regs,  in handle_stack_overflow()  argument
318 	die(message, regs, 0);  in handle_stack_overflow()
370 	if (((long)regs->sp >> P4D_SHIFT) == ESPFIX_PGD_ENTRY &&  in DEFINE_IDTENTRY_DF()
371 		regs->cs == __KERNEL_CS &&  in DEFINE_IDTENTRY_DF()
372 		regs->ip == (unsigned long)native_irq_return_iret)  in DEFINE_IDTENTRY_DF()
375 		unsigned long *p = (unsigned long *)regs->sp;  in DEFINE_IDTENTRY_DF()
401 		regs->ip = (unsigned long)asm_exc_general_protection;  in DEFINE_IDTENTRY_DF()
402 		regs->sp = (unsigned long)&gpregs->orig_ax;  in DEFINE_IDTENTRY_DF()
408 	idtentry_enter_nmi(regs);  in DEFINE_IDTENTRY_DF()
410 	notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);  in DEFINE_IDTENTRY_DF()
455 				      regs, address);  in DEFINE_IDTENTRY_DF()
460 	die("double fault", regs, error_code);  in DEFINE_IDTENTRY_DF()
467 	if (notify_die(DIE_TRAP, "bounds", regs, 0,  in DEFINE_IDTENTRY()
470 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
472 	if (!user_mode(regs))  in DEFINE_IDTENTRY()
473 		die("bounds", regs, 0);  in DEFINE_IDTENTRY()
475 	do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, 0, 0, NULL);  in DEFINE_IDTENTRY()
477 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
491 static enum kernel_gp_hint get_kernel_gp_address(struct pt_regs *regs,  in get_kernel_gp_address()  argument
497 	if (copy_from_kernel_nofault(insn_buf, (void *)regs->ip,  in get_kernel_gp_address()
505 	*addr = (unsigned long)insn_get_addr_ref(&insn, regs);  in get_kernel_gp_address()
533 	cond_local_irq_enable(regs);  in DEFINE_IDTENTRY_ERRORCODE()
536 		if (user_mode(regs) && fixup_umip_exception(regs))  in DEFINE_IDTENTRY_ERRORCODE()
540 	if (v8086_mode(regs)) {  in DEFINE_IDTENTRY_ERRORCODE()
542 		handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);  in DEFINE_IDTENTRY_ERRORCODE()
549 	if (user_mode(regs)) {  in DEFINE_IDTENTRY_ERRORCODE()
553 		show_signal(tsk, SIGSEGV, "", desc, regs, error_code);  in DEFINE_IDTENTRY_ERRORCODE()
558 	if (fixup_exception(regs, X86_TRAP_GP, error_code, 0))  in DEFINE_IDTENTRY_ERRORCODE()
570 	    kprobe_fault_handler(regs, X86_TRAP_GP))  in DEFINE_IDTENTRY_ERRORCODE()
573 	ret = notify_die(DIE_GPF, desc, regs, error_code, X86_TRAP_GP, SIGSEGV);  in DEFINE_IDTENTRY_ERRORCODE()
580 		hint = get_kernel_gp_address(regs, &gp_addr);  in DEFINE_IDTENTRY_ERRORCODE()
595 	die_addr(desc, regs, error_code, gp_addr);  in DEFINE_IDTENTRY_ERRORCODE()
598 	cond_local_irq_disable(regs);  in DEFINE_IDTENTRY_ERRORCODE()
601 static bool do_int3(struct pt_regs *regs)  in do_int3()  argument
606 	if (kgdb_ll_trap(DIE_INT3, "int3", regs, 0, X86_TRAP_BP,  in do_int3()
612 	if (kprobe_int3_handler(regs))  in do_int3()
615 	res = notify_die(DIE_INT3, "int3", regs, 0, X86_TRAP_BP, SIGTRAP);  in do_int3()
620 static void do_int3_user(struct pt_regs *regs)  in do_int3_user()  argument
622 	if (do_int3(regs))  in do_int3_user()
625 	cond_local_irq_enable(regs);  in do_int3_user()
626 	do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, 0, 0, NULL);  in do_int3_user()
627 	cond_local_irq_disable(regs);  in do_int3_user()
637 	if (poke_int3_handler(regs))  in DEFINE_IDTENTRY_RAW()
647 	if (user_mode(regs)) {  in DEFINE_IDTENTRY_RAW()
648 		irqentry_enter_from_user_mode(regs);  in DEFINE_IDTENTRY_RAW()
650 		do_int3_user(regs);  in DEFINE_IDTENTRY_RAW()
652 		irqentry_exit_to_user_mode(regs);  in DEFINE_IDTENTRY_RAW()
654 		bool irq_state = idtentry_enter_nmi(regs);  in DEFINE_IDTENTRY_RAW()
656 		if (!do_int3(regs))  in DEFINE_IDTENTRY_RAW()
657 			die("int3", regs, 0);  in DEFINE_IDTENTRY_RAW()
659 		idtentry_exit_nmi(regs, irq_state);  in DEFINE_IDTENTRY_RAW()
671 	struct pt_regs *regs = (struct pt_regs *)this_cpu_read(cpu_current_top_of_stack) - 1;  in sync_regs()  local
672 	if (regs != eregs)  in sync_regs()
673 		*regs = *eregs;  in sync_regs()
674 	return regs;  in sync_regs()
678 asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *regs)  in vc_switch_off_ist()  argument
688 	if (regs->ip >= (unsigned long)entry_SYSCALL_64 &&  in vc_switch_off_ist()
689 	    regs->ip <  (unsigned long)entry_SYSCALL_64_safe_stack) {  in vc_switch_off_ist()
699 	sp    = regs->sp;  in vc_switch_off_ist()
715 	*regs_ret = *regs;  in vc_switch_off_ist()
723 	struct pt_regs regs;  member
741 	__memcpy(&tmp.regs.ip, (void *)s->regs.sp, 5*8);  in fixup_bad_iret()
744 	__memcpy(&tmp, s, offsetof(struct bad_iret_stack, regs.ip));  in fixup_bad_iret()
749 	BUG_ON(!user_mode(&new_stack->regs));  in fixup_bad_iret()
754 static bool is_sysenter_singlestep(struct pt_regs *regs)  in is_sysenter_singlestep()  argument
765 	return (regs->ip - (unsigned long)__begin_SYSENTER_singlestep_region) <  in is_sysenter_singlestep()
769 	return (regs->ip - (unsigned long)entry_SYSENTER_compat) <  in is_sysenter_singlestep()
824 static bool notify_debug(struct pt_regs *regs, unsigned long *dr6)  in notify_debug()  argument
833 	if (notify_die(DIE_DEBUG, "debug", regs, (long)dr6, 0, SIGTRAP) == NOTIFY_STOP)  in notify_debug()
839 static __always_inline void exc_debug_kernel(struct pt_regs *regs,  in exc_debug_kernel()  argument
854 	bool irq_state = idtentry_enter_nmi(regs);  in exc_debug_kernel()
861 	WARN_ON_ONCE(user_mode(regs));  in exc_debug_kernel()
881 	if ((dr6 & DR_STEP) && is_sysenter_singlestep(regs))  in exc_debug_kernel()
884 	if (kprobe_debug_handler(regs))  in exc_debug_kernel()
893 	if (notify_debug(regs, &dr6))  in exc_debug_kernel()
908 		regs->flags &= ~X86_EFLAGS_TF;  in exc_debug_kernel()
911 	idtentry_exit_nmi(regs, irq_state);  in exc_debug_kernel()
916 static __always_inline void exc_debug_user(struct pt_regs *regs,  in exc_debug_user()  argument
925 	WARN_ON_ONCE(!user_mode(regs));  in exc_debug_user()
936 	irqentry_enter_from_user_mode(regs);  in exc_debug_user()
962 	if (notify_debug(regs, &dr6))  in exc_debug_user()
968 	if (v8086_mode(regs)) {  in exc_debug_user()
969 		handle_vm86_trap((struct kernel_vm86_regs *)regs, 0, X86_TRAP_DB);  in exc_debug_user()
976 		send_sigtrap(regs, 0, get_si_code(dr6));  in exc_debug_user()
982 	irqentry_exit_to_user_mode(regs);  in exc_debug_user()
989 	exc_debug_kernel(regs, debug_read_clear_dr6());  in DEFINE_IDTENTRY_DEBUG()
995 	exc_debug_user(regs, debug_read_clear_dr6());  in DEFINE_IDTENTRY_DEBUG_USER()
1003 	if (user_mode(regs))  in DEFINE_IDTENTRY_RAW()
1004 		exc_debug_user(regs, dr6);  in DEFINE_IDTENTRY_RAW()
1006 		exc_debug_kernel(regs, dr6);  in DEFINE_IDTENTRY_RAW()
1015 static void math_error(struct pt_regs *regs, int trapnr)  in math_error()  argument
1023 	cond_local_irq_enable(regs);  in math_error()
1025 	if (!user_mode(regs)) {  in math_error()
1026 		if (fixup_exception(regs, trapnr, 0, 0))  in math_error()
1032 		if (notify_die(DIE_TRAP, str, regs, 0, trapnr,  in math_error()
1034 			die(str, regs, 0);  in math_error()
1052 			(void __user *)uprobe_get_trap_addr(regs));  in math_error()
1054 	cond_local_irq_disable(regs);  in math_error()
1059 	math_error(regs, X86_TRAP_MF);  in DEFINE_IDTENTRY()
1067 			__exc_general_protection(regs, 0);  in DEFINE_IDTENTRY()
1071 	math_error(regs, X86_TRAP_XF);  in DEFINE_IDTENTRY()
1105 		cond_local_irq_enable(regs);  in DEFINE_IDTENTRY()
1107 		info.regs = regs;  in DEFINE_IDTENTRY()
1110 		cond_local_irq_disable(regs);  in DEFINE_IDTENTRY()
1125 		die("unexpected #NM exception", regs, 0);  in DEFINE_IDTENTRY()
1133 	if (notify_die(DIE_TRAP, "iret exception", regs, 0,  in DEFINE_IDTENTRY_SW()
1135 		do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, 0,  in DEFINE_IDTENTRY_SW()