77 static inline void cond_local_irq_enable(struct pt_regs *regs)  in cond_local_irq_enable()  argument
79 	if (regs->flags & X86_EFLAGS_IF)  in cond_local_irq_enable()
83 static inline void cond_local_irq_disable(struct pt_regs *regs)  in cond_local_irq_disable()  argument
85 	if (regs->flags & X86_EFLAGS_IF)  in cond_local_irq_disable()
95 void ist_enter(struct pt_regs *regs)  in ist_enter()  argument
97 	if (user_mode(regs)) {  in ist_enter()
115 void ist_exit(struct pt_regs *regs)  in ist_exit()  argument
119 	if (!user_mode(regs))  in ist_exit()
136 void ist_begin_non_atomic(struct pt_regs *regs)  in ist_begin_non_atomic()  argument
138 	BUG_ON(!user_mode(regs));  in ist_begin_non_atomic()
173 int fixup_bug(struct pt_regs *regs, int trapnr)  in fixup_bug()  argument
178 	switch (report_bug(regs->ip, regs)) {  in fixup_bug()
184 		regs->ip += LEN_UD2;  in fixup_bug()
193 		  struct pt_regs *regs,	long error_code)  in do_trap_no_signal()  argument
195 	if (v8086_mode(regs)) {  in do_trap_no_signal()
201 			if (!handle_vm86_trap((struct kernel_vm86_regs *) regs,  in do_trap_no_signal()
208 	if (!user_mode(regs)) {  in do_trap_no_signal()
209 		if (fixup_exception(regs, trapnr))  in do_trap_no_signal()
214 		die(str, regs, error_code);  in do_trap_no_signal()
220 static siginfo_t *fill_trap_info(struct pt_regs *regs, int signr, int trapnr,  in fill_trap_info()  argument
232 		siaddr = uprobe_get_trap_addr(regs);  in fill_trap_info()
236 		siaddr = uprobe_get_trap_addr(regs);  in fill_trap_info()
252 do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,  in do_trap()  argument
258 	if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code))  in do_trap()
276 			regs->ip, regs->sp, error_code);  in do_trap()
277 		print_vma_addr(KERN_CONT " in ", regs->ip);  in do_trap()
285 static void do_error_trap(struct pt_regs *regs, long error_code, char *str,  in do_error_trap()  argument
296 	if (!user_mode(regs) && fixup_bug(regs, trapnr))  in do_error_trap()
299 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=  in do_error_trap()
301 		cond_local_irq_enable(regs);  in do_error_trap()
303 		do_trap(trapnr, signr, str, regs, error_code,  in do_error_trap()
304 			fill_trap_info(regs, signr, trapnr, &info));  in do_error_trap()
309 dotraplinkage void do_##name(struct pt_regs *regs, long error_code)	\
311 	do_error_trap(regs, error_code, str, trapnr, signr);		\
325 						struct pt_regs *regs,  in handle_stack_overflow()  argument
331 	die(message, regs, 0);  in handle_stack_overflow()
340 dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)  in do_double_fault()  argument
365 	if (((long)regs->sp >> P4D_SHIFT) == ESPFIX_PGD_ENTRY &&  in do_double_fault()
366 		regs->cs == __KERNEL_CS &&  in do_double_fault()
367 		regs->ip == (unsigned long)native_irq_return_iret)  in do_double_fault()
377 		memmove(&gpregs->ip, (void *)regs->sp, 5*8);  in do_double_fault()
387 		regs->ip = (unsigned long)general_protection;  in do_double_fault()
388 		regs->sp = (unsigned long)&gpregs->orig_ax;  in do_double_fault()
394 	ist_enter(regs);  in do_double_fault()
395 	notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);  in do_double_fault()
440 		handle_stack_overflow("kernel stack overflow (double-fault)", regs, cr2);  in do_double_fault()
444 	df_debug(regs, error_code);  in do_double_fault()
451 		die(str, regs, error_code);  in do_double_fault()
455 dotraplinkage void do_bounds(struct pt_regs *regs, long error_code)  in do_bounds()  argument
461 	if (notify_die(DIE_TRAP, "bounds", regs, error_code,  in do_bounds()
464 	cond_local_irq_enable(regs);  in do_bounds()
466 	if (!user_mode(regs))  in do_bounds()
467 		die("bounds", regs, error_code);  in do_bounds()
496 		info = mpx_generate_siginfo(regs);  in do_bounds()
511 		do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, error_code, info);  in do_bounds()
517 		die("bounds", regs, error_code);  in do_bounds()
530 	do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, error_code, NULL);  in do_bounds()
534 do_general_protection(struct pt_regs *regs, long error_code)  in do_general_protection()  argument
539 	cond_local_irq_enable(regs);  in do_general_protection()
542 		if (user_mode(regs) && fixup_umip_exception(regs))  in do_general_protection()
546 	if (v8086_mode(regs)) {  in do_general_protection()
548 		handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);  in do_general_protection()
553 	if (!user_mode(regs)) {  in do_general_protection()
554 		if (fixup_exception(regs, X86_TRAP_GP))  in do_general_protection()
559 		if (notify_die(DIE_GPF, "general protection fault", regs, error_code,  in do_general_protection()
561 			die("general protection fault", regs, error_code);  in do_general_protection()
572 			regs->ip, regs->sp, error_code);  in do_general_protection()
573 		print_vma_addr(KERN_CONT " in ", regs->ip);  in do_general_protection()
581 dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)  in do_int3()  argument
589 	    ftrace_int3_handler(regs))  in do_int3()
592 	if (poke_int3_handler(regs))  in do_int3()
602 	ist_enter(regs);  in do_int3()
605 	if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,  in do_int3()
611 	if (kprobe_int3_handler(regs))  in do_int3()
615 	if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,  in do_int3()
619 	cond_local_irq_enable(regs);  in do_int3()
620 	do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, NULL);  in do_int3()
621 	cond_local_irq_disable(regs);  in do_int3()
624 	ist_exit(regs);  in do_int3()
636 	struct pt_regs *regs = (struct pt_regs *)this_cpu_read(cpu_current_top_of_stack) - 1;  in sync_regs()  local
637 	if (regs != eregs)  in sync_regs()
638 		*regs = *eregs;  in sync_regs()
639 	return regs;  in sync_regs()
645 	struct pt_regs regs;  member
663 	memmove(&new_stack->regs.ip, (void *)s->regs.sp, 5*8);  in fixup_bad_iret()
666 	memmove(new_stack, s, offsetof(struct bad_iret_stack, regs.ip));  in fixup_bad_iret()
668 	BUG_ON(!user_mode(&new_stack->regs));  in fixup_bad_iret()
674 static bool is_sysenter_singlestep(struct pt_regs *regs)  in is_sysenter_singlestep()  argument
685 	return (regs->ip - (unsigned long)__begin_SYSENTER_singlestep_region) <  in is_sysenter_singlestep()
689 	return (regs->ip - (unsigned long)entry_SYSENTER_compat) <  in is_sysenter_singlestep()
721 dotraplinkage void do_debug(struct pt_regs *regs, long error_code)  in do_debug()  argument
728 	ist_enter(regs);  in do_debug()
754 	if (unlikely(!user_mode(regs) && (dr6 & DR_STEP) &&  in do_debug()
755 		     is_sysenter_singlestep(regs))) {  in do_debug()
771 	if (!dr6 && user_mode(regs))  in do_debug()
778 	if (kprobe_debug_handler(regs))  in do_debug()
782 	if (notify_die(DIE_DEBUG, "debug", regs, (long)&dr6, error_code,  in do_debug()
793 	cond_local_irq_enable(regs);  in do_debug()
795 	if (v8086_mode(regs)) {  in do_debug()
796 		handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code,  in do_debug()
798 		cond_local_irq_disable(regs);  in do_debug()
803 	if (WARN_ON_ONCE((dr6 & DR_STEP) && !user_mode(regs))) {  in do_debug()
812 		regs->flags &= ~X86_EFLAGS_TF;  in do_debug()
816 		send_sigtrap(tsk, regs, error_code, si_code);  in do_debug()
817 	cond_local_irq_disable(regs);  in do_debug()
821 	ist_exit(regs);  in do_debug()
830 static void math_error(struct pt_regs *regs, int error_code, int trapnr)  in math_error()  argument
838 	cond_local_irq_enable(regs);  in math_error()
840 	if (!user_mode(regs)) {  in math_error()
841 		if (fixup_exception(regs, trapnr))  in math_error()
847 		if (notify_die(DIE_TRAP, str, regs, error_code,  in math_error()
849 			die(str, regs, error_code);  in math_error()
863 	info.si_addr		= (void __user *)uprobe_get_trap_addr(regs);  in math_error()
874 dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)  in do_coprocessor_error()  argument
877 	math_error(regs, error_code, X86_TRAP_MF);  in do_coprocessor_error()
881 do_simd_coprocessor_error(struct pt_regs *regs, long error_code)  in do_simd_coprocessor_error()  argument
884 	math_error(regs, error_code, X86_TRAP_XF);  in do_simd_coprocessor_error()
888 do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)  in do_spurious_interrupt_bug()  argument
890 	cond_local_irq_enable(regs);  in do_spurious_interrupt_bug()
894 do_device_not_available(struct pt_regs *regs, long error_code)  in do_device_not_available()  argument
904 		cond_local_irq_enable(regs);  in do_device_not_available()
906 		info.regs = regs;  in do_device_not_available()
923 		die("unexpected #NM exception", regs, error_code);  in do_device_not_available()
929 dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)  in do_iret_error()  argument
941 	if (notify_die(DIE_TRAP, "iret exception", regs, error_code,  in do_iret_error()
943 		do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, error_code,  in do_iret_error()