77 static inline void cond_local_irq_enable(struct pt_regs *regs)  in cond_local_irq_enable()  argument
79 	if (regs->flags & X86_EFLAGS_IF)  in cond_local_irq_enable()
83 static inline void cond_local_irq_disable(struct pt_regs *regs)  in cond_local_irq_disable()  argument
85 	if (regs->flags & X86_EFLAGS_IF)  in cond_local_irq_disable()
95 void ist_enter(struct pt_regs *regs)  in ist_enter()  argument
97 	if (user_mode(regs)) {  in ist_enter()
116 void ist_exit(struct pt_regs *regs)  in ist_exit()  argument
120 	if (!user_mode(regs))  in ist_exit()
137 void ist_begin_non_atomic(struct pt_regs *regs)  in ist_begin_non_atomic()  argument
139 	BUG_ON(!user_mode(regs));  in ist_begin_non_atomic()
174 int fixup_bug(struct pt_regs *regs, int trapnr)  in fixup_bug()  argument
179 	switch (report_bug(regs->ip, regs)) {  in fixup_bug()
185 		regs->ip += LEN_UD2;  in fixup_bug()
194 		  struct pt_regs *regs,	long error_code)  in do_trap_no_signal()  argument
196 	if (v8086_mode(regs)) {  in do_trap_no_signal()
202 			if (!handle_vm86_trap((struct kernel_vm86_regs *) regs,  in do_trap_no_signal()
206 	} else if (!user_mode(regs)) {  in do_trap_no_signal()
207 		if (fixup_exception(regs, trapnr, error_code, 0))  in do_trap_no_signal()
212 		die(str, regs, error_code);  in do_trap_no_signal()
232 			struct pt_regs *regs, long error_code)  in show_signal()  argument
238 			regs->ip, regs->sp, error_code);  in show_signal()
239 		print_vma_addr(KERN_CONT " in ", regs->ip);  in show_signal()
245 do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,  in do_trap()  argument
251 	if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code))  in do_trap()
254 	show_signal(tsk, signr, "trap ", str, regs, error_code);  in do_trap()
263 static void do_error_trap(struct pt_regs *regs, long error_code, char *str,  in do_error_trap()  argument
272 	if (!user_mode(regs) && fixup_bug(regs, trapnr))  in do_error_trap()
275 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=  in do_error_trap()
277 		cond_local_irq_enable(regs);  in do_error_trap()
278 		do_trap(trapnr, signr, str, regs, error_code, sicode, addr);  in do_error_trap()
282 #define IP ((void __user *)uprobe_get_trap_addr(regs))
284 dotraplinkage void do_##name(struct pt_regs *regs, long error_code)	   \
286 	do_error_trap(regs, error_code, str, trapnr, signr, sicode, addr); \
301 						struct pt_regs *regs,  in handle_stack_overflow()  argument
307 	die(message, regs, 0);  in handle_stack_overflow()
316 dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsigned long cr2)  in do_double_fault()  argument
338 	if (((long)regs->sp >> P4D_SHIFT) == ESPFIX_PGD_ENTRY &&  in do_double_fault()
339 		regs->cs == __KERNEL_CS &&  in do_double_fault()
340 		regs->ip == (unsigned long)native_irq_return_iret)  in do_double_fault()
350 		memmove(&gpregs->ip, (void *)regs->sp, 5*8);  in do_double_fault()
364 		regs->ip = (unsigned long)general_protection;  in do_double_fault()
365 		regs->sp = (unsigned long)&gpregs->orig_ax;  in do_double_fault()
371 	ist_enter(regs);  in do_double_fault()
372 	notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);  in do_double_fault()
416 		handle_stack_overflow("kernel stack overflow (double-fault)", regs, cr2);  in do_double_fault()
420 	df_debug(regs, error_code);  in do_double_fault()
427 		die(str, regs, error_code);  in do_double_fault()
431 dotraplinkage void do_bounds(struct pt_regs *regs, long error_code)  in do_bounds()  argument
436 	if (notify_die(DIE_TRAP, "bounds", regs, error_code,  in do_bounds()
439 	cond_local_irq_enable(regs);  in do_bounds()
441 	if (!user_mode(regs))  in do_bounds()
442 		die("bounds", regs, error_code);  in do_bounds()
475 		if (mpx_fault_info(&mpx, regs)) {  in do_bounds()
489 		if (!do_trap_no_signal(tsk, X86_TRAP_BR, "bounds", regs,  in do_bounds()
493 		show_signal(tsk, SIGSEGV, "trap ", "bounds", regs, error_code);  in do_bounds()
501 		die("bounds", regs, error_code);  in do_bounds()
514 	do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, error_code, 0, NULL);  in do_bounds()
518 do_general_protection(struct pt_regs *regs, long error_code)  in do_general_protection()  argument
524 	cond_local_irq_enable(regs);  in do_general_protection()
527 		if (user_mode(regs) && fixup_umip_exception(regs))  in do_general_protection()
531 	if (v8086_mode(regs)) {  in do_general_protection()
533 		handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);  in do_general_protection()
538 	if (!user_mode(regs)) {  in do_general_protection()
539 		if (fixup_exception(regs, X86_TRAP_GP, error_code, 0))  in do_general_protection()
551 		    kprobe_fault_handler(regs, X86_TRAP_GP))  in do_general_protection()
554 		if (notify_die(DIE_GPF, desc, regs, error_code,  in do_general_protection()
556 			die(desc, regs, error_code);  in do_general_protection()
563 	show_signal(tsk, SIGSEGV, "", desc, regs, error_code);  in do_general_protection()
569 dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)  in do_int3()  argument
577 	    ftrace_int3_handler(regs))  in do_int3()
580 	if (poke_int3_handler(regs))  in do_int3()
590 	ist_enter(regs);  in do_int3()
593 	if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,  in do_int3()
599 	if (kprobe_int3_handler(regs))  in do_int3()
603 	if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,  in do_int3()
607 	cond_local_irq_enable(regs);  in do_int3()
608 	do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, 0, NULL);  in do_int3()
609 	cond_local_irq_disable(regs);  in do_int3()
612 	ist_exit(regs);  in do_int3()
624 	struct pt_regs *regs = (struct pt_regs *)this_cpu_read(cpu_current_top_of_stack) - 1;  in sync_regs()  local
625 	if (regs != eregs)  in sync_regs()
626 		*regs = *eregs;  in sync_regs()
627 	return regs;  in sync_regs()
633 	struct pt_regs regs;  member
651 	memmove(&new_stack->regs.ip, (void *)s->regs.sp, 5*8);  in fixup_bad_iret()
654 	memmove(new_stack, s, offsetof(struct bad_iret_stack, regs.ip));  in fixup_bad_iret()
656 	BUG_ON(!user_mode(&new_stack->regs));  in fixup_bad_iret()
662 static bool is_sysenter_singlestep(struct pt_regs *regs)  in is_sysenter_singlestep()  argument
673 	return (regs->ip - (unsigned long)__begin_SYSENTER_singlestep_region) <  in is_sysenter_singlestep()
677 	return (regs->ip - (unsigned long)entry_SYSENTER_compat) <  in is_sysenter_singlestep()
709 dotraplinkage void do_debug(struct pt_regs *regs, long error_code)  in do_debug()  argument
716 	ist_enter(regs);  in do_debug()
742 	if (unlikely(!user_mode(regs) && (dr6 & DR_STEP) &&  in do_debug()
743 		     is_sysenter_singlestep(regs))) {  in do_debug()
759 	if (!dr6 && user_mode(regs))  in do_debug()
766 	if (kprobe_debug_handler(regs))  in do_debug()
770 	if (notify_die(DIE_DEBUG, "debug", regs, (long)&dr6, error_code,  in do_debug()
781 	cond_local_irq_enable(regs);  in do_debug()
783 	if (v8086_mode(regs)) {  in do_debug()
784 		handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code,  in do_debug()
786 		cond_local_irq_disable(regs);  in do_debug()
791 	if (WARN_ON_ONCE((dr6 & DR_STEP) && !user_mode(regs))) {  in do_debug()
800 		regs->flags &= ~X86_EFLAGS_TF;  in do_debug()
804 		send_sigtrap(regs, error_code, si_code);  in do_debug()
805 	cond_local_irq_disable(regs);  in do_debug()
809 	ist_exit(regs);  in do_debug()
818 static void math_error(struct pt_regs *regs, int error_code, int trapnr)  in math_error()  argument
826 	cond_local_irq_enable(regs);  in math_error()
828 	if (!user_mode(regs)) {  in math_error()
829 		if (fixup_exception(regs, trapnr, error_code, 0))  in math_error()
835 		if (notify_die(DIE_TRAP, str, regs, error_code,  in math_error()
837 			die(str, regs, error_code);  in math_error()
855 			(void __user *)uprobe_get_trap_addr(regs));  in math_error()
858 dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)  in do_coprocessor_error()  argument
861 	math_error(regs, error_code, X86_TRAP_MF);  in do_coprocessor_error()
865 do_simd_coprocessor_error(struct pt_regs *regs, long error_code)  in do_simd_coprocessor_error()  argument
868 	math_error(regs, error_code, X86_TRAP_XF);  in do_simd_coprocessor_error()
872 do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)  in do_spurious_interrupt_bug()  argument
874 	cond_local_irq_enable(regs);  in do_spurious_interrupt_bug()
878 do_device_not_available(struct pt_regs *regs, long error_code)  in do_device_not_available()  argument
888 		cond_local_irq_enable(regs);  in do_device_not_available()
890 		info.regs = regs;  in do_device_not_available()
906 		die("unexpected #NM exception", regs, error_code);  in do_device_not_available()
912 dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)  in do_iret_error()  argument
917 	if (notify_die(DIE_TRAP, "iret exception", regs, error_code,  in do_iret_error()
919 		do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, error_code,  in do_iret_error()