51 static void do_audit_syscall_entry(struct pt_regs *regs, u32 arch)  in do_audit_syscall_entry()  argument
55 		audit_syscall_entry(regs->orig_ax, regs->di,  in do_audit_syscall_entry()
56 				    regs->si, regs->dx, regs->r10);  in do_audit_syscall_entry()
60 		audit_syscall_entry(regs->orig_ax, regs->bx,  in do_audit_syscall_entry()
61 				    regs->cx, regs->dx, regs->si);  in do_audit_syscall_entry()
69 static long syscall_trace_enter(struct pt_regs *regs)  in syscall_trace_enter()  argument
78 		BUG_ON(regs != task_pt_regs(current));  in syscall_trace_enter()
83 		ret = tracehook_report_syscall_entry(regs);  in syscall_trace_enter()
96 		sd.nr = regs->orig_ax;  in syscall_trace_enter()
97 		sd.instruction_pointer = regs->ip;  in syscall_trace_enter()
100 			sd.args[0] = regs->di;  in syscall_trace_enter()
101 			sd.args[1] = regs->si;  in syscall_trace_enter()
102 			sd.args[2] = regs->dx;  in syscall_trace_enter()
103 			sd.args[3] = regs->r10;  in syscall_trace_enter()
104 			sd.args[4] = regs->r8;  in syscall_trace_enter()
105 			sd.args[5] = regs->r9;  in syscall_trace_enter()
109 			sd.args[0] = regs->bx;  in syscall_trace_enter()
110 			sd.args[1] = regs->cx;  in syscall_trace_enter()
111 			sd.args[2] = regs->dx;  in syscall_trace_enter()
112 			sd.args[3] = regs->si;  in syscall_trace_enter()
113 			sd.args[4] = regs->di;  in syscall_trace_enter()
114 			sd.args[5] = regs->bp;  in syscall_trace_enter()
124 		trace_sys_enter(regs, regs->orig_ax);  in syscall_trace_enter()
126 	do_audit_syscall_entry(regs, arch);  in syscall_trace_enter()
128 	return ret ?: regs->orig_ax;  in syscall_trace_enter()
135 static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)  in exit_to_usermode_loop()  argument
152 			uprobe_notify_resume(regs);  in exit_to_usermode_loop()
159 			do_signal(regs);  in exit_to_usermode_loop()
163 			tracehook_notify_resume(regs);  in exit_to_usermode_loop()
164 			rseq_handle_notify_resume(NULL, regs);  in exit_to_usermode_loop()
181 __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)  in prepare_exit_to_usermode()  argument
194 		exit_to_usermode_loop(regs, cached_flags);  in prepare_exit_to_usermode()
227 static void syscall_slow_exit_work(struct pt_regs *regs, u32 cached_flags)  in syscall_slow_exit_work()  argument
231 	audit_syscall_exit(regs);  in syscall_slow_exit_work()
234 		trace_sys_exit(regs, regs->ax);  in syscall_slow_exit_work()
246 		tracehook_report_syscall_exit(regs, step);  in syscall_slow_exit_work()
253 __visible inline void syscall_return_slowpath(struct pt_regs *regs)  in syscall_return_slowpath()  argument
261 	    WARN(irqs_disabled(), "syscall %ld left IRQs disabled", regs->orig_ax))  in syscall_return_slowpath()
264 	rseq_syscall(regs);  in syscall_return_slowpath()
271 		syscall_slow_exit_work(regs, cached_flags);  in syscall_return_slowpath()
274 	prepare_exit_to_usermode(regs);  in syscall_return_slowpath()
278 __visible void do_syscall_64(unsigned long nr, struct pt_regs *regs)  in do_syscall_64()  argument
286 		nr = syscall_trace_enter(regs);  in do_syscall_64()
290 		regs->ax = sys_call_table[nr](regs);  in do_syscall_64()
296 		regs->ax = x32_sys_call_table[nr](regs);  in do_syscall_64()
300 	syscall_return_slowpath(regs);  in do_syscall_64()
311 static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs)  in do_syscall_32_irqs_on()  argument
314 	unsigned int nr = (unsigned int)regs->orig_ax;  in do_syscall_32_irqs_on()
327 		nr = syscall_trace_enter(regs);  in do_syscall_32_irqs_on()
333 		regs->ax = ia32_sys_call_table[nr](regs);  in do_syscall_32_irqs_on()
341 		regs->ax = ia32_sys_call_table[nr](  in do_syscall_32_irqs_on()
342 			(unsigned int)regs->bx, (unsigned int)regs->cx,  in do_syscall_32_irqs_on()
343 			(unsigned int)regs->dx, (unsigned int)regs->si,  in do_syscall_32_irqs_on()
344 			(unsigned int)regs->di, (unsigned int)regs->bp);  in do_syscall_32_irqs_on()
348 	syscall_return_slowpath(regs);  in do_syscall_32_irqs_on()
352 __visible void do_int80_syscall_32(struct pt_regs *regs)  in do_int80_syscall_32()  argument
356 	do_syscall_32_irqs_on(regs);  in do_int80_syscall_32()
360 __visible long do_fast_syscall_32(struct pt_regs *regs)  in do_fast_syscall_32()  argument
375 	regs->ip = landing_pad;  in do_fast_syscall_32()
388 		__get_user(*(u32 *)&regs->bp,  in do_fast_syscall_32()
389 			    (u32 __user __force *)(unsigned long)(u32)regs->sp)  in do_fast_syscall_32()
391 		get_user(*(u32 *)&regs->bp,  in do_fast_syscall_32()
392 			 (u32 __user __force *)(unsigned long)(u32)regs->sp)  in do_fast_syscall_32()
398 		regs->ax = -EFAULT;  in do_fast_syscall_32()
399 		prepare_exit_to_usermode(regs);  in do_fast_syscall_32()
404 	do_syscall_32_irqs_on(regs);  in do_fast_syscall_32()
416 	return regs->cs == __USER32_CS && regs->ss == __USER_DS &&  in do_fast_syscall_32()
417 		regs->ip == landing_pad &&  in do_fast_syscall_32()
418 		(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF)) == 0;  in do_fast_syscall_32()
431 		regs->cs == __USER_CS && regs->ss == __USER_DS &&  in do_fast_syscall_32()
432 		regs->ip == landing_pad &&  in do_fast_syscall_32()
433 		(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF | X86_EFLAGS_VM)) == 0;  in do_fast_syscall_32()