1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 *  Copyright (C) 2017  Steven Rostedt, VMware Inc.
4 */
5
6#include <linux/linkage.h>
7#include <asm/page_types.h>
8#include <asm/segment.h>
9#include <asm/export.h>
10#include <asm/ftrace.h>
11#include <asm/nospec-branch.h>
12#include <asm/frame.h>
13#include <asm/asm-offsets.h>
14
15# define function_hook	__fentry__
16EXPORT_SYMBOL(__fentry__)
17
18#ifdef CONFIG_FRAME_POINTER
19# define MCOUNT_FRAME			1	/* using frame = true  */
20#else
21# define MCOUNT_FRAME			0	/* using frame = false */
22#endif
23
24ENTRY(function_hook)
25	ret
26END(function_hook)
27
28ENTRY(ftrace_caller)
29
30#ifdef CONFIG_FRAME_POINTER
31	/*
32	 * Frame pointers are of ip followed by bp.
33	 * Since fentry is an immediate jump, we are left with
34	 * parent-ip, function-ip. We need to add a frame with
35	 * parent-ip followed by ebp.
36	 */
37	pushl	4(%esp)				/* parent ip */
38	pushl	%ebp
39	movl	%esp, %ebp
40	pushl	2*4(%esp)			/* function ip */
41
42	/* For mcount, the function ip is directly above */
43	pushl	%ebp
44	movl	%esp, %ebp
45#endif
46	pushl	%eax
47	pushl	%ecx
48	pushl	%edx
49	pushl	$0				/* Pass NULL as regs pointer */
50
51#ifdef CONFIG_FRAME_POINTER
52	/* Load parent ebp into edx */
53	movl	4*4(%esp), %edx
54#else
55	/* There's no frame pointer, load the appropriate stack addr instead */
56	lea	4*4(%esp), %edx
57#endif
58
59	movl	(MCOUNT_FRAME+4)*4(%esp), %eax	/* load the rip */
60	/* Get the parent ip */
61	movl	4(%edx), %edx			/* edx has ebp */
62
63	movl	function_trace_op, %ecx
64	subl	$MCOUNT_INSN_SIZE, %eax
65
66.globl ftrace_call
67ftrace_call:
68	call	ftrace_stub
69
70	addl	$4, %esp			/* skip NULL pointer */
71	popl	%edx
72	popl	%ecx
73	popl	%eax
74#ifdef CONFIG_FRAME_POINTER
75	popl	%ebp
76	addl	$4,%esp				/* skip function ip */
77	popl	%ebp				/* this is the orig bp */
78	addl	$4, %esp			/* skip parent ip */
79#endif
80.Lftrace_ret:
81#ifdef CONFIG_FUNCTION_GRAPH_TRACER
82.globl ftrace_graph_call
83ftrace_graph_call:
84	jmp	ftrace_stub
85#endif
86
87/* This is weak to keep gas from relaxing the jumps */
88WEAK(ftrace_stub)
89	ret
90END(ftrace_caller)
91
92ENTRY(ftrace_regs_caller)
93	/*
94	 * We're here from an mcount/fentry CALL, and the stack frame looks like:
95	 *
96	 *  <previous context>
97	 *  RET-IP
98	 *
99	 * The purpose of this function is to call out in an emulated INT3
100	 * environment with a stack frame like:
101	 *
102	 *  <previous context>
103	 *  gap / RET-IP
104	 *  gap
105	 *  gap
106	 *  gap
107	 *  pt_regs
108	 *
109	 * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds
110	 */
111	subl	$3*4, %esp	# RET-IP + 3 gaps
112	pushl	%ss		# ss
113	pushl	%esp		# points at ss
114	addl	$5*4, (%esp)	#   make it point at <previous context>
115	pushfl			# flags
116	pushl	$__KERNEL_CS	# cs
117	pushl	7*4(%esp)	# ip <- RET-IP
118	pushl	$0		# orig_eax
119
120	pushl	%gs
121	pushl	%fs
122	pushl	%es
123	pushl	%ds
124
125	pushl	%eax
126	pushl	%ebp
127	pushl	%edi
128	pushl	%esi
129	pushl	%edx
130	pushl	%ecx
131	pushl	%ebx
132
133	ENCODE_FRAME_POINTER
134
135	movl	PT_EIP(%esp), %eax	# 1st argument: IP
136	subl	$MCOUNT_INSN_SIZE, %eax
137	movl	21*4(%esp), %edx	# 2nd argument: parent ip
138	movl	function_trace_op, %ecx	# 3rd argument: ftrace_pos
139	pushl	%esp			# 4th argument: pt_regs
140
141GLOBAL(ftrace_regs_call)
142	call	ftrace_stub
143
144	addl	$4, %esp		# skip 4th argument
145
146	/* place IP below the new SP */
147	movl	PT_OLDESP(%esp), %eax
148	movl	PT_EIP(%esp), %ecx
149	movl	%ecx, -4(%eax)
150
151	/* place EAX below that */
152	movl	PT_EAX(%esp), %ecx
153	movl	%ecx, -8(%eax)
154
155	popl	%ebx
156	popl	%ecx
157	popl	%edx
158	popl	%esi
159	popl	%edi
160	popl	%ebp
161
162	lea	-8(%eax), %esp
163	popl	%eax
164
165	jmp	.Lftrace_ret
166
167#ifdef CONFIG_FUNCTION_GRAPH_TRACER
168ENTRY(ftrace_graph_caller)
169	pushl	%eax
170	pushl	%ecx
171	pushl	%edx
172	movl	3*4(%esp), %eax
173	/* Even with frame pointers, fentry doesn't have one here */
174	lea	4*4(%esp), %edx
175	movl	$0, %ecx
176	subl	$MCOUNT_INSN_SIZE, %eax
177	call	prepare_ftrace_return
178	popl	%edx
179	popl	%ecx
180	popl	%eax
181	ret
182END(ftrace_graph_caller)
183
184.globl return_to_handler
185return_to_handler:
186	pushl	%eax
187	pushl	%edx
188	movl	$0, %eax
189	call	ftrace_return_to_handler
190	movl	%eax, %ecx
191	popl	%edx
192	popl	%eax
193	JMP_NOSPEC %ecx
194#endif
195