1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 *  Copyright (C) 2017  Steven Rostedt, VMware Inc.
4 */
5
6#include <linux/linkage.h>
7#include <asm/page_types.h>
8#include <asm/segment.h>
9#include <asm/export.h>
10#include <asm/ftrace.h>
11#include <asm/nospec-branch.h>
12#include <asm/frame.h>
13#include <asm/asm-offsets.h>
14
15#ifdef CONFIG_FRAME_POINTER
16# define MCOUNT_FRAME			1	/* using frame = true  */
17#else
18# define MCOUNT_FRAME			0	/* using frame = false */
19#endif
20
21SYM_FUNC_START(__fentry__)
22	RET
23SYM_FUNC_END(__fentry__)
24EXPORT_SYMBOL(__fentry__)
25
26SYM_CODE_START(ftrace_caller)
27
28#ifdef CONFIG_FRAME_POINTER
29	/*
30	 * Frame pointers are of ip followed by bp.
31	 * Since fentry is an immediate jump, we are left with
32	 * parent-ip, function-ip. We need to add a frame with
33	 * parent-ip followed by ebp.
34	 */
35	pushl	4(%esp)				/* parent ip */
36	pushl	%ebp
37	movl	%esp, %ebp
38	pushl	2*4(%esp)			/* function ip */
39
40	/* For mcount, the function ip is directly above */
41	pushl	%ebp
42	movl	%esp, %ebp
43#endif
44	pushl	%eax
45	pushl	%ecx
46	pushl	%edx
47	pushl	$0				/* Pass NULL as regs pointer */
48
49#ifdef CONFIG_FRAME_POINTER
50	/* Load parent ebp into edx */
51	movl	4*4(%esp), %edx
52#else
53	/* There's no frame pointer, load the appropriate stack addr instead */
54	lea	4*4(%esp), %edx
55#endif
56
57	movl	(MCOUNT_FRAME+4)*4(%esp), %eax	/* load the rip */
58	/* Get the parent ip */
59	movl	4(%edx), %edx			/* edx has ebp */
60
61	movl	function_trace_op, %ecx
62	subl	$MCOUNT_INSN_SIZE, %eax
63
64.globl ftrace_call
65ftrace_call:
66	call	ftrace_stub
67
68	addl	$4, %esp			/* skip NULL pointer */
69	popl	%edx
70	popl	%ecx
71	popl	%eax
72#ifdef CONFIG_FRAME_POINTER
73	popl	%ebp
74	addl	$4,%esp				/* skip function ip */
75	popl	%ebp				/* this is the orig bp */
76	addl	$4, %esp			/* skip parent ip */
77#endif
78.Lftrace_ret:
79#ifdef CONFIG_FUNCTION_GRAPH_TRACER
80.globl ftrace_graph_call
81ftrace_graph_call:
82	jmp	ftrace_stub
83#endif
84
85/* This is weak to keep gas from relaxing the jumps */
86SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
87	RET
88SYM_CODE_END(ftrace_caller)
89
90SYM_CODE_START(ftrace_regs_caller)
91	/*
92	 * We're here from an mcount/fentry CALL, and the stack frame looks like:
93	 *
94	 *  <previous context>
95	 *  RET-IP
96	 *
97	 * The purpose of this function is to call out in an emulated INT3
98	 * environment with a stack frame like:
99	 *
100	 *  <previous context>
101	 *  gap / RET-IP
102	 *  gap
103	 *  gap
104	 *  gap
105	 *  pt_regs
106	 *
107	 * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds
108	 */
109	subl	$3*4, %esp	# RET-IP + 3 gaps
110	pushl	%ss		# ss
111	pushl	%esp		# points at ss
112	addl	$5*4, (%esp)	#   make it point at <previous context>
113	pushfl			# flags
114	pushl	$__KERNEL_CS	# cs
115	pushl	7*4(%esp)	# ip <- RET-IP
116	pushl	$0		# orig_eax
117
118	pushl	%gs
119	pushl	%fs
120	pushl	%es
121	pushl	%ds
122
123	pushl	%eax
124	pushl	%ebp
125	pushl	%edi
126	pushl	%esi
127	pushl	%edx
128	pushl	%ecx
129	pushl	%ebx
130
131	ENCODE_FRAME_POINTER
132
133	movl	PT_EIP(%esp), %eax	# 1st argument: IP
134	subl	$MCOUNT_INSN_SIZE, %eax
135	movl	21*4(%esp), %edx	# 2nd argument: parent ip
136	movl	function_trace_op, %ecx	# 3rd argument: ftrace_pos
137	pushl	%esp			# 4th argument: pt_regs
138
139SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
140	call	ftrace_stub
141
142	addl	$4, %esp		# skip 4th argument
143
144	/* place IP below the new SP */
145	movl	PT_OLDESP(%esp), %eax
146	movl	PT_EIP(%esp), %ecx
147	movl	%ecx, -4(%eax)
148
149	/* place EAX below that */
150	movl	PT_EAX(%esp), %ecx
151	movl	%ecx, -8(%eax)
152
153	popl	%ebx
154	popl	%ecx
155	popl	%edx
156	popl	%esi
157	popl	%edi
158	popl	%ebp
159
160	lea	-8(%eax), %esp
161	popl	%eax
162
163	jmp	.Lftrace_ret
164SYM_CODE_END(ftrace_regs_caller)
165
166#ifdef CONFIG_FUNCTION_GRAPH_TRACER
167SYM_CODE_START(ftrace_graph_caller)
168	pushl	%eax
169	pushl	%ecx
170	pushl	%edx
171	movl	3*4(%esp), %eax
172	/* Even with frame pointers, fentry doesn't have one here */
173	lea	4*4(%esp), %edx
174	movl	$0, %ecx
175	subl	$MCOUNT_INSN_SIZE, %eax
176	call	prepare_ftrace_return
177	popl	%edx
178	popl	%ecx
179	popl	%eax
180	RET
181SYM_CODE_END(ftrace_graph_caller)
182
183.globl return_to_handler
184return_to_handler:
185	pushl	%eax
186	pushl	%edx
187	movl	$0, %eax
188	call	ftrace_return_to_handler
189	movl	%eax, %ecx
190	popl	%edx
191	popl	%eax
192	JMP_NOSPEC ecx
193#endif
194