1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2014, 2015 Intel Corporation; author Matt Fleming
4 *
5 * Early support for invoking 32-bit EFI services from a 64-bit kernel.
6 *
7 * Because this thunking occurs before ExitBootServices() we have to
8 * restore the firmware's 32-bit GDT and IDT before we make EFI service
9 * calls.
10 *
11 * On the plus side, we don't have to worry about mangling 64-bit
12 * addresses into 32-bits because we're executing with an identity
13 * mapped pagetable and haven't transitioned to 64-bit virtual addresses
14 * yet.
15 */
16
17#include <linux/linkage.h>
18#include <asm/msr.h>
19#include <asm/page_types.h>
20#include <asm/processor-flags.h>
21#include <asm/segment.h>
22
23	.code64
24	.text
25/*
26 * When booting in 64-bit mode on 32-bit EFI firmware, startup_64_mixed_mode()
27 * is the first thing that runs after switching to long mode. Depending on
28 * whether the EFI handover protocol or the compat entry point was used to
29 * enter the kernel, it will either branch to the common 64-bit EFI stub
30 * entrypoint efi_stub_entry() directly, or via the 64-bit EFI PE/COFF
31 * entrypoint efi_pe_entry(). In the former case, the bootloader must provide a
32 * struct bootparams pointer as the third argument, so the presence of such a
33 * pointer is used to disambiguate.
34 *
35 *                                                             +--------------+
36 *  +------------------+     +------------+            +------>| efi_pe_entry |
37 *  | efi32_pe_entry   |---->|            |            |       +-----------+--+
38 *  +------------------+     |            |     +------+----------------+  |
39 *                           | startup_32 |---->| startup_64_mixed_mode |  |
40 *  +------------------+     |            |     +------+----------------+  |
41 *  | efi32_stub_entry |---->|            |            |                   |
42 *  +------------------+     +------------+            |                   |
43 *                                                     V                   |
44 *                           +------------+     +----------------+         |
45 *                           | startup_64 |<----| efi_stub_entry |<--------+
46 *                           +------------+     +----------------+
47 */
48SYM_FUNC_START(startup_64_mixed_mode)
49	lea	efi32_boot_args(%rip), %rdx
50	mov	0(%rdx), %edi
51	mov	4(%rdx), %esi
52#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
53	mov	8(%rdx), %edx		// saved bootparams pointer
54	test	%edx, %edx
55	jnz	efi_stub_entry
56#endif
57	/*
58	 * efi_pe_entry uses MS calling convention, which requires 32 bytes of
59	 * shadow space on the stack even if all arguments are passed in
60	 * registers. We also need an additional 8 bytes for the space that
61	 * would be occupied by the return address, and this also results in
62	 * the correct stack alignment for entry.
63	 */
64	sub	$40, %rsp
65	mov	%rdi, %rcx		// MS calling convention
66	mov	%rsi, %rdx
67	jmp	efi_pe_entry
68SYM_FUNC_END(startup_64_mixed_mode)
69
70SYM_FUNC_START(__efi64_thunk)
71	push	%rbp
72	push	%rbx
73
74	movl	%ds, %eax
75	push	%rax
76	movl	%es, %eax
77	push	%rax
78	movl	%ss, %eax
79	push	%rax
80
81	/* Copy args passed on stack */
82	movq	0x30(%rsp), %rbp
83	movq	0x38(%rsp), %rbx
84	movq	0x40(%rsp), %rax
85
86	/*
87	 * Convert x86-64 ABI params to i386 ABI
88	 */
89	subq	$64, %rsp
90	movl	%esi, 0x0(%rsp)
91	movl	%edx, 0x4(%rsp)
92	movl	%ecx, 0x8(%rsp)
93	movl	%r8d, 0xc(%rsp)
94	movl	%r9d, 0x10(%rsp)
95	movl	%ebp, 0x14(%rsp)
96	movl	%ebx, 0x18(%rsp)
97	movl	%eax, 0x1c(%rsp)
98
99	leaq	0x20(%rsp), %rbx
100	sgdt	(%rbx)
101	sidt	16(%rbx)
102
103	leaq	1f(%rip), %rbp
104
105	/*
106	 * Switch to IDT and GDT with 32-bit segments. These are the firmware
107	 * GDT and IDT that were installed when the kernel started executing.
108	 * The pointers were saved by the efi32_entry() routine below.
109	 *
110	 * Pass the saved DS selector to the 32-bit code, and use far return to
111	 * restore the saved CS selector.
112	 */
113	lidt	efi32_boot_idt(%rip)
114	lgdt	efi32_boot_gdt(%rip)
115
116	movzwl	efi32_boot_ds(%rip), %edx
117	movzwq	efi32_boot_cs(%rip), %rax
118	pushq	%rax
119	leaq	efi_enter32(%rip), %rax
120	pushq	%rax
121	lretq
122
1231:	addq	$64, %rsp
124	movq	%rdi, %rax
125
126	pop	%rbx
127	movl	%ebx, %ss
128	pop	%rbx
129	movl	%ebx, %es
130	pop	%rbx
131	movl	%ebx, %ds
132	/* Clear out 32-bit selector from FS and GS */
133	xorl	%ebx, %ebx
134	movl	%ebx, %fs
135	movl	%ebx, %gs
136
137	pop	%rbx
138	pop	%rbp
139	RET
140SYM_FUNC_END(__efi64_thunk)
141
142	.code32
143#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
144SYM_FUNC_START(efi32_stub_entry)
145	call	1f
1461:	popl	%ecx
147
148	/* Clear BSS */
149	xorl	%eax, %eax
150	leal	(_bss - 1b)(%ecx), %edi
151	leal	(_ebss - 1b)(%ecx), %ecx
152	subl	%edi, %ecx
153	shrl	$2, %ecx
154	cld
155	rep	stosl
156
157	add	$0x4, %esp		/* Discard return address */
158	popl	%ecx
159	popl	%edx
160	popl	%esi
161	jmp	efi32_entry
162SYM_FUNC_END(efi32_stub_entry)
163#endif
164
165/*
166 * EFI service pointer must be in %edi.
167 *
168 * The stack should represent the 32-bit calling convention.
169 */
170SYM_FUNC_START_LOCAL(efi_enter32)
171	/* Load firmware selector into data and stack segment registers */
172	movl	%edx, %ds
173	movl	%edx, %es
174	movl	%edx, %fs
175	movl	%edx, %gs
176	movl	%edx, %ss
177
178	/* Reload pgtables */
179	movl	%cr3, %eax
180	movl	%eax, %cr3
181
182	/* Disable paging */
183	movl	%cr0, %eax
184	btrl	$X86_CR0_PG_BIT, %eax
185	movl	%eax, %cr0
186
187	/* Disable long mode via EFER */
188	movl	$MSR_EFER, %ecx
189	rdmsr
190	btrl	$_EFER_LME, %eax
191	wrmsr
192
193	call	*%edi
194
195	/* We must preserve return value */
196	movl	%eax, %edi
197
198	/*
199	 * Some firmware will return with interrupts enabled. Be sure to
200	 * disable them before we switch GDTs and IDTs.
201	 */
202	cli
203
204	lidtl	16(%ebx)
205	lgdtl	(%ebx)
206
207	movl	%cr4, %eax
208	btsl	$(X86_CR4_PAE_BIT), %eax
209	movl	%eax, %cr4
210
211	movl	%cr3, %eax
212	movl	%eax, %cr3
213
214	movl	$MSR_EFER, %ecx
215	rdmsr
216	btsl	$_EFER_LME, %eax
217	wrmsr
218
219	xorl	%eax, %eax
220	lldt	%ax
221
222	pushl	$__KERNEL_CS
223	pushl	%ebp
224
225	/* Enable paging */
226	movl	%cr0, %eax
227	btsl	$X86_CR0_PG_BIT, %eax
228	movl	%eax, %cr0
229	lret
230SYM_FUNC_END(efi_enter32)
231
232/*
233 * This is the common EFI stub entry point for mixed mode.
234 *
235 * Arguments:	%ecx	image handle
236 * 		%edx	EFI system table pointer
237 *		%esi	struct bootparams pointer (or NULL when not using
238 *			the EFI handover protocol)
239 *
240 * Since this is the point of no return for ordinary execution, no registers
241 * are considered live except for the function parameters. [Note that the EFI
242 * stub may still exit and return to the firmware using the Exit() EFI boot
243 * service.]
244 */
245SYM_FUNC_START_LOCAL(efi32_entry)
246	call	1f
2471:	pop	%ebx
248
249	/* Save firmware GDTR and code/data selectors */
250	sgdtl	(efi32_boot_gdt - 1b)(%ebx)
251	movw	%cs, (efi32_boot_cs - 1b)(%ebx)
252	movw	%ds, (efi32_boot_ds - 1b)(%ebx)
253
254	/* Store firmware IDT descriptor */
255	sidtl	(efi32_boot_idt - 1b)(%ebx)
256
257	/* Store boot arguments */
258	leal	(efi32_boot_args - 1b)(%ebx), %ebx
259	movl	%ecx, 0(%ebx)
260	movl	%edx, 4(%ebx)
261	movl	%esi, 8(%ebx)
262	movb	$0x0, 12(%ebx)          // efi_is64
263
264	/* Disable paging */
265	movl	%cr0, %eax
266	btrl	$X86_CR0_PG_BIT, %eax
267	movl	%eax, %cr0
268
269	jmp	startup_32
270SYM_FUNC_END(efi32_entry)
271
272/*
273 * efi_status_t efi32_pe_entry(efi_handle_t image_handle,
274 *			       efi_system_table_32_t *sys_table)
275 */
276SYM_FUNC_START(efi32_pe_entry)
277	pushl	%ebp
278	movl	%esp, %ebp
279	pushl	%ebx				// save callee-save registers
280	pushl	%edi
281
282	call	verify_cpu			// check for long mode support
283	testl	%eax, %eax
284	movl	$0x80000003, %eax		// EFI_UNSUPPORTED
285	jnz	2f
286
287	movl	8(%ebp), %ecx			// image_handle
288	movl	12(%ebp), %edx			// sys_table
289	xorl	%esi, %esi
290	jmp	efi32_entry			// pass %ecx, %edx, %esi
291						// no other registers remain live
292
2932:	popl	%edi				// restore callee-save registers
294	popl	%ebx
295	leave
296	RET
297SYM_FUNC_END(efi32_pe_entry)
298
299#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
300	.org	efi32_stub_entry + 0x200
301	.code64
302SYM_FUNC_START_NOALIGN(efi64_stub_entry)
303	jmp	efi_handover_entry
304SYM_FUNC_END(efi64_stub_entry)
305#endif
306
307	.data
308	.balign	8
309SYM_DATA_START_LOCAL(efi32_boot_gdt)
310	.word	0
311	.quad	0
312SYM_DATA_END(efi32_boot_gdt)
313
314SYM_DATA_START_LOCAL(efi32_boot_idt)
315	.word	0
316	.quad	0
317SYM_DATA_END(efi32_boot_idt)
318
319SYM_DATA_LOCAL(efi32_boot_cs, .word 0)
320SYM_DATA_LOCAL(efi32_boot_ds, .word 0)
321SYM_DATA_LOCAL(efi32_boot_args, .long 0, 0, 0)
322SYM_DATA(efi_is64, .byte 1)
323