1 /* SPDX-License-Identifier: GPL-2.0 */
2 
3 #ifndef _ASM_X86_CPU_ENTRY_AREA_H
4 #define _ASM_X86_CPU_ENTRY_AREA_H
5 
6 #include <linux/percpu-defs.h>
7 #include <asm/processor.h>
8 #include <asm/intel_ds.h>
9 #include <asm/pgtable_areas.h>
10 
11 #ifdef CONFIG_X86_64
12 
13 #ifdef CONFIG_AMD_MEM_ENCRYPT
14 #define VC_EXCEPTION_STKSZ	EXCEPTION_STKSZ
15 #else
16 #define VC_EXCEPTION_STKSZ	0
17 #endif
18 
19 /* Macro to enforce the same ordering and stack sizes */
20 #define ESTACKS_MEMBERS(guardsize, optional_stack_size)		\
21 	char	DF_stack_guard[guardsize];			\
22 	char	DF_stack[EXCEPTION_STKSZ];			\
23 	char	NMI_stack_guard[guardsize];			\
24 	char	NMI_stack[EXCEPTION_STKSZ];			\
25 	char	DB_stack_guard[guardsize];			\
26 	char	DB_stack[EXCEPTION_STKSZ];			\
27 	char	MCE_stack_guard[guardsize];			\
28 	char	MCE_stack[EXCEPTION_STKSZ];			\
29 	char	VC_stack_guard[guardsize];			\
30 	char	VC_stack[optional_stack_size];			\
31 	char	VC2_stack_guard[guardsize];			\
32 	char	VC2_stack[optional_stack_size];			\
33 	char	IST_top_guard[guardsize];			\
34 
35 /* The exception stacks' physical storage. No guard pages required */
36 struct exception_stacks {
37 	ESTACKS_MEMBERS(0, VC_EXCEPTION_STKSZ)
38 };
39 
40 /* The effective cpu entry area mapping with guard pages. */
41 struct cea_exception_stacks {
42 	ESTACKS_MEMBERS(PAGE_SIZE, EXCEPTION_STKSZ)
43 };
44 
45 /*
46  * The exception stack ordering in [cea_]exception_stacks
47  */
48 enum exception_stack_ordering {
49 	ESTACK_DF,
50 	ESTACK_NMI,
51 	ESTACK_DB,
52 	ESTACK_MCE,
53 	ESTACK_VC,
54 	ESTACK_VC2,
55 	N_EXCEPTION_STACKS
56 };
57 
58 #define CEA_ESTACK_SIZE(st)					\
59 	sizeof(((struct cea_exception_stacks *)0)->st## _stack)
60 
61 #define CEA_ESTACK_BOT(ceastp, st)				\
62 	((unsigned long)&(ceastp)->st## _stack)
63 
64 #define CEA_ESTACK_TOP(ceastp, st)				\
65 	(CEA_ESTACK_BOT(ceastp, st) + CEA_ESTACK_SIZE(st))
66 
67 #define CEA_ESTACK_OFFS(st)					\
68 	offsetof(struct cea_exception_stacks, st## _stack)
69 
70 #define CEA_ESTACK_PAGES					\
71 	(sizeof(struct cea_exception_stacks) / PAGE_SIZE)
72 
73 #endif
74 
75 #ifdef CONFIG_X86_32
76 struct doublefault_stack {
77 	unsigned long stack[(PAGE_SIZE - sizeof(struct x86_hw_tss)) / sizeof(unsigned long)];
78 	struct x86_hw_tss tss;
79 } __aligned(PAGE_SIZE);
80 #endif
81 
82 /*
83  * cpu_entry_area is a percpu region that contains things needed by the CPU
84  * and early entry/exit code.  Real types aren't used for all fields here
85  * to avoid circular header dependencies.
86  *
87  * Every field is a virtual alias of some other allocated backing store.
88  * There is no direct allocation of a struct cpu_entry_area.
89  */
90 struct cpu_entry_area {
91 	char gdt[PAGE_SIZE];
92 
93 	/*
94 	 * The GDT is just below entry_stack and thus serves (on x86_64) as
95 	 * a read-only guard page. On 32-bit the GDT must be writeable, so
96 	 * it needs an extra guard page.
97 	 */
98 #ifdef CONFIG_X86_32
99 	char guard_entry_stack[PAGE_SIZE];
100 #endif
101 	struct entry_stack_page entry_stack_page;
102 
103 #ifdef CONFIG_X86_32
104 	char guard_doublefault_stack[PAGE_SIZE];
105 	struct doublefault_stack doublefault_stack;
106 #endif
107 
108 	/*
109 	 * On x86_64, the TSS is mapped RO.  On x86_32, it's mapped RW because
110 	 * we need task switches to work, and task switches write to the TSS.
111 	 */
112 	struct tss_struct tss;
113 
114 #ifdef CONFIG_X86_64
115 	/*
116 	 * Exception stacks used for IST entries with guard pages.
117 	 */
118 	struct cea_exception_stacks estacks;
119 #endif
120 	/*
121 	 * Per CPU debug store for Intel performance monitoring. Wastes a
122 	 * full page at the moment.
123 	 */
124 	struct debug_store cpu_debug_store;
125 	/*
126 	 * The actual PEBS/BTS buffers must be mapped to user space
127 	 * Reserve enough fixmap PTEs.
128 	 */
129 	struct debug_store_buffers cpu_debug_buffers;
130 };
131 
132 #define CPU_ENTRY_AREA_SIZE		(sizeof(struct cpu_entry_area))
133 #define CPU_ENTRY_AREA_ARRAY_SIZE	(CPU_ENTRY_AREA_SIZE * NR_CPUS)
134 
135 /* Total size includes the readonly IDT mapping page as well: */
136 #define CPU_ENTRY_AREA_TOTAL_SIZE	(CPU_ENTRY_AREA_ARRAY_SIZE + PAGE_SIZE)
137 
138 DECLARE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
139 DECLARE_PER_CPU(struct cea_exception_stacks *, cea_exception_stacks);
140 
141 extern void setup_cpu_entry_areas(void);
142 extern void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags);
143 
144 extern struct cpu_entry_area *get_cpu_entry_area(int cpu);
145 
cpu_entry_stack(int cpu)146 static __always_inline struct entry_stack *cpu_entry_stack(int cpu)
147 {
148 	return &get_cpu_entry_area(cpu)->entry_stack_page.stack;
149 }
150 
151 #define __this_cpu_ist_top_va(name)					\
152 	CEA_ESTACK_TOP(__this_cpu_read(cea_exception_stacks), name)
153 
154 #define __this_cpu_ist_bottom_va(name)					\
155 	CEA_ESTACK_BOT(__this_cpu_read(cea_exception_stacks), name)
156 
157 #endif
158