1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/extable.h>
3 #include <linux/uaccess.h>
4 #include <linux/sched/debug.h>
5 #include <linux/bitfield.h>
6 #include <xen/xen.h>
7 
8 #include <asm/fpu/api.h>
9 #include <asm/sev.h>
10 #include <asm/traps.h>
11 #include <asm/kdebug.h>
12 #include <asm/insn-eval.h>
13 #include <asm/sgx.h>
14 
pt_regs_nr(struct pt_regs * regs,int nr)15 static inline unsigned long *pt_regs_nr(struct pt_regs *regs, int nr)
16 {
17 	int reg_offset = pt_regs_offset(regs, nr);
18 	static unsigned long __dummy;
19 
20 	if (WARN_ON_ONCE(reg_offset < 0))
21 		return &__dummy;
22 
23 	return (unsigned long *)((unsigned long)regs + reg_offset);
24 }
25 
26 static inline unsigned long
ex_fixup_addr(const struct exception_table_entry * x)27 ex_fixup_addr(const struct exception_table_entry *x)
28 {
29 	return (unsigned long)&x->fixup + x->fixup;
30 }
31 
ex_handler_default(const struct exception_table_entry * e,struct pt_regs * regs)32 static bool ex_handler_default(const struct exception_table_entry *e,
33 			       struct pt_regs *regs)
34 {
35 	if (e->data & EX_FLAG_CLEAR_AX)
36 		regs->ax = 0;
37 	if (e->data & EX_FLAG_CLEAR_DX)
38 		regs->dx = 0;
39 
40 	regs->ip = ex_fixup_addr(e);
41 	return true;
42 }
43 
ex_handler_fault(const struct exception_table_entry * fixup,struct pt_regs * regs,int trapnr)44 static bool ex_handler_fault(const struct exception_table_entry *fixup,
45 			     struct pt_regs *regs, int trapnr)
46 {
47 	regs->ax = trapnr;
48 	return ex_handler_default(fixup, regs);
49 }
50 
ex_handler_sgx(const struct exception_table_entry * fixup,struct pt_regs * regs,int trapnr)51 static bool ex_handler_sgx(const struct exception_table_entry *fixup,
52 			   struct pt_regs *regs, int trapnr)
53 {
54 	regs->ax = trapnr | SGX_ENCLS_FAULT_FLAG;
55 	return ex_handler_default(fixup, regs);
56 }
57 
58 /*
59  * Handler for when we fail to restore a task's FPU state.  We should never get
60  * here because the FPU state of a task using the FPU (task->thread.fpu.state)
61  * should always be valid.  However, past bugs have allowed userspace to set
62  * reserved bits in the XSAVE area using PTRACE_SETREGSET or sys_rt_sigreturn().
63  * These caused XRSTOR to fail when switching to the task, leaking the FPU
64  * registers of the task previously executing on the CPU.  Mitigate this class
65  * of vulnerability by restoring from the initial state (essentially, zeroing
66  * out all the FPU registers) if we can't restore from the task's FPU state.
67  */
ex_handler_fprestore(const struct exception_table_entry * fixup,struct pt_regs * regs)68 static bool ex_handler_fprestore(const struct exception_table_entry *fixup,
69 				 struct pt_regs *regs)
70 {
71 	regs->ip = ex_fixup_addr(fixup);
72 
73 	WARN_ONCE(1, "Bad FPU state detected at %pB, reinitializing FPU registers.",
74 		  (void *)instruction_pointer(regs));
75 
76 	fpu_reset_from_exception_fixup();
77 	return true;
78 }
79 
ex_handler_uaccess(const struct exception_table_entry * fixup,struct pt_regs * regs,int trapnr)80 static bool ex_handler_uaccess(const struct exception_table_entry *fixup,
81 			       struct pt_regs *regs, int trapnr)
82 {
83 	WARN_ONCE(trapnr == X86_TRAP_GP, "General protection fault in user access. Non-canonical address?");
84 	return ex_handler_default(fixup, regs);
85 }
86 
ex_handler_copy(const struct exception_table_entry * fixup,struct pt_regs * regs,int trapnr)87 static bool ex_handler_copy(const struct exception_table_entry *fixup,
88 			    struct pt_regs *regs, int trapnr)
89 {
90 	WARN_ONCE(trapnr == X86_TRAP_GP, "General protection fault in user access. Non-canonical address?");
91 	return ex_handler_fault(fixup, regs, trapnr);
92 }
93 
ex_handler_msr(const struct exception_table_entry * fixup,struct pt_regs * regs,bool wrmsr,bool safe,int reg)94 static bool ex_handler_msr(const struct exception_table_entry *fixup,
95 			   struct pt_regs *regs, bool wrmsr, bool safe, int reg)
96 {
97 	if (__ONCE_LITE_IF(!safe && wrmsr)) {
98 		pr_warn("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n",
99 			(unsigned int)regs->cx, (unsigned int)regs->dx,
100 			(unsigned int)regs->ax,  regs->ip, (void *)regs->ip);
101 		show_stack_regs(regs);
102 	}
103 
104 	if (__ONCE_LITE_IF(!safe && !wrmsr)) {
105 		pr_warn("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n",
106 			(unsigned int)regs->cx, regs->ip, (void *)regs->ip);
107 		show_stack_regs(regs);
108 	}
109 
110 	if (!wrmsr) {
111 		/* Pretend that the read succeeded and returned 0. */
112 		regs->ax = 0;
113 		regs->dx = 0;
114 	}
115 
116 	if (safe)
117 		*pt_regs_nr(regs, reg) = -EIO;
118 
119 	return ex_handler_default(fixup, regs);
120 }
121 
ex_handler_clear_fs(const struct exception_table_entry * fixup,struct pt_regs * regs)122 static bool ex_handler_clear_fs(const struct exception_table_entry *fixup,
123 				struct pt_regs *regs)
124 {
125 	if (static_cpu_has(X86_BUG_NULL_SEG))
126 		asm volatile ("mov %0, %%fs" : : "rm" (__USER_DS));
127 	asm volatile ("mov %0, %%fs" : : "rm" (0));
128 	return ex_handler_default(fixup, regs);
129 }
130 
ex_handler_imm_reg(const struct exception_table_entry * fixup,struct pt_regs * regs,int reg,int imm)131 static bool ex_handler_imm_reg(const struct exception_table_entry *fixup,
132 			       struct pt_regs *regs, int reg, int imm)
133 {
134 	*pt_regs_nr(regs, reg) = (long)imm;
135 	return ex_handler_default(fixup, regs);
136 }
137 
ex_handler_ucopy_len(const struct exception_table_entry * fixup,struct pt_regs * regs,int trapnr,int reg,int imm)138 static bool ex_handler_ucopy_len(const struct exception_table_entry *fixup,
139 				  struct pt_regs *regs, int trapnr, int reg, int imm)
140 {
141 	regs->cx = imm * regs->cx + *pt_regs_nr(regs, reg);
142 	return ex_handler_uaccess(fixup, regs, trapnr);
143 }
144 
ex_get_fixup_type(unsigned long ip)145 int ex_get_fixup_type(unsigned long ip)
146 {
147 	const struct exception_table_entry *e = search_exception_tables(ip);
148 
149 	return e ? FIELD_GET(EX_DATA_TYPE_MASK, e->data) : EX_TYPE_NONE;
150 }
151 
fixup_exception(struct pt_regs * regs,int trapnr,unsigned long error_code,unsigned long fault_addr)152 int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code,
153 		    unsigned long fault_addr)
154 {
155 	const struct exception_table_entry *e;
156 	int type, reg, imm;
157 
158 #ifdef CONFIG_PNPBIOS
159 	if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) {
160 		extern u32 pnp_bios_fault_eip, pnp_bios_fault_esp;
161 		extern u32 pnp_bios_is_utter_crap;
162 		pnp_bios_is_utter_crap = 1;
163 		printk(KERN_CRIT "PNPBIOS fault.. attempting recovery.\n");
164 		__asm__ volatile(
165 			"movl %0, %%esp\n\t"
166 			"jmp *%1\n\t"
167 			: : "g" (pnp_bios_fault_esp), "g" (pnp_bios_fault_eip));
168 		panic("do_trap: can't hit this");
169 	}
170 #endif
171 
172 	e = search_exception_tables(regs->ip);
173 	if (!e)
174 		return 0;
175 
176 	type = FIELD_GET(EX_DATA_TYPE_MASK, e->data);
177 	reg  = FIELD_GET(EX_DATA_REG_MASK,  e->data);
178 	imm  = FIELD_GET(EX_DATA_IMM_MASK,  e->data);
179 
180 	switch (type) {
181 	case EX_TYPE_DEFAULT:
182 	case EX_TYPE_DEFAULT_MCE_SAFE:
183 		return ex_handler_default(e, regs);
184 	case EX_TYPE_FAULT:
185 	case EX_TYPE_FAULT_MCE_SAFE:
186 		return ex_handler_fault(e, regs, trapnr);
187 	case EX_TYPE_UACCESS:
188 		return ex_handler_uaccess(e, regs, trapnr);
189 	case EX_TYPE_COPY:
190 		return ex_handler_copy(e, regs, trapnr);
191 	case EX_TYPE_CLEAR_FS:
192 		return ex_handler_clear_fs(e, regs);
193 	case EX_TYPE_FPU_RESTORE:
194 		return ex_handler_fprestore(e, regs);
195 	case EX_TYPE_BPF:
196 		return ex_handler_bpf(e, regs);
197 	case EX_TYPE_WRMSR:
198 		return ex_handler_msr(e, regs, true, false, reg);
199 	case EX_TYPE_RDMSR:
200 		return ex_handler_msr(e, regs, false, false, reg);
201 	case EX_TYPE_WRMSR_SAFE:
202 		return ex_handler_msr(e, regs, true, true, reg);
203 	case EX_TYPE_RDMSR_SAFE:
204 		return ex_handler_msr(e, regs, false, true, reg);
205 	case EX_TYPE_WRMSR_IN_MCE:
206 		ex_handler_msr_mce(regs, true);
207 		break;
208 	case EX_TYPE_RDMSR_IN_MCE:
209 		ex_handler_msr_mce(regs, false);
210 		break;
211 	case EX_TYPE_POP_REG:
212 		regs->sp += sizeof(long);
213 		fallthrough;
214 	case EX_TYPE_IMM_REG:
215 		return ex_handler_imm_reg(e, regs, reg, imm);
216 	case EX_TYPE_FAULT_SGX:
217 		return ex_handler_sgx(e, regs, trapnr);
218 	case EX_TYPE_UCOPY_LEN:
219 		return ex_handler_ucopy_len(e, regs, trapnr, reg, imm);
220 	}
221 	BUG();
222 }
223 
224 extern unsigned int early_recursion_flag;
225 
226 /* Restricted version used during very early boot */
early_fixup_exception(struct pt_regs * regs,int trapnr)227 void __init early_fixup_exception(struct pt_regs *regs, int trapnr)
228 {
229 	/* Ignore early NMIs. */
230 	if (trapnr == X86_TRAP_NMI)
231 		return;
232 
233 	if (early_recursion_flag > 2)
234 		goto halt_loop;
235 
236 	/*
237 	 * Old CPUs leave the high bits of CS on the stack
238 	 * undefined.  I'm not sure which CPUs do this, but at least
239 	 * the 486 DX works this way.
240 	 * Xen pv domains are not using the default __KERNEL_CS.
241 	 */
242 	if (!xen_pv_domain() && regs->cs != __KERNEL_CS)
243 		goto fail;
244 
245 	/*
246 	 * The full exception fixup machinery is available as soon as
247 	 * the early IDT is loaded.  This means that it is the
248 	 * responsibility of extable users to either function correctly
249 	 * when handlers are invoked early or to simply avoid causing
250 	 * exceptions before they're ready to handle them.
251 	 *
252 	 * This is better than filtering which handlers can be used,
253 	 * because refusing to call a handler here is guaranteed to
254 	 * result in a hard-to-debug panic.
255 	 *
256 	 * Keep in mind that not all vectors actually get here.  Early
257 	 * page faults, for example, are special.
258 	 */
259 	if (fixup_exception(regs, trapnr, regs->orig_ax, 0))
260 		return;
261 
262 	if (trapnr == X86_TRAP_UD) {
263 		if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) {
264 			/* Skip the ud2. */
265 			regs->ip += LEN_UD2;
266 			return;
267 		}
268 
269 		/*
270 		 * If this was a BUG and report_bug returns or if this
271 		 * was just a normal #UD, we want to continue onward and
272 		 * crash.
273 		 */
274 	}
275 
276 fail:
277 	early_printk("PANIC: early exception 0x%02x IP %lx:%lx error %lx cr2 0x%lx\n",
278 		     (unsigned)trapnr, (unsigned long)regs->cs, regs->ip,
279 		     regs->orig_ax, read_cr2());
280 
281 	show_regs(regs);
282 
283 halt_loop:
284 	while (true)
285 		halt();
286 }
287