1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Machine check handler.
4  *
5  * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
6  * Rest from unknown author(s).
7  * 2004 Andi Kleen. Rewrote most of it.
8  * Copyright 2008 Intel Corporation
9  * Author: Andi Kleen
10  */
11 
12 #include <linux/thread_info.h>
13 #include <linux/capability.h>
14 #include <linux/miscdevice.h>
15 #include <linux/ratelimit.h>
16 #include <linux/rcupdate.h>
17 #include <linux/kobject.h>
18 #include <linux/uaccess.h>
19 #include <linux/kdebug.h>
20 #include <linux/kernel.h>
21 #include <linux/percpu.h>
22 #include <linux/string.h>
23 #include <linux/device.h>
24 #include <linux/syscore_ops.h>
25 #include <linux/delay.h>
26 #include <linux/ctype.h>
27 #include <linux/sched.h>
28 #include <linux/sysfs.h>
29 #include <linux/types.h>
30 #include <linux/slab.h>
31 #include <linux/init.h>
32 #include <linux/kmod.h>
33 #include <linux/poll.h>
34 #include <linux/nmi.h>
35 #include <linux/cpu.h>
36 #include <linux/ras.h>
37 #include <linux/smp.h>
38 #include <linux/fs.h>
39 #include <linux/mm.h>
40 #include <linux/debugfs.h>
41 #include <linux/irq_work.h>
42 #include <linux/export.h>
43 #include <linux/set_memory.h>
44 #include <linux/sync_core.h>
45 #include <linux/task_work.h>
46 #include <linux/hardirq.h>
47 #include <linux/kexec.h>
48 
49 #include <asm/intel-family.h>
50 #include <asm/processor.h>
51 #include <asm/traps.h>
52 #include <asm/tlbflush.h>
53 #include <asm/mce.h>
54 #include <asm/msr.h>
55 #include <asm/reboot.h>
56 
57 #include "internal.h"
58 
59 /* sysfs synchronization */
60 static DEFINE_MUTEX(mce_sysfs_mutex);
61 
62 #define CREATE_TRACE_POINTS
63 #include <trace/events/mce.h>
64 
65 #define SPINUNIT		100	/* 100ns */
66 
67 DEFINE_PER_CPU(unsigned, mce_exception_count);
68 
69 DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks);
70 
71 DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array);
72 
73 #define ATTR_LEN               16
74 /* One object for each MCE bank, shared by all CPUs */
75 struct mce_bank_dev {
76 	struct device_attribute	attr;			/* device attribute */
77 	char			attrname[ATTR_LEN];	/* attribute name */
78 	u8			bank;			/* bank number */
79 };
80 static struct mce_bank_dev mce_bank_devs[MAX_NR_BANKS];
81 
82 struct mce_vendor_flags mce_flags __read_mostly;
83 
84 struct mca_config mca_cfg __read_mostly = {
85 	.bootlog  = -1,
86 	.monarch_timeout = -1
87 };
88 
89 static DEFINE_PER_CPU(struct mce, mces_seen);
90 static unsigned long mce_need_notify;
91 
92 /*
93  * MCA banks polled by the period polling timer for corrected events.
94  * With Intel CMCI, this only has MCA banks which do not support CMCI (if any).
95  */
96 DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
97 	[0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
98 };
99 
100 /*
101  * MCA banks controlled through firmware first for corrected errors.
102  * This is a global list of banks for which we won't enable CMCI and we
103  * won't poll. Firmware controls these banks and is responsible for
104  * reporting corrected errors through GHES. Uncorrected/recoverable
105  * errors are still notified through a machine check.
106  */
107 mce_banks_t mce_banks_ce_disabled;
108 
109 static struct work_struct mce_work;
110 static struct irq_work mce_irq_work;
111 
112 /*
113  * CPU/chipset specific EDAC code can register a notifier call here to print
114  * MCE errors in a human-readable form.
115  */
116 BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain);
117 
118 /* Do initial initialization of a struct mce */
mce_setup(struct mce * m)119 void mce_setup(struct mce *m)
120 {
121 	memset(m, 0, sizeof(struct mce));
122 	m->cpu = m->extcpu = smp_processor_id();
123 	/* need the internal __ version to avoid deadlocks */
124 	m->time = __ktime_get_real_seconds();
125 	m->cpuvendor = boot_cpu_data.x86_vendor;
126 	m->cpuid = cpuid_eax(1);
127 	m->socketid = cpu_data(m->extcpu).phys_proc_id;
128 	m->apicid = cpu_data(m->extcpu).initial_apicid;
129 	m->mcgcap = __rdmsr(MSR_IA32_MCG_CAP);
130 	m->ppin = cpu_data(m->extcpu).ppin;
131 	m->microcode = boot_cpu_data.microcode;
132 }
133 
134 DEFINE_PER_CPU(struct mce, injectm);
135 EXPORT_PER_CPU_SYMBOL_GPL(injectm);
136 
mce_log(struct mce * m)137 void mce_log(struct mce *m)
138 {
139 	if (!mce_gen_pool_add(m))
140 		irq_work_queue(&mce_irq_work);
141 }
142 EXPORT_SYMBOL_GPL(mce_log);
143 
mce_register_decode_chain(struct notifier_block * nb)144 void mce_register_decode_chain(struct notifier_block *nb)
145 {
146 	if (WARN_ON(nb->priority < MCE_PRIO_LOWEST ||
147 		    nb->priority > MCE_PRIO_HIGHEST))
148 		return;
149 
150 	blocking_notifier_chain_register(&x86_mce_decoder_chain, nb);
151 }
152 EXPORT_SYMBOL_GPL(mce_register_decode_chain);
153 
mce_unregister_decode_chain(struct notifier_block * nb)154 void mce_unregister_decode_chain(struct notifier_block *nb)
155 {
156 	blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb);
157 }
158 EXPORT_SYMBOL_GPL(mce_unregister_decode_chain);
159 
__print_mce(struct mce * m)160 static void __print_mce(struct mce *m)
161 {
162 	pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n",
163 		 m->extcpu,
164 		 (m->mcgstatus & MCG_STATUS_MCIP ? " Exception" : ""),
165 		 m->mcgstatus, m->bank, m->status);
166 
167 	if (m->ip) {
168 		pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ",
169 			!(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
170 			m->cs, m->ip);
171 
172 		if (m->cs == __KERNEL_CS)
173 			pr_cont("{%pS}", (void *)(unsigned long)m->ip);
174 		pr_cont("\n");
175 	}
176 
177 	pr_emerg(HW_ERR "TSC %llx ", m->tsc);
178 	if (m->addr)
179 		pr_cont("ADDR %llx ", m->addr);
180 	if (m->misc)
181 		pr_cont("MISC %llx ", m->misc);
182 	if (m->ppin)
183 		pr_cont("PPIN %llx ", m->ppin);
184 
185 	if (mce_flags.smca) {
186 		if (m->synd)
187 			pr_cont("SYND %llx ", m->synd);
188 		if (m->ipid)
189 			pr_cont("IPID %llx ", m->ipid);
190 	}
191 
192 	pr_cont("\n");
193 
194 	/*
195 	 * Note this output is parsed by external tools and old fields
196 	 * should not be changed.
197 	 */
198 	pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n",
199 		m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid,
200 		m->microcode);
201 }
202 
print_mce(struct mce * m)203 static void print_mce(struct mce *m)
204 {
205 	__print_mce(m);
206 
207 	if (m->cpuvendor != X86_VENDOR_AMD && m->cpuvendor != X86_VENDOR_HYGON)
208 		pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n");
209 }
210 
211 #define PANIC_TIMEOUT 5 /* 5 seconds */
212 
213 static atomic_t mce_panicked;
214 
215 static int fake_panic;
216 static atomic_t mce_fake_panicked;
217 
218 /* Panic in progress. Enable interrupts and wait for final IPI */
wait_for_panic(void)219 static void wait_for_panic(void)
220 {
221 	long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
222 
223 	preempt_disable();
224 	local_irq_enable();
225 	while (timeout-- > 0)
226 		udelay(1);
227 	if (panic_timeout == 0)
228 		panic_timeout = mca_cfg.panic_timeout;
229 	panic("Panicing machine check CPU died");
230 }
231 
mce_panic(const char * msg,struct mce * final,char * exp)232 static noinstr void mce_panic(const char *msg, struct mce *final, char *exp)
233 {
234 	struct llist_node *pending;
235 	struct mce_evt_llist *l;
236 	int apei_err = 0;
237 	struct page *p;
238 
239 	/*
240 	 * Allow instrumentation around external facilities usage. Not that it
241 	 * matters a whole lot since the machine is going to panic anyway.
242 	 */
243 	instrumentation_begin();
244 
245 	if (!fake_panic) {
246 		/*
247 		 * Make sure only one CPU runs in machine check panic
248 		 */
249 		if (atomic_inc_return(&mce_panicked) > 1)
250 			wait_for_panic();
251 		barrier();
252 
253 		bust_spinlocks(1);
254 		console_verbose();
255 	} else {
256 		/* Don't log too much for fake panic */
257 		if (atomic_inc_return(&mce_fake_panicked) > 1)
258 			goto out;
259 	}
260 	pending = mce_gen_pool_prepare_records();
261 	/* First print corrected ones that are still unlogged */
262 	llist_for_each_entry(l, pending, llnode) {
263 		struct mce *m = &l->mce;
264 		if (!(m->status & MCI_STATUS_UC)) {
265 			print_mce(m);
266 			if (!apei_err)
267 				apei_err = apei_write_mce(m);
268 		}
269 	}
270 	/* Now print uncorrected but with the final one last */
271 	llist_for_each_entry(l, pending, llnode) {
272 		struct mce *m = &l->mce;
273 		if (!(m->status & MCI_STATUS_UC))
274 			continue;
275 		if (!final || mce_cmp(m, final)) {
276 			print_mce(m);
277 			if (!apei_err)
278 				apei_err = apei_write_mce(m);
279 		}
280 	}
281 	if (final) {
282 		print_mce(final);
283 		if (!apei_err)
284 			apei_err = apei_write_mce(final);
285 	}
286 	if (exp)
287 		pr_emerg(HW_ERR "Machine check: %s\n", exp);
288 	if (!fake_panic) {
289 		if (panic_timeout == 0)
290 			panic_timeout = mca_cfg.panic_timeout;
291 
292 		/*
293 		 * Kdump skips the poisoned page in order to avoid
294 		 * touching the error bits again. Poison the page even
295 		 * if the error is fatal and the machine is about to
296 		 * panic.
297 		 */
298 		if (kexec_crash_loaded()) {
299 			if (final && (final->status & MCI_STATUS_ADDRV)) {
300 				p = pfn_to_online_page(final->addr >> PAGE_SHIFT);
301 				if (p)
302 					SetPageHWPoison(p);
303 			}
304 		}
305 		panic(msg);
306 	} else
307 		pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
308 
309 out:
310 	instrumentation_end();
311 }
312 
313 /* Support code for software error injection */
314 
msr_to_offset(u32 msr)315 static int msr_to_offset(u32 msr)
316 {
317 	unsigned bank = __this_cpu_read(injectm.bank);
318 
319 	if (msr == mca_cfg.rip_msr)
320 		return offsetof(struct mce, ip);
321 	if (msr == mca_msr_reg(bank, MCA_STATUS))
322 		return offsetof(struct mce, status);
323 	if (msr == mca_msr_reg(bank, MCA_ADDR))
324 		return offsetof(struct mce, addr);
325 	if (msr == mca_msr_reg(bank, MCA_MISC))
326 		return offsetof(struct mce, misc);
327 	if (msr == MSR_IA32_MCG_STATUS)
328 		return offsetof(struct mce, mcgstatus);
329 	return -1;
330 }
331 
ex_handler_msr_mce(struct pt_regs * regs,bool wrmsr)332 void ex_handler_msr_mce(struct pt_regs *regs, bool wrmsr)
333 {
334 	if (wrmsr) {
335 		pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n",
336 			 (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax,
337 			 regs->ip, (void *)regs->ip);
338 	} else {
339 		pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n",
340 			 (unsigned int)regs->cx, regs->ip, (void *)regs->ip);
341 	}
342 
343 	show_stack_regs(regs);
344 
345 	panic("MCA architectural violation!\n");
346 
347 	while (true)
348 		cpu_relax();
349 }
350 
351 /* MSR access wrappers used for error injection */
mce_rdmsrl(u32 msr)352 noinstr u64 mce_rdmsrl(u32 msr)
353 {
354 	DECLARE_ARGS(val, low, high);
355 
356 	if (__this_cpu_read(injectm.finished)) {
357 		int offset;
358 		u64 ret;
359 
360 		instrumentation_begin();
361 
362 		offset = msr_to_offset(msr);
363 		if (offset < 0)
364 			ret = 0;
365 		else
366 			ret = *(u64 *)((char *)this_cpu_ptr(&injectm) + offset);
367 
368 		instrumentation_end();
369 
370 		return ret;
371 	}
372 
373 	/*
374 	 * RDMSR on MCA MSRs should not fault. If they do, this is very much an
375 	 * architectural violation and needs to be reported to hw vendor. Panic
376 	 * the box to not allow any further progress.
377 	 */
378 	asm volatile("1: rdmsr\n"
379 		     "2:\n"
380 		     _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_RDMSR_IN_MCE)
381 		     : EAX_EDX_RET(val, low, high) : "c" (msr));
382 
383 
384 	return EAX_EDX_VAL(val, low, high);
385 }
386 
mce_wrmsrl(u32 msr,u64 v)387 static noinstr void mce_wrmsrl(u32 msr, u64 v)
388 {
389 	u32 low, high;
390 
391 	if (__this_cpu_read(injectm.finished)) {
392 		int offset;
393 
394 		instrumentation_begin();
395 
396 		offset = msr_to_offset(msr);
397 		if (offset >= 0)
398 			*(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v;
399 
400 		instrumentation_end();
401 
402 		return;
403 	}
404 
405 	low  = (u32)v;
406 	high = (u32)(v >> 32);
407 
408 	/* See comment in mce_rdmsrl() */
409 	asm volatile("1: wrmsr\n"
410 		     "2:\n"
411 		     _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR_IN_MCE)
412 		     : : "c" (msr), "a"(low), "d" (high) : "memory");
413 }
414 
415 /*
416  * Collect all global (w.r.t. this processor) status about this machine
417  * check into our "mce" struct so that we can use it later to assess
418  * the severity of the problem as we read per-bank specific details.
419  */
mce_gather_info(struct mce * m,struct pt_regs * regs)420 static noinstr void mce_gather_info(struct mce *m, struct pt_regs *regs)
421 {
422 	/*
423 	 * Enable instrumentation around mce_setup() which calls external
424 	 * facilities.
425 	 */
426 	instrumentation_begin();
427 	mce_setup(m);
428 	instrumentation_end();
429 
430 	m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
431 	if (regs) {
432 		/*
433 		 * Get the address of the instruction at the time of
434 		 * the machine check error.
435 		 */
436 		if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) {
437 			m->ip = regs->ip;
438 			m->cs = regs->cs;
439 
440 			/*
441 			 * When in VM86 mode make the cs look like ring 3
442 			 * always. This is a lie, but it's better than passing
443 			 * the additional vm86 bit around everywhere.
444 			 */
445 			if (v8086_mode(regs))
446 				m->cs |= 3;
447 		}
448 		/* Use accurate RIP reporting if available. */
449 		if (mca_cfg.rip_msr)
450 			m->ip = mce_rdmsrl(mca_cfg.rip_msr);
451 	}
452 }
453 
mce_available(struct cpuinfo_x86 * c)454 int mce_available(struct cpuinfo_x86 *c)
455 {
456 	if (mca_cfg.disabled)
457 		return 0;
458 	return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
459 }
460 
mce_schedule_work(void)461 static void mce_schedule_work(void)
462 {
463 	if (!mce_gen_pool_empty())
464 		schedule_work(&mce_work);
465 }
466 
mce_irq_work_cb(struct irq_work * entry)467 static void mce_irq_work_cb(struct irq_work *entry)
468 {
469 	mce_schedule_work();
470 }
471 
472 /*
473  * Check if the address reported by the CPU is in a format we can parse.
474  * It would be possible to add code for most other cases, but all would
475  * be somewhat complicated (e.g. segment offset would require an instruction
476  * parser). So only support physical addresses up to page granularity for now.
477  */
mce_usable_address(struct mce * m)478 int mce_usable_address(struct mce *m)
479 {
480 	if (!(m->status & MCI_STATUS_ADDRV))
481 		return 0;
482 
483 	/* Checks after this one are Intel/Zhaoxin-specific: */
484 	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL &&
485 	    boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN)
486 		return 1;
487 
488 	if (!(m->status & MCI_STATUS_MISCV))
489 		return 0;
490 
491 	if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)
492 		return 0;
493 
494 	if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)
495 		return 0;
496 
497 	return 1;
498 }
499 EXPORT_SYMBOL_GPL(mce_usable_address);
500 
mce_is_memory_error(struct mce * m)501 bool mce_is_memory_error(struct mce *m)
502 {
503 	switch (m->cpuvendor) {
504 	case X86_VENDOR_AMD:
505 	case X86_VENDOR_HYGON:
506 		return amd_mce_is_memory_error(m);
507 
508 	case X86_VENDOR_INTEL:
509 	case X86_VENDOR_ZHAOXIN:
510 		/*
511 		 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes
512 		 *
513 		 * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for
514 		 * indicating a memory error. Bit 8 is used for indicating a
515 		 * cache hierarchy error. The combination of bit 2 and bit 3
516 		 * is used for indicating a `generic' cache hierarchy error
517 		 * But we can't just blindly check the above bits, because if
518 		 * bit 11 is set, then it is a bus/interconnect error - and
519 		 * either way the above bits just gives more detail on what
520 		 * bus/interconnect error happened. Note that bit 12 can be
521 		 * ignored, as it's the "filter" bit.
522 		 */
523 		return (m->status & 0xef80) == BIT(7) ||
524 		       (m->status & 0xef00) == BIT(8) ||
525 		       (m->status & 0xeffc) == 0xc;
526 
527 	default:
528 		return false;
529 	}
530 }
531 EXPORT_SYMBOL_GPL(mce_is_memory_error);
532 
whole_page(struct mce * m)533 static bool whole_page(struct mce *m)
534 {
535 	if (!mca_cfg.ser || !(m->status & MCI_STATUS_MISCV))
536 		return true;
537 
538 	return MCI_MISC_ADDR_LSB(m->misc) >= PAGE_SHIFT;
539 }
540 
mce_is_correctable(struct mce * m)541 bool mce_is_correctable(struct mce *m)
542 {
543 	if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED)
544 		return false;
545 
546 	if (m->cpuvendor == X86_VENDOR_HYGON && m->status & MCI_STATUS_DEFERRED)
547 		return false;
548 
549 	if (m->status & MCI_STATUS_UC)
550 		return false;
551 
552 	return true;
553 }
554 EXPORT_SYMBOL_GPL(mce_is_correctable);
555 
mce_early_notifier(struct notifier_block * nb,unsigned long val,void * data)556 static int mce_early_notifier(struct notifier_block *nb, unsigned long val,
557 			      void *data)
558 {
559 	struct mce *m = (struct mce *)data;
560 
561 	if (!m)
562 		return NOTIFY_DONE;
563 
564 	/* Emit the trace record: */
565 	trace_mce_record(m);
566 
567 	set_bit(0, &mce_need_notify);
568 
569 	mce_notify_irq();
570 
571 	return NOTIFY_DONE;
572 }
573 
574 static struct notifier_block early_nb = {
575 	.notifier_call	= mce_early_notifier,
576 	.priority	= MCE_PRIO_EARLY,
577 };
578 
uc_decode_notifier(struct notifier_block * nb,unsigned long val,void * data)579 static int uc_decode_notifier(struct notifier_block *nb, unsigned long val,
580 			      void *data)
581 {
582 	struct mce *mce = (struct mce *)data;
583 	unsigned long pfn;
584 
585 	if (!mce || !mce_usable_address(mce))
586 		return NOTIFY_DONE;
587 
588 	if (mce->severity != MCE_AO_SEVERITY &&
589 	    mce->severity != MCE_DEFERRED_SEVERITY)
590 		return NOTIFY_DONE;
591 
592 	pfn = (mce->addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT;
593 	if (!memory_failure(pfn, 0)) {
594 		set_mce_nospec(pfn);
595 		mce->kflags |= MCE_HANDLED_UC;
596 	}
597 
598 	return NOTIFY_OK;
599 }
600 
601 static struct notifier_block mce_uc_nb = {
602 	.notifier_call	= uc_decode_notifier,
603 	.priority	= MCE_PRIO_UC,
604 };
605 
mce_default_notifier(struct notifier_block * nb,unsigned long val,void * data)606 static int mce_default_notifier(struct notifier_block *nb, unsigned long val,
607 				void *data)
608 {
609 	struct mce *m = (struct mce *)data;
610 
611 	if (!m)
612 		return NOTIFY_DONE;
613 
614 	if (mca_cfg.print_all || !m->kflags)
615 		__print_mce(m);
616 
617 	return NOTIFY_DONE;
618 }
619 
620 static struct notifier_block mce_default_nb = {
621 	.notifier_call	= mce_default_notifier,
622 	/* lowest prio, we want it to run last. */
623 	.priority	= MCE_PRIO_LOWEST,
624 };
625 
626 /*
627  * Read ADDR and MISC registers.
628  */
mce_read_aux(struct mce * m,int i)629 static noinstr void mce_read_aux(struct mce *m, int i)
630 {
631 	if (m->status & MCI_STATUS_MISCV)
632 		m->misc = mce_rdmsrl(mca_msr_reg(i, MCA_MISC));
633 
634 	if (m->status & MCI_STATUS_ADDRV) {
635 		m->addr = mce_rdmsrl(mca_msr_reg(i, MCA_ADDR));
636 
637 		/*
638 		 * Mask the reported address by the reported granularity.
639 		 */
640 		if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) {
641 			u8 shift = MCI_MISC_ADDR_LSB(m->misc);
642 			m->addr >>= shift;
643 			m->addr <<= shift;
644 		}
645 
646 		smca_extract_err_addr(m);
647 	}
648 
649 	if (mce_flags.smca) {
650 		m->ipid = mce_rdmsrl(MSR_AMD64_SMCA_MCx_IPID(i));
651 
652 		if (m->status & MCI_STATUS_SYNDV)
653 			m->synd = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND(i));
654 	}
655 }
656 
657 DEFINE_PER_CPU(unsigned, mce_poll_count);
658 
659 /*
660  * Poll for corrected events or events that happened before reset.
661  * Those are just logged through /dev/mcelog.
662  *
663  * This is executed in standard interrupt context.
664  *
665  * Note: spec recommends to panic for fatal unsignalled
666  * errors here. However this would be quite problematic --
667  * we would need to reimplement the Monarch handling and
668  * it would mess up the exclusion between exception handler
669  * and poll handler -- * so we skip this for now.
670  * These cases should not happen anyways, or only when the CPU
671  * is already totally * confused. In this case it's likely it will
672  * not fully execute the machine check handler either.
673  */
machine_check_poll(enum mcp_flags flags,mce_banks_t * b)674 bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
675 {
676 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
677 	bool error_seen = false;
678 	struct mce m;
679 	int i;
680 
681 	this_cpu_inc(mce_poll_count);
682 
683 	mce_gather_info(&m, NULL);
684 
685 	if (flags & MCP_TIMESTAMP)
686 		m.tsc = rdtsc();
687 
688 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
689 		if (!mce_banks[i].ctl || !test_bit(i, *b))
690 			continue;
691 
692 		m.misc = 0;
693 		m.addr = 0;
694 		m.bank = i;
695 
696 		barrier();
697 		m.status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS));
698 
699 		/* If this entry is not valid, ignore it */
700 		if (!(m.status & MCI_STATUS_VAL))
701 			continue;
702 
703 		/*
704 		 * If we are logging everything (at CPU online) or this
705 		 * is a corrected error, then we must log it.
706 		 */
707 		if ((flags & MCP_UC) || !(m.status & MCI_STATUS_UC))
708 			goto log_it;
709 
710 		/*
711 		 * Newer Intel systems that support software error
712 		 * recovery need to make additional checks. Other
713 		 * CPUs should skip over uncorrected errors, but log
714 		 * everything else.
715 		 */
716 		if (!mca_cfg.ser) {
717 			if (m.status & MCI_STATUS_UC)
718 				continue;
719 			goto log_it;
720 		}
721 
722 		/* Log "not enabled" (speculative) errors */
723 		if (!(m.status & MCI_STATUS_EN))
724 			goto log_it;
725 
726 		/*
727 		 * Log UCNA (SDM: 15.6.3 "UCR Error Classification")
728 		 * UC == 1 && PCC == 0 && S == 0
729 		 */
730 		if (!(m.status & MCI_STATUS_PCC) && !(m.status & MCI_STATUS_S))
731 			goto log_it;
732 
733 		/*
734 		 * Skip anything else. Presumption is that our read of this
735 		 * bank is racing with a machine check. Leave the log alone
736 		 * for do_machine_check() to deal with it.
737 		 */
738 		continue;
739 
740 log_it:
741 		error_seen = true;
742 
743 		if (flags & MCP_DONTLOG)
744 			goto clear_it;
745 
746 		mce_read_aux(&m, i);
747 		m.severity = mce_severity(&m, NULL, NULL, false);
748 		/*
749 		 * Don't get the IP here because it's unlikely to
750 		 * have anything to do with the actual error location.
751 		 */
752 
753 		if (mca_cfg.dont_log_ce && !mce_usable_address(&m))
754 			goto clear_it;
755 
756 		if (flags & MCP_QUEUE_LOG)
757 			mce_gen_pool_add(&m);
758 		else
759 			mce_log(&m);
760 
761 clear_it:
762 		/*
763 		 * Clear state for this bank.
764 		 */
765 		mce_wrmsrl(mca_msr_reg(i, MCA_STATUS), 0);
766 	}
767 
768 	/*
769 	 * Don't clear MCG_STATUS here because it's only defined for
770 	 * exceptions.
771 	 */
772 
773 	sync_core();
774 
775 	return error_seen;
776 }
777 EXPORT_SYMBOL_GPL(machine_check_poll);
778 
779 /*
780  * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and
781  * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM
782  * Vol 3B Table 15-20). But this confuses both the code that determines
783  * whether the machine check occurred in kernel or user mode, and also
784  * the severity assessment code. Pretend that EIPV was set, and take the
785  * ip/cs values from the pt_regs that mce_gather_info() ignored earlier.
786  */
787 static __always_inline void
quirk_sandybridge_ifu(int bank,struct mce * m,struct pt_regs * regs)788 quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
789 {
790 	if (bank != 0)
791 		return;
792 	if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0)
793 		return;
794 	if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC|
795 		          MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV|
796 			  MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR|
797 			  MCACOD)) !=
798 			 (MCI_STATUS_UC|MCI_STATUS_EN|
799 			  MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S|
800 			  MCI_STATUS_AR|MCACOD_INSTR))
801 		return;
802 
803 	m->mcgstatus |= MCG_STATUS_EIPV;
804 	m->ip = regs->ip;
805 	m->cs = regs->cs;
806 }
807 
808 /*
809  * Disable fast string copy and return from the MCE handler upon the first SRAR
810  * MCE on bank 1 due to a CPU erratum on Intel Skylake/Cascade Lake/Cooper Lake
811  * CPUs.
812  * The fast string copy instructions ("REP; MOVS*") could consume an
813  * uncorrectable memory error in the cache line _right after_ the desired region
814  * to copy and raise an MCE with RIP pointing to the instruction _after_ the
815  * "REP; MOVS*".
816  * This mitigation addresses the issue completely with the caveat of performance
817  * degradation on the CPU affected. This is still better than the OS crashing on
818  * MCEs raised on an irrelevant process due to "REP; MOVS*" accesses from a
819  * kernel context (e.g., copy_page).
820  *
821  * Returns true when fast string copy on CPU has been disabled.
822  */
quirk_skylake_repmov(void)823 static noinstr bool quirk_skylake_repmov(void)
824 {
825 	u64 mcgstatus   = mce_rdmsrl(MSR_IA32_MCG_STATUS);
826 	u64 misc_enable = mce_rdmsrl(MSR_IA32_MISC_ENABLE);
827 	u64 mc1_status;
828 
829 	/*
830 	 * Apply the quirk only to local machine checks, i.e., no broadcast
831 	 * sync is needed.
832 	 */
833 	if (!(mcgstatus & MCG_STATUS_LMCES) ||
834 	    !(misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING))
835 		return false;
836 
837 	mc1_status = mce_rdmsrl(MSR_IA32_MCx_STATUS(1));
838 
839 	/* Check for a software-recoverable data fetch error. */
840 	if ((mc1_status &
841 	     (MCI_STATUS_VAL | MCI_STATUS_OVER | MCI_STATUS_UC | MCI_STATUS_EN |
842 	      MCI_STATUS_ADDRV | MCI_STATUS_MISCV | MCI_STATUS_PCC |
843 	      MCI_STATUS_AR | MCI_STATUS_S)) ==
844 	     (MCI_STATUS_VAL |                   MCI_STATUS_UC | MCI_STATUS_EN |
845 	      MCI_STATUS_ADDRV | MCI_STATUS_MISCV |
846 	      MCI_STATUS_AR | MCI_STATUS_S)) {
847 		misc_enable &= ~MSR_IA32_MISC_ENABLE_FAST_STRING;
848 		mce_wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
849 		mce_wrmsrl(MSR_IA32_MCx_STATUS(1), 0);
850 
851 		instrumentation_begin();
852 		pr_err_once("Erratum detected, disable fast string copy instructions.\n");
853 		instrumentation_end();
854 
855 		return true;
856 	}
857 
858 	return false;
859 }
860 
861 /*
862  * Some Zen-based Instruction Fetch Units set EIPV=RIPV=0 on poison consumption
863  * errors. This means mce_gather_info() will not save the "ip" and "cs" registers.
864  *
865  * However, the context is still valid, so save the "cs" register for later use.
866  *
867  * The "ip" register is truly unknown, so don't save it or fixup EIPV/RIPV.
868  *
869  * The Instruction Fetch Unit is at MCA bank 1 for all affected systems.
870  */
quirk_zen_ifu(int bank,struct mce * m,struct pt_regs * regs)871 static __always_inline void quirk_zen_ifu(int bank, struct mce *m, struct pt_regs *regs)
872 {
873 	if (bank != 1)
874 		return;
875 	if (!(m->status & MCI_STATUS_POISON))
876 		return;
877 
878 	m->cs = regs->cs;
879 }
880 
881 /*
882  * Do a quick check if any of the events requires a panic.
883  * This decides if we keep the events around or clear them.
884  */
mce_no_way_out(struct mce * m,char ** msg,unsigned long * validp,struct pt_regs * regs)885 static __always_inline int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
886 					  struct pt_regs *regs)
887 {
888 	char *tmp = *msg;
889 	int i;
890 
891 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
892 		m->status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS));
893 		if (!(m->status & MCI_STATUS_VAL))
894 			continue;
895 
896 		arch___set_bit(i, validp);
897 		if (mce_flags.snb_ifu_quirk)
898 			quirk_sandybridge_ifu(i, m, regs);
899 
900 		if (mce_flags.zen_ifu_quirk)
901 			quirk_zen_ifu(i, m, regs);
902 
903 		m->bank = i;
904 		if (mce_severity(m, regs, &tmp, true) >= MCE_PANIC_SEVERITY) {
905 			mce_read_aux(m, i);
906 			*msg = tmp;
907 			return 1;
908 		}
909 	}
910 	return 0;
911 }
912 
913 /*
914  * Variable to establish order between CPUs while scanning.
915  * Each CPU spins initially until executing is equal its number.
916  */
917 static atomic_t mce_executing;
918 
919 /*
920  * Defines order of CPUs on entry. First CPU becomes Monarch.
921  */
922 static atomic_t mce_callin;
923 
924 /*
925  * Track which CPUs entered the MCA broadcast synchronization and which not in
926  * order to print holdouts.
927  */
928 static cpumask_t mce_missing_cpus = CPU_MASK_ALL;
929 
930 /*
931  * Check if a timeout waiting for other CPUs happened.
932  */
mce_timed_out(u64 * t,const char * msg)933 static noinstr int mce_timed_out(u64 *t, const char *msg)
934 {
935 	int ret = 0;
936 
937 	/* Enable instrumentation around calls to external facilities */
938 	instrumentation_begin();
939 
940 	/*
941 	 * The others already did panic for some reason.
942 	 * Bail out like in a timeout.
943 	 * rmb() to tell the compiler that system_state
944 	 * might have been modified by someone else.
945 	 */
946 	rmb();
947 	if (atomic_read(&mce_panicked))
948 		wait_for_panic();
949 	if (!mca_cfg.monarch_timeout)
950 		goto out;
951 	if ((s64)*t < SPINUNIT) {
952 		if (cpumask_and(&mce_missing_cpus, cpu_online_mask, &mce_missing_cpus))
953 			pr_emerg("CPUs not responding to MCE broadcast (may include false positives): %*pbl\n",
954 				 cpumask_pr_args(&mce_missing_cpus));
955 		mce_panic(msg, NULL, NULL);
956 
957 		ret = 1;
958 		goto out;
959 	}
960 	*t -= SPINUNIT;
961 
962 out:
963 	touch_nmi_watchdog();
964 
965 	instrumentation_end();
966 
967 	return ret;
968 }
969 
970 /*
971  * The Monarch's reign.  The Monarch is the CPU who entered
972  * the machine check handler first. It waits for the others to
973  * raise the exception too and then grades them. When any
974  * error is fatal panic. Only then let the others continue.
975  *
976  * The other CPUs entering the MCE handler will be controlled by the
977  * Monarch. They are called Subjects.
978  *
979  * This way we prevent any potential data corruption in a unrecoverable case
980  * and also makes sure always all CPU's errors are examined.
981  *
982  * Also this detects the case of a machine check event coming from outer
983  * space (not detected by any CPUs) In this case some external agent wants
984  * us to shut down, so panic too.
985  *
986  * The other CPUs might still decide to panic if the handler happens
987  * in a unrecoverable place, but in this case the system is in a semi-stable
988  * state and won't corrupt anything by itself. It's ok to let the others
989  * continue for a bit first.
990  *
991  * All the spin loops have timeouts; when a timeout happens a CPU
992  * typically elects itself to be Monarch.
993  */
mce_reign(void)994 static void mce_reign(void)
995 {
996 	int cpu;
997 	struct mce *m = NULL;
998 	int global_worst = 0;
999 	char *msg = NULL;
1000 
1001 	/*
1002 	 * This CPU is the Monarch and the other CPUs have run
1003 	 * through their handlers.
1004 	 * Grade the severity of the errors of all the CPUs.
1005 	 */
1006 	for_each_possible_cpu(cpu) {
1007 		struct mce *mtmp = &per_cpu(mces_seen, cpu);
1008 
1009 		if (mtmp->severity > global_worst) {
1010 			global_worst = mtmp->severity;
1011 			m = &per_cpu(mces_seen, cpu);
1012 		}
1013 	}
1014 
1015 	/*
1016 	 * Cannot recover? Panic here then.
1017 	 * This dumps all the mces in the log buffer and stops the
1018 	 * other CPUs.
1019 	 */
1020 	if (m && global_worst >= MCE_PANIC_SEVERITY) {
1021 		/* call mce_severity() to get "msg" for panic */
1022 		mce_severity(m, NULL, &msg, true);
1023 		mce_panic("Fatal machine check", m, msg);
1024 	}
1025 
1026 	/*
1027 	 * For UC somewhere we let the CPU who detects it handle it.
1028 	 * Also must let continue the others, otherwise the handling
1029 	 * CPU could deadlock on a lock.
1030 	 */
1031 
1032 	/*
1033 	 * No machine check event found. Must be some external
1034 	 * source or one CPU is hung. Panic.
1035 	 */
1036 	if (global_worst <= MCE_KEEP_SEVERITY)
1037 		mce_panic("Fatal machine check from unknown source", NULL, NULL);
1038 
1039 	/*
1040 	 * Now clear all the mces_seen so that they don't reappear on
1041 	 * the next mce.
1042 	 */
1043 	for_each_possible_cpu(cpu)
1044 		memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
1045 }
1046 
1047 static atomic_t global_nwo;
1048 
1049 /*
1050  * Start of Monarch synchronization. This waits until all CPUs have
1051  * entered the exception handler and then determines if any of them
1052  * saw a fatal event that requires panic. Then it executes them
1053  * in the entry order.
1054  * TBD double check parallel CPU hotunplug
1055  */
mce_start(int * no_way_out)1056 static noinstr int mce_start(int *no_way_out)
1057 {
1058 	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
1059 	int order, ret = -1;
1060 
1061 	if (!timeout)
1062 		return ret;
1063 
1064 	raw_atomic_add(*no_way_out, &global_nwo);
1065 	/*
1066 	 * Rely on the implied barrier below, such that global_nwo
1067 	 * is updated before mce_callin.
1068 	 */
1069 	order = raw_atomic_inc_return(&mce_callin);
1070 	arch_cpumask_clear_cpu(smp_processor_id(), &mce_missing_cpus);
1071 
1072 	/* Enable instrumentation around calls to external facilities */
1073 	instrumentation_begin();
1074 
1075 	/*
1076 	 * Wait for everyone.
1077 	 */
1078 	while (raw_atomic_read(&mce_callin) != num_online_cpus()) {
1079 		if (mce_timed_out(&timeout,
1080 				  "Timeout: Not all CPUs entered broadcast exception handler")) {
1081 			raw_atomic_set(&global_nwo, 0);
1082 			goto out;
1083 		}
1084 		ndelay(SPINUNIT);
1085 	}
1086 
1087 	/*
1088 	 * mce_callin should be read before global_nwo
1089 	 */
1090 	smp_rmb();
1091 
1092 	if (order == 1) {
1093 		/*
1094 		 * Monarch: Starts executing now, the others wait.
1095 		 */
1096 		raw_atomic_set(&mce_executing, 1);
1097 	} else {
1098 		/*
1099 		 * Subject: Now start the scanning loop one by one in
1100 		 * the original callin order.
1101 		 * This way when there are any shared banks it will be
1102 		 * only seen by one CPU before cleared, avoiding duplicates.
1103 		 */
1104 		while (raw_atomic_read(&mce_executing) < order) {
1105 			if (mce_timed_out(&timeout,
1106 					  "Timeout: Subject CPUs unable to finish machine check processing")) {
1107 				raw_atomic_set(&global_nwo, 0);
1108 				goto out;
1109 			}
1110 			ndelay(SPINUNIT);
1111 		}
1112 	}
1113 
1114 	/*
1115 	 * Cache the global no_way_out state.
1116 	 */
1117 	*no_way_out = raw_atomic_read(&global_nwo);
1118 
1119 	ret = order;
1120 
1121 out:
1122 	instrumentation_end();
1123 
1124 	return ret;
1125 }
1126 
1127 /*
1128  * Synchronize between CPUs after main scanning loop.
1129  * This invokes the bulk of the Monarch processing.
1130  */
mce_end(int order)1131 static noinstr int mce_end(int order)
1132 {
1133 	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
1134 	int ret = -1;
1135 
1136 	/* Allow instrumentation around external facilities. */
1137 	instrumentation_begin();
1138 
1139 	if (!timeout)
1140 		goto reset;
1141 	if (order < 0)
1142 		goto reset;
1143 
1144 	/*
1145 	 * Allow others to run.
1146 	 */
1147 	atomic_inc(&mce_executing);
1148 
1149 	if (order == 1) {
1150 		/*
1151 		 * Monarch: Wait for everyone to go through their scanning
1152 		 * loops.
1153 		 */
1154 		while (atomic_read(&mce_executing) <= num_online_cpus()) {
1155 			if (mce_timed_out(&timeout,
1156 					  "Timeout: Monarch CPU unable to finish machine check processing"))
1157 				goto reset;
1158 			ndelay(SPINUNIT);
1159 		}
1160 
1161 		mce_reign();
1162 		barrier();
1163 		ret = 0;
1164 	} else {
1165 		/*
1166 		 * Subject: Wait for Monarch to finish.
1167 		 */
1168 		while (atomic_read(&mce_executing) != 0) {
1169 			if (mce_timed_out(&timeout,
1170 					  "Timeout: Monarch CPU did not finish machine check processing"))
1171 				goto reset;
1172 			ndelay(SPINUNIT);
1173 		}
1174 
1175 		/*
1176 		 * Don't reset anything. That's done by the Monarch.
1177 		 */
1178 		ret = 0;
1179 		goto out;
1180 	}
1181 
1182 	/*
1183 	 * Reset all global state.
1184 	 */
1185 reset:
1186 	atomic_set(&global_nwo, 0);
1187 	atomic_set(&mce_callin, 0);
1188 	cpumask_setall(&mce_missing_cpus);
1189 	barrier();
1190 
1191 	/*
1192 	 * Let others run again.
1193 	 */
1194 	atomic_set(&mce_executing, 0);
1195 
1196 out:
1197 	instrumentation_end();
1198 
1199 	return ret;
1200 }
1201 
mce_clear_state(unsigned long * toclear)1202 static __always_inline void mce_clear_state(unsigned long *toclear)
1203 {
1204 	int i;
1205 
1206 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1207 		if (arch_test_bit(i, toclear))
1208 			mce_wrmsrl(mca_msr_reg(i, MCA_STATUS), 0);
1209 	}
1210 }
1211 
1212 /*
1213  * Cases where we avoid rendezvous handler timeout:
1214  * 1) If this CPU is offline.
1215  *
1216  * 2) If crashing_cpu was set, e.g. we're entering kdump and we need to
1217  *  skip those CPUs which remain looping in the 1st kernel - see
1218  *  crash_nmi_callback().
1219  *
1220  * Note: there still is a small window between kexec-ing and the new,
1221  * kdump kernel establishing a new #MC handler where a broadcasted MCE
1222  * might not get handled properly.
1223  */
mce_check_crashing_cpu(void)1224 static noinstr bool mce_check_crashing_cpu(void)
1225 {
1226 	unsigned int cpu = smp_processor_id();
1227 
1228 	if (arch_cpu_is_offline(cpu) ||
1229 	    (crashing_cpu != -1 && crashing_cpu != cpu)) {
1230 		u64 mcgstatus;
1231 
1232 		mcgstatus = __rdmsr(MSR_IA32_MCG_STATUS);
1233 
1234 		if (boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) {
1235 			if (mcgstatus & MCG_STATUS_LMCES)
1236 				return false;
1237 		}
1238 
1239 		if (mcgstatus & MCG_STATUS_RIPV) {
1240 			__wrmsr(MSR_IA32_MCG_STATUS, 0, 0);
1241 			return true;
1242 		}
1243 	}
1244 	return false;
1245 }
1246 
1247 static __always_inline int
__mc_scan_banks(struct mce * m,struct pt_regs * regs,struct mce * final,unsigned long * toclear,unsigned long * valid_banks,int no_way_out,int * worst)1248 __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *final,
1249 		unsigned long *toclear, unsigned long *valid_banks, int no_way_out,
1250 		int *worst)
1251 {
1252 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1253 	struct mca_config *cfg = &mca_cfg;
1254 	int severity, i, taint = 0;
1255 
1256 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1257 		arch___clear_bit(i, toclear);
1258 		if (!arch_test_bit(i, valid_banks))
1259 			continue;
1260 
1261 		if (!mce_banks[i].ctl)
1262 			continue;
1263 
1264 		m->misc = 0;
1265 		m->addr = 0;
1266 		m->bank = i;
1267 
1268 		m->status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS));
1269 		if (!(m->status & MCI_STATUS_VAL))
1270 			continue;
1271 
1272 		/*
1273 		 * Corrected or non-signaled errors are handled by
1274 		 * machine_check_poll(). Leave them alone, unless this panics.
1275 		 */
1276 		if (!(m->status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
1277 			!no_way_out)
1278 			continue;
1279 
1280 		/* Set taint even when machine check was not enabled. */
1281 		taint++;
1282 
1283 		severity = mce_severity(m, regs, NULL, true);
1284 
1285 		/*
1286 		 * When machine check was for corrected/deferred handler don't
1287 		 * touch, unless we're panicking.
1288 		 */
1289 		if ((severity == MCE_KEEP_SEVERITY ||
1290 		     severity == MCE_UCNA_SEVERITY) && !no_way_out)
1291 			continue;
1292 
1293 		arch___set_bit(i, toclear);
1294 
1295 		/* Machine check event was not enabled. Clear, but ignore. */
1296 		if (severity == MCE_NO_SEVERITY)
1297 			continue;
1298 
1299 		mce_read_aux(m, i);
1300 
1301 		/* assuming valid severity level != 0 */
1302 		m->severity = severity;
1303 
1304 		/*
1305 		 * Enable instrumentation around the mce_log() call which is
1306 		 * done in #MC context, where instrumentation is disabled.
1307 		 */
1308 		instrumentation_begin();
1309 		mce_log(m);
1310 		instrumentation_end();
1311 
1312 		if (severity > *worst) {
1313 			*final = *m;
1314 			*worst = severity;
1315 		}
1316 	}
1317 
1318 	/* mce_clear_state will clear *final, save locally for use later */
1319 	*m = *final;
1320 
1321 	return taint;
1322 }
1323 
kill_me_now(struct callback_head * ch)1324 static void kill_me_now(struct callback_head *ch)
1325 {
1326 	struct task_struct *p = container_of(ch, struct task_struct, mce_kill_me);
1327 
1328 	p->mce_count = 0;
1329 	force_sig(SIGBUS);
1330 }
1331 
kill_me_maybe(struct callback_head * cb)1332 static void kill_me_maybe(struct callback_head *cb)
1333 {
1334 	struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me);
1335 	int flags = MF_ACTION_REQUIRED;
1336 	unsigned long pfn;
1337 	int ret;
1338 
1339 	p->mce_count = 0;
1340 	pr_err("Uncorrected hardware memory error in user-access at %llx", p->mce_addr);
1341 
1342 	if (!p->mce_ripv)
1343 		flags |= MF_MUST_KILL;
1344 
1345 	pfn = (p->mce_addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT;
1346 	ret = memory_failure(pfn, flags);
1347 	if (!ret) {
1348 		set_mce_nospec(pfn);
1349 		sync_core();
1350 		return;
1351 	}
1352 
1353 	/*
1354 	 * -EHWPOISON from memory_failure() means that it already sent SIGBUS
1355 	 * to the current process with the proper error info,
1356 	 * -EOPNOTSUPP means hwpoison_filter() filtered the error event,
1357 	 *
1358 	 * In both cases, no further processing is required.
1359 	 */
1360 	if (ret == -EHWPOISON || ret == -EOPNOTSUPP)
1361 		return;
1362 
1363 	pr_err("Memory error not recovered");
1364 	kill_me_now(cb);
1365 }
1366 
kill_me_never(struct callback_head * cb)1367 static void kill_me_never(struct callback_head *cb)
1368 {
1369 	struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me);
1370 	unsigned long pfn;
1371 
1372 	p->mce_count = 0;
1373 	pr_err("Kernel accessed poison in user space at %llx\n", p->mce_addr);
1374 	pfn = (p->mce_addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT;
1375 	if (!memory_failure(pfn, 0))
1376 		set_mce_nospec(pfn);
1377 }
1378 
queue_task_work(struct mce * m,char * msg,void (* func)(struct callback_head *))1379 static void queue_task_work(struct mce *m, char *msg, void (*func)(struct callback_head *))
1380 {
1381 	int count = ++current->mce_count;
1382 
1383 	/* First call, save all the details */
1384 	if (count == 1) {
1385 		current->mce_addr = m->addr;
1386 		current->mce_kflags = m->kflags;
1387 		current->mce_ripv = !!(m->mcgstatus & MCG_STATUS_RIPV);
1388 		current->mce_whole_page = whole_page(m);
1389 		current->mce_kill_me.func = func;
1390 	}
1391 
1392 	/* Ten is likely overkill. Don't expect more than two faults before task_work() */
1393 	if (count > 10)
1394 		mce_panic("Too many consecutive machine checks while accessing user data", m, msg);
1395 
1396 	/* Second or later call, make sure page address matches the one from first call */
1397 	if (count > 1 && (current->mce_addr >> PAGE_SHIFT) != (m->addr >> PAGE_SHIFT))
1398 		mce_panic("Consecutive machine checks to different user pages", m, msg);
1399 
1400 	/* Do not call task_work_add() more than once */
1401 	if (count > 1)
1402 		return;
1403 
1404 	task_work_add(current, &current->mce_kill_me, TWA_RESUME);
1405 }
1406 
1407 /* Handle unconfigured int18 (should never happen) */
unexpected_machine_check(struct pt_regs * regs)1408 static noinstr void unexpected_machine_check(struct pt_regs *regs)
1409 {
1410 	instrumentation_begin();
1411 	pr_err("CPU#%d: Unexpected int18 (Machine Check)\n",
1412 	       smp_processor_id());
1413 	instrumentation_end();
1414 }
1415 
1416 /*
1417  * The actual machine check handler. This only handles real exceptions when
1418  * something got corrupted coming in through int 18.
1419  *
1420  * This is executed in #MC context not subject to normal locking rules.
1421  * This implies that most kernel services cannot be safely used. Don't even
1422  * think about putting a printk in there!
1423  *
1424  * On Intel systems this is entered on all CPUs in parallel through
1425  * MCE broadcast. However some CPUs might be broken beyond repair,
1426  * so be always careful when synchronizing with others.
1427  *
1428  * Tracing and kprobes are disabled: if we interrupted a kernel context
1429  * with IF=1, we need to minimize stack usage.  There are also recursion
1430  * issues: if the machine check was due to a failure of the memory
1431  * backing the user stack, tracing that reads the user stack will cause
1432  * potentially infinite recursion.
1433  *
1434  * Currently, the #MC handler calls out to a number of external facilities
1435  * and, therefore, allows instrumentation around them. The optimal thing to
1436  * have would be to do the absolutely minimal work required in #MC context
1437  * and have instrumentation disabled only around that. Further processing can
1438  * then happen in process context where instrumentation is allowed. Achieving
1439  * that requires careful auditing and modifications. Until then, the code
1440  * allows instrumentation temporarily, where required. *
1441  */
do_machine_check(struct pt_regs * regs)1442 noinstr void do_machine_check(struct pt_regs *regs)
1443 {
1444 	int worst = 0, order, no_way_out, kill_current_task, lmce, taint = 0;
1445 	DECLARE_BITMAP(valid_banks, MAX_NR_BANKS) = { 0 };
1446 	DECLARE_BITMAP(toclear, MAX_NR_BANKS) = { 0 };
1447 	struct mce m, *final;
1448 	char *msg = NULL;
1449 
1450 	if (unlikely(mce_flags.p5))
1451 		return pentium_machine_check(regs);
1452 	else if (unlikely(mce_flags.winchip))
1453 		return winchip_machine_check(regs);
1454 	else if (unlikely(!mca_cfg.initialized))
1455 		return unexpected_machine_check(regs);
1456 
1457 	if (mce_flags.skx_repmov_quirk && quirk_skylake_repmov())
1458 		goto clear;
1459 
1460 	/*
1461 	 * Establish sequential order between the CPUs entering the machine
1462 	 * check handler.
1463 	 */
1464 	order = -1;
1465 
1466 	/*
1467 	 * If no_way_out gets set, there is no safe way to recover from this
1468 	 * MCE.
1469 	 */
1470 	no_way_out = 0;
1471 
1472 	/*
1473 	 * If kill_current_task is not set, there might be a way to recover from this
1474 	 * error.
1475 	 */
1476 	kill_current_task = 0;
1477 
1478 	/*
1479 	 * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES
1480 	 * on Intel.
1481 	 */
1482 	lmce = 1;
1483 
1484 	this_cpu_inc(mce_exception_count);
1485 
1486 	mce_gather_info(&m, regs);
1487 	m.tsc = rdtsc();
1488 
1489 	final = this_cpu_ptr(&mces_seen);
1490 	*final = m;
1491 
1492 	no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs);
1493 
1494 	barrier();
1495 
1496 	/*
1497 	 * When no restart IP might need to kill or panic.
1498 	 * Assume the worst for now, but if we find the
1499 	 * severity is MCE_AR_SEVERITY we have other options.
1500 	 */
1501 	if (!(m.mcgstatus & MCG_STATUS_RIPV))
1502 		kill_current_task = 1;
1503 	/*
1504 	 * Check if this MCE is signaled to only this logical processor,
1505 	 * on Intel, Zhaoxin only.
1506 	 */
1507 	if (m.cpuvendor == X86_VENDOR_INTEL ||
1508 	    m.cpuvendor == X86_VENDOR_ZHAOXIN)
1509 		lmce = m.mcgstatus & MCG_STATUS_LMCES;
1510 
1511 	/*
1512 	 * Local machine check may already know that we have to panic.
1513 	 * Broadcast machine check begins rendezvous in mce_start()
1514 	 * Go through all banks in exclusion of the other CPUs. This way we
1515 	 * don't report duplicated events on shared banks because the first one
1516 	 * to see it will clear it.
1517 	 */
1518 	if (lmce) {
1519 		if (no_way_out)
1520 			mce_panic("Fatal local machine check", &m, msg);
1521 	} else {
1522 		order = mce_start(&no_way_out);
1523 	}
1524 
1525 	taint = __mc_scan_banks(&m, regs, final, toclear, valid_banks, no_way_out, &worst);
1526 
1527 	if (!no_way_out)
1528 		mce_clear_state(toclear);
1529 
1530 	/*
1531 	 * Do most of the synchronization with other CPUs.
1532 	 * When there's any problem use only local no_way_out state.
1533 	 */
1534 	if (!lmce) {
1535 		if (mce_end(order) < 0) {
1536 			if (!no_way_out)
1537 				no_way_out = worst >= MCE_PANIC_SEVERITY;
1538 
1539 			if (no_way_out)
1540 				mce_panic("Fatal machine check on current CPU", &m, msg);
1541 		}
1542 	} else {
1543 		/*
1544 		 * If there was a fatal machine check we should have
1545 		 * already called mce_panic earlier in this function.
1546 		 * Since we re-read the banks, we might have found
1547 		 * something new. Check again to see if we found a
1548 		 * fatal error. We call "mce_severity()" again to
1549 		 * make sure we have the right "msg".
1550 		 */
1551 		if (worst >= MCE_PANIC_SEVERITY) {
1552 			mce_severity(&m, regs, &msg, true);
1553 			mce_panic("Local fatal machine check!", &m, msg);
1554 		}
1555 	}
1556 
1557 	/*
1558 	 * Enable instrumentation around the external facilities like task_work_add()
1559 	 * (via queue_task_work()), fixup_exception() etc. For now, that is. Fixing this
1560 	 * properly would need a lot more involved reorganization.
1561 	 */
1562 	instrumentation_begin();
1563 
1564 	if (taint)
1565 		add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
1566 
1567 	if (worst != MCE_AR_SEVERITY && !kill_current_task)
1568 		goto out;
1569 
1570 	/* Fault was in user mode and we need to take some action */
1571 	if ((m.cs & 3) == 3) {
1572 		/* If this triggers there is no way to recover. Die hard. */
1573 		BUG_ON(!on_thread_stack() || !user_mode(regs));
1574 
1575 		if (!mce_usable_address(&m))
1576 			queue_task_work(&m, msg, kill_me_now);
1577 		else
1578 			queue_task_work(&m, msg, kill_me_maybe);
1579 
1580 	} else {
1581 		/*
1582 		 * Handle an MCE which has happened in kernel space but from
1583 		 * which the kernel can recover: ex_has_fault_handler() has
1584 		 * already verified that the rIP at which the error happened is
1585 		 * a rIP from which the kernel can recover (by jumping to
1586 		 * recovery code specified in _ASM_EXTABLE_FAULT()) and the
1587 		 * corresponding exception handler which would do that is the
1588 		 * proper one.
1589 		 */
1590 		if (m.kflags & MCE_IN_KERNEL_RECOV) {
1591 			if (!fixup_exception(regs, X86_TRAP_MC, 0, 0))
1592 				mce_panic("Failed kernel mode recovery", &m, msg);
1593 		}
1594 
1595 		if (m.kflags & MCE_IN_KERNEL_COPYIN)
1596 			queue_task_work(&m, msg, kill_me_never);
1597 	}
1598 
1599 out:
1600 	instrumentation_end();
1601 
1602 clear:
1603 	mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1604 }
1605 EXPORT_SYMBOL_GPL(do_machine_check);
1606 
1607 #ifndef CONFIG_MEMORY_FAILURE
memory_failure(unsigned long pfn,int flags)1608 int memory_failure(unsigned long pfn, int flags)
1609 {
1610 	/* mce_severity() should not hand us an ACTION_REQUIRED error */
1611 	BUG_ON(flags & MF_ACTION_REQUIRED);
1612 	pr_err("Uncorrected memory error in page 0x%lx ignored\n"
1613 	       "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n",
1614 	       pfn);
1615 
1616 	return 0;
1617 }
1618 #endif
1619 
1620 /*
1621  * Periodic polling timer for "silent" machine check errors.  If the
1622  * poller finds an MCE, poll 2x faster.  When the poller finds no more
1623  * errors, poll 2x slower (up to check_interval seconds).
1624  */
1625 static unsigned long check_interval = INITIAL_CHECK_INTERVAL;
1626 
1627 static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */
1628 static DEFINE_PER_CPU(struct timer_list, mce_timer);
1629 
mce_adjust_timer_default(unsigned long interval)1630 static unsigned long mce_adjust_timer_default(unsigned long interval)
1631 {
1632 	return interval;
1633 }
1634 
1635 static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default;
1636 
__start_timer(struct timer_list * t,unsigned long interval)1637 static void __start_timer(struct timer_list *t, unsigned long interval)
1638 {
1639 	unsigned long when = jiffies + interval;
1640 	unsigned long flags;
1641 
1642 	local_irq_save(flags);
1643 
1644 	if (!timer_pending(t) || time_before(when, t->expires))
1645 		mod_timer(t, round_jiffies(when));
1646 
1647 	local_irq_restore(flags);
1648 }
1649 
mc_poll_banks_default(void)1650 static void mc_poll_banks_default(void)
1651 {
1652 	machine_check_poll(0, this_cpu_ptr(&mce_poll_banks));
1653 }
1654 
1655 void (*mc_poll_banks)(void) = mc_poll_banks_default;
1656 
mce_timer_fn(struct timer_list * t)1657 static void mce_timer_fn(struct timer_list *t)
1658 {
1659 	struct timer_list *cpu_t = this_cpu_ptr(&mce_timer);
1660 	unsigned long iv;
1661 
1662 	WARN_ON(cpu_t != t);
1663 
1664 	iv = __this_cpu_read(mce_next_interval);
1665 
1666 	if (mce_available(this_cpu_ptr(&cpu_info))) {
1667 		mc_poll_banks();
1668 
1669 		if (mce_intel_cmci_poll()) {
1670 			iv = mce_adjust_timer(iv);
1671 			goto done;
1672 		}
1673 	}
1674 
1675 	/*
1676 	 * Alert userspace if needed. If we logged an MCE, reduce the polling
1677 	 * interval, otherwise increase the polling interval.
1678 	 */
1679 	if (mce_notify_irq())
1680 		iv = max(iv / 2, (unsigned long) HZ/100);
1681 	else
1682 		iv = min(iv * 2, round_jiffies_relative(check_interval * HZ));
1683 
1684 done:
1685 	__this_cpu_write(mce_next_interval, iv);
1686 	__start_timer(t, iv);
1687 }
1688 
1689 /*
1690  * Ensure that the timer is firing in @interval from now.
1691  */
mce_timer_kick(unsigned long interval)1692 void mce_timer_kick(unsigned long interval)
1693 {
1694 	struct timer_list *t = this_cpu_ptr(&mce_timer);
1695 	unsigned long iv = __this_cpu_read(mce_next_interval);
1696 
1697 	__start_timer(t, interval);
1698 
1699 	if (interval < iv)
1700 		__this_cpu_write(mce_next_interval, interval);
1701 }
1702 
1703 /* Must not be called in IRQ context where del_timer_sync() can deadlock */
mce_timer_delete_all(void)1704 static void mce_timer_delete_all(void)
1705 {
1706 	int cpu;
1707 
1708 	for_each_online_cpu(cpu)
1709 		del_timer_sync(&per_cpu(mce_timer, cpu));
1710 }
1711 
1712 /*
1713  * Notify the user(s) about new machine check events.
1714  * Can be called from interrupt context, but not from machine check/NMI
1715  * context.
1716  */
mce_notify_irq(void)1717 int mce_notify_irq(void)
1718 {
1719 	/* Not more than two messages every minute */
1720 	static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);
1721 
1722 	if (test_and_clear_bit(0, &mce_need_notify)) {
1723 		mce_work_trigger();
1724 
1725 		if (__ratelimit(&ratelimit))
1726 			pr_info(HW_ERR "Machine check events logged\n");
1727 
1728 		return 1;
1729 	}
1730 	return 0;
1731 }
1732 EXPORT_SYMBOL_GPL(mce_notify_irq);
1733 
__mcheck_cpu_mce_banks_init(void)1734 static void __mcheck_cpu_mce_banks_init(void)
1735 {
1736 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1737 	u8 n_banks = this_cpu_read(mce_num_banks);
1738 	int i;
1739 
1740 	for (i = 0; i < n_banks; i++) {
1741 		struct mce_bank *b = &mce_banks[i];
1742 
1743 		/*
1744 		 * Init them all, __mcheck_cpu_apply_quirks() is going to apply
1745 		 * the required vendor quirks before
1746 		 * __mcheck_cpu_init_clear_banks() does the final bank setup.
1747 		 */
1748 		b->ctl = -1ULL;
1749 		b->init = true;
1750 	}
1751 }
1752 
1753 /*
1754  * Initialize Machine Checks for a CPU.
1755  */
__mcheck_cpu_cap_init(void)1756 static void __mcheck_cpu_cap_init(void)
1757 {
1758 	u64 cap;
1759 	u8 b;
1760 
1761 	rdmsrl(MSR_IA32_MCG_CAP, cap);
1762 
1763 	b = cap & MCG_BANKCNT_MASK;
1764 
1765 	if (b > MAX_NR_BANKS) {
1766 		pr_warn("CPU%d: Using only %u machine check banks out of %u\n",
1767 			smp_processor_id(), MAX_NR_BANKS, b);
1768 		b = MAX_NR_BANKS;
1769 	}
1770 
1771 	this_cpu_write(mce_num_banks, b);
1772 
1773 	__mcheck_cpu_mce_banks_init();
1774 
1775 	/* Use accurate RIP reporting if available. */
1776 	if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1777 		mca_cfg.rip_msr = MSR_IA32_MCG_EIP;
1778 
1779 	if (cap & MCG_SER_P)
1780 		mca_cfg.ser = 1;
1781 }
1782 
__mcheck_cpu_init_generic(void)1783 static void __mcheck_cpu_init_generic(void)
1784 {
1785 	enum mcp_flags m_fl = 0;
1786 	mce_banks_t all_banks;
1787 	u64 cap;
1788 
1789 	if (!mca_cfg.bootlog)
1790 		m_fl = MCP_DONTLOG;
1791 
1792 	/*
1793 	 * Log the machine checks left over from the previous reset. Log them
1794 	 * only, do not start processing them. That will happen in mcheck_late_init()
1795 	 * when all consumers have been registered on the notifier chain.
1796 	 */
1797 	bitmap_fill(all_banks, MAX_NR_BANKS);
1798 	machine_check_poll(MCP_UC | MCP_QUEUE_LOG | m_fl, &all_banks);
1799 
1800 	cr4_set_bits(X86_CR4_MCE);
1801 
1802 	rdmsrl(MSR_IA32_MCG_CAP, cap);
1803 	if (cap & MCG_CTL_P)
1804 		wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
1805 }
1806 
__mcheck_cpu_init_clear_banks(void)1807 static void __mcheck_cpu_init_clear_banks(void)
1808 {
1809 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1810 	int i;
1811 
1812 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1813 		struct mce_bank *b = &mce_banks[i];
1814 
1815 		if (!b->init)
1816 			continue;
1817 		wrmsrl(mca_msr_reg(i, MCA_CTL), b->ctl);
1818 		wrmsrl(mca_msr_reg(i, MCA_STATUS), 0);
1819 	}
1820 }
1821 
1822 /*
1823  * Do a final check to see if there are any unused/RAZ banks.
1824  *
1825  * This must be done after the banks have been initialized and any quirks have
1826  * been applied.
1827  *
1828  * Do not call this from any user-initiated flows, e.g. CPU hotplug or sysfs.
1829  * Otherwise, a user who disables a bank will not be able to re-enable it
1830  * without a system reboot.
1831  */
__mcheck_cpu_check_banks(void)1832 static void __mcheck_cpu_check_banks(void)
1833 {
1834 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1835 	u64 msrval;
1836 	int i;
1837 
1838 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1839 		struct mce_bank *b = &mce_banks[i];
1840 
1841 		if (!b->init)
1842 			continue;
1843 
1844 		rdmsrl(mca_msr_reg(i, MCA_CTL), msrval);
1845 		b->init = !!msrval;
1846 	}
1847 }
1848 
1849 /* Add per CPU specific workarounds here */
__mcheck_cpu_apply_quirks(struct cpuinfo_x86 * c)1850 static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1851 {
1852 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1853 	struct mca_config *cfg = &mca_cfg;
1854 
1855 	if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
1856 		pr_info("unknown CPU type - not enabling MCE support\n");
1857 		return -EOPNOTSUPP;
1858 	}
1859 
1860 	/* This should be disabled by the BIOS, but isn't always */
1861 	if (c->x86_vendor == X86_VENDOR_AMD) {
1862 		if (c->x86 == 15 && this_cpu_read(mce_num_banks) > 4) {
1863 			/*
1864 			 * disable GART TBL walk error reporting, which
1865 			 * trips off incorrectly with the IOMMU & 3ware
1866 			 * & Cerberus:
1867 			 */
1868 			clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
1869 		}
1870 		if (c->x86 < 0x11 && cfg->bootlog < 0) {
1871 			/*
1872 			 * Lots of broken BIOS around that don't clear them
1873 			 * by default and leave crap in there. Don't log:
1874 			 */
1875 			cfg->bootlog = 0;
1876 		}
1877 		/*
1878 		 * Various K7s with broken bank 0 around. Always disable
1879 		 * by default.
1880 		 */
1881 		if (c->x86 == 6 && this_cpu_read(mce_num_banks) > 0)
1882 			mce_banks[0].ctl = 0;
1883 
1884 		/*
1885 		 * overflow_recov is supported for F15h Models 00h-0fh
1886 		 * even though we don't have a CPUID bit for it.
1887 		 */
1888 		if (c->x86 == 0x15 && c->x86_model <= 0xf)
1889 			mce_flags.overflow_recov = 1;
1890 
1891 		if (c->x86 >= 0x17 && c->x86 <= 0x1A)
1892 			mce_flags.zen_ifu_quirk = 1;
1893 
1894 	}
1895 
1896 	if (c->x86_vendor == X86_VENDOR_INTEL) {
1897 		/*
1898 		 * SDM documents that on family 6 bank 0 should not be written
1899 		 * because it aliases to another special BIOS controlled
1900 		 * register.
1901 		 * But it's not aliased anymore on model 0x1a+
1902 		 * Don't ignore bank 0 completely because there could be a
1903 		 * valid event later, merely don't write CTL0.
1904 		 */
1905 
1906 		if (c->x86 == 6 && c->x86_model < 0x1A && this_cpu_read(mce_num_banks) > 0)
1907 			mce_banks[0].init = false;
1908 
1909 		/*
1910 		 * All newer Intel systems support MCE broadcasting. Enable
1911 		 * synchronization with a one second timeout.
1912 		 */
1913 		if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
1914 			cfg->monarch_timeout < 0)
1915 			cfg->monarch_timeout = USEC_PER_SEC;
1916 
1917 		/*
1918 		 * There are also broken BIOSes on some Pentium M and
1919 		 * earlier systems:
1920 		 */
1921 		if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0)
1922 			cfg->bootlog = 0;
1923 
1924 		if (c->x86 == 6 && c->x86_model == 45)
1925 			mce_flags.snb_ifu_quirk = 1;
1926 
1927 		/*
1928 		 * Skylake, Cascacde Lake and Cooper Lake require a quirk on
1929 		 * rep movs.
1930 		 */
1931 		if (c->x86 == 6 && c->x86_model == INTEL_FAM6_SKYLAKE_X)
1932 			mce_flags.skx_repmov_quirk = 1;
1933 	}
1934 
1935 	if (c->x86_vendor == X86_VENDOR_ZHAOXIN) {
1936 		/*
1937 		 * All newer Zhaoxin CPUs support MCE broadcasting. Enable
1938 		 * synchronization with a one second timeout.
1939 		 */
1940 		if (c->x86 > 6 || (c->x86_model == 0x19 || c->x86_model == 0x1f)) {
1941 			if (cfg->monarch_timeout < 0)
1942 				cfg->monarch_timeout = USEC_PER_SEC;
1943 		}
1944 	}
1945 
1946 	if (cfg->monarch_timeout < 0)
1947 		cfg->monarch_timeout = 0;
1948 	if (cfg->bootlog != 0)
1949 		cfg->panic_timeout = 30;
1950 
1951 	return 0;
1952 }
1953 
__mcheck_cpu_ancient_init(struct cpuinfo_x86 * c)1954 static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
1955 {
1956 	if (c->x86 != 5)
1957 		return 0;
1958 
1959 	switch (c->x86_vendor) {
1960 	case X86_VENDOR_INTEL:
1961 		intel_p5_mcheck_init(c);
1962 		mce_flags.p5 = 1;
1963 		return 1;
1964 	case X86_VENDOR_CENTAUR:
1965 		winchip_mcheck_init(c);
1966 		mce_flags.winchip = 1;
1967 		return 1;
1968 	default:
1969 		return 0;
1970 	}
1971 
1972 	return 0;
1973 }
1974 
1975 /*
1976  * Init basic CPU features needed for early decoding of MCEs.
1977  */
__mcheck_cpu_init_early(struct cpuinfo_x86 * c)1978 static void __mcheck_cpu_init_early(struct cpuinfo_x86 *c)
1979 {
1980 	if (c->x86_vendor == X86_VENDOR_AMD || c->x86_vendor == X86_VENDOR_HYGON) {
1981 		mce_flags.overflow_recov = !!cpu_has(c, X86_FEATURE_OVERFLOW_RECOV);
1982 		mce_flags.succor	 = !!cpu_has(c, X86_FEATURE_SUCCOR);
1983 		mce_flags.smca		 = !!cpu_has(c, X86_FEATURE_SMCA);
1984 		mce_flags.amd_threshold	 = 1;
1985 	}
1986 }
1987 
mce_centaur_feature_init(struct cpuinfo_x86 * c)1988 static void mce_centaur_feature_init(struct cpuinfo_x86 *c)
1989 {
1990 	struct mca_config *cfg = &mca_cfg;
1991 
1992 	 /*
1993 	  * All newer Centaur CPUs support MCE broadcasting. Enable
1994 	  * synchronization with a one second timeout.
1995 	  */
1996 	if ((c->x86 == 6 && c->x86_model == 0xf && c->x86_stepping >= 0xe) ||
1997 	     c->x86 > 6) {
1998 		if (cfg->monarch_timeout < 0)
1999 			cfg->monarch_timeout = USEC_PER_SEC;
2000 	}
2001 }
2002 
mce_zhaoxin_feature_init(struct cpuinfo_x86 * c)2003 static void mce_zhaoxin_feature_init(struct cpuinfo_x86 *c)
2004 {
2005 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
2006 
2007 	/*
2008 	 * These CPUs have MCA bank 8 which reports only one error type called
2009 	 * SVAD (System View Address Decoder). The reporting of that error is
2010 	 * controlled by IA32_MC8.CTL.0.
2011 	 *
2012 	 * If enabled, prefetching on these CPUs will cause SVAD MCE when
2013 	 * virtual machines start and result in a system  panic. Always disable
2014 	 * bank 8 SVAD error by default.
2015 	 */
2016 	if ((c->x86 == 7 && c->x86_model == 0x1b) ||
2017 	    (c->x86_model == 0x19 || c->x86_model == 0x1f)) {
2018 		if (this_cpu_read(mce_num_banks) > 8)
2019 			mce_banks[8].ctl = 0;
2020 	}
2021 
2022 	intel_init_cmci();
2023 	intel_init_lmce();
2024 	mce_adjust_timer = cmci_intel_adjust_timer;
2025 }
2026 
mce_zhaoxin_feature_clear(struct cpuinfo_x86 * c)2027 static void mce_zhaoxin_feature_clear(struct cpuinfo_x86 *c)
2028 {
2029 	intel_clear_lmce();
2030 }
2031 
__mcheck_cpu_init_vendor(struct cpuinfo_x86 * c)2032 static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
2033 {
2034 	switch (c->x86_vendor) {
2035 	case X86_VENDOR_INTEL:
2036 		mce_intel_feature_init(c);
2037 		mce_adjust_timer = cmci_intel_adjust_timer;
2038 		break;
2039 
2040 	case X86_VENDOR_AMD: {
2041 		mce_amd_feature_init(c);
2042 		break;
2043 		}
2044 
2045 	case X86_VENDOR_HYGON:
2046 		mce_hygon_feature_init(c);
2047 		break;
2048 
2049 	case X86_VENDOR_CENTAUR:
2050 		mce_centaur_feature_init(c);
2051 		break;
2052 
2053 	case X86_VENDOR_ZHAOXIN:
2054 		mce_zhaoxin_feature_init(c);
2055 		break;
2056 
2057 	default:
2058 		break;
2059 	}
2060 }
2061 
__mcheck_cpu_clear_vendor(struct cpuinfo_x86 * c)2062 static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c)
2063 {
2064 	switch (c->x86_vendor) {
2065 	case X86_VENDOR_INTEL:
2066 		mce_intel_feature_clear(c);
2067 		break;
2068 
2069 	case X86_VENDOR_ZHAOXIN:
2070 		mce_zhaoxin_feature_clear(c);
2071 		break;
2072 
2073 	default:
2074 		break;
2075 	}
2076 }
2077 
mce_start_timer(struct timer_list * t)2078 static void mce_start_timer(struct timer_list *t)
2079 {
2080 	unsigned long iv = check_interval * HZ;
2081 
2082 	if (mca_cfg.ignore_ce || !iv)
2083 		return;
2084 
2085 	this_cpu_write(mce_next_interval, iv);
2086 	__start_timer(t, iv);
2087 }
2088 
__mcheck_cpu_setup_timer(void)2089 static void __mcheck_cpu_setup_timer(void)
2090 {
2091 	struct timer_list *t = this_cpu_ptr(&mce_timer);
2092 
2093 	timer_setup(t, mce_timer_fn, TIMER_PINNED);
2094 }
2095 
__mcheck_cpu_init_timer(void)2096 static void __mcheck_cpu_init_timer(void)
2097 {
2098 	struct timer_list *t = this_cpu_ptr(&mce_timer);
2099 
2100 	timer_setup(t, mce_timer_fn, TIMER_PINNED);
2101 	mce_start_timer(t);
2102 }
2103 
filter_mce(struct mce * m)2104 bool filter_mce(struct mce *m)
2105 {
2106 	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
2107 		return amd_filter_mce(m);
2108 	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
2109 		return intel_filter_mce(m);
2110 
2111 	return false;
2112 }
2113 
exc_machine_check_kernel(struct pt_regs * regs)2114 static __always_inline void exc_machine_check_kernel(struct pt_regs *regs)
2115 {
2116 	irqentry_state_t irq_state;
2117 
2118 	WARN_ON_ONCE(user_mode(regs));
2119 
2120 	/*
2121 	 * Only required when from kernel mode. See
2122 	 * mce_check_crashing_cpu() for details.
2123 	 */
2124 	if (mca_cfg.initialized && mce_check_crashing_cpu())
2125 		return;
2126 
2127 	irq_state = irqentry_nmi_enter(regs);
2128 
2129 	do_machine_check(regs);
2130 
2131 	irqentry_nmi_exit(regs, irq_state);
2132 }
2133 
exc_machine_check_user(struct pt_regs * regs)2134 static __always_inline void exc_machine_check_user(struct pt_regs *regs)
2135 {
2136 	irqentry_enter_from_user_mode(regs);
2137 
2138 	do_machine_check(regs);
2139 
2140 	irqentry_exit_to_user_mode(regs);
2141 }
2142 
2143 #ifdef CONFIG_X86_64
2144 /* MCE hit kernel mode */
DEFINE_IDTENTRY_MCE(exc_machine_check)2145 DEFINE_IDTENTRY_MCE(exc_machine_check)
2146 {
2147 	unsigned long dr7;
2148 
2149 	dr7 = local_db_save();
2150 	exc_machine_check_kernel(regs);
2151 	local_db_restore(dr7);
2152 }
2153 
2154 /* The user mode variant. */
DEFINE_IDTENTRY_MCE_USER(exc_machine_check)2155 DEFINE_IDTENTRY_MCE_USER(exc_machine_check)
2156 {
2157 	unsigned long dr7;
2158 
2159 	dr7 = local_db_save();
2160 	exc_machine_check_user(regs);
2161 	local_db_restore(dr7);
2162 }
2163 #else
2164 /* 32bit unified entry point */
DEFINE_IDTENTRY_RAW(exc_machine_check)2165 DEFINE_IDTENTRY_RAW(exc_machine_check)
2166 {
2167 	unsigned long dr7;
2168 
2169 	dr7 = local_db_save();
2170 	if (user_mode(regs))
2171 		exc_machine_check_user(regs);
2172 	else
2173 		exc_machine_check_kernel(regs);
2174 	local_db_restore(dr7);
2175 }
2176 #endif
2177 
2178 /*
2179  * Called for each booted CPU to set up machine checks.
2180  * Must be called with preempt off:
2181  */
mcheck_cpu_init(struct cpuinfo_x86 * c)2182 void mcheck_cpu_init(struct cpuinfo_x86 *c)
2183 {
2184 	if (mca_cfg.disabled)
2185 		return;
2186 
2187 	if (__mcheck_cpu_ancient_init(c))
2188 		return;
2189 
2190 	if (!mce_available(c))
2191 		return;
2192 
2193 	__mcheck_cpu_cap_init();
2194 
2195 	if (__mcheck_cpu_apply_quirks(c) < 0) {
2196 		mca_cfg.disabled = 1;
2197 		return;
2198 	}
2199 
2200 	if (mce_gen_pool_init()) {
2201 		mca_cfg.disabled = 1;
2202 		pr_emerg("Couldn't allocate MCE records pool!\n");
2203 		return;
2204 	}
2205 
2206 	mca_cfg.initialized = 1;
2207 
2208 	__mcheck_cpu_init_early(c);
2209 	__mcheck_cpu_init_generic();
2210 	__mcheck_cpu_init_vendor(c);
2211 	__mcheck_cpu_init_clear_banks();
2212 	__mcheck_cpu_check_banks();
2213 	__mcheck_cpu_setup_timer();
2214 }
2215 
2216 /*
2217  * Called for each booted CPU to clear some machine checks opt-ins
2218  */
mcheck_cpu_clear(struct cpuinfo_x86 * c)2219 void mcheck_cpu_clear(struct cpuinfo_x86 *c)
2220 {
2221 	if (mca_cfg.disabled)
2222 		return;
2223 
2224 	if (!mce_available(c))
2225 		return;
2226 
2227 	/*
2228 	 * Possibly to clear general settings generic to x86
2229 	 * __mcheck_cpu_clear_generic(c);
2230 	 */
2231 	__mcheck_cpu_clear_vendor(c);
2232 
2233 }
2234 
__mce_disable_bank(void * arg)2235 static void __mce_disable_bank(void *arg)
2236 {
2237 	int bank = *((int *)arg);
2238 	__clear_bit(bank, this_cpu_ptr(mce_poll_banks));
2239 	cmci_disable_bank(bank);
2240 }
2241 
mce_disable_bank(int bank)2242 void mce_disable_bank(int bank)
2243 {
2244 	if (bank >= this_cpu_read(mce_num_banks)) {
2245 		pr_warn(FW_BUG
2246 			"Ignoring request to disable invalid MCA bank %d.\n",
2247 			bank);
2248 		return;
2249 	}
2250 	set_bit(bank, mce_banks_ce_disabled);
2251 	on_each_cpu(__mce_disable_bank, &bank, 1);
2252 }
2253 
2254 /*
2255  * mce=off Disables machine check
2256  * mce=no_cmci Disables CMCI
2257  * mce=no_lmce Disables LMCE
2258  * mce=dont_log_ce Clears corrected events silently, no log created for CEs.
2259  * mce=print_all Print all machine check logs to console
2260  * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared.
2261  * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above)
2262  *	monarchtimeout is how long to wait for other CPUs on machine
2263  *	check, or 0 to not wait
2264  * mce=bootlog Log MCEs from before booting. Disabled by default on AMD Fam10h
2265 	and older.
2266  * mce=nobootlog Don't log MCEs from before booting.
2267  * mce=bios_cmci_threshold Don't program the CMCI threshold
2268  * mce=recovery force enable copy_mc_fragile()
2269  */
mcheck_enable(char * str)2270 static int __init mcheck_enable(char *str)
2271 {
2272 	struct mca_config *cfg = &mca_cfg;
2273 
2274 	if (*str == 0) {
2275 		enable_p5_mce();
2276 		return 1;
2277 	}
2278 	if (*str == '=')
2279 		str++;
2280 	if (!strcmp(str, "off"))
2281 		cfg->disabled = 1;
2282 	else if (!strcmp(str, "no_cmci"))
2283 		cfg->cmci_disabled = true;
2284 	else if (!strcmp(str, "no_lmce"))
2285 		cfg->lmce_disabled = 1;
2286 	else if (!strcmp(str, "dont_log_ce"))
2287 		cfg->dont_log_ce = true;
2288 	else if (!strcmp(str, "print_all"))
2289 		cfg->print_all = true;
2290 	else if (!strcmp(str, "ignore_ce"))
2291 		cfg->ignore_ce = true;
2292 	else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
2293 		cfg->bootlog = (str[0] == 'b');
2294 	else if (!strcmp(str, "bios_cmci_threshold"))
2295 		cfg->bios_cmci_threshold = 1;
2296 	else if (!strcmp(str, "recovery"))
2297 		cfg->recovery = 1;
2298 	else if (isdigit(str[0]))
2299 		get_option(&str, &(cfg->monarch_timeout));
2300 	else {
2301 		pr_info("mce argument %s ignored. Please use /sys\n", str);
2302 		return 0;
2303 	}
2304 	return 1;
2305 }
2306 __setup("mce", mcheck_enable);
2307 
mcheck_init(void)2308 int __init mcheck_init(void)
2309 {
2310 	mce_register_decode_chain(&early_nb);
2311 	mce_register_decode_chain(&mce_uc_nb);
2312 	mce_register_decode_chain(&mce_default_nb);
2313 
2314 	INIT_WORK(&mce_work, mce_gen_pool_process);
2315 	init_irq_work(&mce_irq_work, mce_irq_work_cb);
2316 
2317 	return 0;
2318 }
2319 
2320 /*
2321  * mce_syscore: PM support
2322  */
2323 
2324 /*
2325  * Disable machine checks on suspend and shutdown. We can't really handle
2326  * them later.
2327  */
mce_disable_error_reporting(void)2328 static void mce_disable_error_reporting(void)
2329 {
2330 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
2331 	int i;
2332 
2333 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
2334 		struct mce_bank *b = &mce_banks[i];
2335 
2336 		if (b->init)
2337 			wrmsrl(mca_msr_reg(i, MCA_CTL), 0);
2338 	}
2339 	return;
2340 }
2341 
vendor_disable_error_reporting(void)2342 static void vendor_disable_error_reporting(void)
2343 {
2344 	/*
2345 	 * Don't clear on Intel or AMD or Hygon or Zhaoxin CPUs. Some of these
2346 	 * MSRs are socket-wide. Disabling them for just a single offlined CPU
2347 	 * is bad, since it will inhibit reporting for all shared resources on
2348 	 * the socket like the last level cache (LLC), the integrated memory
2349 	 * controller (iMC), etc.
2350 	 */
2351 	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL ||
2352 	    boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ||
2353 	    boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
2354 	    boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN)
2355 		return;
2356 
2357 	mce_disable_error_reporting();
2358 }
2359 
mce_syscore_suspend(void)2360 static int mce_syscore_suspend(void)
2361 {
2362 	vendor_disable_error_reporting();
2363 	return 0;
2364 }
2365 
mce_syscore_shutdown(void)2366 static void mce_syscore_shutdown(void)
2367 {
2368 	vendor_disable_error_reporting();
2369 }
2370 
2371 /*
2372  * On resume clear all MCE state. Don't want to see leftovers from the BIOS.
2373  * Only one CPU is active at this time, the others get re-added later using
2374  * CPU hotplug:
2375  */
mce_syscore_resume(void)2376 static void mce_syscore_resume(void)
2377 {
2378 	__mcheck_cpu_init_generic();
2379 	__mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info));
2380 	__mcheck_cpu_init_clear_banks();
2381 }
2382 
2383 static struct syscore_ops mce_syscore_ops = {
2384 	.suspend	= mce_syscore_suspend,
2385 	.shutdown	= mce_syscore_shutdown,
2386 	.resume		= mce_syscore_resume,
2387 };
2388 
2389 /*
2390  * mce_device: Sysfs support
2391  */
2392 
mce_cpu_restart(void * data)2393 static void mce_cpu_restart(void *data)
2394 {
2395 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2396 		return;
2397 	__mcheck_cpu_init_generic();
2398 	__mcheck_cpu_init_clear_banks();
2399 	__mcheck_cpu_init_timer();
2400 }
2401 
2402 /* Reinit MCEs after user configuration changes */
mce_restart(void)2403 static void mce_restart(void)
2404 {
2405 	mce_timer_delete_all();
2406 	on_each_cpu(mce_cpu_restart, NULL, 1);
2407 	mce_schedule_work();
2408 }
2409 
2410 /* Toggle features for corrected errors */
mce_disable_cmci(void * data)2411 static void mce_disable_cmci(void *data)
2412 {
2413 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2414 		return;
2415 	cmci_clear();
2416 }
2417 
mce_enable_ce(void * all)2418 static void mce_enable_ce(void *all)
2419 {
2420 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2421 		return;
2422 	cmci_reenable();
2423 	cmci_recheck();
2424 	if (all)
2425 		__mcheck_cpu_init_timer();
2426 }
2427 
2428 static struct bus_type mce_subsys = {
2429 	.name		= "machinecheck",
2430 	.dev_name	= "machinecheck",
2431 };
2432 
2433 DEFINE_PER_CPU(struct device *, mce_device);
2434 
attr_to_bank(struct device_attribute * attr)2435 static inline struct mce_bank_dev *attr_to_bank(struct device_attribute *attr)
2436 {
2437 	return container_of(attr, struct mce_bank_dev, attr);
2438 }
2439 
show_bank(struct device * s,struct device_attribute * attr,char * buf)2440 static ssize_t show_bank(struct device *s, struct device_attribute *attr,
2441 			 char *buf)
2442 {
2443 	u8 bank = attr_to_bank(attr)->bank;
2444 	struct mce_bank *b;
2445 
2446 	if (bank >= per_cpu(mce_num_banks, s->id))
2447 		return -EINVAL;
2448 
2449 	b = &per_cpu(mce_banks_array, s->id)[bank];
2450 
2451 	if (!b->init)
2452 		return -ENODEV;
2453 
2454 	return sprintf(buf, "%llx\n", b->ctl);
2455 }
2456 
set_bank(struct device * s,struct device_attribute * attr,const char * buf,size_t size)2457 static ssize_t set_bank(struct device *s, struct device_attribute *attr,
2458 			const char *buf, size_t size)
2459 {
2460 	u8 bank = attr_to_bank(attr)->bank;
2461 	struct mce_bank *b;
2462 	u64 new;
2463 
2464 	if (kstrtou64(buf, 0, &new) < 0)
2465 		return -EINVAL;
2466 
2467 	if (bank >= per_cpu(mce_num_banks, s->id))
2468 		return -EINVAL;
2469 
2470 	b = &per_cpu(mce_banks_array, s->id)[bank];
2471 
2472 	if (!b->init)
2473 		return -ENODEV;
2474 
2475 	b->ctl = new;
2476 	mce_restart();
2477 
2478 	return size;
2479 }
2480 
set_ignore_ce(struct device * s,struct device_attribute * attr,const char * buf,size_t size)2481 static ssize_t set_ignore_ce(struct device *s,
2482 			     struct device_attribute *attr,
2483 			     const char *buf, size_t size)
2484 {
2485 	u64 new;
2486 
2487 	if (kstrtou64(buf, 0, &new) < 0)
2488 		return -EINVAL;
2489 
2490 	mutex_lock(&mce_sysfs_mutex);
2491 	if (mca_cfg.ignore_ce ^ !!new) {
2492 		if (new) {
2493 			/* disable ce features */
2494 			mce_timer_delete_all();
2495 			on_each_cpu(mce_disable_cmci, NULL, 1);
2496 			mca_cfg.ignore_ce = true;
2497 		} else {
2498 			/* enable ce features */
2499 			mca_cfg.ignore_ce = false;
2500 			on_each_cpu(mce_enable_ce, (void *)1, 1);
2501 		}
2502 	}
2503 	mutex_unlock(&mce_sysfs_mutex);
2504 
2505 	return size;
2506 }
2507 
set_cmci_disabled(struct device * s,struct device_attribute * attr,const char * buf,size_t size)2508 static ssize_t set_cmci_disabled(struct device *s,
2509 				 struct device_attribute *attr,
2510 				 const char *buf, size_t size)
2511 {
2512 	u64 new;
2513 
2514 	if (kstrtou64(buf, 0, &new) < 0)
2515 		return -EINVAL;
2516 
2517 	mutex_lock(&mce_sysfs_mutex);
2518 	if (mca_cfg.cmci_disabled ^ !!new) {
2519 		if (new) {
2520 			/* disable cmci */
2521 			on_each_cpu(mce_disable_cmci, NULL, 1);
2522 			mca_cfg.cmci_disabled = true;
2523 		} else {
2524 			/* enable cmci */
2525 			mca_cfg.cmci_disabled = false;
2526 			on_each_cpu(mce_enable_ce, NULL, 1);
2527 		}
2528 	}
2529 	mutex_unlock(&mce_sysfs_mutex);
2530 
2531 	return size;
2532 }
2533 
store_int_with_restart(struct device * s,struct device_attribute * attr,const char * buf,size_t size)2534 static ssize_t store_int_with_restart(struct device *s,
2535 				      struct device_attribute *attr,
2536 				      const char *buf, size_t size)
2537 {
2538 	unsigned long old_check_interval = check_interval;
2539 	ssize_t ret = device_store_ulong(s, attr, buf, size);
2540 
2541 	if (check_interval == old_check_interval)
2542 		return ret;
2543 
2544 	mutex_lock(&mce_sysfs_mutex);
2545 	mce_restart();
2546 	mutex_unlock(&mce_sysfs_mutex);
2547 
2548 	return ret;
2549 }
2550 
2551 static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout);
2552 static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
2553 static DEVICE_BOOL_ATTR(print_all, 0644, mca_cfg.print_all);
2554 
2555 static struct dev_ext_attribute dev_attr_check_interval = {
2556 	__ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
2557 	&check_interval
2558 };
2559 
2560 static struct dev_ext_attribute dev_attr_ignore_ce = {
2561 	__ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
2562 	&mca_cfg.ignore_ce
2563 };
2564 
2565 static struct dev_ext_attribute dev_attr_cmci_disabled = {
2566 	__ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
2567 	&mca_cfg.cmci_disabled
2568 };
2569 
2570 static struct device_attribute *mce_device_attrs[] = {
2571 	&dev_attr_check_interval.attr,
2572 #ifdef CONFIG_X86_MCELOG_LEGACY
2573 	&dev_attr_trigger,
2574 #endif
2575 	&dev_attr_monarch_timeout.attr,
2576 	&dev_attr_dont_log_ce.attr,
2577 	&dev_attr_print_all.attr,
2578 	&dev_attr_ignore_ce.attr,
2579 	&dev_attr_cmci_disabled.attr,
2580 	NULL
2581 };
2582 
2583 static cpumask_var_t mce_device_initialized;
2584 
mce_device_release(struct device * dev)2585 static void mce_device_release(struct device *dev)
2586 {
2587 	kfree(dev);
2588 }
2589 
2590 /* Per CPU device init. All of the CPUs still share the same bank device: */
mce_device_create(unsigned int cpu)2591 static int mce_device_create(unsigned int cpu)
2592 {
2593 	struct device *dev;
2594 	int err;
2595 	int i, j;
2596 
2597 	if (!mce_available(&boot_cpu_data))
2598 		return -EIO;
2599 
2600 	dev = per_cpu(mce_device, cpu);
2601 	if (dev)
2602 		return 0;
2603 
2604 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2605 	if (!dev)
2606 		return -ENOMEM;
2607 	dev->id  = cpu;
2608 	dev->bus = &mce_subsys;
2609 	dev->release = &mce_device_release;
2610 
2611 	err = device_register(dev);
2612 	if (err) {
2613 		put_device(dev);
2614 		return err;
2615 	}
2616 
2617 	for (i = 0; mce_device_attrs[i]; i++) {
2618 		err = device_create_file(dev, mce_device_attrs[i]);
2619 		if (err)
2620 			goto error;
2621 	}
2622 	for (j = 0; j < per_cpu(mce_num_banks, cpu); j++) {
2623 		err = device_create_file(dev, &mce_bank_devs[j].attr);
2624 		if (err)
2625 			goto error2;
2626 	}
2627 	cpumask_set_cpu(cpu, mce_device_initialized);
2628 	per_cpu(mce_device, cpu) = dev;
2629 
2630 	return 0;
2631 error2:
2632 	while (--j >= 0)
2633 		device_remove_file(dev, &mce_bank_devs[j].attr);
2634 error:
2635 	while (--i >= 0)
2636 		device_remove_file(dev, mce_device_attrs[i]);
2637 
2638 	device_unregister(dev);
2639 
2640 	return err;
2641 }
2642 
mce_device_remove(unsigned int cpu)2643 static void mce_device_remove(unsigned int cpu)
2644 {
2645 	struct device *dev = per_cpu(mce_device, cpu);
2646 	int i;
2647 
2648 	if (!cpumask_test_cpu(cpu, mce_device_initialized))
2649 		return;
2650 
2651 	for (i = 0; mce_device_attrs[i]; i++)
2652 		device_remove_file(dev, mce_device_attrs[i]);
2653 
2654 	for (i = 0; i < per_cpu(mce_num_banks, cpu); i++)
2655 		device_remove_file(dev, &mce_bank_devs[i].attr);
2656 
2657 	device_unregister(dev);
2658 	cpumask_clear_cpu(cpu, mce_device_initialized);
2659 	per_cpu(mce_device, cpu) = NULL;
2660 }
2661 
2662 /* Make sure there are no machine checks on offlined CPUs. */
mce_disable_cpu(void)2663 static void mce_disable_cpu(void)
2664 {
2665 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2666 		return;
2667 
2668 	if (!cpuhp_tasks_frozen)
2669 		cmci_clear();
2670 
2671 	vendor_disable_error_reporting();
2672 }
2673 
mce_reenable_cpu(void)2674 static void mce_reenable_cpu(void)
2675 {
2676 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
2677 	int i;
2678 
2679 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2680 		return;
2681 
2682 	if (!cpuhp_tasks_frozen)
2683 		cmci_reenable();
2684 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
2685 		struct mce_bank *b = &mce_banks[i];
2686 
2687 		if (b->init)
2688 			wrmsrl(mca_msr_reg(i, MCA_CTL), b->ctl);
2689 	}
2690 }
2691 
mce_cpu_dead(unsigned int cpu)2692 static int mce_cpu_dead(unsigned int cpu)
2693 {
2694 	mce_intel_hcpu_update(cpu);
2695 
2696 	/* intentionally ignoring frozen here */
2697 	if (!cpuhp_tasks_frozen)
2698 		cmci_rediscover();
2699 	return 0;
2700 }
2701 
mce_cpu_online(unsigned int cpu)2702 static int mce_cpu_online(unsigned int cpu)
2703 {
2704 	struct timer_list *t = this_cpu_ptr(&mce_timer);
2705 	int ret;
2706 
2707 	mce_device_create(cpu);
2708 
2709 	ret = mce_threshold_create_device(cpu);
2710 	if (ret) {
2711 		mce_device_remove(cpu);
2712 		return ret;
2713 	}
2714 	mce_reenable_cpu();
2715 	mce_start_timer(t);
2716 	return 0;
2717 }
2718 
mce_cpu_pre_down(unsigned int cpu)2719 static int mce_cpu_pre_down(unsigned int cpu)
2720 {
2721 	struct timer_list *t = this_cpu_ptr(&mce_timer);
2722 
2723 	mce_disable_cpu();
2724 	del_timer_sync(t);
2725 	mce_threshold_remove_device(cpu);
2726 	mce_device_remove(cpu);
2727 	return 0;
2728 }
2729 
mce_init_banks(void)2730 static __init void mce_init_banks(void)
2731 {
2732 	int i;
2733 
2734 	for (i = 0; i < MAX_NR_BANKS; i++) {
2735 		struct mce_bank_dev *b = &mce_bank_devs[i];
2736 		struct device_attribute *a = &b->attr;
2737 
2738 		b->bank = i;
2739 
2740 		sysfs_attr_init(&a->attr);
2741 		a->attr.name	= b->attrname;
2742 		snprintf(b->attrname, ATTR_LEN, "bank%d", i);
2743 
2744 		a->attr.mode	= 0644;
2745 		a->show		= show_bank;
2746 		a->store	= set_bank;
2747 	}
2748 }
2749 
2750 /*
2751  * When running on XEN, this initcall is ordered against the XEN mcelog
2752  * initcall:
2753  *
2754  *   device_initcall(xen_late_init_mcelog);
2755  *   device_initcall_sync(mcheck_init_device);
2756  */
mcheck_init_device(void)2757 static __init int mcheck_init_device(void)
2758 {
2759 	int err;
2760 
2761 	/*
2762 	 * Check if we have a spare virtual bit. This will only become
2763 	 * a problem if/when we move beyond 5-level page tables.
2764 	 */
2765 	MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63);
2766 
2767 	if (!mce_available(&boot_cpu_data)) {
2768 		err = -EIO;
2769 		goto err_out;
2770 	}
2771 
2772 	if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) {
2773 		err = -ENOMEM;
2774 		goto err_out;
2775 	}
2776 
2777 	mce_init_banks();
2778 
2779 	err = subsys_system_register(&mce_subsys, NULL);
2780 	if (err)
2781 		goto err_out_mem;
2782 
2783 	err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL,
2784 				mce_cpu_dead);
2785 	if (err)
2786 		goto err_out_mem;
2787 
2788 	/*
2789 	 * Invokes mce_cpu_online() on all CPUs which are online when
2790 	 * the state is installed.
2791 	 */
2792 	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
2793 				mce_cpu_online, mce_cpu_pre_down);
2794 	if (err < 0)
2795 		goto err_out_online;
2796 
2797 	register_syscore_ops(&mce_syscore_ops);
2798 
2799 	return 0;
2800 
2801 err_out_online:
2802 	cpuhp_remove_state(CPUHP_X86_MCE_DEAD);
2803 
2804 err_out_mem:
2805 	free_cpumask_var(mce_device_initialized);
2806 
2807 err_out:
2808 	pr_err("Unable to init MCE device (rc: %d)\n", err);
2809 
2810 	return err;
2811 }
2812 device_initcall_sync(mcheck_init_device);
2813 
2814 /*
2815  * Old style boot options parsing. Only for compatibility.
2816  */
mcheck_disable(char * str)2817 static int __init mcheck_disable(char *str)
2818 {
2819 	mca_cfg.disabled = 1;
2820 	return 1;
2821 }
2822 __setup("nomce", mcheck_disable);
2823 
2824 #ifdef CONFIG_DEBUG_FS
mce_get_debugfs_dir(void)2825 struct dentry *mce_get_debugfs_dir(void)
2826 {
2827 	static struct dentry *dmce;
2828 
2829 	if (!dmce)
2830 		dmce = debugfs_create_dir("mce", NULL);
2831 
2832 	return dmce;
2833 }
2834 
mce_reset(void)2835 static void mce_reset(void)
2836 {
2837 	atomic_set(&mce_fake_panicked, 0);
2838 	atomic_set(&mce_executing, 0);
2839 	atomic_set(&mce_callin, 0);
2840 	atomic_set(&global_nwo, 0);
2841 	cpumask_setall(&mce_missing_cpus);
2842 }
2843 
fake_panic_get(void * data,u64 * val)2844 static int fake_panic_get(void *data, u64 *val)
2845 {
2846 	*val = fake_panic;
2847 	return 0;
2848 }
2849 
fake_panic_set(void * data,u64 val)2850 static int fake_panic_set(void *data, u64 val)
2851 {
2852 	mce_reset();
2853 	fake_panic = val;
2854 	return 0;
2855 }
2856 
2857 DEFINE_DEBUGFS_ATTRIBUTE(fake_panic_fops, fake_panic_get, fake_panic_set,
2858 			 "%llu\n");
2859 
mcheck_debugfs_init(void)2860 static void __init mcheck_debugfs_init(void)
2861 {
2862 	struct dentry *dmce;
2863 
2864 	dmce = mce_get_debugfs_dir();
2865 	debugfs_create_file_unsafe("fake_panic", 0444, dmce, NULL,
2866 				   &fake_panic_fops);
2867 }
2868 #else
mcheck_debugfs_init(void)2869 static void __init mcheck_debugfs_init(void) { }
2870 #endif
2871 
mcheck_late_init(void)2872 static int __init mcheck_late_init(void)
2873 {
2874 	if (mca_cfg.recovery)
2875 		enable_copy_mc_fragile();
2876 
2877 	mcheck_debugfs_init();
2878 
2879 	/*
2880 	 * Flush out everything that has been logged during early boot, now that
2881 	 * everything has been initialized (workqueues, decoders, ...).
2882 	 */
2883 	mce_schedule_work();
2884 
2885 	return 0;
2886 }
2887 late_initcall(mcheck_late_init);
2888