1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Machine check handler.
4 *
5 * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
6 * Rest from unknown author(s).
7 * 2004 Andi Kleen. Rewrote most of it.
8 * Copyright 2008 Intel Corporation
9 * Author: Andi Kleen
10 */
11
12 #include <linux/thread_info.h>
13 #include <linux/capability.h>
14 #include <linux/miscdevice.h>
15 #include <linux/ratelimit.h>
16 #include <linux/rcupdate.h>
17 #include <linux/kobject.h>
18 #include <linux/uaccess.h>
19 #include <linux/kdebug.h>
20 #include <linux/kernel.h>
21 #include <linux/percpu.h>
22 #include <linux/string.h>
23 #include <linux/device.h>
24 #include <linux/syscore_ops.h>
25 #include <linux/delay.h>
26 #include <linux/ctype.h>
27 #include <linux/sched.h>
28 #include <linux/sysfs.h>
29 #include <linux/types.h>
30 #include <linux/slab.h>
31 #include <linux/init.h>
32 #include <linux/kmod.h>
33 #include <linux/poll.h>
34 #include <linux/nmi.h>
35 #include <linux/cpu.h>
36 #include <linux/ras.h>
37 #include <linux/smp.h>
38 #include <linux/fs.h>
39 #include <linux/mm.h>
40 #include <linux/debugfs.h>
41 #include <linux/irq_work.h>
42 #include <linux/export.h>
43 #include <linux/set_memory.h>
44 #include <linux/sync_core.h>
45 #include <linux/task_work.h>
46 #include <linux/hardirq.h>
47 #include <linux/kexec.h>
48
49 #include <asm/intel-family.h>
50 #include <asm/processor.h>
51 #include <asm/traps.h>
52 #include <asm/tlbflush.h>
53 #include <asm/mce.h>
54 #include <asm/msr.h>
55 #include <asm/reboot.h>
56
57 #include "internal.h"
58
59 /* sysfs synchronization */
60 static DEFINE_MUTEX(mce_sysfs_mutex);
61
62 #define CREATE_TRACE_POINTS
63 #include <trace/events/mce.h>
64
65 #define SPINUNIT 100 /* 100ns */
66
67 DEFINE_PER_CPU(unsigned, mce_exception_count);
68
69 DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks);
70
71 DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array);
72
73 #define ATTR_LEN 16
74 /* One object for each MCE bank, shared by all CPUs */
75 struct mce_bank_dev {
76 struct device_attribute attr; /* device attribute */
77 char attrname[ATTR_LEN]; /* attribute name */
78 u8 bank; /* bank number */
79 };
80 static struct mce_bank_dev mce_bank_devs[MAX_NR_BANKS];
81
82 struct mce_vendor_flags mce_flags __read_mostly;
83
84 struct mca_config mca_cfg __read_mostly = {
85 .bootlog = -1,
86 .monarch_timeout = -1
87 };
88
89 static DEFINE_PER_CPU(struct mce, mces_seen);
90 static unsigned long mce_need_notify;
91
92 /*
93 * MCA banks polled by the period polling timer for corrected events.
94 * With Intel CMCI, this only has MCA banks which do not support CMCI (if any).
95 */
96 DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
97 [0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
98 };
99
100 /*
101 * MCA banks controlled through firmware first for corrected errors.
102 * This is a global list of banks for which we won't enable CMCI and we
103 * won't poll. Firmware controls these banks and is responsible for
104 * reporting corrected errors through GHES. Uncorrected/recoverable
105 * errors are still notified through a machine check.
106 */
107 mce_banks_t mce_banks_ce_disabled;
108
109 static struct work_struct mce_work;
110 static struct irq_work mce_irq_work;
111
112 /*
113 * CPU/chipset specific EDAC code can register a notifier call here to print
114 * MCE errors in a human-readable form.
115 */
116 BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain);
117
118 /* Do initial initialization of a struct mce */
mce_setup(struct mce * m)119 void mce_setup(struct mce *m)
120 {
121 memset(m, 0, sizeof(struct mce));
122 m->cpu = m->extcpu = smp_processor_id();
123 /* need the internal __ version to avoid deadlocks */
124 m->time = __ktime_get_real_seconds();
125 m->cpuvendor = boot_cpu_data.x86_vendor;
126 m->cpuid = cpuid_eax(1);
127 m->socketid = cpu_data(m->extcpu).phys_proc_id;
128 m->apicid = cpu_data(m->extcpu).initial_apicid;
129 m->mcgcap = __rdmsr(MSR_IA32_MCG_CAP);
130 m->ppin = cpu_data(m->extcpu).ppin;
131 m->microcode = boot_cpu_data.microcode;
132 }
133
134 DEFINE_PER_CPU(struct mce, injectm);
135 EXPORT_PER_CPU_SYMBOL_GPL(injectm);
136
mce_log(struct mce * m)137 void mce_log(struct mce *m)
138 {
139 if (!mce_gen_pool_add(m))
140 irq_work_queue(&mce_irq_work);
141 }
142 EXPORT_SYMBOL_GPL(mce_log);
143
mce_register_decode_chain(struct notifier_block * nb)144 void mce_register_decode_chain(struct notifier_block *nb)
145 {
146 if (WARN_ON(nb->priority < MCE_PRIO_LOWEST ||
147 nb->priority > MCE_PRIO_HIGHEST))
148 return;
149
150 blocking_notifier_chain_register(&x86_mce_decoder_chain, nb);
151 }
152 EXPORT_SYMBOL_GPL(mce_register_decode_chain);
153
mce_unregister_decode_chain(struct notifier_block * nb)154 void mce_unregister_decode_chain(struct notifier_block *nb)
155 {
156 blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb);
157 }
158 EXPORT_SYMBOL_GPL(mce_unregister_decode_chain);
159
__print_mce(struct mce * m)160 static void __print_mce(struct mce *m)
161 {
162 pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n",
163 m->extcpu,
164 (m->mcgstatus & MCG_STATUS_MCIP ? " Exception" : ""),
165 m->mcgstatus, m->bank, m->status);
166
167 if (m->ip) {
168 pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ",
169 !(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
170 m->cs, m->ip);
171
172 if (m->cs == __KERNEL_CS)
173 pr_cont("{%pS}", (void *)(unsigned long)m->ip);
174 pr_cont("\n");
175 }
176
177 pr_emerg(HW_ERR "TSC %llx ", m->tsc);
178 if (m->addr)
179 pr_cont("ADDR %llx ", m->addr);
180 if (m->misc)
181 pr_cont("MISC %llx ", m->misc);
182 if (m->ppin)
183 pr_cont("PPIN %llx ", m->ppin);
184
185 if (mce_flags.smca) {
186 if (m->synd)
187 pr_cont("SYND %llx ", m->synd);
188 if (m->ipid)
189 pr_cont("IPID %llx ", m->ipid);
190 }
191
192 pr_cont("\n");
193
194 /*
195 * Note this output is parsed by external tools and old fields
196 * should not be changed.
197 */
198 pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n",
199 m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid,
200 m->microcode);
201 }
202
print_mce(struct mce * m)203 static void print_mce(struct mce *m)
204 {
205 __print_mce(m);
206
207 if (m->cpuvendor != X86_VENDOR_AMD && m->cpuvendor != X86_VENDOR_HYGON)
208 pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n");
209 }
210
211 #define PANIC_TIMEOUT 5 /* 5 seconds */
212
213 static atomic_t mce_panicked;
214
215 static int fake_panic;
216 static atomic_t mce_fake_panicked;
217
218 /* Panic in progress. Enable interrupts and wait for final IPI */
wait_for_panic(void)219 static void wait_for_panic(void)
220 {
221 long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
222
223 preempt_disable();
224 local_irq_enable();
225 while (timeout-- > 0)
226 udelay(1);
227 if (panic_timeout == 0)
228 panic_timeout = mca_cfg.panic_timeout;
229 panic("Panicing machine check CPU died");
230 }
231
mce_panic(const char * msg,struct mce * final,char * exp)232 static noinstr void mce_panic(const char *msg, struct mce *final, char *exp)
233 {
234 struct llist_node *pending;
235 struct mce_evt_llist *l;
236 int apei_err = 0;
237 struct page *p;
238
239 /*
240 * Allow instrumentation around external facilities usage. Not that it
241 * matters a whole lot since the machine is going to panic anyway.
242 */
243 instrumentation_begin();
244
245 if (!fake_panic) {
246 /*
247 * Make sure only one CPU runs in machine check panic
248 */
249 if (atomic_inc_return(&mce_panicked) > 1)
250 wait_for_panic();
251 barrier();
252
253 bust_spinlocks(1);
254 console_verbose();
255 } else {
256 /* Don't log too much for fake panic */
257 if (atomic_inc_return(&mce_fake_panicked) > 1)
258 goto out;
259 }
260 pending = mce_gen_pool_prepare_records();
261 /* First print corrected ones that are still unlogged */
262 llist_for_each_entry(l, pending, llnode) {
263 struct mce *m = &l->mce;
264 if (!(m->status & MCI_STATUS_UC)) {
265 print_mce(m);
266 if (!apei_err)
267 apei_err = apei_write_mce(m);
268 }
269 }
270 /* Now print uncorrected but with the final one last */
271 llist_for_each_entry(l, pending, llnode) {
272 struct mce *m = &l->mce;
273 if (!(m->status & MCI_STATUS_UC))
274 continue;
275 if (!final || mce_cmp(m, final)) {
276 print_mce(m);
277 if (!apei_err)
278 apei_err = apei_write_mce(m);
279 }
280 }
281 if (final) {
282 print_mce(final);
283 if (!apei_err)
284 apei_err = apei_write_mce(final);
285 }
286 if (exp)
287 pr_emerg(HW_ERR "Machine check: %s\n", exp);
288 if (!fake_panic) {
289 if (panic_timeout == 0)
290 panic_timeout = mca_cfg.panic_timeout;
291
292 /*
293 * Kdump skips the poisoned page in order to avoid
294 * touching the error bits again. Poison the page even
295 * if the error is fatal and the machine is about to
296 * panic.
297 */
298 if (kexec_crash_loaded()) {
299 if (final && (final->status & MCI_STATUS_ADDRV)) {
300 p = pfn_to_online_page(final->addr >> PAGE_SHIFT);
301 if (p)
302 SetPageHWPoison(p);
303 }
304 }
305 panic(msg);
306 } else
307 pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
308
309 out:
310 instrumentation_end();
311 }
312
313 /* Support code for software error injection */
314
msr_to_offset(u32 msr)315 static int msr_to_offset(u32 msr)
316 {
317 unsigned bank = __this_cpu_read(injectm.bank);
318
319 if (msr == mca_cfg.rip_msr)
320 return offsetof(struct mce, ip);
321 if (msr == mca_msr_reg(bank, MCA_STATUS))
322 return offsetof(struct mce, status);
323 if (msr == mca_msr_reg(bank, MCA_ADDR))
324 return offsetof(struct mce, addr);
325 if (msr == mca_msr_reg(bank, MCA_MISC))
326 return offsetof(struct mce, misc);
327 if (msr == MSR_IA32_MCG_STATUS)
328 return offsetof(struct mce, mcgstatus);
329 return -1;
330 }
331
ex_handler_msr_mce(struct pt_regs * regs,bool wrmsr)332 void ex_handler_msr_mce(struct pt_regs *regs, bool wrmsr)
333 {
334 if (wrmsr) {
335 pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n",
336 (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax,
337 regs->ip, (void *)regs->ip);
338 } else {
339 pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n",
340 (unsigned int)regs->cx, regs->ip, (void *)regs->ip);
341 }
342
343 show_stack_regs(regs);
344
345 panic("MCA architectural violation!\n");
346
347 while (true)
348 cpu_relax();
349 }
350
351 /* MSR access wrappers used for error injection */
mce_rdmsrl(u32 msr)352 noinstr u64 mce_rdmsrl(u32 msr)
353 {
354 DECLARE_ARGS(val, low, high);
355
356 if (__this_cpu_read(injectm.finished)) {
357 int offset;
358 u64 ret;
359
360 instrumentation_begin();
361
362 offset = msr_to_offset(msr);
363 if (offset < 0)
364 ret = 0;
365 else
366 ret = *(u64 *)((char *)this_cpu_ptr(&injectm) + offset);
367
368 instrumentation_end();
369
370 return ret;
371 }
372
373 /*
374 * RDMSR on MCA MSRs should not fault. If they do, this is very much an
375 * architectural violation and needs to be reported to hw vendor. Panic
376 * the box to not allow any further progress.
377 */
378 asm volatile("1: rdmsr\n"
379 "2:\n"
380 _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_RDMSR_IN_MCE)
381 : EAX_EDX_RET(val, low, high) : "c" (msr));
382
383
384 return EAX_EDX_VAL(val, low, high);
385 }
386
mce_wrmsrl(u32 msr,u64 v)387 static noinstr void mce_wrmsrl(u32 msr, u64 v)
388 {
389 u32 low, high;
390
391 if (__this_cpu_read(injectm.finished)) {
392 int offset;
393
394 instrumentation_begin();
395
396 offset = msr_to_offset(msr);
397 if (offset >= 0)
398 *(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v;
399
400 instrumentation_end();
401
402 return;
403 }
404
405 low = (u32)v;
406 high = (u32)(v >> 32);
407
408 /* See comment in mce_rdmsrl() */
409 asm volatile("1: wrmsr\n"
410 "2:\n"
411 _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR_IN_MCE)
412 : : "c" (msr), "a"(low), "d" (high) : "memory");
413 }
414
415 /*
416 * Collect all global (w.r.t. this processor) status about this machine
417 * check into our "mce" struct so that we can use it later to assess
418 * the severity of the problem as we read per-bank specific details.
419 */
mce_gather_info(struct mce * m,struct pt_regs * regs)420 static noinstr void mce_gather_info(struct mce *m, struct pt_regs *regs)
421 {
422 /*
423 * Enable instrumentation around mce_setup() which calls external
424 * facilities.
425 */
426 instrumentation_begin();
427 mce_setup(m);
428 instrumentation_end();
429
430 m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
431 if (regs) {
432 /*
433 * Get the address of the instruction at the time of
434 * the machine check error.
435 */
436 if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) {
437 m->ip = regs->ip;
438 m->cs = regs->cs;
439
440 /*
441 * When in VM86 mode make the cs look like ring 3
442 * always. This is a lie, but it's better than passing
443 * the additional vm86 bit around everywhere.
444 */
445 if (v8086_mode(regs))
446 m->cs |= 3;
447 }
448 /* Use accurate RIP reporting if available. */
449 if (mca_cfg.rip_msr)
450 m->ip = mce_rdmsrl(mca_cfg.rip_msr);
451 }
452 }
453
mce_available(struct cpuinfo_x86 * c)454 int mce_available(struct cpuinfo_x86 *c)
455 {
456 if (mca_cfg.disabled)
457 return 0;
458 return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
459 }
460
mce_schedule_work(void)461 static void mce_schedule_work(void)
462 {
463 if (!mce_gen_pool_empty())
464 schedule_work(&mce_work);
465 }
466
mce_irq_work_cb(struct irq_work * entry)467 static void mce_irq_work_cb(struct irq_work *entry)
468 {
469 mce_schedule_work();
470 }
471
472 /*
473 * Check if the address reported by the CPU is in a format we can parse.
474 * It would be possible to add code for most other cases, but all would
475 * be somewhat complicated (e.g. segment offset would require an instruction
476 * parser). So only support physical addresses up to page granularity for now.
477 */
mce_usable_address(struct mce * m)478 int mce_usable_address(struct mce *m)
479 {
480 if (!(m->status & MCI_STATUS_ADDRV))
481 return 0;
482
483 /* Checks after this one are Intel/Zhaoxin-specific: */
484 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL &&
485 boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN)
486 return 1;
487
488 if (!(m->status & MCI_STATUS_MISCV))
489 return 0;
490
491 if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)
492 return 0;
493
494 if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)
495 return 0;
496
497 return 1;
498 }
499 EXPORT_SYMBOL_GPL(mce_usable_address);
500
mce_is_memory_error(struct mce * m)501 bool mce_is_memory_error(struct mce *m)
502 {
503 switch (m->cpuvendor) {
504 case X86_VENDOR_AMD:
505 case X86_VENDOR_HYGON:
506 return amd_mce_is_memory_error(m);
507
508 case X86_VENDOR_INTEL:
509 case X86_VENDOR_ZHAOXIN:
510 /*
511 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes
512 *
513 * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for
514 * indicating a memory error. Bit 8 is used for indicating a
515 * cache hierarchy error. The combination of bit 2 and bit 3
516 * is used for indicating a `generic' cache hierarchy error
517 * But we can't just blindly check the above bits, because if
518 * bit 11 is set, then it is a bus/interconnect error - and
519 * either way the above bits just gives more detail on what
520 * bus/interconnect error happened. Note that bit 12 can be
521 * ignored, as it's the "filter" bit.
522 */
523 return (m->status & 0xef80) == BIT(7) ||
524 (m->status & 0xef00) == BIT(8) ||
525 (m->status & 0xeffc) == 0xc;
526
527 default:
528 return false;
529 }
530 }
531 EXPORT_SYMBOL_GPL(mce_is_memory_error);
532
whole_page(struct mce * m)533 static bool whole_page(struct mce *m)
534 {
535 if (!mca_cfg.ser || !(m->status & MCI_STATUS_MISCV))
536 return true;
537
538 return MCI_MISC_ADDR_LSB(m->misc) >= PAGE_SHIFT;
539 }
540
mce_is_correctable(struct mce * m)541 bool mce_is_correctable(struct mce *m)
542 {
543 if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED)
544 return false;
545
546 if (m->cpuvendor == X86_VENDOR_HYGON && m->status & MCI_STATUS_DEFERRED)
547 return false;
548
549 if (m->status & MCI_STATUS_UC)
550 return false;
551
552 return true;
553 }
554 EXPORT_SYMBOL_GPL(mce_is_correctable);
555
mce_early_notifier(struct notifier_block * nb,unsigned long val,void * data)556 static int mce_early_notifier(struct notifier_block *nb, unsigned long val,
557 void *data)
558 {
559 struct mce *m = (struct mce *)data;
560
561 if (!m)
562 return NOTIFY_DONE;
563
564 /* Emit the trace record: */
565 trace_mce_record(m);
566
567 set_bit(0, &mce_need_notify);
568
569 mce_notify_irq();
570
571 return NOTIFY_DONE;
572 }
573
574 static struct notifier_block early_nb = {
575 .notifier_call = mce_early_notifier,
576 .priority = MCE_PRIO_EARLY,
577 };
578
uc_decode_notifier(struct notifier_block * nb,unsigned long val,void * data)579 static int uc_decode_notifier(struct notifier_block *nb, unsigned long val,
580 void *data)
581 {
582 struct mce *mce = (struct mce *)data;
583 unsigned long pfn;
584
585 if (!mce || !mce_usable_address(mce))
586 return NOTIFY_DONE;
587
588 if (mce->severity != MCE_AO_SEVERITY &&
589 mce->severity != MCE_DEFERRED_SEVERITY)
590 return NOTIFY_DONE;
591
592 pfn = (mce->addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT;
593 if (!memory_failure(pfn, 0)) {
594 set_mce_nospec(pfn);
595 mce->kflags |= MCE_HANDLED_UC;
596 }
597
598 return NOTIFY_OK;
599 }
600
601 static struct notifier_block mce_uc_nb = {
602 .notifier_call = uc_decode_notifier,
603 .priority = MCE_PRIO_UC,
604 };
605
mce_default_notifier(struct notifier_block * nb,unsigned long val,void * data)606 static int mce_default_notifier(struct notifier_block *nb, unsigned long val,
607 void *data)
608 {
609 struct mce *m = (struct mce *)data;
610
611 if (!m)
612 return NOTIFY_DONE;
613
614 if (mca_cfg.print_all || !m->kflags)
615 __print_mce(m);
616
617 return NOTIFY_DONE;
618 }
619
620 static struct notifier_block mce_default_nb = {
621 .notifier_call = mce_default_notifier,
622 /* lowest prio, we want it to run last. */
623 .priority = MCE_PRIO_LOWEST,
624 };
625
626 /*
627 * Read ADDR and MISC registers.
628 */
mce_read_aux(struct mce * m,int i)629 static noinstr void mce_read_aux(struct mce *m, int i)
630 {
631 if (m->status & MCI_STATUS_MISCV)
632 m->misc = mce_rdmsrl(mca_msr_reg(i, MCA_MISC));
633
634 if (m->status & MCI_STATUS_ADDRV) {
635 m->addr = mce_rdmsrl(mca_msr_reg(i, MCA_ADDR));
636
637 /*
638 * Mask the reported address by the reported granularity.
639 */
640 if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) {
641 u8 shift = MCI_MISC_ADDR_LSB(m->misc);
642 m->addr >>= shift;
643 m->addr <<= shift;
644 }
645
646 smca_extract_err_addr(m);
647 }
648
649 if (mce_flags.smca) {
650 m->ipid = mce_rdmsrl(MSR_AMD64_SMCA_MCx_IPID(i));
651
652 if (m->status & MCI_STATUS_SYNDV)
653 m->synd = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND(i));
654 }
655 }
656
657 DEFINE_PER_CPU(unsigned, mce_poll_count);
658
659 /*
660 * Poll for corrected events or events that happened before reset.
661 * Those are just logged through /dev/mcelog.
662 *
663 * This is executed in standard interrupt context.
664 *
665 * Note: spec recommends to panic for fatal unsignalled
666 * errors here. However this would be quite problematic --
667 * we would need to reimplement the Monarch handling and
668 * it would mess up the exclusion between exception handler
669 * and poll handler -- * so we skip this for now.
670 * These cases should not happen anyways, or only when the CPU
671 * is already totally * confused. In this case it's likely it will
672 * not fully execute the machine check handler either.
673 */
machine_check_poll(enum mcp_flags flags,mce_banks_t * b)674 bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
675 {
676 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
677 bool error_seen = false;
678 struct mce m;
679 int i;
680
681 this_cpu_inc(mce_poll_count);
682
683 mce_gather_info(&m, NULL);
684
685 if (flags & MCP_TIMESTAMP)
686 m.tsc = rdtsc();
687
688 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
689 if (!mce_banks[i].ctl || !test_bit(i, *b))
690 continue;
691
692 m.misc = 0;
693 m.addr = 0;
694 m.bank = i;
695
696 barrier();
697 m.status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS));
698
699 /* If this entry is not valid, ignore it */
700 if (!(m.status & MCI_STATUS_VAL))
701 continue;
702
703 /*
704 * If we are logging everything (at CPU online) or this
705 * is a corrected error, then we must log it.
706 */
707 if ((flags & MCP_UC) || !(m.status & MCI_STATUS_UC))
708 goto log_it;
709
710 /*
711 * Newer Intel systems that support software error
712 * recovery need to make additional checks. Other
713 * CPUs should skip over uncorrected errors, but log
714 * everything else.
715 */
716 if (!mca_cfg.ser) {
717 if (m.status & MCI_STATUS_UC)
718 continue;
719 goto log_it;
720 }
721
722 /* Log "not enabled" (speculative) errors */
723 if (!(m.status & MCI_STATUS_EN))
724 goto log_it;
725
726 /*
727 * Log UCNA (SDM: 15.6.3 "UCR Error Classification")
728 * UC == 1 && PCC == 0 && S == 0
729 */
730 if (!(m.status & MCI_STATUS_PCC) && !(m.status & MCI_STATUS_S))
731 goto log_it;
732
733 /*
734 * Skip anything else. Presumption is that our read of this
735 * bank is racing with a machine check. Leave the log alone
736 * for do_machine_check() to deal with it.
737 */
738 continue;
739
740 log_it:
741 error_seen = true;
742
743 if (flags & MCP_DONTLOG)
744 goto clear_it;
745
746 mce_read_aux(&m, i);
747 m.severity = mce_severity(&m, NULL, NULL, false);
748 /*
749 * Don't get the IP here because it's unlikely to
750 * have anything to do with the actual error location.
751 */
752
753 if (mca_cfg.dont_log_ce && !mce_usable_address(&m))
754 goto clear_it;
755
756 if (flags & MCP_QUEUE_LOG)
757 mce_gen_pool_add(&m);
758 else
759 mce_log(&m);
760
761 clear_it:
762 /*
763 * Clear state for this bank.
764 */
765 mce_wrmsrl(mca_msr_reg(i, MCA_STATUS), 0);
766 }
767
768 /*
769 * Don't clear MCG_STATUS here because it's only defined for
770 * exceptions.
771 */
772
773 sync_core();
774
775 return error_seen;
776 }
777 EXPORT_SYMBOL_GPL(machine_check_poll);
778
779 /*
780 * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and
781 * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM
782 * Vol 3B Table 15-20). But this confuses both the code that determines
783 * whether the machine check occurred in kernel or user mode, and also
784 * the severity assessment code. Pretend that EIPV was set, and take the
785 * ip/cs values from the pt_regs that mce_gather_info() ignored earlier.
786 */
787 static __always_inline void
quirk_sandybridge_ifu(int bank,struct mce * m,struct pt_regs * regs)788 quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
789 {
790 if (bank != 0)
791 return;
792 if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0)
793 return;
794 if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC|
795 MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV|
796 MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR|
797 MCACOD)) !=
798 (MCI_STATUS_UC|MCI_STATUS_EN|
799 MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S|
800 MCI_STATUS_AR|MCACOD_INSTR))
801 return;
802
803 m->mcgstatus |= MCG_STATUS_EIPV;
804 m->ip = regs->ip;
805 m->cs = regs->cs;
806 }
807
808 /*
809 * Disable fast string copy and return from the MCE handler upon the first SRAR
810 * MCE on bank 1 due to a CPU erratum on Intel Skylake/Cascade Lake/Cooper Lake
811 * CPUs.
812 * The fast string copy instructions ("REP; MOVS*") could consume an
813 * uncorrectable memory error in the cache line _right after_ the desired region
814 * to copy and raise an MCE with RIP pointing to the instruction _after_ the
815 * "REP; MOVS*".
816 * This mitigation addresses the issue completely with the caveat of performance
817 * degradation on the CPU affected. This is still better than the OS crashing on
818 * MCEs raised on an irrelevant process due to "REP; MOVS*" accesses from a
819 * kernel context (e.g., copy_page).
820 *
821 * Returns true when fast string copy on CPU has been disabled.
822 */
quirk_skylake_repmov(void)823 static noinstr bool quirk_skylake_repmov(void)
824 {
825 u64 mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
826 u64 misc_enable = mce_rdmsrl(MSR_IA32_MISC_ENABLE);
827 u64 mc1_status;
828
829 /*
830 * Apply the quirk only to local machine checks, i.e., no broadcast
831 * sync is needed.
832 */
833 if (!(mcgstatus & MCG_STATUS_LMCES) ||
834 !(misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING))
835 return false;
836
837 mc1_status = mce_rdmsrl(MSR_IA32_MCx_STATUS(1));
838
839 /* Check for a software-recoverable data fetch error. */
840 if ((mc1_status &
841 (MCI_STATUS_VAL | MCI_STATUS_OVER | MCI_STATUS_UC | MCI_STATUS_EN |
842 MCI_STATUS_ADDRV | MCI_STATUS_MISCV | MCI_STATUS_PCC |
843 MCI_STATUS_AR | MCI_STATUS_S)) ==
844 (MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN |
845 MCI_STATUS_ADDRV | MCI_STATUS_MISCV |
846 MCI_STATUS_AR | MCI_STATUS_S)) {
847 misc_enable &= ~MSR_IA32_MISC_ENABLE_FAST_STRING;
848 mce_wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
849 mce_wrmsrl(MSR_IA32_MCx_STATUS(1), 0);
850
851 instrumentation_begin();
852 pr_err_once("Erratum detected, disable fast string copy instructions.\n");
853 instrumentation_end();
854
855 return true;
856 }
857
858 return false;
859 }
860
861 /*
862 * Some Zen-based Instruction Fetch Units set EIPV=RIPV=0 on poison consumption
863 * errors. This means mce_gather_info() will not save the "ip" and "cs" registers.
864 *
865 * However, the context is still valid, so save the "cs" register for later use.
866 *
867 * The "ip" register is truly unknown, so don't save it or fixup EIPV/RIPV.
868 *
869 * The Instruction Fetch Unit is at MCA bank 1 for all affected systems.
870 */
quirk_zen_ifu(int bank,struct mce * m,struct pt_regs * regs)871 static __always_inline void quirk_zen_ifu(int bank, struct mce *m, struct pt_regs *regs)
872 {
873 if (bank != 1)
874 return;
875 if (!(m->status & MCI_STATUS_POISON))
876 return;
877
878 m->cs = regs->cs;
879 }
880
881 /*
882 * Do a quick check if any of the events requires a panic.
883 * This decides if we keep the events around or clear them.
884 */
mce_no_way_out(struct mce * m,char ** msg,unsigned long * validp,struct pt_regs * regs)885 static __always_inline int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
886 struct pt_regs *regs)
887 {
888 char *tmp = *msg;
889 int i;
890
891 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
892 m->status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS));
893 if (!(m->status & MCI_STATUS_VAL))
894 continue;
895
896 arch___set_bit(i, validp);
897 if (mce_flags.snb_ifu_quirk)
898 quirk_sandybridge_ifu(i, m, regs);
899
900 if (mce_flags.zen_ifu_quirk)
901 quirk_zen_ifu(i, m, regs);
902
903 m->bank = i;
904 if (mce_severity(m, regs, &tmp, true) >= MCE_PANIC_SEVERITY) {
905 mce_read_aux(m, i);
906 *msg = tmp;
907 return 1;
908 }
909 }
910 return 0;
911 }
912
913 /*
914 * Variable to establish order between CPUs while scanning.
915 * Each CPU spins initially until executing is equal its number.
916 */
917 static atomic_t mce_executing;
918
919 /*
920 * Defines order of CPUs on entry. First CPU becomes Monarch.
921 */
922 static atomic_t mce_callin;
923
924 /*
925 * Track which CPUs entered the MCA broadcast synchronization and which not in
926 * order to print holdouts.
927 */
928 static cpumask_t mce_missing_cpus = CPU_MASK_ALL;
929
930 /*
931 * Check if a timeout waiting for other CPUs happened.
932 */
mce_timed_out(u64 * t,const char * msg)933 static noinstr int mce_timed_out(u64 *t, const char *msg)
934 {
935 int ret = 0;
936
937 /* Enable instrumentation around calls to external facilities */
938 instrumentation_begin();
939
940 /*
941 * The others already did panic for some reason.
942 * Bail out like in a timeout.
943 * rmb() to tell the compiler that system_state
944 * might have been modified by someone else.
945 */
946 rmb();
947 if (atomic_read(&mce_panicked))
948 wait_for_panic();
949 if (!mca_cfg.monarch_timeout)
950 goto out;
951 if ((s64)*t < SPINUNIT) {
952 if (cpumask_and(&mce_missing_cpus, cpu_online_mask, &mce_missing_cpus))
953 pr_emerg("CPUs not responding to MCE broadcast (may include false positives): %*pbl\n",
954 cpumask_pr_args(&mce_missing_cpus));
955 mce_panic(msg, NULL, NULL);
956
957 ret = 1;
958 goto out;
959 }
960 *t -= SPINUNIT;
961
962 out:
963 touch_nmi_watchdog();
964
965 instrumentation_end();
966
967 return ret;
968 }
969
970 /*
971 * The Monarch's reign. The Monarch is the CPU who entered
972 * the machine check handler first. It waits for the others to
973 * raise the exception too and then grades them. When any
974 * error is fatal panic. Only then let the others continue.
975 *
976 * The other CPUs entering the MCE handler will be controlled by the
977 * Monarch. They are called Subjects.
978 *
979 * This way we prevent any potential data corruption in a unrecoverable case
980 * and also makes sure always all CPU's errors are examined.
981 *
982 * Also this detects the case of a machine check event coming from outer
983 * space (not detected by any CPUs) In this case some external agent wants
984 * us to shut down, so panic too.
985 *
986 * The other CPUs might still decide to panic if the handler happens
987 * in a unrecoverable place, but in this case the system is in a semi-stable
988 * state and won't corrupt anything by itself. It's ok to let the others
989 * continue for a bit first.
990 *
991 * All the spin loops have timeouts; when a timeout happens a CPU
992 * typically elects itself to be Monarch.
993 */
mce_reign(void)994 static void mce_reign(void)
995 {
996 int cpu;
997 struct mce *m = NULL;
998 int global_worst = 0;
999 char *msg = NULL;
1000
1001 /*
1002 * This CPU is the Monarch and the other CPUs have run
1003 * through their handlers.
1004 * Grade the severity of the errors of all the CPUs.
1005 */
1006 for_each_possible_cpu(cpu) {
1007 struct mce *mtmp = &per_cpu(mces_seen, cpu);
1008
1009 if (mtmp->severity > global_worst) {
1010 global_worst = mtmp->severity;
1011 m = &per_cpu(mces_seen, cpu);
1012 }
1013 }
1014
1015 /*
1016 * Cannot recover? Panic here then.
1017 * This dumps all the mces in the log buffer and stops the
1018 * other CPUs.
1019 */
1020 if (m && global_worst >= MCE_PANIC_SEVERITY) {
1021 /* call mce_severity() to get "msg" for panic */
1022 mce_severity(m, NULL, &msg, true);
1023 mce_panic("Fatal machine check", m, msg);
1024 }
1025
1026 /*
1027 * For UC somewhere we let the CPU who detects it handle it.
1028 * Also must let continue the others, otherwise the handling
1029 * CPU could deadlock on a lock.
1030 */
1031
1032 /*
1033 * No machine check event found. Must be some external
1034 * source or one CPU is hung. Panic.
1035 */
1036 if (global_worst <= MCE_KEEP_SEVERITY)
1037 mce_panic("Fatal machine check from unknown source", NULL, NULL);
1038
1039 /*
1040 * Now clear all the mces_seen so that they don't reappear on
1041 * the next mce.
1042 */
1043 for_each_possible_cpu(cpu)
1044 memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
1045 }
1046
1047 static atomic_t global_nwo;
1048
1049 /*
1050 * Start of Monarch synchronization. This waits until all CPUs have
1051 * entered the exception handler and then determines if any of them
1052 * saw a fatal event that requires panic. Then it executes them
1053 * in the entry order.
1054 * TBD double check parallel CPU hotunplug
1055 */
mce_start(int * no_way_out)1056 static noinstr int mce_start(int *no_way_out)
1057 {
1058 u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
1059 int order, ret = -1;
1060
1061 if (!timeout)
1062 return ret;
1063
1064 raw_atomic_add(*no_way_out, &global_nwo);
1065 /*
1066 * Rely on the implied barrier below, such that global_nwo
1067 * is updated before mce_callin.
1068 */
1069 order = raw_atomic_inc_return(&mce_callin);
1070 arch_cpumask_clear_cpu(smp_processor_id(), &mce_missing_cpus);
1071
1072 /* Enable instrumentation around calls to external facilities */
1073 instrumentation_begin();
1074
1075 /*
1076 * Wait for everyone.
1077 */
1078 while (raw_atomic_read(&mce_callin) != num_online_cpus()) {
1079 if (mce_timed_out(&timeout,
1080 "Timeout: Not all CPUs entered broadcast exception handler")) {
1081 raw_atomic_set(&global_nwo, 0);
1082 goto out;
1083 }
1084 ndelay(SPINUNIT);
1085 }
1086
1087 /*
1088 * mce_callin should be read before global_nwo
1089 */
1090 smp_rmb();
1091
1092 if (order == 1) {
1093 /*
1094 * Monarch: Starts executing now, the others wait.
1095 */
1096 raw_atomic_set(&mce_executing, 1);
1097 } else {
1098 /*
1099 * Subject: Now start the scanning loop one by one in
1100 * the original callin order.
1101 * This way when there are any shared banks it will be
1102 * only seen by one CPU before cleared, avoiding duplicates.
1103 */
1104 while (raw_atomic_read(&mce_executing) < order) {
1105 if (mce_timed_out(&timeout,
1106 "Timeout: Subject CPUs unable to finish machine check processing")) {
1107 raw_atomic_set(&global_nwo, 0);
1108 goto out;
1109 }
1110 ndelay(SPINUNIT);
1111 }
1112 }
1113
1114 /*
1115 * Cache the global no_way_out state.
1116 */
1117 *no_way_out = raw_atomic_read(&global_nwo);
1118
1119 ret = order;
1120
1121 out:
1122 instrumentation_end();
1123
1124 return ret;
1125 }
1126
1127 /*
1128 * Synchronize between CPUs after main scanning loop.
1129 * This invokes the bulk of the Monarch processing.
1130 */
mce_end(int order)1131 static noinstr int mce_end(int order)
1132 {
1133 u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
1134 int ret = -1;
1135
1136 /* Allow instrumentation around external facilities. */
1137 instrumentation_begin();
1138
1139 if (!timeout)
1140 goto reset;
1141 if (order < 0)
1142 goto reset;
1143
1144 /*
1145 * Allow others to run.
1146 */
1147 atomic_inc(&mce_executing);
1148
1149 if (order == 1) {
1150 /*
1151 * Monarch: Wait for everyone to go through their scanning
1152 * loops.
1153 */
1154 while (atomic_read(&mce_executing) <= num_online_cpus()) {
1155 if (mce_timed_out(&timeout,
1156 "Timeout: Monarch CPU unable to finish machine check processing"))
1157 goto reset;
1158 ndelay(SPINUNIT);
1159 }
1160
1161 mce_reign();
1162 barrier();
1163 ret = 0;
1164 } else {
1165 /*
1166 * Subject: Wait for Monarch to finish.
1167 */
1168 while (atomic_read(&mce_executing) != 0) {
1169 if (mce_timed_out(&timeout,
1170 "Timeout: Monarch CPU did not finish machine check processing"))
1171 goto reset;
1172 ndelay(SPINUNIT);
1173 }
1174
1175 /*
1176 * Don't reset anything. That's done by the Monarch.
1177 */
1178 ret = 0;
1179 goto out;
1180 }
1181
1182 /*
1183 * Reset all global state.
1184 */
1185 reset:
1186 atomic_set(&global_nwo, 0);
1187 atomic_set(&mce_callin, 0);
1188 cpumask_setall(&mce_missing_cpus);
1189 barrier();
1190
1191 /*
1192 * Let others run again.
1193 */
1194 atomic_set(&mce_executing, 0);
1195
1196 out:
1197 instrumentation_end();
1198
1199 return ret;
1200 }
1201
mce_clear_state(unsigned long * toclear)1202 static __always_inline void mce_clear_state(unsigned long *toclear)
1203 {
1204 int i;
1205
1206 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1207 if (arch_test_bit(i, toclear))
1208 mce_wrmsrl(mca_msr_reg(i, MCA_STATUS), 0);
1209 }
1210 }
1211
1212 /*
1213 * Cases where we avoid rendezvous handler timeout:
1214 * 1) If this CPU is offline.
1215 *
1216 * 2) If crashing_cpu was set, e.g. we're entering kdump and we need to
1217 * skip those CPUs which remain looping in the 1st kernel - see
1218 * crash_nmi_callback().
1219 *
1220 * Note: there still is a small window between kexec-ing and the new,
1221 * kdump kernel establishing a new #MC handler where a broadcasted MCE
1222 * might not get handled properly.
1223 */
mce_check_crashing_cpu(void)1224 static noinstr bool mce_check_crashing_cpu(void)
1225 {
1226 unsigned int cpu = smp_processor_id();
1227
1228 if (arch_cpu_is_offline(cpu) ||
1229 (crashing_cpu != -1 && crashing_cpu != cpu)) {
1230 u64 mcgstatus;
1231
1232 mcgstatus = __rdmsr(MSR_IA32_MCG_STATUS);
1233
1234 if (boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) {
1235 if (mcgstatus & MCG_STATUS_LMCES)
1236 return false;
1237 }
1238
1239 if (mcgstatus & MCG_STATUS_RIPV) {
1240 __wrmsr(MSR_IA32_MCG_STATUS, 0, 0);
1241 return true;
1242 }
1243 }
1244 return false;
1245 }
1246
1247 static __always_inline int
__mc_scan_banks(struct mce * m,struct pt_regs * regs,struct mce * final,unsigned long * toclear,unsigned long * valid_banks,int no_way_out,int * worst)1248 __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *final,
1249 unsigned long *toclear, unsigned long *valid_banks, int no_way_out,
1250 int *worst)
1251 {
1252 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1253 struct mca_config *cfg = &mca_cfg;
1254 int severity, i, taint = 0;
1255
1256 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1257 arch___clear_bit(i, toclear);
1258 if (!arch_test_bit(i, valid_banks))
1259 continue;
1260
1261 if (!mce_banks[i].ctl)
1262 continue;
1263
1264 m->misc = 0;
1265 m->addr = 0;
1266 m->bank = i;
1267
1268 m->status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS));
1269 if (!(m->status & MCI_STATUS_VAL))
1270 continue;
1271
1272 /*
1273 * Corrected or non-signaled errors are handled by
1274 * machine_check_poll(). Leave them alone, unless this panics.
1275 */
1276 if (!(m->status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
1277 !no_way_out)
1278 continue;
1279
1280 /* Set taint even when machine check was not enabled. */
1281 taint++;
1282
1283 severity = mce_severity(m, regs, NULL, true);
1284
1285 /*
1286 * When machine check was for corrected/deferred handler don't
1287 * touch, unless we're panicking.
1288 */
1289 if ((severity == MCE_KEEP_SEVERITY ||
1290 severity == MCE_UCNA_SEVERITY) && !no_way_out)
1291 continue;
1292
1293 arch___set_bit(i, toclear);
1294
1295 /* Machine check event was not enabled. Clear, but ignore. */
1296 if (severity == MCE_NO_SEVERITY)
1297 continue;
1298
1299 mce_read_aux(m, i);
1300
1301 /* assuming valid severity level != 0 */
1302 m->severity = severity;
1303
1304 /*
1305 * Enable instrumentation around the mce_log() call which is
1306 * done in #MC context, where instrumentation is disabled.
1307 */
1308 instrumentation_begin();
1309 mce_log(m);
1310 instrumentation_end();
1311
1312 if (severity > *worst) {
1313 *final = *m;
1314 *worst = severity;
1315 }
1316 }
1317
1318 /* mce_clear_state will clear *final, save locally for use later */
1319 *m = *final;
1320
1321 return taint;
1322 }
1323
kill_me_now(struct callback_head * ch)1324 static void kill_me_now(struct callback_head *ch)
1325 {
1326 struct task_struct *p = container_of(ch, struct task_struct, mce_kill_me);
1327
1328 p->mce_count = 0;
1329 force_sig(SIGBUS);
1330 }
1331
kill_me_maybe(struct callback_head * cb)1332 static void kill_me_maybe(struct callback_head *cb)
1333 {
1334 struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me);
1335 int flags = MF_ACTION_REQUIRED;
1336 unsigned long pfn;
1337 int ret;
1338
1339 p->mce_count = 0;
1340 pr_err("Uncorrected hardware memory error in user-access at %llx", p->mce_addr);
1341
1342 if (!p->mce_ripv)
1343 flags |= MF_MUST_KILL;
1344
1345 pfn = (p->mce_addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT;
1346 ret = memory_failure(pfn, flags);
1347 if (!ret) {
1348 set_mce_nospec(pfn);
1349 sync_core();
1350 return;
1351 }
1352
1353 /*
1354 * -EHWPOISON from memory_failure() means that it already sent SIGBUS
1355 * to the current process with the proper error info,
1356 * -EOPNOTSUPP means hwpoison_filter() filtered the error event,
1357 *
1358 * In both cases, no further processing is required.
1359 */
1360 if (ret == -EHWPOISON || ret == -EOPNOTSUPP)
1361 return;
1362
1363 pr_err("Memory error not recovered");
1364 kill_me_now(cb);
1365 }
1366
kill_me_never(struct callback_head * cb)1367 static void kill_me_never(struct callback_head *cb)
1368 {
1369 struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me);
1370 unsigned long pfn;
1371
1372 p->mce_count = 0;
1373 pr_err("Kernel accessed poison in user space at %llx\n", p->mce_addr);
1374 pfn = (p->mce_addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT;
1375 if (!memory_failure(pfn, 0))
1376 set_mce_nospec(pfn);
1377 }
1378
queue_task_work(struct mce * m,char * msg,void (* func)(struct callback_head *))1379 static void queue_task_work(struct mce *m, char *msg, void (*func)(struct callback_head *))
1380 {
1381 int count = ++current->mce_count;
1382
1383 /* First call, save all the details */
1384 if (count == 1) {
1385 current->mce_addr = m->addr;
1386 current->mce_kflags = m->kflags;
1387 current->mce_ripv = !!(m->mcgstatus & MCG_STATUS_RIPV);
1388 current->mce_whole_page = whole_page(m);
1389 current->mce_kill_me.func = func;
1390 }
1391
1392 /* Ten is likely overkill. Don't expect more than two faults before task_work() */
1393 if (count > 10)
1394 mce_panic("Too many consecutive machine checks while accessing user data", m, msg);
1395
1396 /* Second or later call, make sure page address matches the one from first call */
1397 if (count > 1 && (current->mce_addr >> PAGE_SHIFT) != (m->addr >> PAGE_SHIFT))
1398 mce_panic("Consecutive machine checks to different user pages", m, msg);
1399
1400 /* Do not call task_work_add() more than once */
1401 if (count > 1)
1402 return;
1403
1404 task_work_add(current, ¤t->mce_kill_me, TWA_RESUME);
1405 }
1406
1407 /* Handle unconfigured int18 (should never happen) */
unexpected_machine_check(struct pt_regs * regs)1408 static noinstr void unexpected_machine_check(struct pt_regs *regs)
1409 {
1410 instrumentation_begin();
1411 pr_err("CPU#%d: Unexpected int18 (Machine Check)\n",
1412 smp_processor_id());
1413 instrumentation_end();
1414 }
1415
1416 /*
1417 * The actual machine check handler. This only handles real exceptions when
1418 * something got corrupted coming in through int 18.
1419 *
1420 * This is executed in #MC context not subject to normal locking rules.
1421 * This implies that most kernel services cannot be safely used. Don't even
1422 * think about putting a printk in there!
1423 *
1424 * On Intel systems this is entered on all CPUs in parallel through
1425 * MCE broadcast. However some CPUs might be broken beyond repair,
1426 * so be always careful when synchronizing with others.
1427 *
1428 * Tracing and kprobes are disabled: if we interrupted a kernel context
1429 * with IF=1, we need to minimize stack usage. There are also recursion
1430 * issues: if the machine check was due to a failure of the memory
1431 * backing the user stack, tracing that reads the user stack will cause
1432 * potentially infinite recursion.
1433 *
1434 * Currently, the #MC handler calls out to a number of external facilities
1435 * and, therefore, allows instrumentation around them. The optimal thing to
1436 * have would be to do the absolutely minimal work required in #MC context
1437 * and have instrumentation disabled only around that. Further processing can
1438 * then happen in process context where instrumentation is allowed. Achieving
1439 * that requires careful auditing and modifications. Until then, the code
1440 * allows instrumentation temporarily, where required. *
1441 */
do_machine_check(struct pt_regs * regs)1442 noinstr void do_machine_check(struct pt_regs *regs)
1443 {
1444 int worst = 0, order, no_way_out, kill_current_task, lmce, taint = 0;
1445 DECLARE_BITMAP(valid_banks, MAX_NR_BANKS) = { 0 };
1446 DECLARE_BITMAP(toclear, MAX_NR_BANKS) = { 0 };
1447 struct mce m, *final;
1448 char *msg = NULL;
1449
1450 if (unlikely(mce_flags.p5))
1451 return pentium_machine_check(regs);
1452 else if (unlikely(mce_flags.winchip))
1453 return winchip_machine_check(regs);
1454 else if (unlikely(!mca_cfg.initialized))
1455 return unexpected_machine_check(regs);
1456
1457 if (mce_flags.skx_repmov_quirk && quirk_skylake_repmov())
1458 goto clear;
1459
1460 /*
1461 * Establish sequential order between the CPUs entering the machine
1462 * check handler.
1463 */
1464 order = -1;
1465
1466 /*
1467 * If no_way_out gets set, there is no safe way to recover from this
1468 * MCE.
1469 */
1470 no_way_out = 0;
1471
1472 /*
1473 * If kill_current_task is not set, there might be a way to recover from this
1474 * error.
1475 */
1476 kill_current_task = 0;
1477
1478 /*
1479 * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES
1480 * on Intel.
1481 */
1482 lmce = 1;
1483
1484 this_cpu_inc(mce_exception_count);
1485
1486 mce_gather_info(&m, regs);
1487 m.tsc = rdtsc();
1488
1489 final = this_cpu_ptr(&mces_seen);
1490 *final = m;
1491
1492 no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs);
1493
1494 barrier();
1495
1496 /*
1497 * When no restart IP might need to kill or panic.
1498 * Assume the worst for now, but if we find the
1499 * severity is MCE_AR_SEVERITY we have other options.
1500 */
1501 if (!(m.mcgstatus & MCG_STATUS_RIPV))
1502 kill_current_task = 1;
1503 /*
1504 * Check if this MCE is signaled to only this logical processor,
1505 * on Intel, Zhaoxin only.
1506 */
1507 if (m.cpuvendor == X86_VENDOR_INTEL ||
1508 m.cpuvendor == X86_VENDOR_ZHAOXIN)
1509 lmce = m.mcgstatus & MCG_STATUS_LMCES;
1510
1511 /*
1512 * Local machine check may already know that we have to panic.
1513 * Broadcast machine check begins rendezvous in mce_start()
1514 * Go through all banks in exclusion of the other CPUs. This way we
1515 * don't report duplicated events on shared banks because the first one
1516 * to see it will clear it.
1517 */
1518 if (lmce) {
1519 if (no_way_out)
1520 mce_panic("Fatal local machine check", &m, msg);
1521 } else {
1522 order = mce_start(&no_way_out);
1523 }
1524
1525 taint = __mc_scan_banks(&m, regs, final, toclear, valid_banks, no_way_out, &worst);
1526
1527 if (!no_way_out)
1528 mce_clear_state(toclear);
1529
1530 /*
1531 * Do most of the synchronization with other CPUs.
1532 * When there's any problem use only local no_way_out state.
1533 */
1534 if (!lmce) {
1535 if (mce_end(order) < 0) {
1536 if (!no_way_out)
1537 no_way_out = worst >= MCE_PANIC_SEVERITY;
1538
1539 if (no_way_out)
1540 mce_panic("Fatal machine check on current CPU", &m, msg);
1541 }
1542 } else {
1543 /*
1544 * If there was a fatal machine check we should have
1545 * already called mce_panic earlier in this function.
1546 * Since we re-read the banks, we might have found
1547 * something new. Check again to see if we found a
1548 * fatal error. We call "mce_severity()" again to
1549 * make sure we have the right "msg".
1550 */
1551 if (worst >= MCE_PANIC_SEVERITY) {
1552 mce_severity(&m, regs, &msg, true);
1553 mce_panic("Local fatal machine check!", &m, msg);
1554 }
1555 }
1556
1557 /*
1558 * Enable instrumentation around the external facilities like task_work_add()
1559 * (via queue_task_work()), fixup_exception() etc. For now, that is. Fixing this
1560 * properly would need a lot more involved reorganization.
1561 */
1562 instrumentation_begin();
1563
1564 if (taint)
1565 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
1566
1567 if (worst != MCE_AR_SEVERITY && !kill_current_task)
1568 goto out;
1569
1570 /* Fault was in user mode and we need to take some action */
1571 if ((m.cs & 3) == 3) {
1572 /* If this triggers there is no way to recover. Die hard. */
1573 BUG_ON(!on_thread_stack() || !user_mode(regs));
1574
1575 if (!mce_usable_address(&m))
1576 queue_task_work(&m, msg, kill_me_now);
1577 else
1578 queue_task_work(&m, msg, kill_me_maybe);
1579
1580 } else {
1581 /*
1582 * Handle an MCE which has happened in kernel space but from
1583 * which the kernel can recover: ex_has_fault_handler() has
1584 * already verified that the rIP at which the error happened is
1585 * a rIP from which the kernel can recover (by jumping to
1586 * recovery code specified in _ASM_EXTABLE_FAULT()) and the
1587 * corresponding exception handler which would do that is the
1588 * proper one.
1589 */
1590 if (m.kflags & MCE_IN_KERNEL_RECOV) {
1591 if (!fixup_exception(regs, X86_TRAP_MC, 0, 0))
1592 mce_panic("Failed kernel mode recovery", &m, msg);
1593 }
1594
1595 if (m.kflags & MCE_IN_KERNEL_COPYIN)
1596 queue_task_work(&m, msg, kill_me_never);
1597 }
1598
1599 out:
1600 instrumentation_end();
1601
1602 clear:
1603 mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1604 }
1605 EXPORT_SYMBOL_GPL(do_machine_check);
1606
1607 #ifndef CONFIG_MEMORY_FAILURE
memory_failure(unsigned long pfn,int flags)1608 int memory_failure(unsigned long pfn, int flags)
1609 {
1610 /* mce_severity() should not hand us an ACTION_REQUIRED error */
1611 BUG_ON(flags & MF_ACTION_REQUIRED);
1612 pr_err("Uncorrected memory error in page 0x%lx ignored\n"
1613 "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n",
1614 pfn);
1615
1616 return 0;
1617 }
1618 #endif
1619
1620 /*
1621 * Periodic polling timer for "silent" machine check errors. If the
1622 * poller finds an MCE, poll 2x faster. When the poller finds no more
1623 * errors, poll 2x slower (up to check_interval seconds).
1624 */
1625 static unsigned long check_interval = INITIAL_CHECK_INTERVAL;
1626
1627 static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */
1628 static DEFINE_PER_CPU(struct timer_list, mce_timer);
1629
mce_adjust_timer_default(unsigned long interval)1630 static unsigned long mce_adjust_timer_default(unsigned long interval)
1631 {
1632 return interval;
1633 }
1634
1635 static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default;
1636
__start_timer(struct timer_list * t,unsigned long interval)1637 static void __start_timer(struct timer_list *t, unsigned long interval)
1638 {
1639 unsigned long when = jiffies + interval;
1640 unsigned long flags;
1641
1642 local_irq_save(flags);
1643
1644 if (!timer_pending(t) || time_before(when, t->expires))
1645 mod_timer(t, round_jiffies(when));
1646
1647 local_irq_restore(flags);
1648 }
1649
mc_poll_banks_default(void)1650 static void mc_poll_banks_default(void)
1651 {
1652 machine_check_poll(0, this_cpu_ptr(&mce_poll_banks));
1653 }
1654
1655 void (*mc_poll_banks)(void) = mc_poll_banks_default;
1656
mce_timer_fn(struct timer_list * t)1657 static void mce_timer_fn(struct timer_list *t)
1658 {
1659 struct timer_list *cpu_t = this_cpu_ptr(&mce_timer);
1660 unsigned long iv;
1661
1662 WARN_ON(cpu_t != t);
1663
1664 iv = __this_cpu_read(mce_next_interval);
1665
1666 if (mce_available(this_cpu_ptr(&cpu_info))) {
1667 mc_poll_banks();
1668
1669 if (mce_intel_cmci_poll()) {
1670 iv = mce_adjust_timer(iv);
1671 goto done;
1672 }
1673 }
1674
1675 /*
1676 * Alert userspace if needed. If we logged an MCE, reduce the polling
1677 * interval, otherwise increase the polling interval.
1678 */
1679 if (mce_notify_irq())
1680 iv = max(iv / 2, (unsigned long) HZ/100);
1681 else
1682 iv = min(iv * 2, round_jiffies_relative(check_interval * HZ));
1683
1684 done:
1685 __this_cpu_write(mce_next_interval, iv);
1686 __start_timer(t, iv);
1687 }
1688
1689 /*
1690 * Ensure that the timer is firing in @interval from now.
1691 */
mce_timer_kick(unsigned long interval)1692 void mce_timer_kick(unsigned long interval)
1693 {
1694 struct timer_list *t = this_cpu_ptr(&mce_timer);
1695 unsigned long iv = __this_cpu_read(mce_next_interval);
1696
1697 __start_timer(t, interval);
1698
1699 if (interval < iv)
1700 __this_cpu_write(mce_next_interval, interval);
1701 }
1702
1703 /* Must not be called in IRQ context where del_timer_sync() can deadlock */
mce_timer_delete_all(void)1704 static void mce_timer_delete_all(void)
1705 {
1706 int cpu;
1707
1708 for_each_online_cpu(cpu)
1709 del_timer_sync(&per_cpu(mce_timer, cpu));
1710 }
1711
1712 /*
1713 * Notify the user(s) about new machine check events.
1714 * Can be called from interrupt context, but not from machine check/NMI
1715 * context.
1716 */
mce_notify_irq(void)1717 int mce_notify_irq(void)
1718 {
1719 /* Not more than two messages every minute */
1720 static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);
1721
1722 if (test_and_clear_bit(0, &mce_need_notify)) {
1723 mce_work_trigger();
1724
1725 if (__ratelimit(&ratelimit))
1726 pr_info(HW_ERR "Machine check events logged\n");
1727
1728 return 1;
1729 }
1730 return 0;
1731 }
1732 EXPORT_SYMBOL_GPL(mce_notify_irq);
1733
__mcheck_cpu_mce_banks_init(void)1734 static void __mcheck_cpu_mce_banks_init(void)
1735 {
1736 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1737 u8 n_banks = this_cpu_read(mce_num_banks);
1738 int i;
1739
1740 for (i = 0; i < n_banks; i++) {
1741 struct mce_bank *b = &mce_banks[i];
1742
1743 /*
1744 * Init them all, __mcheck_cpu_apply_quirks() is going to apply
1745 * the required vendor quirks before
1746 * __mcheck_cpu_init_clear_banks() does the final bank setup.
1747 */
1748 b->ctl = -1ULL;
1749 b->init = true;
1750 }
1751 }
1752
1753 /*
1754 * Initialize Machine Checks for a CPU.
1755 */
__mcheck_cpu_cap_init(void)1756 static void __mcheck_cpu_cap_init(void)
1757 {
1758 u64 cap;
1759 u8 b;
1760
1761 rdmsrl(MSR_IA32_MCG_CAP, cap);
1762
1763 b = cap & MCG_BANKCNT_MASK;
1764
1765 if (b > MAX_NR_BANKS) {
1766 pr_warn("CPU%d: Using only %u machine check banks out of %u\n",
1767 smp_processor_id(), MAX_NR_BANKS, b);
1768 b = MAX_NR_BANKS;
1769 }
1770
1771 this_cpu_write(mce_num_banks, b);
1772
1773 __mcheck_cpu_mce_banks_init();
1774
1775 /* Use accurate RIP reporting if available. */
1776 if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1777 mca_cfg.rip_msr = MSR_IA32_MCG_EIP;
1778
1779 if (cap & MCG_SER_P)
1780 mca_cfg.ser = 1;
1781 }
1782
__mcheck_cpu_init_generic(void)1783 static void __mcheck_cpu_init_generic(void)
1784 {
1785 enum mcp_flags m_fl = 0;
1786 mce_banks_t all_banks;
1787 u64 cap;
1788
1789 if (!mca_cfg.bootlog)
1790 m_fl = MCP_DONTLOG;
1791
1792 /*
1793 * Log the machine checks left over from the previous reset. Log them
1794 * only, do not start processing them. That will happen in mcheck_late_init()
1795 * when all consumers have been registered on the notifier chain.
1796 */
1797 bitmap_fill(all_banks, MAX_NR_BANKS);
1798 machine_check_poll(MCP_UC | MCP_QUEUE_LOG | m_fl, &all_banks);
1799
1800 cr4_set_bits(X86_CR4_MCE);
1801
1802 rdmsrl(MSR_IA32_MCG_CAP, cap);
1803 if (cap & MCG_CTL_P)
1804 wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
1805 }
1806
__mcheck_cpu_init_clear_banks(void)1807 static void __mcheck_cpu_init_clear_banks(void)
1808 {
1809 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1810 int i;
1811
1812 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1813 struct mce_bank *b = &mce_banks[i];
1814
1815 if (!b->init)
1816 continue;
1817 wrmsrl(mca_msr_reg(i, MCA_CTL), b->ctl);
1818 wrmsrl(mca_msr_reg(i, MCA_STATUS), 0);
1819 }
1820 }
1821
1822 /*
1823 * Do a final check to see if there are any unused/RAZ banks.
1824 *
1825 * This must be done after the banks have been initialized and any quirks have
1826 * been applied.
1827 *
1828 * Do not call this from any user-initiated flows, e.g. CPU hotplug or sysfs.
1829 * Otherwise, a user who disables a bank will not be able to re-enable it
1830 * without a system reboot.
1831 */
__mcheck_cpu_check_banks(void)1832 static void __mcheck_cpu_check_banks(void)
1833 {
1834 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1835 u64 msrval;
1836 int i;
1837
1838 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1839 struct mce_bank *b = &mce_banks[i];
1840
1841 if (!b->init)
1842 continue;
1843
1844 rdmsrl(mca_msr_reg(i, MCA_CTL), msrval);
1845 b->init = !!msrval;
1846 }
1847 }
1848
1849 /* Add per CPU specific workarounds here */
__mcheck_cpu_apply_quirks(struct cpuinfo_x86 * c)1850 static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1851 {
1852 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1853 struct mca_config *cfg = &mca_cfg;
1854
1855 if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
1856 pr_info("unknown CPU type - not enabling MCE support\n");
1857 return -EOPNOTSUPP;
1858 }
1859
1860 /* This should be disabled by the BIOS, but isn't always */
1861 if (c->x86_vendor == X86_VENDOR_AMD) {
1862 if (c->x86 == 15 && this_cpu_read(mce_num_banks) > 4) {
1863 /*
1864 * disable GART TBL walk error reporting, which
1865 * trips off incorrectly with the IOMMU & 3ware
1866 * & Cerberus:
1867 */
1868 clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
1869 }
1870 if (c->x86 < 0x11 && cfg->bootlog < 0) {
1871 /*
1872 * Lots of broken BIOS around that don't clear them
1873 * by default and leave crap in there. Don't log:
1874 */
1875 cfg->bootlog = 0;
1876 }
1877 /*
1878 * Various K7s with broken bank 0 around. Always disable
1879 * by default.
1880 */
1881 if (c->x86 == 6 && this_cpu_read(mce_num_banks) > 0)
1882 mce_banks[0].ctl = 0;
1883
1884 /*
1885 * overflow_recov is supported for F15h Models 00h-0fh
1886 * even though we don't have a CPUID bit for it.
1887 */
1888 if (c->x86 == 0x15 && c->x86_model <= 0xf)
1889 mce_flags.overflow_recov = 1;
1890
1891 if (c->x86 >= 0x17 && c->x86 <= 0x1A)
1892 mce_flags.zen_ifu_quirk = 1;
1893
1894 }
1895
1896 if (c->x86_vendor == X86_VENDOR_INTEL) {
1897 /*
1898 * SDM documents that on family 6 bank 0 should not be written
1899 * because it aliases to another special BIOS controlled
1900 * register.
1901 * But it's not aliased anymore on model 0x1a+
1902 * Don't ignore bank 0 completely because there could be a
1903 * valid event later, merely don't write CTL0.
1904 */
1905
1906 if (c->x86 == 6 && c->x86_model < 0x1A && this_cpu_read(mce_num_banks) > 0)
1907 mce_banks[0].init = false;
1908
1909 /*
1910 * All newer Intel systems support MCE broadcasting. Enable
1911 * synchronization with a one second timeout.
1912 */
1913 if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
1914 cfg->monarch_timeout < 0)
1915 cfg->monarch_timeout = USEC_PER_SEC;
1916
1917 /*
1918 * There are also broken BIOSes on some Pentium M and
1919 * earlier systems:
1920 */
1921 if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0)
1922 cfg->bootlog = 0;
1923
1924 if (c->x86 == 6 && c->x86_model == 45)
1925 mce_flags.snb_ifu_quirk = 1;
1926
1927 /*
1928 * Skylake, Cascacde Lake and Cooper Lake require a quirk on
1929 * rep movs.
1930 */
1931 if (c->x86 == 6 && c->x86_model == INTEL_FAM6_SKYLAKE_X)
1932 mce_flags.skx_repmov_quirk = 1;
1933 }
1934
1935 if (c->x86_vendor == X86_VENDOR_ZHAOXIN) {
1936 /*
1937 * All newer Zhaoxin CPUs support MCE broadcasting. Enable
1938 * synchronization with a one second timeout.
1939 */
1940 if (c->x86 > 6 || (c->x86_model == 0x19 || c->x86_model == 0x1f)) {
1941 if (cfg->monarch_timeout < 0)
1942 cfg->monarch_timeout = USEC_PER_SEC;
1943 }
1944 }
1945
1946 if (cfg->monarch_timeout < 0)
1947 cfg->monarch_timeout = 0;
1948 if (cfg->bootlog != 0)
1949 cfg->panic_timeout = 30;
1950
1951 return 0;
1952 }
1953
__mcheck_cpu_ancient_init(struct cpuinfo_x86 * c)1954 static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
1955 {
1956 if (c->x86 != 5)
1957 return 0;
1958
1959 switch (c->x86_vendor) {
1960 case X86_VENDOR_INTEL:
1961 intel_p5_mcheck_init(c);
1962 mce_flags.p5 = 1;
1963 return 1;
1964 case X86_VENDOR_CENTAUR:
1965 winchip_mcheck_init(c);
1966 mce_flags.winchip = 1;
1967 return 1;
1968 default:
1969 return 0;
1970 }
1971
1972 return 0;
1973 }
1974
1975 /*
1976 * Init basic CPU features needed for early decoding of MCEs.
1977 */
__mcheck_cpu_init_early(struct cpuinfo_x86 * c)1978 static void __mcheck_cpu_init_early(struct cpuinfo_x86 *c)
1979 {
1980 if (c->x86_vendor == X86_VENDOR_AMD || c->x86_vendor == X86_VENDOR_HYGON) {
1981 mce_flags.overflow_recov = !!cpu_has(c, X86_FEATURE_OVERFLOW_RECOV);
1982 mce_flags.succor = !!cpu_has(c, X86_FEATURE_SUCCOR);
1983 mce_flags.smca = !!cpu_has(c, X86_FEATURE_SMCA);
1984 mce_flags.amd_threshold = 1;
1985 }
1986 }
1987
mce_centaur_feature_init(struct cpuinfo_x86 * c)1988 static void mce_centaur_feature_init(struct cpuinfo_x86 *c)
1989 {
1990 struct mca_config *cfg = &mca_cfg;
1991
1992 /*
1993 * All newer Centaur CPUs support MCE broadcasting. Enable
1994 * synchronization with a one second timeout.
1995 */
1996 if ((c->x86 == 6 && c->x86_model == 0xf && c->x86_stepping >= 0xe) ||
1997 c->x86 > 6) {
1998 if (cfg->monarch_timeout < 0)
1999 cfg->monarch_timeout = USEC_PER_SEC;
2000 }
2001 }
2002
mce_zhaoxin_feature_init(struct cpuinfo_x86 * c)2003 static void mce_zhaoxin_feature_init(struct cpuinfo_x86 *c)
2004 {
2005 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
2006
2007 /*
2008 * These CPUs have MCA bank 8 which reports only one error type called
2009 * SVAD (System View Address Decoder). The reporting of that error is
2010 * controlled by IA32_MC8.CTL.0.
2011 *
2012 * If enabled, prefetching on these CPUs will cause SVAD MCE when
2013 * virtual machines start and result in a system panic. Always disable
2014 * bank 8 SVAD error by default.
2015 */
2016 if ((c->x86 == 7 && c->x86_model == 0x1b) ||
2017 (c->x86_model == 0x19 || c->x86_model == 0x1f)) {
2018 if (this_cpu_read(mce_num_banks) > 8)
2019 mce_banks[8].ctl = 0;
2020 }
2021
2022 intel_init_cmci();
2023 intel_init_lmce();
2024 mce_adjust_timer = cmci_intel_adjust_timer;
2025 }
2026
mce_zhaoxin_feature_clear(struct cpuinfo_x86 * c)2027 static void mce_zhaoxin_feature_clear(struct cpuinfo_x86 *c)
2028 {
2029 intel_clear_lmce();
2030 }
2031
__mcheck_cpu_init_vendor(struct cpuinfo_x86 * c)2032 static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
2033 {
2034 switch (c->x86_vendor) {
2035 case X86_VENDOR_INTEL:
2036 mce_intel_feature_init(c);
2037 mce_adjust_timer = cmci_intel_adjust_timer;
2038 break;
2039
2040 case X86_VENDOR_AMD: {
2041 mce_amd_feature_init(c);
2042 break;
2043 }
2044
2045 case X86_VENDOR_HYGON:
2046 mce_hygon_feature_init(c);
2047 break;
2048
2049 case X86_VENDOR_CENTAUR:
2050 mce_centaur_feature_init(c);
2051 break;
2052
2053 case X86_VENDOR_ZHAOXIN:
2054 mce_zhaoxin_feature_init(c);
2055 break;
2056
2057 default:
2058 break;
2059 }
2060 }
2061
__mcheck_cpu_clear_vendor(struct cpuinfo_x86 * c)2062 static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c)
2063 {
2064 switch (c->x86_vendor) {
2065 case X86_VENDOR_INTEL:
2066 mce_intel_feature_clear(c);
2067 break;
2068
2069 case X86_VENDOR_ZHAOXIN:
2070 mce_zhaoxin_feature_clear(c);
2071 break;
2072
2073 default:
2074 break;
2075 }
2076 }
2077
mce_start_timer(struct timer_list * t)2078 static void mce_start_timer(struct timer_list *t)
2079 {
2080 unsigned long iv = check_interval * HZ;
2081
2082 if (mca_cfg.ignore_ce || !iv)
2083 return;
2084
2085 this_cpu_write(mce_next_interval, iv);
2086 __start_timer(t, iv);
2087 }
2088
__mcheck_cpu_setup_timer(void)2089 static void __mcheck_cpu_setup_timer(void)
2090 {
2091 struct timer_list *t = this_cpu_ptr(&mce_timer);
2092
2093 timer_setup(t, mce_timer_fn, TIMER_PINNED);
2094 }
2095
__mcheck_cpu_init_timer(void)2096 static void __mcheck_cpu_init_timer(void)
2097 {
2098 struct timer_list *t = this_cpu_ptr(&mce_timer);
2099
2100 timer_setup(t, mce_timer_fn, TIMER_PINNED);
2101 mce_start_timer(t);
2102 }
2103
filter_mce(struct mce * m)2104 bool filter_mce(struct mce *m)
2105 {
2106 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
2107 return amd_filter_mce(m);
2108 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
2109 return intel_filter_mce(m);
2110
2111 return false;
2112 }
2113
exc_machine_check_kernel(struct pt_regs * regs)2114 static __always_inline void exc_machine_check_kernel(struct pt_regs *regs)
2115 {
2116 irqentry_state_t irq_state;
2117
2118 WARN_ON_ONCE(user_mode(regs));
2119
2120 /*
2121 * Only required when from kernel mode. See
2122 * mce_check_crashing_cpu() for details.
2123 */
2124 if (mca_cfg.initialized && mce_check_crashing_cpu())
2125 return;
2126
2127 irq_state = irqentry_nmi_enter(regs);
2128
2129 do_machine_check(regs);
2130
2131 irqentry_nmi_exit(regs, irq_state);
2132 }
2133
exc_machine_check_user(struct pt_regs * regs)2134 static __always_inline void exc_machine_check_user(struct pt_regs *regs)
2135 {
2136 irqentry_enter_from_user_mode(regs);
2137
2138 do_machine_check(regs);
2139
2140 irqentry_exit_to_user_mode(regs);
2141 }
2142
2143 #ifdef CONFIG_X86_64
2144 /* MCE hit kernel mode */
DEFINE_IDTENTRY_MCE(exc_machine_check)2145 DEFINE_IDTENTRY_MCE(exc_machine_check)
2146 {
2147 unsigned long dr7;
2148
2149 dr7 = local_db_save();
2150 exc_machine_check_kernel(regs);
2151 local_db_restore(dr7);
2152 }
2153
2154 /* The user mode variant. */
DEFINE_IDTENTRY_MCE_USER(exc_machine_check)2155 DEFINE_IDTENTRY_MCE_USER(exc_machine_check)
2156 {
2157 unsigned long dr7;
2158
2159 dr7 = local_db_save();
2160 exc_machine_check_user(regs);
2161 local_db_restore(dr7);
2162 }
2163 #else
2164 /* 32bit unified entry point */
DEFINE_IDTENTRY_RAW(exc_machine_check)2165 DEFINE_IDTENTRY_RAW(exc_machine_check)
2166 {
2167 unsigned long dr7;
2168
2169 dr7 = local_db_save();
2170 if (user_mode(regs))
2171 exc_machine_check_user(regs);
2172 else
2173 exc_machine_check_kernel(regs);
2174 local_db_restore(dr7);
2175 }
2176 #endif
2177
2178 /*
2179 * Called for each booted CPU to set up machine checks.
2180 * Must be called with preempt off:
2181 */
mcheck_cpu_init(struct cpuinfo_x86 * c)2182 void mcheck_cpu_init(struct cpuinfo_x86 *c)
2183 {
2184 if (mca_cfg.disabled)
2185 return;
2186
2187 if (__mcheck_cpu_ancient_init(c))
2188 return;
2189
2190 if (!mce_available(c))
2191 return;
2192
2193 __mcheck_cpu_cap_init();
2194
2195 if (__mcheck_cpu_apply_quirks(c) < 0) {
2196 mca_cfg.disabled = 1;
2197 return;
2198 }
2199
2200 if (mce_gen_pool_init()) {
2201 mca_cfg.disabled = 1;
2202 pr_emerg("Couldn't allocate MCE records pool!\n");
2203 return;
2204 }
2205
2206 mca_cfg.initialized = 1;
2207
2208 __mcheck_cpu_init_early(c);
2209 __mcheck_cpu_init_generic();
2210 __mcheck_cpu_init_vendor(c);
2211 __mcheck_cpu_init_clear_banks();
2212 __mcheck_cpu_check_banks();
2213 __mcheck_cpu_setup_timer();
2214 }
2215
2216 /*
2217 * Called for each booted CPU to clear some machine checks opt-ins
2218 */
mcheck_cpu_clear(struct cpuinfo_x86 * c)2219 void mcheck_cpu_clear(struct cpuinfo_x86 *c)
2220 {
2221 if (mca_cfg.disabled)
2222 return;
2223
2224 if (!mce_available(c))
2225 return;
2226
2227 /*
2228 * Possibly to clear general settings generic to x86
2229 * __mcheck_cpu_clear_generic(c);
2230 */
2231 __mcheck_cpu_clear_vendor(c);
2232
2233 }
2234
__mce_disable_bank(void * arg)2235 static void __mce_disable_bank(void *arg)
2236 {
2237 int bank = *((int *)arg);
2238 __clear_bit(bank, this_cpu_ptr(mce_poll_banks));
2239 cmci_disable_bank(bank);
2240 }
2241
mce_disable_bank(int bank)2242 void mce_disable_bank(int bank)
2243 {
2244 if (bank >= this_cpu_read(mce_num_banks)) {
2245 pr_warn(FW_BUG
2246 "Ignoring request to disable invalid MCA bank %d.\n",
2247 bank);
2248 return;
2249 }
2250 set_bit(bank, mce_banks_ce_disabled);
2251 on_each_cpu(__mce_disable_bank, &bank, 1);
2252 }
2253
2254 /*
2255 * mce=off Disables machine check
2256 * mce=no_cmci Disables CMCI
2257 * mce=no_lmce Disables LMCE
2258 * mce=dont_log_ce Clears corrected events silently, no log created for CEs.
2259 * mce=print_all Print all machine check logs to console
2260 * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared.
2261 * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above)
2262 * monarchtimeout is how long to wait for other CPUs on machine
2263 * check, or 0 to not wait
2264 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD Fam10h
2265 and older.
2266 * mce=nobootlog Don't log MCEs from before booting.
2267 * mce=bios_cmci_threshold Don't program the CMCI threshold
2268 * mce=recovery force enable copy_mc_fragile()
2269 */
mcheck_enable(char * str)2270 static int __init mcheck_enable(char *str)
2271 {
2272 struct mca_config *cfg = &mca_cfg;
2273
2274 if (*str == 0) {
2275 enable_p5_mce();
2276 return 1;
2277 }
2278 if (*str == '=')
2279 str++;
2280 if (!strcmp(str, "off"))
2281 cfg->disabled = 1;
2282 else if (!strcmp(str, "no_cmci"))
2283 cfg->cmci_disabled = true;
2284 else if (!strcmp(str, "no_lmce"))
2285 cfg->lmce_disabled = 1;
2286 else if (!strcmp(str, "dont_log_ce"))
2287 cfg->dont_log_ce = true;
2288 else if (!strcmp(str, "print_all"))
2289 cfg->print_all = true;
2290 else if (!strcmp(str, "ignore_ce"))
2291 cfg->ignore_ce = true;
2292 else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
2293 cfg->bootlog = (str[0] == 'b');
2294 else if (!strcmp(str, "bios_cmci_threshold"))
2295 cfg->bios_cmci_threshold = 1;
2296 else if (!strcmp(str, "recovery"))
2297 cfg->recovery = 1;
2298 else if (isdigit(str[0]))
2299 get_option(&str, &(cfg->monarch_timeout));
2300 else {
2301 pr_info("mce argument %s ignored. Please use /sys\n", str);
2302 return 0;
2303 }
2304 return 1;
2305 }
2306 __setup("mce", mcheck_enable);
2307
mcheck_init(void)2308 int __init mcheck_init(void)
2309 {
2310 mce_register_decode_chain(&early_nb);
2311 mce_register_decode_chain(&mce_uc_nb);
2312 mce_register_decode_chain(&mce_default_nb);
2313
2314 INIT_WORK(&mce_work, mce_gen_pool_process);
2315 init_irq_work(&mce_irq_work, mce_irq_work_cb);
2316
2317 return 0;
2318 }
2319
2320 /*
2321 * mce_syscore: PM support
2322 */
2323
2324 /*
2325 * Disable machine checks on suspend and shutdown. We can't really handle
2326 * them later.
2327 */
mce_disable_error_reporting(void)2328 static void mce_disable_error_reporting(void)
2329 {
2330 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
2331 int i;
2332
2333 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
2334 struct mce_bank *b = &mce_banks[i];
2335
2336 if (b->init)
2337 wrmsrl(mca_msr_reg(i, MCA_CTL), 0);
2338 }
2339 return;
2340 }
2341
vendor_disable_error_reporting(void)2342 static void vendor_disable_error_reporting(void)
2343 {
2344 /*
2345 * Don't clear on Intel or AMD or Hygon or Zhaoxin CPUs. Some of these
2346 * MSRs are socket-wide. Disabling them for just a single offlined CPU
2347 * is bad, since it will inhibit reporting for all shared resources on
2348 * the socket like the last level cache (LLC), the integrated memory
2349 * controller (iMC), etc.
2350 */
2351 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL ||
2352 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ||
2353 boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
2354 boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN)
2355 return;
2356
2357 mce_disable_error_reporting();
2358 }
2359
mce_syscore_suspend(void)2360 static int mce_syscore_suspend(void)
2361 {
2362 vendor_disable_error_reporting();
2363 return 0;
2364 }
2365
mce_syscore_shutdown(void)2366 static void mce_syscore_shutdown(void)
2367 {
2368 vendor_disable_error_reporting();
2369 }
2370
2371 /*
2372 * On resume clear all MCE state. Don't want to see leftovers from the BIOS.
2373 * Only one CPU is active at this time, the others get re-added later using
2374 * CPU hotplug:
2375 */
mce_syscore_resume(void)2376 static void mce_syscore_resume(void)
2377 {
2378 __mcheck_cpu_init_generic();
2379 __mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info));
2380 __mcheck_cpu_init_clear_banks();
2381 }
2382
2383 static struct syscore_ops mce_syscore_ops = {
2384 .suspend = mce_syscore_suspend,
2385 .shutdown = mce_syscore_shutdown,
2386 .resume = mce_syscore_resume,
2387 };
2388
2389 /*
2390 * mce_device: Sysfs support
2391 */
2392
mce_cpu_restart(void * data)2393 static void mce_cpu_restart(void *data)
2394 {
2395 if (!mce_available(raw_cpu_ptr(&cpu_info)))
2396 return;
2397 __mcheck_cpu_init_generic();
2398 __mcheck_cpu_init_clear_banks();
2399 __mcheck_cpu_init_timer();
2400 }
2401
2402 /* Reinit MCEs after user configuration changes */
mce_restart(void)2403 static void mce_restart(void)
2404 {
2405 mce_timer_delete_all();
2406 on_each_cpu(mce_cpu_restart, NULL, 1);
2407 mce_schedule_work();
2408 }
2409
2410 /* Toggle features for corrected errors */
mce_disable_cmci(void * data)2411 static void mce_disable_cmci(void *data)
2412 {
2413 if (!mce_available(raw_cpu_ptr(&cpu_info)))
2414 return;
2415 cmci_clear();
2416 }
2417
mce_enable_ce(void * all)2418 static void mce_enable_ce(void *all)
2419 {
2420 if (!mce_available(raw_cpu_ptr(&cpu_info)))
2421 return;
2422 cmci_reenable();
2423 cmci_recheck();
2424 if (all)
2425 __mcheck_cpu_init_timer();
2426 }
2427
2428 static struct bus_type mce_subsys = {
2429 .name = "machinecheck",
2430 .dev_name = "machinecheck",
2431 };
2432
2433 DEFINE_PER_CPU(struct device *, mce_device);
2434
attr_to_bank(struct device_attribute * attr)2435 static inline struct mce_bank_dev *attr_to_bank(struct device_attribute *attr)
2436 {
2437 return container_of(attr, struct mce_bank_dev, attr);
2438 }
2439
show_bank(struct device * s,struct device_attribute * attr,char * buf)2440 static ssize_t show_bank(struct device *s, struct device_attribute *attr,
2441 char *buf)
2442 {
2443 u8 bank = attr_to_bank(attr)->bank;
2444 struct mce_bank *b;
2445
2446 if (bank >= per_cpu(mce_num_banks, s->id))
2447 return -EINVAL;
2448
2449 b = &per_cpu(mce_banks_array, s->id)[bank];
2450
2451 if (!b->init)
2452 return -ENODEV;
2453
2454 return sprintf(buf, "%llx\n", b->ctl);
2455 }
2456
set_bank(struct device * s,struct device_attribute * attr,const char * buf,size_t size)2457 static ssize_t set_bank(struct device *s, struct device_attribute *attr,
2458 const char *buf, size_t size)
2459 {
2460 u8 bank = attr_to_bank(attr)->bank;
2461 struct mce_bank *b;
2462 u64 new;
2463
2464 if (kstrtou64(buf, 0, &new) < 0)
2465 return -EINVAL;
2466
2467 if (bank >= per_cpu(mce_num_banks, s->id))
2468 return -EINVAL;
2469
2470 b = &per_cpu(mce_banks_array, s->id)[bank];
2471
2472 if (!b->init)
2473 return -ENODEV;
2474
2475 b->ctl = new;
2476 mce_restart();
2477
2478 return size;
2479 }
2480
set_ignore_ce(struct device * s,struct device_attribute * attr,const char * buf,size_t size)2481 static ssize_t set_ignore_ce(struct device *s,
2482 struct device_attribute *attr,
2483 const char *buf, size_t size)
2484 {
2485 u64 new;
2486
2487 if (kstrtou64(buf, 0, &new) < 0)
2488 return -EINVAL;
2489
2490 mutex_lock(&mce_sysfs_mutex);
2491 if (mca_cfg.ignore_ce ^ !!new) {
2492 if (new) {
2493 /* disable ce features */
2494 mce_timer_delete_all();
2495 on_each_cpu(mce_disable_cmci, NULL, 1);
2496 mca_cfg.ignore_ce = true;
2497 } else {
2498 /* enable ce features */
2499 mca_cfg.ignore_ce = false;
2500 on_each_cpu(mce_enable_ce, (void *)1, 1);
2501 }
2502 }
2503 mutex_unlock(&mce_sysfs_mutex);
2504
2505 return size;
2506 }
2507
set_cmci_disabled(struct device * s,struct device_attribute * attr,const char * buf,size_t size)2508 static ssize_t set_cmci_disabled(struct device *s,
2509 struct device_attribute *attr,
2510 const char *buf, size_t size)
2511 {
2512 u64 new;
2513
2514 if (kstrtou64(buf, 0, &new) < 0)
2515 return -EINVAL;
2516
2517 mutex_lock(&mce_sysfs_mutex);
2518 if (mca_cfg.cmci_disabled ^ !!new) {
2519 if (new) {
2520 /* disable cmci */
2521 on_each_cpu(mce_disable_cmci, NULL, 1);
2522 mca_cfg.cmci_disabled = true;
2523 } else {
2524 /* enable cmci */
2525 mca_cfg.cmci_disabled = false;
2526 on_each_cpu(mce_enable_ce, NULL, 1);
2527 }
2528 }
2529 mutex_unlock(&mce_sysfs_mutex);
2530
2531 return size;
2532 }
2533
store_int_with_restart(struct device * s,struct device_attribute * attr,const char * buf,size_t size)2534 static ssize_t store_int_with_restart(struct device *s,
2535 struct device_attribute *attr,
2536 const char *buf, size_t size)
2537 {
2538 unsigned long old_check_interval = check_interval;
2539 ssize_t ret = device_store_ulong(s, attr, buf, size);
2540
2541 if (check_interval == old_check_interval)
2542 return ret;
2543
2544 mutex_lock(&mce_sysfs_mutex);
2545 mce_restart();
2546 mutex_unlock(&mce_sysfs_mutex);
2547
2548 return ret;
2549 }
2550
2551 static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout);
2552 static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
2553 static DEVICE_BOOL_ATTR(print_all, 0644, mca_cfg.print_all);
2554
2555 static struct dev_ext_attribute dev_attr_check_interval = {
2556 __ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
2557 &check_interval
2558 };
2559
2560 static struct dev_ext_attribute dev_attr_ignore_ce = {
2561 __ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
2562 &mca_cfg.ignore_ce
2563 };
2564
2565 static struct dev_ext_attribute dev_attr_cmci_disabled = {
2566 __ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
2567 &mca_cfg.cmci_disabled
2568 };
2569
2570 static struct device_attribute *mce_device_attrs[] = {
2571 &dev_attr_check_interval.attr,
2572 #ifdef CONFIG_X86_MCELOG_LEGACY
2573 &dev_attr_trigger,
2574 #endif
2575 &dev_attr_monarch_timeout.attr,
2576 &dev_attr_dont_log_ce.attr,
2577 &dev_attr_print_all.attr,
2578 &dev_attr_ignore_ce.attr,
2579 &dev_attr_cmci_disabled.attr,
2580 NULL
2581 };
2582
2583 static cpumask_var_t mce_device_initialized;
2584
mce_device_release(struct device * dev)2585 static void mce_device_release(struct device *dev)
2586 {
2587 kfree(dev);
2588 }
2589
2590 /* Per CPU device init. All of the CPUs still share the same bank device: */
mce_device_create(unsigned int cpu)2591 static int mce_device_create(unsigned int cpu)
2592 {
2593 struct device *dev;
2594 int err;
2595 int i, j;
2596
2597 if (!mce_available(&boot_cpu_data))
2598 return -EIO;
2599
2600 dev = per_cpu(mce_device, cpu);
2601 if (dev)
2602 return 0;
2603
2604 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2605 if (!dev)
2606 return -ENOMEM;
2607 dev->id = cpu;
2608 dev->bus = &mce_subsys;
2609 dev->release = &mce_device_release;
2610
2611 err = device_register(dev);
2612 if (err) {
2613 put_device(dev);
2614 return err;
2615 }
2616
2617 for (i = 0; mce_device_attrs[i]; i++) {
2618 err = device_create_file(dev, mce_device_attrs[i]);
2619 if (err)
2620 goto error;
2621 }
2622 for (j = 0; j < per_cpu(mce_num_banks, cpu); j++) {
2623 err = device_create_file(dev, &mce_bank_devs[j].attr);
2624 if (err)
2625 goto error2;
2626 }
2627 cpumask_set_cpu(cpu, mce_device_initialized);
2628 per_cpu(mce_device, cpu) = dev;
2629
2630 return 0;
2631 error2:
2632 while (--j >= 0)
2633 device_remove_file(dev, &mce_bank_devs[j].attr);
2634 error:
2635 while (--i >= 0)
2636 device_remove_file(dev, mce_device_attrs[i]);
2637
2638 device_unregister(dev);
2639
2640 return err;
2641 }
2642
mce_device_remove(unsigned int cpu)2643 static void mce_device_remove(unsigned int cpu)
2644 {
2645 struct device *dev = per_cpu(mce_device, cpu);
2646 int i;
2647
2648 if (!cpumask_test_cpu(cpu, mce_device_initialized))
2649 return;
2650
2651 for (i = 0; mce_device_attrs[i]; i++)
2652 device_remove_file(dev, mce_device_attrs[i]);
2653
2654 for (i = 0; i < per_cpu(mce_num_banks, cpu); i++)
2655 device_remove_file(dev, &mce_bank_devs[i].attr);
2656
2657 device_unregister(dev);
2658 cpumask_clear_cpu(cpu, mce_device_initialized);
2659 per_cpu(mce_device, cpu) = NULL;
2660 }
2661
2662 /* Make sure there are no machine checks on offlined CPUs. */
mce_disable_cpu(void)2663 static void mce_disable_cpu(void)
2664 {
2665 if (!mce_available(raw_cpu_ptr(&cpu_info)))
2666 return;
2667
2668 if (!cpuhp_tasks_frozen)
2669 cmci_clear();
2670
2671 vendor_disable_error_reporting();
2672 }
2673
mce_reenable_cpu(void)2674 static void mce_reenable_cpu(void)
2675 {
2676 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
2677 int i;
2678
2679 if (!mce_available(raw_cpu_ptr(&cpu_info)))
2680 return;
2681
2682 if (!cpuhp_tasks_frozen)
2683 cmci_reenable();
2684 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
2685 struct mce_bank *b = &mce_banks[i];
2686
2687 if (b->init)
2688 wrmsrl(mca_msr_reg(i, MCA_CTL), b->ctl);
2689 }
2690 }
2691
mce_cpu_dead(unsigned int cpu)2692 static int mce_cpu_dead(unsigned int cpu)
2693 {
2694 mce_intel_hcpu_update(cpu);
2695
2696 /* intentionally ignoring frozen here */
2697 if (!cpuhp_tasks_frozen)
2698 cmci_rediscover();
2699 return 0;
2700 }
2701
mce_cpu_online(unsigned int cpu)2702 static int mce_cpu_online(unsigned int cpu)
2703 {
2704 struct timer_list *t = this_cpu_ptr(&mce_timer);
2705 int ret;
2706
2707 mce_device_create(cpu);
2708
2709 ret = mce_threshold_create_device(cpu);
2710 if (ret) {
2711 mce_device_remove(cpu);
2712 return ret;
2713 }
2714 mce_reenable_cpu();
2715 mce_start_timer(t);
2716 return 0;
2717 }
2718
mce_cpu_pre_down(unsigned int cpu)2719 static int mce_cpu_pre_down(unsigned int cpu)
2720 {
2721 struct timer_list *t = this_cpu_ptr(&mce_timer);
2722
2723 mce_disable_cpu();
2724 del_timer_sync(t);
2725 mce_threshold_remove_device(cpu);
2726 mce_device_remove(cpu);
2727 return 0;
2728 }
2729
mce_init_banks(void)2730 static __init void mce_init_banks(void)
2731 {
2732 int i;
2733
2734 for (i = 0; i < MAX_NR_BANKS; i++) {
2735 struct mce_bank_dev *b = &mce_bank_devs[i];
2736 struct device_attribute *a = &b->attr;
2737
2738 b->bank = i;
2739
2740 sysfs_attr_init(&a->attr);
2741 a->attr.name = b->attrname;
2742 snprintf(b->attrname, ATTR_LEN, "bank%d", i);
2743
2744 a->attr.mode = 0644;
2745 a->show = show_bank;
2746 a->store = set_bank;
2747 }
2748 }
2749
2750 /*
2751 * When running on XEN, this initcall is ordered against the XEN mcelog
2752 * initcall:
2753 *
2754 * device_initcall(xen_late_init_mcelog);
2755 * device_initcall_sync(mcheck_init_device);
2756 */
mcheck_init_device(void)2757 static __init int mcheck_init_device(void)
2758 {
2759 int err;
2760
2761 /*
2762 * Check if we have a spare virtual bit. This will only become
2763 * a problem if/when we move beyond 5-level page tables.
2764 */
2765 MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63);
2766
2767 if (!mce_available(&boot_cpu_data)) {
2768 err = -EIO;
2769 goto err_out;
2770 }
2771
2772 if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) {
2773 err = -ENOMEM;
2774 goto err_out;
2775 }
2776
2777 mce_init_banks();
2778
2779 err = subsys_system_register(&mce_subsys, NULL);
2780 if (err)
2781 goto err_out_mem;
2782
2783 err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL,
2784 mce_cpu_dead);
2785 if (err)
2786 goto err_out_mem;
2787
2788 /*
2789 * Invokes mce_cpu_online() on all CPUs which are online when
2790 * the state is installed.
2791 */
2792 err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
2793 mce_cpu_online, mce_cpu_pre_down);
2794 if (err < 0)
2795 goto err_out_online;
2796
2797 register_syscore_ops(&mce_syscore_ops);
2798
2799 return 0;
2800
2801 err_out_online:
2802 cpuhp_remove_state(CPUHP_X86_MCE_DEAD);
2803
2804 err_out_mem:
2805 free_cpumask_var(mce_device_initialized);
2806
2807 err_out:
2808 pr_err("Unable to init MCE device (rc: %d)\n", err);
2809
2810 return err;
2811 }
2812 device_initcall_sync(mcheck_init_device);
2813
2814 /*
2815 * Old style boot options parsing. Only for compatibility.
2816 */
mcheck_disable(char * str)2817 static int __init mcheck_disable(char *str)
2818 {
2819 mca_cfg.disabled = 1;
2820 return 1;
2821 }
2822 __setup("nomce", mcheck_disable);
2823
2824 #ifdef CONFIG_DEBUG_FS
mce_get_debugfs_dir(void)2825 struct dentry *mce_get_debugfs_dir(void)
2826 {
2827 static struct dentry *dmce;
2828
2829 if (!dmce)
2830 dmce = debugfs_create_dir("mce", NULL);
2831
2832 return dmce;
2833 }
2834
mce_reset(void)2835 static void mce_reset(void)
2836 {
2837 atomic_set(&mce_fake_panicked, 0);
2838 atomic_set(&mce_executing, 0);
2839 atomic_set(&mce_callin, 0);
2840 atomic_set(&global_nwo, 0);
2841 cpumask_setall(&mce_missing_cpus);
2842 }
2843
fake_panic_get(void * data,u64 * val)2844 static int fake_panic_get(void *data, u64 *val)
2845 {
2846 *val = fake_panic;
2847 return 0;
2848 }
2849
fake_panic_set(void * data,u64 val)2850 static int fake_panic_set(void *data, u64 val)
2851 {
2852 mce_reset();
2853 fake_panic = val;
2854 return 0;
2855 }
2856
2857 DEFINE_DEBUGFS_ATTRIBUTE(fake_panic_fops, fake_panic_get, fake_panic_set,
2858 "%llu\n");
2859
mcheck_debugfs_init(void)2860 static void __init mcheck_debugfs_init(void)
2861 {
2862 struct dentry *dmce;
2863
2864 dmce = mce_get_debugfs_dir();
2865 debugfs_create_file_unsafe("fake_panic", 0444, dmce, NULL,
2866 &fake_panic_fops);
2867 }
2868 #else
mcheck_debugfs_init(void)2869 static void __init mcheck_debugfs_init(void) { }
2870 #endif
2871
mcheck_late_init(void)2872 static int __init mcheck_late_init(void)
2873 {
2874 if (mca_cfg.recovery)
2875 enable_copy_mc_fragile();
2876
2877 mcheck_debugfs_init();
2878
2879 /*
2880 * Flush out everything that has been logged during early boot, now that
2881 * everything has been initialized (workqueues, decoders, ...).
2882 */
2883 mce_schedule_work();
2884
2885 return 0;
2886 }
2887 late_initcall(mcheck_late_init);
2888