1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *	Local APIC handling, local APIC timers
4  *
5  *	(c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6  *
7  *	Fixes
8  *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs;
9  *					thanks to Eric Gilmore
10  *					and Rolf G. Tews
11  *					for testing these extensively.
12  *	Maciej W. Rozycki	:	Various updates and fixes.
13  *	Mikael Pettersson	:	Power Management for UP-APIC.
14  *	Pavel Machek and
15  *	Mikael Pettersson	:	PM converted to driver model.
16  */
17 
18 #include <linux/perf_event.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/mc146818rtc.h>
21 #include <linux/acpi_pmtmr.h>
22 #include <linux/clockchips.h>
23 #include <linux/interrupt.h>
24 #include <linux/memblock.h>
25 #include <linux/ftrace.h>
26 #include <linux/ioport.h>
27 #include <linux/export.h>
28 #include <linux/syscore_ops.h>
29 #include <linux/delay.h>
30 #include <linux/timex.h>
31 #include <linux/i8253.h>
32 #include <linux/dmar.h>
33 #include <linux/init.h>
34 #include <linux/cpu.h>
35 #include <linux/dmi.h>
36 #include <linux/smp.h>
37 #include <linux/mm.h>
38 
39 #include <xen/xen.h>
40 
41 #include <asm/trace/irq_vectors.h>
42 #include <asm/irq_remapping.h>
43 #include <asm/pc-conf-reg.h>
44 #include <asm/perf_event.h>
45 #include <asm/x86_init.h>
46 #include <linux/atomic.h>
47 #include <asm/barrier.h>
48 #include <asm/mpspec.h>
49 #include <asm/i8259.h>
50 #include <asm/proto.h>
51 #include <asm/traps.h>
52 #include <asm/apic.h>
53 #include <asm/acpi.h>
54 #include <asm/io_apic.h>
55 #include <asm/desc.h>
56 #include <asm/hpet.h>
57 #include <asm/mtrr.h>
58 #include <asm/time.h>
59 #include <asm/smp.h>
60 #include <asm/mce.h>
61 #include <asm/tsc.h>
62 #include <asm/hypervisor.h>
63 #include <asm/cpu_device_id.h>
64 #include <asm/intel-family.h>
65 #include <asm/irq_regs.h>
66 #include <asm/cpu.h>
67 
68 #include "local.h"
69 
70 unsigned int num_processors;
71 
72 unsigned disabled_cpus;
73 
74 /* Processor that is doing the boot up */
75 unsigned int boot_cpu_physical_apicid __ro_after_init = -1U;
76 EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
77 
78 u8 boot_cpu_apic_version __ro_after_init;
79 
80 /*
81  * Bitmask of physically existing CPUs:
82  */
83 physid_mask_t phys_cpu_present_map;
84 
85 /*
86  * Processor to be disabled specified by kernel parameter
87  * disable_cpu_apicid=<int>, mostly used for the kdump 2nd kernel to
88  * avoid undefined behaviour caused by sending INIT from AP to BSP.
89  */
90 static unsigned int disabled_cpu_apicid __ro_after_init = BAD_APICID;
91 
92 /*
93  * This variable controls which CPUs receive external NMIs.  By default,
94  * external NMIs are delivered only to the BSP.
95  */
96 static int apic_extnmi __ro_after_init = APIC_EXTNMI_BSP;
97 
98 /*
99  * Hypervisor supports 15 bits of APIC ID in MSI Extended Destination ID
100  */
101 static bool virt_ext_dest_id __ro_after_init;
102 
103 /* For parallel bootup. */
104 unsigned long apic_mmio_base __ro_after_init;
105 
apic_accessible(void)106 static inline bool apic_accessible(void)
107 {
108 	return x2apic_mode || apic_mmio_base;
109 }
110 
111 /*
112  * Map cpu index to physical APIC ID
113  */
114 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID);
115 DEFINE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid, U32_MAX);
116 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
117 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_acpiid);
118 
119 #ifdef CONFIG_X86_32
120 /* Local APIC was disabled by the BIOS and enabled by the kernel */
121 static int enabled_via_apicbase __ro_after_init;
122 
123 /*
124  * Handle interrupt mode configuration register (IMCR).
125  * This register controls whether the interrupt signals
126  * that reach the BSP come from the master PIC or from the
127  * local APIC. Before entering Symmetric I/O Mode, either
128  * the BIOS or the operating system must switch out of
129  * PIC Mode by changing the IMCR.
130  */
imcr_pic_to_apic(void)131 static inline void imcr_pic_to_apic(void)
132 {
133 	/* NMI and 8259 INTR go through APIC */
134 	pc_conf_set(PC_CONF_MPS_IMCR, 0x01);
135 }
136 
imcr_apic_to_pic(void)137 static inline void imcr_apic_to_pic(void)
138 {
139 	/* NMI and 8259 INTR go directly to BSP */
140 	pc_conf_set(PC_CONF_MPS_IMCR, 0x00);
141 }
142 #endif
143 
144 /*
145  * Knob to control our willingness to enable the local APIC.
146  *
147  * +1=force-enable
148  */
149 static int force_enable_local_apic __initdata;
150 
151 /*
152  * APIC command line parameters
153  */
parse_lapic(char * arg)154 static int __init parse_lapic(char *arg)
155 {
156 	if (IS_ENABLED(CONFIG_X86_32) && !arg)
157 		force_enable_local_apic = 1;
158 	else if (arg && !strncmp(arg, "notscdeadline", 13))
159 		setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
160 	return 0;
161 }
162 early_param("lapic", parse_lapic);
163 
164 #ifdef CONFIG_X86_64
165 static int apic_calibrate_pmtmr __initdata;
setup_apicpmtimer(char * s)166 static __init int setup_apicpmtimer(char *s)
167 {
168 	apic_calibrate_pmtmr = 1;
169 	notsc_setup(NULL);
170 	return 1;
171 }
172 __setup("apicpmtimer", setup_apicpmtimer);
173 #endif
174 
175 static unsigned long mp_lapic_addr __ro_after_init;
176 bool apic_is_disabled __ro_after_init;
177 /* Disable local APIC timer from the kernel commandline or via dmi quirk */
178 static int disable_apic_timer __initdata;
179 /* Local APIC timer works in C2 */
180 int local_apic_timer_c2_ok __ro_after_init;
181 EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
182 
183 /*
184  * Debug level, exported for io_apic.c
185  */
186 int apic_verbosity __ro_after_init;
187 
188 int pic_mode __ro_after_init;
189 
190 /* Have we found an MP table */
191 int smp_found_config __ro_after_init;
192 
193 static struct resource lapic_resource = {
194 	.name = "Local APIC",
195 	.flags = IORESOURCE_MEM | IORESOURCE_BUSY,
196 };
197 
198 unsigned int lapic_timer_period = 0;
199 
200 static void apic_pm_activate(void);
201 
202 /*
203  * Get the LAPIC version
204  */
lapic_get_version(void)205 static inline int lapic_get_version(void)
206 {
207 	return GET_APIC_VERSION(apic_read(APIC_LVR));
208 }
209 
210 /*
211  * Check, if the APIC is integrated or a separate chip
212  */
lapic_is_integrated(void)213 static inline int lapic_is_integrated(void)
214 {
215 	return APIC_INTEGRATED(lapic_get_version());
216 }
217 
218 /*
219  * Check, whether this is a modern or a first generation APIC
220  */
modern_apic(void)221 static int modern_apic(void)
222 {
223 	/* AMD systems use old APIC versions, so check the CPU */
224 	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
225 	    boot_cpu_data.x86 >= 0xf)
226 		return 1;
227 
228 	/* Hygon systems use modern APIC */
229 	if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
230 		return 1;
231 
232 	return lapic_get_version() >= 0x14;
233 }
234 
235 /*
236  * right after this call apic become NOOP driven
237  * so apic->write/read doesn't do anything
238  */
apic_disable(void)239 static void __init apic_disable(void)
240 {
241 	apic_install_driver(&apic_noop);
242 }
243 
native_apic_icr_write(u32 low,u32 id)244 void native_apic_icr_write(u32 low, u32 id)
245 {
246 	unsigned long flags;
247 
248 	local_irq_save(flags);
249 	apic_write(APIC_ICR2, SET_XAPIC_DEST_FIELD(id));
250 	apic_write(APIC_ICR, low);
251 	local_irq_restore(flags);
252 }
253 
native_apic_icr_read(void)254 u64 native_apic_icr_read(void)
255 {
256 	u32 icr1, icr2;
257 
258 	icr2 = apic_read(APIC_ICR2);
259 	icr1 = apic_read(APIC_ICR);
260 
261 	return icr1 | ((u64)icr2 << 32);
262 }
263 
264 #ifdef CONFIG_X86_32
265 /**
266  * get_physical_broadcast - Get number of physical broadcast IDs
267  */
get_physical_broadcast(void)268 int get_physical_broadcast(void)
269 {
270 	return modern_apic() ? 0xff : 0xf;
271 }
272 #endif
273 
274 /**
275  * lapic_get_maxlvt - get the maximum number of local vector table entries
276  */
lapic_get_maxlvt(void)277 int lapic_get_maxlvt(void)
278 {
279 	/*
280 	 * - we always have APIC integrated on 64bit mode
281 	 * - 82489DXs do not report # of LVT entries
282 	 */
283 	return lapic_is_integrated() ? GET_APIC_MAXLVT(apic_read(APIC_LVR)) : 2;
284 }
285 
286 /*
287  * Local APIC timer
288  */
289 
290 /* Clock divisor */
291 #define APIC_DIVISOR 16
292 #define TSC_DIVISOR  8
293 
294 /* i82489DX specific */
295 #define		I82489DX_BASE_DIVIDER		(((0x2) << 18))
296 
297 /*
298  * This function sets up the local APIC timer, with a timeout of
299  * 'clocks' APIC bus clock. During calibration we actually call
300  * this function twice on the boot CPU, once with a bogus timeout
301  * value, second time for real. The other (noncalibrating) CPUs
302  * call this function only once, with the real, calibrated value.
303  *
304  * We do reads before writes even if unnecessary, to get around the
305  * P5 APIC double write bug.
306  */
__setup_APIC_LVTT(unsigned int clocks,int oneshot,int irqen)307 static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
308 {
309 	unsigned int lvtt_value, tmp_value;
310 
311 	lvtt_value = LOCAL_TIMER_VECTOR;
312 	if (!oneshot)
313 		lvtt_value |= APIC_LVT_TIMER_PERIODIC;
314 	else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
315 		lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE;
316 
317 	/*
318 	 * The i82489DX APIC uses bit 18 and 19 for the base divider.  This
319 	 * overlaps with bit 18 on integrated APICs, but is not documented
320 	 * in the SDM. No problem though. i82489DX equipped systems do not
321 	 * have TSC deadline timer.
322 	 */
323 	if (!lapic_is_integrated())
324 		lvtt_value |= I82489DX_BASE_DIVIDER;
325 
326 	if (!irqen)
327 		lvtt_value |= APIC_LVT_MASKED;
328 
329 	apic_write(APIC_LVTT, lvtt_value);
330 
331 	if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) {
332 		/*
333 		 * See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode,
334 		 * writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized.
335 		 * According to Intel, MFENCE can do the serialization here.
336 		 */
337 		asm volatile("mfence" : : : "memory");
338 		return;
339 	}
340 
341 	/*
342 	 * Divide PICLK by 16
343 	 */
344 	tmp_value = apic_read(APIC_TDCR);
345 	apic_write(APIC_TDCR,
346 		(tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
347 		APIC_TDR_DIV_16);
348 
349 	if (!oneshot)
350 		apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
351 }
352 
353 /*
354  * Setup extended LVT, AMD specific
355  *
356  * Software should use the LVT offsets the BIOS provides.  The offsets
357  * are determined by the subsystems using it like those for MCE
358  * threshold or IBS.  On K8 only offset 0 (APIC500) and MCE interrupts
359  * are supported. Beginning with family 10h at least 4 offsets are
360  * available.
361  *
362  * Since the offsets must be consistent for all cores, we keep track
363  * of the LVT offsets in software and reserve the offset for the same
364  * vector also to be used on other cores. An offset is freed by
365  * setting the entry to APIC_EILVT_MASKED.
366  *
367  * If the BIOS is right, there should be no conflicts. Otherwise a
368  * "[Firmware Bug]: ..." error message is generated. However, if
369  * software does not properly determines the offsets, it is not
370  * necessarily a BIOS bug.
371  */
372 
373 static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
374 
eilvt_entry_is_changeable(unsigned int old,unsigned int new)375 static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
376 {
377 	return (old & APIC_EILVT_MASKED)
378 		|| (new == APIC_EILVT_MASKED)
379 		|| ((new & ~APIC_EILVT_MASKED) == old);
380 }
381 
reserve_eilvt_offset(int offset,unsigned int new)382 static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
383 {
384 	unsigned int rsvd, vector;
385 
386 	if (offset >= APIC_EILVT_NR_MAX)
387 		return ~0;
388 
389 	rsvd = atomic_read(&eilvt_offsets[offset]);
390 	do {
391 		vector = rsvd & ~APIC_EILVT_MASKED;	/* 0: unassigned */
392 		if (vector && !eilvt_entry_is_changeable(vector, new))
393 			/* may not change if vectors are different */
394 			return rsvd;
395 	} while (!atomic_try_cmpxchg(&eilvt_offsets[offset], &rsvd, new));
396 
397 	rsvd = new & ~APIC_EILVT_MASKED;
398 	if (rsvd && rsvd != vector)
399 		pr_info("LVT offset %d assigned for vector 0x%02x\n",
400 			offset, rsvd);
401 
402 	return new;
403 }
404 
405 /*
406  * If mask=1, the LVT entry does not generate interrupts while mask=0
407  * enables the vector. See also the BKDGs. Must be called with
408  * preemption disabled.
409  */
410 
setup_APIC_eilvt(u8 offset,u8 vector,u8 msg_type,u8 mask)411 int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
412 {
413 	unsigned long reg = APIC_EILVTn(offset);
414 	unsigned int new, old, reserved;
415 
416 	new = (mask << 16) | (msg_type << 8) | vector;
417 	old = apic_read(reg);
418 	reserved = reserve_eilvt_offset(offset, new);
419 
420 	if (reserved != new) {
421 		pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
422 		       "vector 0x%x, but the register is already in use for "
423 		       "vector 0x%x on another cpu\n",
424 		       smp_processor_id(), reg, offset, new, reserved);
425 		return -EINVAL;
426 	}
427 
428 	if (!eilvt_entry_is_changeable(old, new)) {
429 		pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
430 		       "vector 0x%x, but the register is already in use for "
431 		       "vector 0x%x on this cpu\n",
432 		       smp_processor_id(), reg, offset, new, old);
433 		return -EBUSY;
434 	}
435 
436 	apic_write(reg, new);
437 
438 	return 0;
439 }
440 EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
441 
442 /*
443  * Program the next event, relative to now
444  */
lapic_next_event(unsigned long delta,struct clock_event_device * evt)445 static int lapic_next_event(unsigned long delta,
446 			    struct clock_event_device *evt)
447 {
448 	apic_write(APIC_TMICT, delta);
449 	return 0;
450 }
451 
lapic_next_deadline(unsigned long delta,struct clock_event_device * evt)452 static int lapic_next_deadline(unsigned long delta,
453 			       struct clock_event_device *evt)
454 {
455 	u64 tsc;
456 
457 	/* This MSR is special and need a special fence: */
458 	weak_wrmsr_fence();
459 
460 	tsc = rdtsc();
461 	wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
462 	return 0;
463 }
464 
lapic_timer_shutdown(struct clock_event_device * evt)465 static int lapic_timer_shutdown(struct clock_event_device *evt)
466 {
467 	unsigned int v;
468 
469 	/* Lapic used as dummy for broadcast ? */
470 	if (evt->features & CLOCK_EVT_FEAT_DUMMY)
471 		return 0;
472 
473 	v = apic_read(APIC_LVTT);
474 	v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
475 	apic_write(APIC_LVTT, v);
476 	apic_write(APIC_TMICT, 0);
477 	return 0;
478 }
479 
480 static inline int
lapic_timer_set_periodic_oneshot(struct clock_event_device * evt,bool oneshot)481 lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot)
482 {
483 	/* Lapic used as dummy for broadcast ? */
484 	if (evt->features & CLOCK_EVT_FEAT_DUMMY)
485 		return 0;
486 
487 	__setup_APIC_LVTT(lapic_timer_period, oneshot, 1);
488 	return 0;
489 }
490 
lapic_timer_set_periodic(struct clock_event_device * evt)491 static int lapic_timer_set_periodic(struct clock_event_device *evt)
492 {
493 	return lapic_timer_set_periodic_oneshot(evt, false);
494 }
495 
lapic_timer_set_oneshot(struct clock_event_device * evt)496 static int lapic_timer_set_oneshot(struct clock_event_device *evt)
497 {
498 	return lapic_timer_set_periodic_oneshot(evt, true);
499 }
500 
501 /*
502  * Local APIC timer broadcast function
503  */
lapic_timer_broadcast(const struct cpumask * mask)504 static void lapic_timer_broadcast(const struct cpumask *mask)
505 {
506 #ifdef CONFIG_SMP
507 	__apic_send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
508 #endif
509 }
510 
511 
512 /*
513  * The local apic timer can be used for any function which is CPU local.
514  */
515 static struct clock_event_device lapic_clockevent = {
516 	.name				= "lapic",
517 	.features			= CLOCK_EVT_FEAT_PERIODIC |
518 					  CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP
519 					  | CLOCK_EVT_FEAT_DUMMY,
520 	.shift				= 32,
521 	.set_state_shutdown		= lapic_timer_shutdown,
522 	.set_state_periodic		= lapic_timer_set_periodic,
523 	.set_state_oneshot		= lapic_timer_set_oneshot,
524 	.set_state_oneshot_stopped	= lapic_timer_shutdown,
525 	.set_next_event			= lapic_next_event,
526 	.broadcast			= lapic_timer_broadcast,
527 	.rating				= 100,
528 	.irq				= -1,
529 };
530 static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
531 
532 static const struct x86_cpu_id deadline_match[] __initconst = {
533 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x2, 0x2), 0x3a), /* EP */
534 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x4, 0x4), 0x0f), /* EX */
535 
536 	X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_X,	0x0b000020),
537 
538 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x2, 0x2), 0x00000011),
539 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x3, 0x3), 0x0700000e),
540 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x4, 0x4), 0x0f00000c),
541 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x5, 0x5), 0x0e000003),
542 
543 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x3, 0x3), 0x01000136),
544 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x4, 0x4), 0x02000014),
545 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x5, 0xf), 0),
546 
547 	X86_MATCH_INTEL_FAM6_MODEL( HASWELL,		0x22),
548 	X86_MATCH_INTEL_FAM6_MODEL( HASWELL_L,		0x20),
549 	X86_MATCH_INTEL_FAM6_MODEL( HASWELL_G,		0x17),
550 
551 	X86_MATCH_INTEL_FAM6_MODEL( BROADWELL,		0x25),
552 	X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_G,	0x17),
553 
554 	X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE_L,		0xb2),
555 	X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE,		0xb2),
556 
557 	X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE_L,		0x52),
558 	X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE,		0x52),
559 
560 	{},
561 };
562 
apic_validate_deadline_timer(void)563 static __init bool apic_validate_deadline_timer(void)
564 {
565 	const struct x86_cpu_id *m;
566 	u32 rev;
567 
568 	if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
569 		return false;
570 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
571 		return true;
572 
573 	m = x86_match_cpu(deadline_match);
574 	if (!m)
575 		return true;
576 
577 	rev = (u32)m->driver_data;
578 
579 	if (boot_cpu_data.microcode >= rev)
580 		return true;
581 
582 	setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
583 	pr_err(FW_BUG "TSC_DEADLINE disabled due to Errata; "
584 	       "please update microcode to version: 0x%x (or later)\n", rev);
585 	return false;
586 }
587 
588 /*
589  * Setup the local APIC timer for this CPU. Copy the initialized values
590  * of the boot CPU and register the clock event in the framework.
591  */
setup_APIC_timer(void)592 static void setup_APIC_timer(void)
593 {
594 	struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
595 
596 	if (this_cpu_has(X86_FEATURE_ARAT)) {
597 		lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
598 		/* Make LAPIC timer preferable over percpu HPET */
599 		lapic_clockevent.rating = 150;
600 	}
601 
602 	memcpy(levt, &lapic_clockevent, sizeof(*levt));
603 	levt->cpumask = cpumask_of(smp_processor_id());
604 
605 	if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
606 		levt->name = "lapic-deadline";
607 		levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC |
608 				    CLOCK_EVT_FEAT_DUMMY);
609 		levt->set_next_event = lapic_next_deadline;
610 		clockevents_config_and_register(levt,
611 						tsc_khz * (1000 / TSC_DIVISOR),
612 						0xF, ~0UL);
613 	} else
614 		clockevents_register_device(levt);
615 }
616 
617 /*
618  * Install the updated TSC frequency from recalibration at the TSC
619  * deadline clockevent devices.
620  */
__lapic_update_tsc_freq(void * info)621 static void __lapic_update_tsc_freq(void *info)
622 {
623 	struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
624 
625 	if (!this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
626 		return;
627 
628 	clockevents_update_freq(levt, tsc_khz * (1000 / TSC_DIVISOR));
629 }
630 
lapic_update_tsc_freq(void)631 void lapic_update_tsc_freq(void)
632 {
633 	/*
634 	 * The clockevent device's ->mult and ->shift can both be
635 	 * changed. In order to avoid races, schedule the frequency
636 	 * update code on each CPU.
637 	 */
638 	on_each_cpu(__lapic_update_tsc_freq, NULL, 0);
639 }
640 
641 /*
642  * In this functions we calibrate APIC bus clocks to the external timer.
643  *
644  * We want to do the calibration only once since we want to have local timer
645  * irqs synchronous. CPUs connected by the same APIC bus have the very same bus
646  * frequency.
647  *
648  * This was previously done by reading the PIT/HPET and waiting for a wrap
649  * around to find out, that a tick has elapsed. I have a box, where the PIT
650  * readout is broken, so it never gets out of the wait loop again. This was
651  * also reported by others.
652  *
653  * Monitoring the jiffies value is inaccurate and the clockevents
654  * infrastructure allows us to do a simple substitution of the interrupt
655  * handler.
656  *
657  * The calibration routine also uses the pm_timer when possible, as the PIT
658  * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
659  * back to normal later in the boot process).
660  */
661 
662 #define LAPIC_CAL_LOOPS		(HZ/10)
663 
664 static __initdata int lapic_cal_loops = -1;
665 static __initdata long lapic_cal_t1, lapic_cal_t2;
666 static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
667 static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2;
668 static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
669 
670 /*
671  * Temporary interrupt handler and polled calibration function.
672  */
lapic_cal_handler(struct clock_event_device * dev)673 static void __init lapic_cal_handler(struct clock_event_device *dev)
674 {
675 	unsigned long long tsc = 0;
676 	long tapic = apic_read(APIC_TMCCT);
677 	unsigned long pm = acpi_pm_read_early();
678 
679 	if (boot_cpu_has(X86_FEATURE_TSC))
680 		tsc = rdtsc();
681 
682 	switch (lapic_cal_loops++) {
683 	case 0:
684 		lapic_cal_t1 = tapic;
685 		lapic_cal_tsc1 = tsc;
686 		lapic_cal_pm1 = pm;
687 		lapic_cal_j1 = jiffies;
688 		break;
689 
690 	case LAPIC_CAL_LOOPS:
691 		lapic_cal_t2 = tapic;
692 		lapic_cal_tsc2 = tsc;
693 		if (pm < lapic_cal_pm1)
694 			pm += ACPI_PM_OVRRUN;
695 		lapic_cal_pm2 = pm;
696 		lapic_cal_j2 = jiffies;
697 		break;
698 	}
699 }
700 
701 static int __init
calibrate_by_pmtimer(long deltapm,long * delta,long * deltatsc)702 calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc)
703 {
704 	const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
705 	const long pm_thresh = pm_100ms / 100;
706 	unsigned long mult;
707 	u64 res;
708 
709 #ifndef CONFIG_X86_PM_TIMER
710 	return -1;
711 #endif
712 
713 	apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm);
714 
715 	/* Check, if the PM timer is available */
716 	if (!deltapm)
717 		return -1;
718 
719 	mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22);
720 
721 	if (deltapm > (pm_100ms - pm_thresh) &&
722 	    deltapm < (pm_100ms + pm_thresh)) {
723 		apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n");
724 		return 0;
725 	}
726 
727 	res = (((u64)deltapm) *  mult) >> 22;
728 	do_div(res, 1000000);
729 	pr_warn("APIC calibration not consistent "
730 		"with PM-Timer: %ldms instead of 100ms\n", (long)res);
731 
732 	/* Correct the lapic counter value */
733 	res = (((u64)(*delta)) * pm_100ms);
734 	do_div(res, deltapm);
735 	pr_info("APIC delta adjusted to PM-Timer: "
736 		"%lu (%ld)\n", (unsigned long)res, *delta);
737 	*delta = (long)res;
738 
739 	/* Correct the tsc counter value */
740 	if (boot_cpu_has(X86_FEATURE_TSC)) {
741 		res = (((u64)(*deltatsc)) * pm_100ms);
742 		do_div(res, deltapm);
743 		apic_printk(APIC_VERBOSE, "TSC delta adjusted to "
744 					  "PM-Timer: %lu (%ld)\n",
745 					(unsigned long)res, *deltatsc);
746 		*deltatsc = (long)res;
747 	}
748 
749 	return 0;
750 }
751 
lapic_init_clockevent(void)752 static int __init lapic_init_clockevent(void)
753 {
754 	if (!lapic_timer_period)
755 		return -1;
756 
757 	/* Calculate the scaled math multiplication factor */
758 	lapic_clockevent.mult = div_sc(lapic_timer_period/APIC_DIVISOR,
759 					TICK_NSEC, lapic_clockevent.shift);
760 	lapic_clockevent.max_delta_ns =
761 		clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent);
762 	lapic_clockevent.max_delta_ticks = 0x7FFFFFFF;
763 	lapic_clockevent.min_delta_ns =
764 		clockevent_delta2ns(0xF, &lapic_clockevent);
765 	lapic_clockevent.min_delta_ticks = 0xF;
766 
767 	return 0;
768 }
769 
apic_needs_pit(void)770 bool __init apic_needs_pit(void)
771 {
772 	/*
773 	 * If the frequencies are not known, PIT is required for both TSC
774 	 * and apic timer calibration.
775 	 */
776 	if (!tsc_khz || !cpu_khz)
777 		return true;
778 
779 	/* Is there an APIC at all or is it disabled? */
780 	if (!boot_cpu_has(X86_FEATURE_APIC) || apic_is_disabled)
781 		return true;
782 
783 	/*
784 	 * If interrupt delivery mode is legacy PIC or virtual wire without
785 	 * configuration, the local APIC timer wont be set up. Make sure
786 	 * that the PIT is initialized.
787 	 */
788 	if (apic_intr_mode == APIC_PIC ||
789 	    apic_intr_mode == APIC_VIRTUAL_WIRE_NO_CONFIG)
790 		return true;
791 
792 	/* Virt guests may lack ARAT, but still have DEADLINE */
793 	if (!boot_cpu_has(X86_FEATURE_ARAT))
794 		return true;
795 
796 	/* Deadline timer is based on TSC so no further PIT action required */
797 	if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
798 		return false;
799 
800 	/* APIC timer disabled? */
801 	if (disable_apic_timer)
802 		return true;
803 	/*
804 	 * The APIC timer frequency is known already, no PIT calibration
805 	 * required. If unknown, let the PIT be initialized.
806 	 */
807 	return lapic_timer_period == 0;
808 }
809 
calibrate_APIC_clock(void)810 static int __init calibrate_APIC_clock(void)
811 {
812 	struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
813 	u64 tsc_perj = 0, tsc_start = 0;
814 	unsigned long jif_start;
815 	unsigned long deltaj;
816 	long delta, deltatsc;
817 	int pm_referenced = 0;
818 
819 	if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
820 		return 0;
821 
822 	/*
823 	 * Check if lapic timer has already been calibrated by platform
824 	 * specific routine, such as tsc calibration code. If so just fill
825 	 * in the clockevent structure and return.
826 	 */
827 	if (!lapic_init_clockevent()) {
828 		apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n",
829 			    lapic_timer_period);
830 		/*
831 		 * Direct calibration methods must have an always running
832 		 * local APIC timer, no need for broadcast timer.
833 		 */
834 		lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
835 		return 0;
836 	}
837 
838 	apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n"
839 		    "calibrating APIC timer ...\n");
840 
841 	/*
842 	 * There are platforms w/o global clockevent devices. Instead of
843 	 * making the calibration conditional on that, use a polling based
844 	 * approach everywhere.
845 	 */
846 	local_irq_disable();
847 
848 	/*
849 	 * Setup the APIC counter to maximum. There is no way the lapic
850 	 * can underflow in the 100ms detection time frame
851 	 */
852 	__setup_APIC_LVTT(0xffffffff, 0, 0);
853 
854 	/*
855 	 * Methods to terminate the calibration loop:
856 	 *  1) Global clockevent if available (jiffies)
857 	 *  2) TSC if available and frequency is known
858 	 */
859 	jif_start = READ_ONCE(jiffies);
860 
861 	if (tsc_khz) {
862 		tsc_start = rdtsc();
863 		tsc_perj = div_u64((u64)tsc_khz * 1000, HZ);
864 	}
865 
866 	/*
867 	 * Enable interrupts so the tick can fire, if a global
868 	 * clockevent device is available
869 	 */
870 	local_irq_enable();
871 
872 	while (lapic_cal_loops <= LAPIC_CAL_LOOPS) {
873 		/* Wait for a tick to elapse */
874 		while (1) {
875 			if (tsc_khz) {
876 				u64 tsc_now = rdtsc();
877 				if ((tsc_now - tsc_start) >= tsc_perj) {
878 					tsc_start += tsc_perj;
879 					break;
880 				}
881 			} else {
882 				unsigned long jif_now = READ_ONCE(jiffies);
883 
884 				if (time_after(jif_now, jif_start)) {
885 					jif_start = jif_now;
886 					break;
887 				}
888 			}
889 			cpu_relax();
890 		}
891 
892 		/* Invoke the calibration routine */
893 		local_irq_disable();
894 		lapic_cal_handler(NULL);
895 		local_irq_enable();
896 	}
897 
898 	local_irq_disable();
899 
900 	/* Build delta t1-t2 as apic timer counts down */
901 	delta = lapic_cal_t1 - lapic_cal_t2;
902 	apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta);
903 
904 	deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
905 
906 	/* we trust the PM based calibration if possible */
907 	pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1,
908 					&delta, &deltatsc);
909 
910 	lapic_timer_period = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
911 	lapic_init_clockevent();
912 
913 	apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta);
914 	apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult);
915 	apic_printk(APIC_VERBOSE, "..... calibration result: %u\n",
916 		    lapic_timer_period);
917 
918 	if (boot_cpu_has(X86_FEATURE_TSC)) {
919 		apic_printk(APIC_VERBOSE, "..... CPU clock speed is "
920 			    "%ld.%04ld MHz.\n",
921 			    (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ),
922 			    (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ));
923 	}
924 
925 	apic_printk(APIC_VERBOSE, "..... host bus clock speed is "
926 		    "%u.%04u MHz.\n",
927 		    lapic_timer_period / (1000000 / HZ),
928 		    lapic_timer_period % (1000000 / HZ));
929 
930 	/*
931 	 * Do a sanity check on the APIC calibration result
932 	 */
933 	if (lapic_timer_period < (1000000 / HZ)) {
934 		local_irq_enable();
935 		pr_warn("APIC frequency too slow, disabling apic timer\n");
936 		return -1;
937 	}
938 
939 	levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
940 
941 	/*
942 	 * PM timer calibration failed or not turned on so lets try APIC
943 	 * timer based calibration, if a global clockevent device is
944 	 * available.
945 	 */
946 	if (!pm_referenced && global_clock_event) {
947 		apic_printk(APIC_VERBOSE, "... verify APIC timer\n");
948 
949 		/*
950 		 * Setup the apic timer manually
951 		 */
952 		levt->event_handler = lapic_cal_handler;
953 		lapic_timer_set_periodic(levt);
954 		lapic_cal_loops = -1;
955 
956 		/* Let the interrupts run */
957 		local_irq_enable();
958 
959 		while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
960 			cpu_relax();
961 
962 		/* Stop the lapic timer */
963 		local_irq_disable();
964 		lapic_timer_shutdown(levt);
965 
966 		/* Jiffies delta */
967 		deltaj = lapic_cal_j2 - lapic_cal_j1;
968 		apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj);
969 
970 		/* Check, if the jiffies result is consistent */
971 		if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
972 			apic_printk(APIC_VERBOSE, "... jiffies result ok\n");
973 		else
974 			levt->features |= CLOCK_EVT_FEAT_DUMMY;
975 	}
976 	local_irq_enable();
977 
978 	if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
979 		pr_warn("APIC timer disabled due to verification failure\n");
980 		return -1;
981 	}
982 
983 	return 0;
984 }
985 
986 /*
987  * Setup the boot APIC
988  *
989  * Calibrate and verify the result.
990  */
setup_boot_APIC_clock(void)991 void __init setup_boot_APIC_clock(void)
992 {
993 	/*
994 	 * The local apic timer can be disabled via the kernel
995 	 * commandline or from the CPU detection code. Register the lapic
996 	 * timer as a dummy clock event source on SMP systems, so the
997 	 * broadcast mechanism is used. On UP systems simply ignore it.
998 	 */
999 	if (disable_apic_timer) {
1000 		pr_info("Disabling APIC timer\n");
1001 		/* No broadcast on UP ! */
1002 		if (num_possible_cpus() > 1) {
1003 			lapic_clockevent.mult = 1;
1004 			setup_APIC_timer();
1005 		}
1006 		return;
1007 	}
1008 
1009 	if (calibrate_APIC_clock()) {
1010 		/* No broadcast on UP ! */
1011 		if (num_possible_cpus() > 1)
1012 			setup_APIC_timer();
1013 		return;
1014 	}
1015 
1016 	/*
1017 	 * If nmi_watchdog is set to IO_APIC, we need the
1018 	 * PIT/HPET going.  Otherwise register lapic as a dummy
1019 	 * device.
1020 	 */
1021 	lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
1022 
1023 	/* Setup the lapic or request the broadcast */
1024 	setup_APIC_timer();
1025 	amd_e400_c1e_apic_setup();
1026 }
1027 
setup_secondary_APIC_clock(void)1028 void setup_secondary_APIC_clock(void)
1029 {
1030 	setup_APIC_timer();
1031 	amd_e400_c1e_apic_setup();
1032 }
1033 
1034 /*
1035  * The guts of the apic timer interrupt
1036  */
local_apic_timer_interrupt(void)1037 static void local_apic_timer_interrupt(void)
1038 {
1039 	struct clock_event_device *evt = this_cpu_ptr(&lapic_events);
1040 
1041 	/*
1042 	 * Normally we should not be here till LAPIC has been initialized but
1043 	 * in some cases like kdump, its possible that there is a pending LAPIC
1044 	 * timer interrupt from previous kernel's context and is delivered in
1045 	 * new kernel the moment interrupts are enabled.
1046 	 *
1047 	 * Interrupts are enabled early and LAPIC is setup much later, hence
1048 	 * its possible that when we get here evt->event_handler is NULL.
1049 	 * Check for event_handler being NULL and discard the interrupt as
1050 	 * spurious.
1051 	 */
1052 	if (!evt->event_handler) {
1053 		pr_warn("Spurious LAPIC timer interrupt on cpu %d\n",
1054 			smp_processor_id());
1055 		/* Switch it off */
1056 		lapic_timer_shutdown(evt);
1057 		return;
1058 	}
1059 
1060 	/*
1061 	 * the NMI deadlock-detector uses this.
1062 	 */
1063 	inc_irq_stat(apic_timer_irqs);
1064 
1065 	evt->event_handler(evt);
1066 }
1067 
1068 /*
1069  * Local APIC timer interrupt. This is the most natural way for doing
1070  * local interrupts, but local timer interrupts can be emulated by
1071  * broadcast interrupts too. [in case the hw doesn't support APIC timers]
1072  *
1073  * [ if a single-CPU system runs an SMP kernel then we call the local
1074  *   interrupt as well. Thus we cannot inline the local irq ... ]
1075  */
DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt)1076 DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt)
1077 {
1078 	struct pt_regs *old_regs = set_irq_regs(regs);
1079 
1080 	apic_eoi();
1081 	trace_local_timer_entry(LOCAL_TIMER_VECTOR);
1082 	local_apic_timer_interrupt();
1083 	trace_local_timer_exit(LOCAL_TIMER_VECTOR);
1084 
1085 	set_irq_regs(old_regs);
1086 }
1087 
1088 /*
1089  * Local APIC start and shutdown
1090  */
1091 
1092 /**
1093  * clear_local_APIC - shutdown the local APIC
1094  *
1095  * This is called, when a CPU is disabled and before rebooting, so the state of
1096  * the local APIC has no dangling leftovers. Also used to cleanout any BIOS
1097  * leftovers during boot.
1098  */
clear_local_APIC(void)1099 void clear_local_APIC(void)
1100 {
1101 	int maxlvt;
1102 	u32 v;
1103 
1104 	if (!apic_accessible())
1105 		return;
1106 
1107 	maxlvt = lapic_get_maxlvt();
1108 	/*
1109 	 * Masking an LVT entry can trigger a local APIC error
1110 	 * if the vector is zero. Mask LVTERR first to prevent this.
1111 	 */
1112 	if (maxlvt >= 3) {
1113 		v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
1114 		apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
1115 	}
1116 	/*
1117 	 * Careful: we have to set masks only first to deassert
1118 	 * any level-triggered sources.
1119 	 */
1120 	v = apic_read(APIC_LVTT);
1121 	apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
1122 	v = apic_read(APIC_LVT0);
1123 	apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
1124 	v = apic_read(APIC_LVT1);
1125 	apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
1126 	if (maxlvt >= 4) {
1127 		v = apic_read(APIC_LVTPC);
1128 		apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
1129 	}
1130 
1131 	/* lets not touch this if we didn't frob it */
1132 #ifdef CONFIG_X86_THERMAL_VECTOR
1133 	if (maxlvt >= 5) {
1134 		v = apic_read(APIC_LVTTHMR);
1135 		apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED);
1136 	}
1137 #endif
1138 #ifdef CONFIG_X86_MCE_INTEL
1139 	if (maxlvt >= 6) {
1140 		v = apic_read(APIC_LVTCMCI);
1141 		if (!(v & APIC_LVT_MASKED))
1142 			apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED);
1143 	}
1144 #endif
1145 
1146 	/*
1147 	 * Clean APIC state for other OSs:
1148 	 */
1149 	apic_write(APIC_LVTT, APIC_LVT_MASKED);
1150 	apic_write(APIC_LVT0, APIC_LVT_MASKED);
1151 	apic_write(APIC_LVT1, APIC_LVT_MASKED);
1152 	if (maxlvt >= 3)
1153 		apic_write(APIC_LVTERR, APIC_LVT_MASKED);
1154 	if (maxlvt >= 4)
1155 		apic_write(APIC_LVTPC, APIC_LVT_MASKED);
1156 
1157 	/* Integrated APIC (!82489DX) ? */
1158 	if (lapic_is_integrated()) {
1159 		if (maxlvt > 3)
1160 			/* Clear ESR due to Pentium errata 3AP and 11AP */
1161 			apic_write(APIC_ESR, 0);
1162 		apic_read(APIC_ESR);
1163 	}
1164 }
1165 
1166 /**
1167  * apic_soft_disable - Clears and software disables the local APIC on hotplug
1168  *
1169  * Contrary to disable_local_APIC() this does not touch the enable bit in
1170  * MSR_IA32_APICBASE. Clearing that bit on systems based on the 3 wire APIC
1171  * bus would require a hardware reset as the APIC would lose track of bus
1172  * arbitration. On systems with FSB delivery APICBASE could be disabled,
1173  * but it has to be guaranteed that no interrupt is sent to the APIC while
1174  * in that state and it's not clear from the SDM whether it still responds
1175  * to INIT/SIPI messages. Stay on the safe side and use software disable.
1176  */
apic_soft_disable(void)1177 void apic_soft_disable(void)
1178 {
1179 	u32 value;
1180 
1181 	clear_local_APIC();
1182 
1183 	/* Soft disable APIC (implies clearing of registers for 82489DX!). */
1184 	value = apic_read(APIC_SPIV);
1185 	value &= ~APIC_SPIV_APIC_ENABLED;
1186 	apic_write(APIC_SPIV, value);
1187 }
1188 
1189 /**
1190  * disable_local_APIC - clear and disable the local APIC
1191  */
disable_local_APIC(void)1192 void disable_local_APIC(void)
1193 {
1194 	if (!apic_accessible())
1195 		return;
1196 
1197 	apic_soft_disable();
1198 
1199 #ifdef CONFIG_X86_32
1200 	/*
1201 	 * When LAPIC was disabled by the BIOS and enabled by the kernel,
1202 	 * restore the disabled state.
1203 	 */
1204 	if (enabled_via_apicbase) {
1205 		unsigned int l, h;
1206 
1207 		rdmsr(MSR_IA32_APICBASE, l, h);
1208 		l &= ~MSR_IA32_APICBASE_ENABLE;
1209 		wrmsr(MSR_IA32_APICBASE, l, h);
1210 	}
1211 #endif
1212 }
1213 
1214 /*
1215  * If Linux enabled the LAPIC against the BIOS default disable it down before
1216  * re-entering the BIOS on shutdown.  Otherwise the BIOS may get confused and
1217  * not power-off.  Additionally clear all LVT entries before disable_local_APIC
1218  * for the case where Linux didn't enable the LAPIC.
1219  */
lapic_shutdown(void)1220 void lapic_shutdown(void)
1221 {
1222 	unsigned long flags;
1223 
1224 	if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
1225 		return;
1226 
1227 	local_irq_save(flags);
1228 
1229 #ifdef CONFIG_X86_32
1230 	if (!enabled_via_apicbase)
1231 		clear_local_APIC();
1232 	else
1233 #endif
1234 		disable_local_APIC();
1235 
1236 
1237 	local_irq_restore(flags);
1238 }
1239 
1240 /**
1241  * sync_Arb_IDs - synchronize APIC bus arbitration IDs
1242  */
sync_Arb_IDs(void)1243 void __init sync_Arb_IDs(void)
1244 {
1245 	/*
1246 	 * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
1247 	 * needed on AMD.
1248 	 */
1249 	if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1250 		return;
1251 
1252 	/*
1253 	 * Wait for idle.
1254 	 */
1255 	apic_wait_icr_idle();
1256 
1257 	apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
1258 	apic_write(APIC_ICR, APIC_DEST_ALLINC |
1259 			APIC_INT_LEVELTRIG | APIC_DM_INIT);
1260 }
1261 
1262 enum apic_intr_mode_id apic_intr_mode __ro_after_init;
1263 
__apic_intr_mode_select(void)1264 static int __init __apic_intr_mode_select(void)
1265 {
1266 	/* Check kernel option */
1267 	if (apic_is_disabled) {
1268 		pr_info("APIC disabled via kernel command line\n");
1269 		return APIC_PIC;
1270 	}
1271 
1272 	/* Check BIOS */
1273 #ifdef CONFIG_X86_64
1274 	/* On 64-bit, the APIC must be integrated, Check local APIC only */
1275 	if (!boot_cpu_has(X86_FEATURE_APIC)) {
1276 		apic_is_disabled = true;
1277 		pr_info("APIC disabled by BIOS\n");
1278 		return APIC_PIC;
1279 	}
1280 #else
1281 	/* On 32-bit, the APIC may be integrated APIC or 82489DX */
1282 
1283 	/* Neither 82489DX nor integrated APIC ? */
1284 	if (!boot_cpu_has(X86_FEATURE_APIC) && !smp_found_config) {
1285 		apic_is_disabled = true;
1286 		return APIC_PIC;
1287 	}
1288 
1289 	/* If the BIOS pretends there is an integrated APIC ? */
1290 	if (!boot_cpu_has(X86_FEATURE_APIC) &&
1291 		APIC_INTEGRATED(boot_cpu_apic_version)) {
1292 		apic_is_disabled = true;
1293 		pr_err(FW_BUG "Local APIC not detected, force emulation\n");
1294 		return APIC_PIC;
1295 	}
1296 #endif
1297 
1298 	/* Check MP table or ACPI MADT configuration */
1299 	if (!smp_found_config) {
1300 		disable_ioapic_support();
1301 		if (!acpi_lapic) {
1302 			pr_info("APIC: ACPI MADT or MP tables are not detected\n");
1303 			return APIC_VIRTUAL_WIRE_NO_CONFIG;
1304 		}
1305 		return APIC_VIRTUAL_WIRE;
1306 	}
1307 
1308 #ifdef CONFIG_SMP
1309 	/* If SMP should be disabled, then really disable it! */
1310 	if (!setup_max_cpus) {
1311 		pr_info("APIC: SMP mode deactivated\n");
1312 		return APIC_SYMMETRIC_IO_NO_ROUTING;
1313 	}
1314 #endif
1315 
1316 	return APIC_SYMMETRIC_IO;
1317 }
1318 
1319 /* Select the interrupt delivery mode for the BSP */
apic_intr_mode_select(void)1320 void __init apic_intr_mode_select(void)
1321 {
1322 	apic_intr_mode = __apic_intr_mode_select();
1323 }
1324 
1325 /*
1326  * An initial setup of the virtual wire mode.
1327  */
init_bsp_APIC(void)1328 void __init init_bsp_APIC(void)
1329 {
1330 	unsigned int value;
1331 
1332 	/*
1333 	 * Don't do the setup now if we have a SMP BIOS as the
1334 	 * through-I/O-APIC virtual wire mode might be active.
1335 	 */
1336 	if (smp_found_config || !boot_cpu_has(X86_FEATURE_APIC))
1337 		return;
1338 
1339 	/*
1340 	 * Do not trust the local APIC being empty at bootup.
1341 	 */
1342 	clear_local_APIC();
1343 
1344 	/*
1345 	 * Enable APIC.
1346 	 */
1347 	value = apic_read(APIC_SPIV);
1348 	value &= ~APIC_VECTOR_MASK;
1349 	value |= APIC_SPIV_APIC_ENABLED;
1350 
1351 #ifdef CONFIG_X86_32
1352 	/* This bit is reserved on P4/Xeon and should be cleared */
1353 	if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
1354 	    (boot_cpu_data.x86 == 15))
1355 		value &= ~APIC_SPIV_FOCUS_DISABLED;
1356 	else
1357 #endif
1358 		value |= APIC_SPIV_FOCUS_DISABLED;
1359 	value |= SPURIOUS_APIC_VECTOR;
1360 	apic_write(APIC_SPIV, value);
1361 
1362 	/*
1363 	 * Set up the virtual wire mode.
1364 	 */
1365 	apic_write(APIC_LVT0, APIC_DM_EXTINT);
1366 	value = APIC_DM_NMI;
1367 	if (!lapic_is_integrated())		/* 82489DX */
1368 		value |= APIC_LVT_LEVEL_TRIGGER;
1369 	if (apic_extnmi == APIC_EXTNMI_NONE)
1370 		value |= APIC_LVT_MASKED;
1371 	apic_write(APIC_LVT1, value);
1372 }
1373 
1374 static void __init apic_bsp_setup(bool upmode);
1375 
1376 /* Init the interrupt delivery mode for the BSP */
apic_intr_mode_init(void)1377 void __init apic_intr_mode_init(void)
1378 {
1379 	bool upmode = IS_ENABLED(CONFIG_UP_LATE_INIT);
1380 
1381 	switch (apic_intr_mode) {
1382 	case APIC_PIC:
1383 		pr_info("APIC: Keep in PIC mode(8259)\n");
1384 		return;
1385 	case APIC_VIRTUAL_WIRE:
1386 		pr_info("APIC: Switch to virtual wire mode setup\n");
1387 		break;
1388 	case APIC_VIRTUAL_WIRE_NO_CONFIG:
1389 		pr_info("APIC: Switch to virtual wire mode setup with no configuration\n");
1390 		upmode = true;
1391 		break;
1392 	case APIC_SYMMETRIC_IO:
1393 		pr_info("APIC: Switch to symmetric I/O mode setup\n");
1394 		break;
1395 	case APIC_SYMMETRIC_IO_NO_ROUTING:
1396 		pr_info("APIC: Switch to symmetric I/O mode setup in no SMP routine\n");
1397 		break;
1398 	}
1399 
1400 	x86_64_probe_apic();
1401 
1402 	x86_32_install_bigsmp();
1403 
1404 	if (x86_platform.apic_post_init)
1405 		x86_platform.apic_post_init();
1406 
1407 	apic_bsp_setup(upmode);
1408 }
1409 
lapic_setup_esr(void)1410 static void lapic_setup_esr(void)
1411 {
1412 	unsigned int oldvalue, value, maxlvt;
1413 
1414 	if (!lapic_is_integrated()) {
1415 		pr_info("No ESR for 82489DX.\n");
1416 		return;
1417 	}
1418 
1419 	if (apic->disable_esr) {
1420 		/*
1421 		 * Something untraceable is creating bad interrupts on
1422 		 * secondary quads ... for the moment, just leave the
1423 		 * ESR disabled - we can't do anything useful with the
1424 		 * errors anyway - mbligh
1425 		 */
1426 		pr_info("Leaving ESR disabled.\n");
1427 		return;
1428 	}
1429 
1430 	maxlvt = lapic_get_maxlvt();
1431 	if (maxlvt > 3)		/* Due to the Pentium erratum 3AP. */
1432 		apic_write(APIC_ESR, 0);
1433 	oldvalue = apic_read(APIC_ESR);
1434 
1435 	/* enables sending errors */
1436 	value = ERROR_APIC_VECTOR;
1437 	apic_write(APIC_LVTERR, value);
1438 
1439 	/*
1440 	 * spec says clear errors after enabling vector.
1441 	 */
1442 	if (maxlvt > 3)
1443 		apic_write(APIC_ESR, 0);
1444 	value = apic_read(APIC_ESR);
1445 	if (value != oldvalue)
1446 		apic_printk(APIC_VERBOSE, "ESR value before enabling "
1447 			"vector: 0x%08x  after: 0x%08x\n",
1448 			oldvalue, value);
1449 }
1450 
1451 #define APIC_IR_REGS		APIC_ISR_NR
1452 #define APIC_IR_BITS		(APIC_IR_REGS * 32)
1453 #define APIC_IR_MAPSIZE		(APIC_IR_BITS / BITS_PER_LONG)
1454 
1455 union apic_ir {
1456 	unsigned long	map[APIC_IR_MAPSIZE];
1457 	u32		regs[APIC_IR_REGS];
1458 };
1459 
apic_check_and_ack(union apic_ir * irr,union apic_ir * isr)1460 static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr)
1461 {
1462 	int i, bit;
1463 
1464 	/* Read the IRRs */
1465 	for (i = 0; i < APIC_IR_REGS; i++)
1466 		irr->regs[i] = apic_read(APIC_IRR + i * 0x10);
1467 
1468 	/* Read the ISRs */
1469 	for (i = 0; i < APIC_IR_REGS; i++)
1470 		isr->regs[i] = apic_read(APIC_ISR + i * 0x10);
1471 
1472 	/*
1473 	 * If the ISR map is not empty. ACK the APIC and run another round
1474 	 * to verify whether a pending IRR has been unblocked and turned
1475 	 * into a ISR.
1476 	 */
1477 	if (!bitmap_empty(isr->map, APIC_IR_BITS)) {
1478 		/*
1479 		 * There can be multiple ISR bits set when a high priority
1480 		 * interrupt preempted a lower priority one. Issue an ACK
1481 		 * per set bit.
1482 		 */
1483 		for_each_set_bit(bit, isr->map, APIC_IR_BITS)
1484 			apic_eoi();
1485 		return true;
1486 	}
1487 
1488 	return !bitmap_empty(irr->map, APIC_IR_BITS);
1489 }
1490 
1491 /*
1492  * After a crash, we no longer service the interrupts and a pending
1493  * interrupt from previous kernel might still have ISR bit set.
1494  *
1495  * Most probably by now the CPU has serviced that pending interrupt and it
1496  * might not have done the apic_eoi() because it thought, interrupt
1497  * came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear
1498  * the ISR bit and cpu thinks it has already serviced the interrupt. Hence
1499  * a vector might get locked. It was noticed for timer irq (vector
1500  * 0x31). Issue an extra EOI to clear ISR.
1501  *
1502  * If there are pending IRR bits they turn into ISR bits after a higher
1503  * priority ISR bit has been acked.
1504  */
apic_pending_intr_clear(void)1505 static void apic_pending_intr_clear(void)
1506 {
1507 	union apic_ir irr, isr;
1508 	unsigned int i;
1509 
1510 	/* 512 loops are way oversized and give the APIC a chance to obey. */
1511 	for (i = 0; i < 512; i++) {
1512 		if (!apic_check_and_ack(&irr, &isr))
1513 			return;
1514 	}
1515 	/* Dump the IRR/ISR content if that failed */
1516 	pr_warn("APIC: Stale IRR: %256pb ISR: %256pb\n", irr.map, isr.map);
1517 }
1518 
1519 /**
1520  * setup_local_APIC - setup the local APIC
1521  *
1522  * Used to setup local APIC while initializing BSP or bringing up APs.
1523  * Always called with preemption disabled.
1524  */
setup_local_APIC(void)1525 static void setup_local_APIC(void)
1526 {
1527 	int cpu = smp_processor_id();
1528 	unsigned int value;
1529 
1530 	if (apic_is_disabled) {
1531 		disable_ioapic_support();
1532 		return;
1533 	}
1534 
1535 	/*
1536 	 * If this comes from kexec/kcrash the APIC might be enabled in
1537 	 * SPIV. Soft disable it before doing further initialization.
1538 	 */
1539 	value = apic_read(APIC_SPIV);
1540 	value &= ~APIC_SPIV_APIC_ENABLED;
1541 	apic_write(APIC_SPIV, value);
1542 
1543 #ifdef CONFIG_X86_32
1544 	/* Pound the ESR really hard over the head with a big hammer - mbligh */
1545 	if (lapic_is_integrated() && apic->disable_esr) {
1546 		apic_write(APIC_ESR, 0);
1547 		apic_write(APIC_ESR, 0);
1548 		apic_write(APIC_ESR, 0);
1549 		apic_write(APIC_ESR, 0);
1550 	}
1551 #endif
1552 	/* Validate that the APIC is registered if required */
1553 	BUG_ON(apic->apic_id_registered && !apic->apic_id_registered());
1554 
1555 	/*
1556 	 * Intel recommends to set DFR, LDR and TPR before enabling
1557 	 * an APIC.  See e.g. "AP-388 82489DX User's Manual" (Intel
1558 	 * document number 292116).
1559 	 *
1560 	 * Except for APICs which operate in physical destination mode.
1561 	 */
1562 	if (apic->init_apic_ldr)
1563 		apic->init_apic_ldr();
1564 
1565 	/*
1566 	 * Set Task Priority to 'accept all except vectors 0-31'.  An APIC
1567 	 * vector in the 16-31 range could be delivered if TPR == 0, but we
1568 	 * would think it's an exception and terrible things will happen.  We
1569 	 * never change this later on.
1570 	 */
1571 	value = apic_read(APIC_TASKPRI);
1572 	value &= ~APIC_TPRI_MASK;
1573 	value |= 0x10;
1574 	apic_write(APIC_TASKPRI, value);
1575 
1576 	/* Clear eventually stale ISR/IRR bits */
1577 	apic_pending_intr_clear();
1578 
1579 	/*
1580 	 * Now that we are all set up, enable the APIC
1581 	 */
1582 	value = apic_read(APIC_SPIV);
1583 	value &= ~APIC_VECTOR_MASK;
1584 	/*
1585 	 * Enable APIC
1586 	 */
1587 	value |= APIC_SPIV_APIC_ENABLED;
1588 
1589 #ifdef CONFIG_X86_32
1590 	/*
1591 	 * Some unknown Intel IO/APIC (or APIC) errata is biting us with
1592 	 * certain networking cards. If high frequency interrupts are
1593 	 * happening on a particular IOAPIC pin, plus the IOAPIC routing
1594 	 * entry is masked/unmasked at a high rate as well then sooner or
1595 	 * later IOAPIC line gets 'stuck', no more interrupts are received
1596 	 * from the device. If focus CPU is disabled then the hang goes
1597 	 * away, oh well :-(
1598 	 *
1599 	 * [ This bug can be reproduced easily with a level-triggered
1600 	 *   PCI Ne2000 networking cards and PII/PIII processors, dual
1601 	 *   BX chipset. ]
1602 	 */
1603 	/*
1604 	 * Actually disabling the focus CPU check just makes the hang less
1605 	 * frequent as it makes the interrupt distribution model be more
1606 	 * like LRU than MRU (the short-term load is more even across CPUs).
1607 	 */
1608 
1609 	/*
1610 	 * - enable focus processor (bit==0)
1611 	 * - 64bit mode always use processor focus
1612 	 *   so no need to set it
1613 	 */
1614 	value &= ~APIC_SPIV_FOCUS_DISABLED;
1615 #endif
1616 
1617 	/*
1618 	 * Set spurious IRQ vector
1619 	 */
1620 	value |= SPURIOUS_APIC_VECTOR;
1621 	apic_write(APIC_SPIV, value);
1622 
1623 	perf_events_lapic_init();
1624 
1625 	/*
1626 	 * Set up LVT0, LVT1:
1627 	 *
1628 	 * set up through-local-APIC on the boot CPU's LINT0. This is not
1629 	 * strictly necessary in pure symmetric-IO mode, but sometimes
1630 	 * we delegate interrupts to the 8259A.
1631 	 */
1632 	/*
1633 	 * TODO: set up through-local-APIC from through-I/O-APIC? --macro
1634 	 */
1635 	value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
1636 	if (!cpu && (pic_mode || !value || ioapic_is_disabled)) {
1637 		value = APIC_DM_EXTINT;
1638 		apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu);
1639 	} else {
1640 		value = APIC_DM_EXTINT | APIC_LVT_MASKED;
1641 		apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu);
1642 	}
1643 	apic_write(APIC_LVT0, value);
1644 
1645 	/*
1646 	 * Only the BSP sees the LINT1 NMI signal by default. This can be
1647 	 * modified by apic_extnmi= boot option.
1648 	 */
1649 	if ((!cpu && apic_extnmi != APIC_EXTNMI_NONE) ||
1650 	    apic_extnmi == APIC_EXTNMI_ALL)
1651 		value = APIC_DM_NMI;
1652 	else
1653 		value = APIC_DM_NMI | APIC_LVT_MASKED;
1654 
1655 	/* Is 82489DX ? */
1656 	if (!lapic_is_integrated())
1657 		value |= APIC_LVT_LEVEL_TRIGGER;
1658 	apic_write(APIC_LVT1, value);
1659 
1660 #ifdef CONFIG_X86_MCE_INTEL
1661 	/* Recheck CMCI information after local APIC is up on CPU #0 */
1662 	if (!cpu)
1663 		cmci_recheck();
1664 #endif
1665 }
1666 
end_local_APIC_setup(void)1667 static void end_local_APIC_setup(void)
1668 {
1669 	lapic_setup_esr();
1670 
1671 #ifdef CONFIG_X86_32
1672 	{
1673 		unsigned int value;
1674 		/* Disable the local apic timer */
1675 		value = apic_read(APIC_LVTT);
1676 		value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
1677 		apic_write(APIC_LVTT, value);
1678 	}
1679 #endif
1680 
1681 	apic_pm_activate();
1682 }
1683 
1684 /*
1685  * APIC setup function for application processors. Called from smpboot.c
1686  */
apic_ap_setup(void)1687 void apic_ap_setup(void)
1688 {
1689 	setup_local_APIC();
1690 	end_local_APIC_setup();
1691 }
1692 
1693 static __init void cpu_set_boot_apic(void);
1694 
apic_read_boot_cpu_id(bool x2apic)1695 static __init void apic_read_boot_cpu_id(bool x2apic)
1696 {
1697 	/*
1698 	 * This can be invoked from check_x2apic() before the APIC has been
1699 	 * selected. But that code knows for sure that the BIOS enabled
1700 	 * X2APIC.
1701 	 */
1702 	if (x2apic) {
1703 		boot_cpu_physical_apicid = native_apic_msr_read(APIC_ID);
1704 		boot_cpu_apic_version = GET_APIC_VERSION(native_apic_msr_read(APIC_LVR));
1705 	} else {
1706 		boot_cpu_physical_apicid = read_apic_id();
1707 		boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
1708 	}
1709 	cpu_set_boot_apic();
1710 }
1711 
1712 #ifdef CONFIG_X86_X2APIC
1713 int x2apic_mode;
1714 EXPORT_SYMBOL_GPL(x2apic_mode);
1715 
1716 enum {
1717 	X2APIC_OFF,
1718 	X2APIC_DISABLED,
1719 	/* All states below here have X2APIC enabled */
1720 	X2APIC_ON,
1721 	X2APIC_ON_LOCKED
1722 };
1723 static int x2apic_state;
1724 
x2apic_hw_locked(void)1725 static bool x2apic_hw_locked(void)
1726 {
1727 	u64 ia32_cap;
1728 	u64 msr;
1729 
1730 	ia32_cap = x86_read_arch_cap_msr();
1731 	if (ia32_cap & ARCH_CAP_XAPIC_DISABLE) {
1732 		rdmsrl(MSR_IA32_XAPIC_DISABLE_STATUS, msr);
1733 		return (msr & LEGACY_XAPIC_DISABLED);
1734 	}
1735 	return false;
1736 }
1737 
__x2apic_disable(void)1738 static void __x2apic_disable(void)
1739 {
1740 	u64 msr;
1741 
1742 	if (!boot_cpu_has(X86_FEATURE_APIC))
1743 		return;
1744 
1745 	rdmsrl(MSR_IA32_APICBASE, msr);
1746 	if (!(msr & X2APIC_ENABLE))
1747 		return;
1748 	/* Disable xapic and x2apic first and then reenable xapic mode */
1749 	wrmsrl(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
1750 	wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE);
1751 	printk_once(KERN_INFO "x2apic disabled\n");
1752 }
1753 
__x2apic_enable(void)1754 static void __x2apic_enable(void)
1755 {
1756 	u64 msr;
1757 
1758 	rdmsrl(MSR_IA32_APICBASE, msr);
1759 	if (msr & X2APIC_ENABLE)
1760 		return;
1761 	wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE);
1762 	printk_once(KERN_INFO "x2apic enabled\n");
1763 }
1764 
setup_nox2apic(char * str)1765 static int __init setup_nox2apic(char *str)
1766 {
1767 	if (x2apic_enabled()) {
1768 		int apicid = native_apic_msr_read(APIC_ID);
1769 
1770 		if (apicid >= 255) {
1771 			pr_warn("Apicid: %08x, cannot enforce nox2apic\n",
1772 				apicid);
1773 			return 0;
1774 		}
1775 		if (x2apic_hw_locked()) {
1776 			pr_warn("APIC locked in x2apic mode, can't disable\n");
1777 			return 0;
1778 		}
1779 		pr_warn("x2apic already enabled.\n");
1780 		__x2apic_disable();
1781 	}
1782 	setup_clear_cpu_cap(X86_FEATURE_X2APIC);
1783 	x2apic_state = X2APIC_DISABLED;
1784 	x2apic_mode = 0;
1785 	return 0;
1786 }
1787 early_param("nox2apic", setup_nox2apic);
1788 
1789 /* Called from cpu_init() to enable x2apic on (secondary) cpus */
x2apic_setup(void)1790 void x2apic_setup(void)
1791 {
1792 	/*
1793 	 * Try to make the AP's APIC state match that of the BSP,  but if the
1794 	 * BSP is unlocked and the AP is locked then there is a state mismatch.
1795 	 * Warn about the mismatch in case a GP fault occurs due to a locked AP
1796 	 * trying to be turned off.
1797 	 */
1798 	if (x2apic_state != X2APIC_ON_LOCKED && x2apic_hw_locked())
1799 		pr_warn("x2apic lock mismatch between BSP and AP.\n");
1800 	/*
1801 	 * If x2apic is not in ON or LOCKED state, disable it if already enabled
1802 	 * from BIOS.
1803 	 */
1804 	if (x2apic_state < X2APIC_ON) {
1805 		__x2apic_disable();
1806 		return;
1807 	}
1808 	__x2apic_enable();
1809 }
1810 
1811 static __init void apic_set_fixmap(void);
1812 
x2apic_disable(void)1813 static __init void x2apic_disable(void)
1814 {
1815 	u32 x2apic_id, state = x2apic_state;
1816 
1817 	x2apic_mode = 0;
1818 	x2apic_state = X2APIC_DISABLED;
1819 
1820 	if (state != X2APIC_ON)
1821 		return;
1822 
1823 	x2apic_id = read_apic_id();
1824 	if (x2apic_id >= 255)
1825 		panic("Cannot disable x2apic, id: %08x\n", x2apic_id);
1826 
1827 	if (x2apic_hw_locked()) {
1828 		pr_warn("Cannot disable locked x2apic, id: %08x\n", x2apic_id);
1829 		return;
1830 	}
1831 
1832 	__x2apic_disable();
1833 	apic_set_fixmap();
1834 }
1835 
x2apic_enable(void)1836 static __init void x2apic_enable(void)
1837 {
1838 	if (x2apic_state != X2APIC_OFF)
1839 		return;
1840 
1841 	x2apic_mode = 1;
1842 	x2apic_state = X2APIC_ON;
1843 	__x2apic_enable();
1844 }
1845 
try_to_enable_x2apic(int remap_mode)1846 static __init void try_to_enable_x2apic(int remap_mode)
1847 {
1848 	if (x2apic_state == X2APIC_DISABLED)
1849 		return;
1850 
1851 	if (remap_mode != IRQ_REMAP_X2APIC_MODE) {
1852 		u32 apic_limit = 255;
1853 
1854 		/*
1855 		 * Using X2APIC without IR is not architecturally supported
1856 		 * on bare metal but may be supported in guests.
1857 		 */
1858 		if (!x86_init.hyper.x2apic_available()) {
1859 			pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n");
1860 			x2apic_disable();
1861 			return;
1862 		}
1863 
1864 		/*
1865 		 * If the hypervisor supports extended destination ID in
1866 		 * MSI, that increases the maximum APIC ID that can be
1867 		 * used for non-remapped IRQ domains.
1868 		 */
1869 		if (x86_init.hyper.msi_ext_dest_id()) {
1870 			virt_ext_dest_id = 1;
1871 			apic_limit = 32767;
1872 		}
1873 
1874 		/*
1875 		 * Without IR, all CPUs can be addressed by IOAPIC/MSI only
1876 		 * in physical mode, and CPUs with an APIC ID that cannot
1877 		 * be addressed must not be brought online.
1878 		 */
1879 		x2apic_set_max_apicid(apic_limit);
1880 		x2apic_phys = 1;
1881 	}
1882 	x2apic_enable();
1883 }
1884 
check_x2apic(void)1885 void __init check_x2apic(void)
1886 {
1887 	if (x2apic_enabled()) {
1888 		pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n");
1889 		x2apic_mode = 1;
1890 		if (x2apic_hw_locked())
1891 			x2apic_state = X2APIC_ON_LOCKED;
1892 		else
1893 			x2apic_state = X2APIC_ON;
1894 		apic_read_boot_cpu_id(true);
1895 	} else if (!boot_cpu_has(X86_FEATURE_X2APIC)) {
1896 		x2apic_state = X2APIC_DISABLED;
1897 	}
1898 }
1899 #else /* CONFIG_X86_X2APIC */
check_x2apic(void)1900 void __init check_x2apic(void)
1901 {
1902 	if (!apic_is_x2apic_enabled())
1903 		return;
1904 	/*
1905 	 * Checkme: Can we simply turn off x2APIC here instead of disabling the APIC?
1906 	 */
1907 	pr_err("Kernel does not support x2APIC, please recompile with CONFIG_X86_X2APIC.\n");
1908 	pr_err("Disabling APIC, expect reduced performance and functionality.\n");
1909 
1910 	apic_is_disabled = true;
1911 	setup_clear_cpu_cap(X86_FEATURE_APIC);
1912 }
1913 
try_to_enable_x2apic(int remap_mode)1914 static inline void try_to_enable_x2apic(int remap_mode) { }
__x2apic_enable(void)1915 static inline void __x2apic_enable(void) { }
1916 #endif /* !CONFIG_X86_X2APIC */
1917 
enable_IR_x2apic(void)1918 void __init enable_IR_x2apic(void)
1919 {
1920 	unsigned long flags;
1921 	int ret, ir_stat;
1922 
1923 	if (ioapic_is_disabled) {
1924 		pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
1925 		return;
1926 	}
1927 
1928 	ir_stat = irq_remapping_prepare();
1929 	if (ir_stat < 0 && !x2apic_supported())
1930 		return;
1931 
1932 	ret = save_ioapic_entries();
1933 	if (ret) {
1934 		pr_info("Saving IO-APIC state failed: %d\n", ret);
1935 		return;
1936 	}
1937 
1938 	local_irq_save(flags);
1939 	legacy_pic->mask_all();
1940 	mask_ioapic_entries();
1941 
1942 	/* If irq_remapping_prepare() succeeded, try to enable it */
1943 	if (ir_stat >= 0)
1944 		ir_stat = irq_remapping_enable();
1945 	/* ir_stat contains the remap mode or an error code */
1946 	try_to_enable_x2apic(ir_stat);
1947 
1948 	if (ir_stat < 0)
1949 		restore_ioapic_entries();
1950 	legacy_pic->restore_mask();
1951 	local_irq_restore(flags);
1952 }
1953 
1954 #ifdef CONFIG_X86_64
1955 /*
1956  * Detect and enable local APICs on non-SMP boards.
1957  * Original code written by Keir Fraser.
1958  * On AMD64 we trust the BIOS - if it says no APIC it is likely
1959  * not correctly set up (usually the APIC timer won't work etc.)
1960  */
detect_init_APIC(void)1961 static bool __init detect_init_APIC(void)
1962 {
1963 	if (!boot_cpu_has(X86_FEATURE_APIC)) {
1964 		pr_info("No local APIC present\n");
1965 		return false;
1966 	}
1967 
1968 	register_lapic_address(APIC_DEFAULT_PHYS_BASE);
1969 	return true;
1970 }
1971 #else
1972 
apic_verify(unsigned long addr)1973 static bool __init apic_verify(unsigned long addr)
1974 {
1975 	u32 features, h, l;
1976 
1977 	/*
1978 	 * The APIC feature bit should now be enabled
1979 	 * in `cpuid'
1980 	 */
1981 	features = cpuid_edx(1);
1982 	if (!(features & (1 << X86_FEATURE_APIC))) {
1983 		pr_warn("Could not enable APIC!\n");
1984 		return false;
1985 	}
1986 	set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
1987 
1988 	/* The BIOS may have set up the APIC at some other address */
1989 	if (boot_cpu_data.x86 >= 6) {
1990 		rdmsr(MSR_IA32_APICBASE, l, h);
1991 		if (l & MSR_IA32_APICBASE_ENABLE)
1992 			addr = l & MSR_IA32_APICBASE_BASE;
1993 	}
1994 
1995 	register_lapic_address(addr);
1996 	pr_info("Found and enabled local APIC!\n");
1997 	return true;
1998 }
1999 
apic_force_enable(unsigned long addr)2000 bool __init apic_force_enable(unsigned long addr)
2001 {
2002 	u32 h, l;
2003 
2004 	if (apic_is_disabled)
2005 		return false;
2006 
2007 	/*
2008 	 * Some BIOSes disable the local APIC in the APIC_BASE
2009 	 * MSR. This can only be done in software for Intel P6 or later
2010 	 * and AMD K7 (Model > 1) or later.
2011 	 */
2012 	if (boot_cpu_data.x86 >= 6) {
2013 		rdmsr(MSR_IA32_APICBASE, l, h);
2014 		if (!(l & MSR_IA32_APICBASE_ENABLE)) {
2015 			pr_info("Local APIC disabled by BIOS -- reenabling.\n");
2016 			l &= ~MSR_IA32_APICBASE_BASE;
2017 			l |= MSR_IA32_APICBASE_ENABLE | addr;
2018 			wrmsr(MSR_IA32_APICBASE, l, h);
2019 			enabled_via_apicbase = 1;
2020 		}
2021 	}
2022 	return apic_verify(addr);
2023 }
2024 
2025 /*
2026  * Detect and initialize APIC
2027  */
detect_init_APIC(void)2028 static bool __init detect_init_APIC(void)
2029 {
2030 	/* Disabled by kernel option? */
2031 	if (apic_is_disabled)
2032 		return false;
2033 
2034 	switch (boot_cpu_data.x86_vendor) {
2035 	case X86_VENDOR_AMD:
2036 		if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
2037 		    (boot_cpu_data.x86 >= 15))
2038 			break;
2039 		goto no_apic;
2040 	case X86_VENDOR_HYGON:
2041 		break;
2042 	case X86_VENDOR_INTEL:
2043 		if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
2044 		    (boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC)))
2045 			break;
2046 		goto no_apic;
2047 	default:
2048 		goto no_apic;
2049 	}
2050 
2051 	if (!boot_cpu_has(X86_FEATURE_APIC)) {
2052 		/*
2053 		 * Over-ride BIOS and try to enable the local APIC only if
2054 		 * "lapic" specified.
2055 		 */
2056 		if (!force_enable_local_apic) {
2057 			pr_info("Local APIC disabled by BIOS -- "
2058 				"you can enable it with \"lapic\"\n");
2059 			return false;
2060 		}
2061 		if (!apic_force_enable(APIC_DEFAULT_PHYS_BASE))
2062 			return false;
2063 	} else {
2064 		if (!apic_verify(APIC_DEFAULT_PHYS_BASE))
2065 			return false;
2066 	}
2067 
2068 	apic_pm_activate();
2069 
2070 	return true;
2071 
2072 no_apic:
2073 	pr_info("No local APIC present or hardware disabled\n");
2074 	return false;
2075 }
2076 #endif
2077 
2078 /**
2079  * init_apic_mappings - initialize APIC mappings
2080  */
init_apic_mappings(void)2081 void __init init_apic_mappings(void)
2082 {
2083 	if (apic_validate_deadline_timer())
2084 		pr_info("TSC deadline timer available\n");
2085 
2086 	if (x2apic_mode)
2087 		return;
2088 
2089 	if (!smp_found_config) {
2090 		if (!detect_init_APIC()) {
2091 			pr_info("APIC: disable apic facility\n");
2092 			apic_disable();
2093 		}
2094 		num_processors = 1;
2095 	}
2096 }
2097 
apic_set_fixmap(void)2098 static __init void apic_set_fixmap(void)
2099 {
2100 	set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr);
2101 	apic_mmio_base = APIC_BASE;
2102 	apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
2103 		    apic_mmio_base, mp_lapic_addr);
2104 	apic_read_boot_cpu_id(false);
2105 }
2106 
register_lapic_address(unsigned long address)2107 void __init register_lapic_address(unsigned long address)
2108 {
2109 	/* This should only happen once */
2110 	WARN_ON_ONCE(mp_lapic_addr);
2111 	mp_lapic_addr = address;
2112 
2113 	if (!x2apic_mode)
2114 		apic_set_fixmap();
2115 }
2116 
2117 /*
2118  * Local APIC interrupts
2119  */
2120 
2121 /*
2122  * Common handling code for spurious_interrupt and spurious_vector entry
2123  * points below. No point in allowing the compiler to inline it twice.
2124  */
handle_spurious_interrupt(u8 vector)2125 static noinline void handle_spurious_interrupt(u8 vector)
2126 {
2127 	u32 v;
2128 
2129 	trace_spurious_apic_entry(vector);
2130 
2131 	inc_irq_stat(irq_spurious_count);
2132 
2133 	/*
2134 	 * If this is a spurious interrupt then do not acknowledge
2135 	 */
2136 	if (vector == SPURIOUS_APIC_VECTOR) {
2137 		/* See SDM vol 3 */
2138 		pr_info("Spurious APIC interrupt (vector 0xFF) on CPU#%d, should never happen.\n",
2139 			smp_processor_id());
2140 		goto out;
2141 	}
2142 
2143 	/*
2144 	 * If it is a vectored one, verify it's set in the ISR. If set,
2145 	 * acknowledge it.
2146 	 */
2147 	v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1));
2148 	if (v & (1 << (vector & 0x1f))) {
2149 		pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n",
2150 			vector, smp_processor_id());
2151 		apic_eoi();
2152 	} else {
2153 		pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n",
2154 			vector, smp_processor_id());
2155 	}
2156 out:
2157 	trace_spurious_apic_exit(vector);
2158 }
2159 
2160 /**
2161  * spurious_interrupt - Catch all for interrupts raised on unused vectors
2162  * @regs:	Pointer to pt_regs on stack
2163  * @vector:	The vector number
2164  *
2165  * This is invoked from ASM entry code to catch all interrupts which
2166  * trigger on an entry which is routed to the common_spurious idtentry
2167  * point.
2168  */
DEFINE_IDTENTRY_IRQ(spurious_interrupt)2169 DEFINE_IDTENTRY_IRQ(spurious_interrupt)
2170 {
2171 	handle_spurious_interrupt(vector);
2172 }
2173 
DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt)2174 DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt)
2175 {
2176 	handle_spurious_interrupt(SPURIOUS_APIC_VECTOR);
2177 }
2178 
2179 /*
2180  * This interrupt should never happen with our APIC/SMP architecture
2181  */
DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)2182 DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)
2183 {
2184 	static const char * const error_interrupt_reason[] = {
2185 		"Send CS error",		/* APIC Error Bit 0 */
2186 		"Receive CS error",		/* APIC Error Bit 1 */
2187 		"Send accept error",		/* APIC Error Bit 2 */
2188 		"Receive accept error",		/* APIC Error Bit 3 */
2189 		"Redirectable IPI",		/* APIC Error Bit 4 */
2190 		"Send illegal vector",		/* APIC Error Bit 5 */
2191 		"Received illegal vector",	/* APIC Error Bit 6 */
2192 		"Illegal register address",	/* APIC Error Bit 7 */
2193 	};
2194 	u32 v, i = 0;
2195 
2196 	trace_error_apic_entry(ERROR_APIC_VECTOR);
2197 
2198 	/* First tickle the hardware, only then report what went on. -- REW */
2199 	if (lapic_get_maxlvt() > 3)	/* Due to the Pentium erratum 3AP. */
2200 		apic_write(APIC_ESR, 0);
2201 	v = apic_read(APIC_ESR);
2202 	apic_eoi();
2203 	atomic_inc(&irq_err_count);
2204 
2205 	apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x",
2206 		    smp_processor_id(), v);
2207 
2208 	v &= 0xff;
2209 	while (v) {
2210 		if (v & 0x1)
2211 			apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]);
2212 		i++;
2213 		v >>= 1;
2214 	}
2215 
2216 	apic_printk(APIC_DEBUG, KERN_CONT "\n");
2217 
2218 	trace_error_apic_exit(ERROR_APIC_VECTOR);
2219 }
2220 
2221 /**
2222  * connect_bsp_APIC - attach the APIC to the interrupt system
2223  */
connect_bsp_APIC(void)2224 static void __init connect_bsp_APIC(void)
2225 {
2226 #ifdef CONFIG_X86_32
2227 	if (pic_mode) {
2228 		/*
2229 		 * Do not trust the local APIC being empty at bootup.
2230 		 */
2231 		clear_local_APIC();
2232 		/*
2233 		 * PIC mode, enable APIC mode in the IMCR, i.e.  connect BSP's
2234 		 * local APIC to INT and NMI lines.
2235 		 */
2236 		apic_printk(APIC_VERBOSE, "leaving PIC mode, "
2237 				"enabling APIC mode.\n");
2238 		imcr_pic_to_apic();
2239 	}
2240 #endif
2241 }
2242 
2243 /**
2244  * disconnect_bsp_APIC - detach the APIC from the interrupt system
2245  * @virt_wire_setup:	indicates, whether virtual wire mode is selected
2246  *
2247  * Virtual wire mode is necessary to deliver legacy interrupts even when the
2248  * APIC is disabled.
2249  */
disconnect_bsp_APIC(int virt_wire_setup)2250 void disconnect_bsp_APIC(int virt_wire_setup)
2251 {
2252 	unsigned int value;
2253 
2254 #ifdef CONFIG_X86_32
2255 	if (pic_mode) {
2256 		/*
2257 		 * Put the board back into PIC mode (has an effect only on
2258 		 * certain older boards).  Note that APIC interrupts, including
2259 		 * IPIs, won't work beyond this point!  The only exception are
2260 		 * INIT IPIs.
2261 		 */
2262 		apic_printk(APIC_VERBOSE, "disabling APIC mode, "
2263 				"entering PIC mode.\n");
2264 		imcr_apic_to_pic();
2265 		return;
2266 	}
2267 #endif
2268 
2269 	/* Go back to Virtual Wire compatibility mode */
2270 
2271 	/* For the spurious interrupt use vector F, and enable it */
2272 	value = apic_read(APIC_SPIV);
2273 	value &= ~APIC_VECTOR_MASK;
2274 	value |= APIC_SPIV_APIC_ENABLED;
2275 	value |= 0xf;
2276 	apic_write(APIC_SPIV, value);
2277 
2278 	if (!virt_wire_setup) {
2279 		/*
2280 		 * For LVT0 make it edge triggered, active high,
2281 		 * external and enabled
2282 		 */
2283 		value = apic_read(APIC_LVT0);
2284 		value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2285 			APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2286 			APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2287 		value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2288 		value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
2289 		apic_write(APIC_LVT0, value);
2290 	} else {
2291 		/* Disable LVT0 */
2292 		apic_write(APIC_LVT0, APIC_LVT_MASKED);
2293 	}
2294 
2295 	/*
2296 	 * For LVT1 make it edge triggered, active high,
2297 	 * nmi and enabled
2298 	 */
2299 	value = apic_read(APIC_LVT1);
2300 	value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2301 			APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2302 			APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2303 	value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2304 	value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
2305 	apic_write(APIC_LVT1, value);
2306 }
2307 
2308 /*
2309  * The number of allocated logical CPU IDs. Since logical CPU IDs are allocated
2310  * contiguously, it equals to current allocated max logical CPU ID plus 1.
2311  * All allocated CPU IDs should be in the [0, nr_logical_cpuids) range,
2312  * so the maximum of nr_logical_cpuids is nr_cpu_ids.
2313  *
2314  * NOTE: Reserve 0 for BSP.
2315  */
2316 static int nr_logical_cpuids = 1;
2317 
2318 /*
2319  * Used to store mapping between logical CPU IDs and APIC IDs.
2320  */
2321 int cpuid_to_apicid[] = {
2322 	[0 ... NR_CPUS - 1] = -1,
2323 };
2324 
arch_match_cpu_phys_id(int cpu,u64 phys_id)2325 bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
2326 {
2327 	return phys_id == cpuid_to_apicid[cpu];
2328 }
2329 
2330 #ifdef CONFIG_SMP
cpu_mark_primary_thread(unsigned int cpu,unsigned int apicid)2331 static void cpu_mark_primary_thread(unsigned int cpu, unsigned int apicid)
2332 {
2333 	/* Isolate the SMT bit(s) in the APICID and check for 0 */
2334 	u32 mask = (1U << (fls(smp_num_siblings) - 1)) - 1;
2335 
2336 	if (smp_num_siblings == 1 || !(apicid & mask))
2337 		cpumask_set_cpu(cpu, &__cpu_primary_thread_mask);
2338 }
2339 
2340 /*
2341  * Due to the utter mess of CPUID evaluation smp_num_siblings is not valid
2342  * during early boot. Initialize the primary thread mask before SMP
2343  * bringup.
2344  */
smp_init_primary_thread_mask(void)2345 static int __init smp_init_primary_thread_mask(void)
2346 {
2347 	unsigned int cpu;
2348 
2349 	/*
2350 	 * XEN/PV provides either none or useless topology information.
2351 	 * Pretend that all vCPUs are primary threads.
2352 	 */
2353 	if (xen_pv_domain()) {
2354 		cpumask_copy(&__cpu_primary_thread_mask, cpu_possible_mask);
2355 		return 0;
2356 	}
2357 
2358 	for (cpu = 0; cpu < nr_logical_cpuids; cpu++)
2359 		cpu_mark_primary_thread(cpu, cpuid_to_apicid[cpu]);
2360 	return 0;
2361 }
2362 early_initcall(smp_init_primary_thread_mask);
2363 #else
cpu_mark_primary_thread(unsigned int cpu,unsigned int apicid)2364 static inline void cpu_mark_primary_thread(unsigned int cpu, unsigned int apicid) { }
2365 #endif
2366 
2367 /*
2368  * Should use this API to allocate logical CPU IDs to keep nr_logical_cpuids
2369  * and cpuid_to_apicid[] synchronized.
2370  */
allocate_logical_cpuid(int apicid)2371 static int allocate_logical_cpuid(int apicid)
2372 {
2373 	int i;
2374 
2375 	/*
2376 	 * cpuid <-> apicid mapping is persistent, so when a cpu is up,
2377 	 * check if the kernel has allocated a cpuid for it.
2378 	 */
2379 	for (i = 0; i < nr_logical_cpuids; i++) {
2380 		if (cpuid_to_apicid[i] == apicid)
2381 			return i;
2382 	}
2383 
2384 	/* Allocate a new cpuid. */
2385 	if (nr_logical_cpuids >= nr_cpu_ids) {
2386 		WARN_ONCE(1, "APIC: NR_CPUS/possible_cpus limit of %u reached. "
2387 			     "Processor %d/0x%x and the rest are ignored.\n",
2388 			     nr_cpu_ids, nr_logical_cpuids, apicid);
2389 		return -EINVAL;
2390 	}
2391 
2392 	cpuid_to_apicid[nr_logical_cpuids] = apicid;
2393 	return nr_logical_cpuids++;
2394 }
2395 
cpu_update_apic(int cpu,int apicid)2396 static void cpu_update_apic(int cpu, int apicid)
2397 {
2398 #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
2399 	early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
2400 #endif
2401 	set_cpu_possible(cpu, true);
2402 	physid_set(apicid, phys_cpu_present_map);
2403 	set_cpu_present(cpu, true);
2404 	num_processors++;
2405 
2406 	if (system_state != SYSTEM_BOOTING)
2407 		cpu_mark_primary_thread(cpu, apicid);
2408 }
2409 
cpu_set_boot_apic(void)2410 static __init void cpu_set_boot_apic(void)
2411 {
2412 	cpuid_to_apicid[0] = boot_cpu_physical_apicid;
2413 	cpu_update_apic(0, boot_cpu_physical_apicid);
2414 	x86_32_probe_bigsmp_early();
2415 }
2416 
generic_processor_info(int apicid)2417 int generic_processor_info(int apicid)
2418 {
2419 	int cpu, max = nr_cpu_ids;
2420 
2421 	/* The boot CPU must be set before MADT/MPTABLE parsing happens */
2422 	if (cpuid_to_apicid[0] == BAD_APICID)
2423 		panic("Boot CPU APIC not registered yet\n");
2424 
2425 	if (apicid == boot_cpu_physical_apicid)
2426 		return 0;
2427 
2428 	if (disabled_cpu_apicid == apicid) {
2429 		int thiscpu = num_processors + disabled_cpus;
2430 
2431 		pr_warn("APIC: Disabling requested cpu. Processor %d/0x%x ignored.\n",
2432 			thiscpu, apicid);
2433 
2434 		disabled_cpus++;
2435 		return -ENODEV;
2436 	}
2437 
2438 	if (num_processors >= nr_cpu_ids) {
2439 		int thiscpu = max + disabled_cpus;
2440 
2441 		pr_warn("APIC: NR_CPUS/possible_cpus limit of %i reached. "
2442 			"Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
2443 
2444 		disabled_cpus++;
2445 		return -EINVAL;
2446 	}
2447 
2448 	cpu = allocate_logical_cpuid(apicid);
2449 	if (cpu < 0) {
2450 		disabled_cpus++;
2451 		return -EINVAL;
2452 	}
2453 
2454 	cpu_update_apic(cpu, apicid);
2455 	return cpu;
2456 }
2457 
2458 
__irq_msi_compose_msg(struct irq_cfg * cfg,struct msi_msg * msg,bool dmar)2459 void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg,
2460 			   bool dmar)
2461 {
2462 	memset(msg, 0, sizeof(*msg));
2463 
2464 	msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;
2465 	msg->arch_addr_lo.dest_mode_logical = apic->dest_mode_logical;
2466 	msg->arch_addr_lo.destid_0_7 = cfg->dest_apicid & 0xFF;
2467 
2468 	msg->arch_data.delivery_mode = APIC_DELIVERY_MODE_FIXED;
2469 	msg->arch_data.vector = cfg->vector;
2470 
2471 	msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;
2472 	/*
2473 	 * Only the IOMMU itself can use the trick of putting destination
2474 	 * APIC ID into the high bits of the address. Anything else would
2475 	 * just be writing to memory if it tried that, and needs IR to
2476 	 * address APICs which can't be addressed in the normal 32-bit
2477 	 * address range at 0xFFExxxxx. That is typically just 8 bits, but
2478 	 * some hypervisors allow the extended destination ID field in bits
2479 	 * 5-11 to be used, giving support for 15 bits of APIC IDs in total.
2480 	 */
2481 	if (dmar)
2482 		msg->arch_addr_hi.destid_8_31 = cfg->dest_apicid >> 8;
2483 	else if (virt_ext_dest_id && cfg->dest_apicid < 0x8000)
2484 		msg->arch_addr_lo.virt_destid_8_14 = cfg->dest_apicid >> 8;
2485 	else
2486 		WARN_ON_ONCE(cfg->dest_apicid > 0xFF);
2487 }
2488 
x86_msi_msg_get_destid(struct msi_msg * msg,bool extid)2489 u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid)
2490 {
2491 	u32 dest = msg->arch_addr_lo.destid_0_7;
2492 
2493 	if (extid)
2494 		dest |= msg->arch_addr_hi.destid_8_31 << 8;
2495 	return dest;
2496 }
2497 EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid);
2498 
apic_bsp_up_setup(void)2499 static void __init apic_bsp_up_setup(void)
2500 {
2501 #ifdef CONFIG_X86_64
2502 	apic_write(APIC_ID, apic->set_apic_id(boot_cpu_physical_apicid));
2503 #endif
2504 	physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
2505 }
2506 
2507 /**
2508  * apic_bsp_setup - Setup function for local apic and io-apic
2509  * @upmode:		Force UP mode (for APIC_init_uniprocessor)
2510  */
apic_bsp_setup(bool upmode)2511 static void __init apic_bsp_setup(bool upmode)
2512 {
2513 	connect_bsp_APIC();
2514 	if (upmode)
2515 		apic_bsp_up_setup();
2516 	setup_local_APIC();
2517 
2518 	enable_IO_APIC();
2519 	end_local_APIC_setup();
2520 	irq_remap_enable_fault_handling();
2521 	setup_IO_APIC();
2522 	lapic_update_legacy_vectors();
2523 }
2524 
2525 #ifdef CONFIG_UP_LATE_INIT
up_late_init(void)2526 void __init up_late_init(void)
2527 {
2528 	if (apic_intr_mode == APIC_PIC)
2529 		return;
2530 
2531 	/* Setup local timer */
2532 	x86_init.timers.setup_percpu_clockev();
2533 }
2534 #endif
2535 
2536 /*
2537  * Power management
2538  */
2539 #ifdef CONFIG_PM
2540 
2541 static struct {
2542 	/*
2543 	 * 'active' is true if the local APIC was enabled by us and
2544 	 * not the BIOS; this signifies that we are also responsible
2545 	 * for disabling it before entering apm/acpi suspend
2546 	 */
2547 	int active;
2548 	/* r/w apic fields */
2549 	unsigned int apic_id;
2550 	unsigned int apic_taskpri;
2551 	unsigned int apic_ldr;
2552 	unsigned int apic_dfr;
2553 	unsigned int apic_spiv;
2554 	unsigned int apic_lvtt;
2555 	unsigned int apic_lvtpc;
2556 	unsigned int apic_lvt0;
2557 	unsigned int apic_lvt1;
2558 	unsigned int apic_lvterr;
2559 	unsigned int apic_tmict;
2560 	unsigned int apic_tdcr;
2561 	unsigned int apic_thmr;
2562 	unsigned int apic_cmci;
2563 } apic_pm_state;
2564 
lapic_suspend(void)2565 static int lapic_suspend(void)
2566 {
2567 	unsigned long flags;
2568 	int maxlvt;
2569 
2570 	if (!apic_pm_state.active)
2571 		return 0;
2572 
2573 	maxlvt = lapic_get_maxlvt();
2574 
2575 	apic_pm_state.apic_id = apic_read(APIC_ID);
2576 	apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
2577 	apic_pm_state.apic_ldr = apic_read(APIC_LDR);
2578 	apic_pm_state.apic_dfr = apic_read(APIC_DFR);
2579 	apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
2580 	apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
2581 	if (maxlvt >= 4)
2582 		apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
2583 	apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
2584 	apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
2585 	apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
2586 	apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
2587 	apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
2588 #ifdef CONFIG_X86_THERMAL_VECTOR
2589 	if (maxlvt >= 5)
2590 		apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
2591 #endif
2592 #ifdef CONFIG_X86_MCE_INTEL
2593 	if (maxlvt >= 6)
2594 		apic_pm_state.apic_cmci = apic_read(APIC_LVTCMCI);
2595 #endif
2596 
2597 	local_irq_save(flags);
2598 
2599 	/*
2600 	 * Mask IOAPIC before disabling the local APIC to prevent stale IRR
2601 	 * entries on some implementations.
2602 	 */
2603 	mask_ioapic_entries();
2604 
2605 	disable_local_APIC();
2606 
2607 	irq_remapping_disable();
2608 
2609 	local_irq_restore(flags);
2610 	return 0;
2611 }
2612 
lapic_resume(void)2613 static void lapic_resume(void)
2614 {
2615 	unsigned int l, h;
2616 	unsigned long flags;
2617 	int maxlvt;
2618 
2619 	if (!apic_pm_state.active)
2620 		return;
2621 
2622 	local_irq_save(flags);
2623 
2624 	/*
2625 	 * IO-APIC and PIC have their own resume routines.
2626 	 * We just mask them here to make sure the interrupt
2627 	 * subsystem is completely quiet while we enable x2apic
2628 	 * and interrupt-remapping.
2629 	 */
2630 	mask_ioapic_entries();
2631 	legacy_pic->mask_all();
2632 
2633 	if (x2apic_mode) {
2634 		__x2apic_enable();
2635 	} else {
2636 		/*
2637 		 * Make sure the APICBASE points to the right address
2638 		 *
2639 		 * FIXME! This will be wrong if we ever support suspend on
2640 		 * SMP! We'll need to do this as part of the CPU restore!
2641 		 */
2642 		if (boot_cpu_data.x86 >= 6) {
2643 			rdmsr(MSR_IA32_APICBASE, l, h);
2644 			l &= ~MSR_IA32_APICBASE_BASE;
2645 			l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
2646 			wrmsr(MSR_IA32_APICBASE, l, h);
2647 		}
2648 	}
2649 
2650 	maxlvt = lapic_get_maxlvt();
2651 	apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
2652 	apic_write(APIC_ID, apic_pm_state.apic_id);
2653 	apic_write(APIC_DFR, apic_pm_state.apic_dfr);
2654 	apic_write(APIC_LDR, apic_pm_state.apic_ldr);
2655 	apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
2656 	apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
2657 	apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
2658 	apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
2659 #ifdef CONFIG_X86_THERMAL_VECTOR
2660 	if (maxlvt >= 5)
2661 		apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
2662 #endif
2663 #ifdef CONFIG_X86_MCE_INTEL
2664 	if (maxlvt >= 6)
2665 		apic_write(APIC_LVTCMCI, apic_pm_state.apic_cmci);
2666 #endif
2667 	if (maxlvt >= 4)
2668 		apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
2669 	apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
2670 	apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
2671 	apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
2672 	apic_write(APIC_ESR, 0);
2673 	apic_read(APIC_ESR);
2674 	apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
2675 	apic_write(APIC_ESR, 0);
2676 	apic_read(APIC_ESR);
2677 
2678 	irq_remapping_reenable(x2apic_mode);
2679 
2680 	local_irq_restore(flags);
2681 }
2682 
2683 /*
2684  * This device has no shutdown method - fully functioning local APICs
2685  * are needed on every CPU up until machine_halt/restart/poweroff.
2686  */
2687 
2688 static struct syscore_ops lapic_syscore_ops = {
2689 	.resume		= lapic_resume,
2690 	.suspend	= lapic_suspend,
2691 };
2692 
apic_pm_activate(void)2693 static void apic_pm_activate(void)
2694 {
2695 	apic_pm_state.active = 1;
2696 }
2697 
init_lapic_sysfs(void)2698 static int __init init_lapic_sysfs(void)
2699 {
2700 	/* XXX: remove suspend/resume procs if !apic_pm_state.active? */
2701 	if (boot_cpu_has(X86_FEATURE_APIC))
2702 		register_syscore_ops(&lapic_syscore_ops);
2703 
2704 	return 0;
2705 }
2706 
2707 /* local apic needs to resume before other devices access its registers. */
2708 core_initcall(init_lapic_sysfs);
2709 
2710 #else	/* CONFIG_PM */
2711 
apic_pm_activate(void)2712 static void apic_pm_activate(void) { }
2713 
2714 #endif	/* CONFIG_PM */
2715 
2716 #ifdef CONFIG_X86_64
2717 
2718 static int multi_checked;
2719 static int multi;
2720 
set_multi(const struct dmi_system_id * d)2721 static int set_multi(const struct dmi_system_id *d)
2722 {
2723 	if (multi)
2724 		return 0;
2725 	pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
2726 	multi = 1;
2727 	return 0;
2728 }
2729 
2730 static const struct dmi_system_id multi_dmi_table[] = {
2731 	{
2732 		.callback = set_multi,
2733 		.ident = "IBM System Summit2",
2734 		.matches = {
2735 			DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
2736 			DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
2737 		},
2738 	},
2739 	{}
2740 };
2741 
dmi_check_multi(void)2742 static void dmi_check_multi(void)
2743 {
2744 	if (multi_checked)
2745 		return;
2746 
2747 	dmi_check_system(multi_dmi_table);
2748 	multi_checked = 1;
2749 }
2750 
2751 /*
2752  * apic_is_clustered_box() -- Check if we can expect good TSC
2753  *
2754  * Thus far, the major user of this is IBM's Summit2 series:
2755  * Clustered boxes may have unsynced TSC problems if they are
2756  * multi-chassis.
2757  * Use DMI to check them
2758  */
apic_is_clustered_box(void)2759 int apic_is_clustered_box(void)
2760 {
2761 	dmi_check_multi();
2762 	return multi;
2763 }
2764 #endif
2765 
2766 /*
2767  * APIC command line parameters
2768  */
setup_disableapic(char * arg)2769 static int __init setup_disableapic(char *arg)
2770 {
2771 	apic_is_disabled = true;
2772 	setup_clear_cpu_cap(X86_FEATURE_APIC);
2773 	return 0;
2774 }
2775 early_param("disableapic", setup_disableapic);
2776 
2777 /* same as disableapic, for compatibility */
setup_nolapic(char * arg)2778 static int __init setup_nolapic(char *arg)
2779 {
2780 	return setup_disableapic(arg);
2781 }
2782 early_param("nolapic", setup_nolapic);
2783 
parse_lapic_timer_c2_ok(char * arg)2784 static int __init parse_lapic_timer_c2_ok(char *arg)
2785 {
2786 	local_apic_timer_c2_ok = 1;
2787 	return 0;
2788 }
2789 early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
2790 
parse_disable_apic_timer(char * arg)2791 static int __init parse_disable_apic_timer(char *arg)
2792 {
2793 	disable_apic_timer = 1;
2794 	return 0;
2795 }
2796 early_param("noapictimer", parse_disable_apic_timer);
2797 
parse_nolapic_timer(char * arg)2798 static int __init parse_nolapic_timer(char *arg)
2799 {
2800 	disable_apic_timer = 1;
2801 	return 0;
2802 }
2803 early_param("nolapic_timer", parse_nolapic_timer);
2804 
apic_set_verbosity(char * arg)2805 static int __init apic_set_verbosity(char *arg)
2806 {
2807 	if (!arg)  {
2808 		if (IS_ENABLED(CONFIG_X86_32))
2809 			return -EINVAL;
2810 
2811 		ioapic_is_disabled = false;
2812 		return 0;
2813 	}
2814 
2815 	if (strcmp("debug", arg) == 0)
2816 		apic_verbosity = APIC_DEBUG;
2817 	else if (strcmp("verbose", arg) == 0)
2818 		apic_verbosity = APIC_VERBOSE;
2819 #ifdef CONFIG_X86_64
2820 	else {
2821 		pr_warn("APIC Verbosity level %s not recognised"
2822 			" use apic=verbose or apic=debug\n", arg);
2823 		return -EINVAL;
2824 	}
2825 #endif
2826 
2827 	return 0;
2828 }
2829 early_param("apic", apic_set_verbosity);
2830 
lapic_insert_resource(void)2831 static int __init lapic_insert_resource(void)
2832 {
2833 	if (!apic_mmio_base)
2834 		return -1;
2835 
2836 	/* Put local APIC into the resource map. */
2837 	lapic_resource.start = apic_mmio_base;
2838 	lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
2839 	insert_resource(&iomem_resource, &lapic_resource);
2840 
2841 	return 0;
2842 }
2843 
2844 /*
2845  * need call insert after e820__reserve_resources()
2846  * that is using request_resource
2847  */
2848 late_initcall(lapic_insert_resource);
2849 
apic_set_disabled_cpu_apicid(char * arg)2850 static int __init apic_set_disabled_cpu_apicid(char *arg)
2851 {
2852 	if (!arg || !get_option(&arg, &disabled_cpu_apicid))
2853 		return -EINVAL;
2854 
2855 	return 0;
2856 }
2857 early_param("disable_cpu_apicid", apic_set_disabled_cpu_apicid);
2858 
apic_set_extnmi(char * arg)2859 static int __init apic_set_extnmi(char *arg)
2860 {
2861 	if (!arg)
2862 		return -EINVAL;
2863 
2864 	if (!strncmp("all", arg, 3))
2865 		apic_extnmi = APIC_EXTNMI_ALL;
2866 	else if (!strncmp("none", arg, 4))
2867 		apic_extnmi = APIC_EXTNMI_NONE;
2868 	else if (!strncmp("bsp", arg, 3))
2869 		apic_extnmi = APIC_EXTNMI_BSP;
2870 	else {
2871 		pr_warn("Unknown external NMI delivery mode `%s' ignored\n", arg);
2872 		return -EINVAL;
2873 	}
2874 
2875 	return 0;
2876 }
2877 early_param("apic_extnmi", apic_set_extnmi);
2878