1 // SPDX-License-Identifier: GPL-2.0-only
2 /* cpu_feature_enabled() cannot be used this early */
3 #define USE_EARLY_PGTABLE_L5
4 
5 #include <linux/memblock.h>
6 #include <linux/linkage.h>
7 #include <linux/bitops.h>
8 #include <linux/kernel.h>
9 #include <linux/export.h>
10 #include <linux/percpu.h>
11 #include <linux/string.h>
12 #include <linux/ctype.h>
13 #include <linux/delay.h>
14 #include <linux/sched/mm.h>
15 #include <linux/sched/clock.h>
16 #include <linux/sched/task.h>
17 #include <linux/sched/smt.h>
18 #include <linux/init.h>
19 #include <linux/kprobes.h>
20 #include <linux/kgdb.h>
21 #include <linux/smp.h>
22 #include <linux/io.h>
23 #include <linux/syscore_ops.h>
24 #include <linux/pgtable.h>
25 
26 #include <asm/cmdline.h>
27 #include <asm/stackprotector.h>
28 #include <asm/perf_event.h>
29 #include <asm/mmu_context.h>
30 #include <asm/doublefault.h>
31 #include <asm/archrandom.h>
32 #include <asm/hypervisor.h>
33 #include <asm/processor.h>
34 #include <asm/tlbflush.h>
35 #include <asm/debugreg.h>
36 #include <asm/sections.h>
37 #include <asm/vsyscall.h>
38 #include <linux/topology.h>
39 #include <linux/cpumask.h>
40 #include <linux/atomic.h>
41 #include <asm/proto.h>
42 #include <asm/setup.h>
43 #include <asm/apic.h>
44 #include <asm/desc.h>
45 #include <asm/fpu/api.h>
46 #include <asm/mtrr.h>
47 #include <asm/hwcap2.h>
48 #include <linux/numa.h>
49 #include <asm/numa.h>
50 #include <asm/asm.h>
51 #include <asm/bugs.h>
52 #include <asm/cpu.h>
53 #include <asm/mce.h>
54 #include <asm/msr.h>
55 #include <asm/memtype.h>
56 #include <asm/microcode.h>
57 #include <asm/microcode_intel.h>
58 #include <asm/intel-family.h>
59 #include <asm/cpu_device_id.h>
60 #include <asm/uv/uv.h>
61 #include <asm/sigframe.h>
62 #include <asm/traps.h>
63 #include <asm/sev.h>
64 
65 #include "cpu.h"
66 
67 u32 elf_hwcap2 __read_mostly;
68 
69 /* all of these masks are initialized in setup_cpu_local_masks() */
70 cpumask_var_t cpu_initialized_mask;
71 cpumask_var_t cpu_callout_mask;
72 cpumask_var_t cpu_callin_mask;
73 
74 /* representing cpus for which sibling maps can be computed */
75 cpumask_var_t cpu_sibling_setup_mask;
76 
77 /* Number of siblings per CPU package */
78 int smp_num_siblings = 1;
79 EXPORT_SYMBOL(smp_num_siblings);
80 
81 /* Last level cache ID of each logical CPU */
82 DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
83 
get_llc_id(unsigned int cpu)84 u16 get_llc_id(unsigned int cpu)
85 {
86 	return per_cpu(cpu_llc_id, cpu);
87 }
88 EXPORT_SYMBOL_GPL(get_llc_id);
89 
90 /* L2 cache ID of each logical CPU */
91 DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_l2c_id) = BAD_APICID;
92 
93 static struct ppin_info {
94 	int	feature;
95 	int	msr_ppin_ctl;
96 	int	msr_ppin;
97 } ppin_info[] = {
98 	[X86_VENDOR_INTEL] = {
99 		.feature = X86_FEATURE_INTEL_PPIN,
100 		.msr_ppin_ctl = MSR_PPIN_CTL,
101 		.msr_ppin = MSR_PPIN
102 	},
103 	[X86_VENDOR_AMD] = {
104 		.feature = X86_FEATURE_AMD_PPIN,
105 		.msr_ppin_ctl = MSR_AMD_PPIN_CTL,
106 		.msr_ppin = MSR_AMD_PPIN
107 	},
108 };
109 
110 static const struct x86_cpu_id ppin_cpuids[] = {
111 	X86_MATCH_FEATURE(X86_FEATURE_AMD_PPIN, &ppin_info[X86_VENDOR_AMD]),
112 	X86_MATCH_FEATURE(X86_FEATURE_INTEL_PPIN, &ppin_info[X86_VENDOR_INTEL]),
113 
114 	/* Legacy models without CPUID enumeration */
115 	X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &ppin_info[X86_VENDOR_INTEL]),
116 	X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &ppin_info[X86_VENDOR_INTEL]),
117 	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &ppin_info[X86_VENDOR_INTEL]),
118 	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &ppin_info[X86_VENDOR_INTEL]),
119 	X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &ppin_info[X86_VENDOR_INTEL]),
120 	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &ppin_info[X86_VENDOR_INTEL]),
121 	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &ppin_info[X86_VENDOR_INTEL]),
122 	X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
123 	X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &ppin_info[X86_VENDOR_INTEL]),
124 	X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &ppin_info[X86_VENDOR_INTEL]),
125 
126 	{}
127 };
128 
ppin_init(struct cpuinfo_x86 * c)129 static void ppin_init(struct cpuinfo_x86 *c)
130 {
131 	const struct x86_cpu_id *id;
132 	unsigned long long val;
133 	struct ppin_info *info;
134 
135 	id = x86_match_cpu(ppin_cpuids);
136 	if (!id)
137 		return;
138 
139 	/*
140 	 * Testing the presence of the MSR is not enough. Need to check
141 	 * that the PPIN_CTL allows reading of the PPIN.
142 	 */
143 	info = (struct ppin_info *)id->driver_data;
144 
145 	if (rdmsrl_safe(info->msr_ppin_ctl, &val))
146 		goto clear_ppin;
147 
148 	if ((val & 3UL) == 1UL) {
149 		/* PPIN locked in disabled mode */
150 		goto clear_ppin;
151 	}
152 
153 	/* If PPIN is disabled, try to enable */
154 	if (!(val & 2UL)) {
155 		wrmsrl_safe(info->msr_ppin_ctl,  val | 2UL);
156 		rdmsrl_safe(info->msr_ppin_ctl, &val);
157 	}
158 
159 	/* Is the enable bit set? */
160 	if (val & 2UL) {
161 		c->ppin = __rdmsr(info->msr_ppin);
162 		set_cpu_cap(c, info->feature);
163 		return;
164 	}
165 
166 clear_ppin:
167 	clear_cpu_cap(c, info->feature);
168 }
169 
170 /* correctly size the local cpu masks */
setup_cpu_local_masks(void)171 void __init setup_cpu_local_masks(void)
172 {
173 	alloc_bootmem_cpumask_var(&cpu_initialized_mask);
174 	alloc_bootmem_cpumask_var(&cpu_callin_mask);
175 	alloc_bootmem_cpumask_var(&cpu_callout_mask);
176 	alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
177 }
178 
default_init(struct cpuinfo_x86 * c)179 static void default_init(struct cpuinfo_x86 *c)
180 {
181 #ifdef CONFIG_X86_64
182 	cpu_detect_cache_sizes(c);
183 #else
184 	/* Not much we can do here... */
185 	/* Check if at least it has cpuid */
186 	if (c->cpuid_level == -1) {
187 		/* No cpuid. It must be an ancient CPU */
188 		if (c->x86 == 4)
189 			strcpy(c->x86_model_id, "486");
190 		else if (c->x86 == 3)
191 			strcpy(c->x86_model_id, "386");
192 	}
193 #endif
194 }
195 
196 static const struct cpu_dev default_cpu = {
197 	.c_init		= default_init,
198 	.c_vendor	= "Unknown",
199 	.c_x86_vendor	= X86_VENDOR_UNKNOWN,
200 };
201 
202 static const struct cpu_dev *this_cpu = &default_cpu;
203 
204 DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
205 #ifdef CONFIG_X86_64
206 	/*
207 	 * We need valid kernel segments for data and code in long mode too
208 	 * IRET will check the segment types  kkeil 2000/10/28
209 	 * Also sysret mandates a special GDT layout
210 	 *
211 	 * TLS descriptors are currently at a different place compared to i386.
212 	 * Hopefully nobody expects them at a fixed place (Wine?)
213 	 */
214 	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
215 	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
216 	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
217 	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
218 	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
219 	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
220 #else
221 	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
222 	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
223 	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
224 	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
225 	/*
226 	 * Segments used for calling PnP BIOS have byte granularity.
227 	 * They code segments and data segments have fixed 64k limits,
228 	 * the transfer segment sizes are set at run time.
229 	 */
230 	/* 32-bit code */
231 	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
232 	/* 16-bit code */
233 	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
234 	/* 16-bit data */
235 	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(0x0092, 0, 0xffff),
236 	/* 16-bit data */
237 	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(0x0092, 0, 0),
238 	/* 16-bit data */
239 	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(0x0092, 0, 0),
240 	/*
241 	 * The APM segments have byte granularity and their bases
242 	 * are set at run time.  All have 64k limits.
243 	 */
244 	/* 32-bit code */
245 	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(0x409a, 0, 0xffff),
246 	/* 16-bit code */
247 	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(0x009a, 0, 0xffff),
248 	/* data */
249 	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(0x4092, 0, 0xffff),
250 
251 	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
252 	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
253 #endif
254 } };
255 EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
256 
257 #ifdef CONFIG_X86_64
x86_nopcid_setup(char * s)258 static int __init x86_nopcid_setup(char *s)
259 {
260 	/* nopcid doesn't accept parameters */
261 	if (s)
262 		return -EINVAL;
263 
264 	/* do not emit a message if the feature is not present */
265 	if (!boot_cpu_has(X86_FEATURE_PCID))
266 		return 0;
267 
268 	setup_clear_cpu_cap(X86_FEATURE_PCID);
269 	pr_info("nopcid: PCID feature disabled\n");
270 	return 0;
271 }
272 early_param("nopcid", x86_nopcid_setup);
273 #endif
274 
x86_noinvpcid_setup(char * s)275 static int __init x86_noinvpcid_setup(char *s)
276 {
277 	/* noinvpcid doesn't accept parameters */
278 	if (s)
279 		return -EINVAL;
280 
281 	/* do not emit a message if the feature is not present */
282 	if (!boot_cpu_has(X86_FEATURE_INVPCID))
283 		return 0;
284 
285 	setup_clear_cpu_cap(X86_FEATURE_INVPCID);
286 	pr_info("noinvpcid: INVPCID feature disabled\n");
287 	return 0;
288 }
289 early_param("noinvpcid", x86_noinvpcid_setup);
290 
291 #ifdef CONFIG_X86_32
292 static int cachesize_override = -1;
293 static int disable_x86_serial_nr = 1;
294 
cachesize_setup(char * str)295 static int __init cachesize_setup(char *str)
296 {
297 	get_option(&str, &cachesize_override);
298 	return 1;
299 }
300 __setup("cachesize=", cachesize_setup);
301 
302 /* Standard macro to see if a specific flag is changeable */
flag_is_changeable_p(u32 flag)303 static inline int flag_is_changeable_p(u32 flag)
304 {
305 	u32 f1, f2;
306 
307 	/*
308 	 * Cyrix and IDT cpus allow disabling of CPUID
309 	 * so the code below may return different results
310 	 * when it is executed before and after enabling
311 	 * the CPUID. Add "volatile" to not allow gcc to
312 	 * optimize the subsequent calls to this function.
313 	 */
314 	asm volatile ("pushfl		\n\t"
315 		      "pushfl		\n\t"
316 		      "popl %0		\n\t"
317 		      "movl %0, %1	\n\t"
318 		      "xorl %2, %0	\n\t"
319 		      "pushl %0		\n\t"
320 		      "popfl		\n\t"
321 		      "pushfl		\n\t"
322 		      "popl %0		\n\t"
323 		      "popfl		\n\t"
324 
325 		      : "=&r" (f1), "=&r" (f2)
326 		      : "ir" (flag));
327 
328 	return ((f1^f2) & flag) != 0;
329 }
330 
331 /* Probe for the CPUID instruction */
have_cpuid_p(void)332 int have_cpuid_p(void)
333 {
334 	return flag_is_changeable_p(X86_EFLAGS_ID);
335 }
336 
squash_the_stupid_serial_number(struct cpuinfo_x86 * c)337 static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
338 {
339 	unsigned long lo, hi;
340 
341 	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
342 		return;
343 
344 	/* Disable processor serial number: */
345 
346 	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
347 	lo |= 0x200000;
348 	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
349 
350 	pr_notice("CPU serial number disabled.\n");
351 	clear_cpu_cap(c, X86_FEATURE_PN);
352 
353 	/* Disabling the serial number may affect the cpuid level */
354 	c->cpuid_level = cpuid_eax(0);
355 }
356 
x86_serial_nr_setup(char * s)357 static int __init x86_serial_nr_setup(char *s)
358 {
359 	disable_x86_serial_nr = 0;
360 	return 1;
361 }
362 __setup("serialnumber", x86_serial_nr_setup);
363 #else
flag_is_changeable_p(u32 flag)364 static inline int flag_is_changeable_p(u32 flag)
365 {
366 	return 1;
367 }
squash_the_stupid_serial_number(struct cpuinfo_x86 * c)368 static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
369 {
370 }
371 #endif
372 
setup_smep(struct cpuinfo_x86 * c)373 static __always_inline void setup_smep(struct cpuinfo_x86 *c)
374 {
375 	if (cpu_has(c, X86_FEATURE_SMEP))
376 		cr4_set_bits(X86_CR4_SMEP);
377 }
378 
setup_smap(struct cpuinfo_x86 * c)379 static __always_inline void setup_smap(struct cpuinfo_x86 *c)
380 {
381 	unsigned long eflags = native_save_fl();
382 
383 	/* This should have been cleared long ago */
384 	BUG_ON(eflags & X86_EFLAGS_AC);
385 
386 	if (cpu_has(c, X86_FEATURE_SMAP))
387 		cr4_set_bits(X86_CR4_SMAP);
388 }
389 
setup_umip(struct cpuinfo_x86 * c)390 static __always_inline void setup_umip(struct cpuinfo_x86 *c)
391 {
392 	/* Check the boot processor, plus build option for UMIP. */
393 	if (!cpu_feature_enabled(X86_FEATURE_UMIP))
394 		goto out;
395 
396 	/* Check the current processor's cpuid bits. */
397 	if (!cpu_has(c, X86_FEATURE_UMIP))
398 		goto out;
399 
400 	cr4_set_bits(X86_CR4_UMIP);
401 
402 	pr_info_once("x86/cpu: User Mode Instruction Prevention (UMIP) activated\n");
403 
404 	return;
405 
406 out:
407 	/*
408 	 * Make sure UMIP is disabled in case it was enabled in a
409 	 * previous boot (e.g., via kexec).
410 	 */
411 	cr4_clear_bits(X86_CR4_UMIP);
412 }
413 
414 /* These bits should not change their value after CPU init is finished. */
415 static const unsigned long cr4_pinned_mask =
416 	X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP |
417 	X86_CR4_FSGSBASE | X86_CR4_CET;
418 static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
419 static unsigned long cr4_pinned_bits __ro_after_init;
420 
native_write_cr0(unsigned long val)421 void native_write_cr0(unsigned long val)
422 {
423 	unsigned long bits_missing = 0;
424 
425 set_register:
426 	asm volatile("mov %0,%%cr0": "+r" (val) : : "memory");
427 
428 	if (static_branch_likely(&cr_pinning)) {
429 		if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
430 			bits_missing = X86_CR0_WP;
431 			val |= bits_missing;
432 			goto set_register;
433 		}
434 		/* Warn after we've set the missing bits. */
435 		WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
436 	}
437 }
438 EXPORT_SYMBOL(native_write_cr0);
439 
native_write_cr4(unsigned long val)440 void __no_profile native_write_cr4(unsigned long val)
441 {
442 	unsigned long bits_changed = 0;
443 
444 set_register:
445 	asm volatile("mov %0,%%cr4": "+r" (val) : : "memory");
446 
447 	if (static_branch_likely(&cr_pinning)) {
448 		if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {
449 			bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits;
450 			val = (val & ~cr4_pinned_mask) | cr4_pinned_bits;
451 			goto set_register;
452 		}
453 		/* Warn after we've corrected the changed bits. */
454 		WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n",
455 			  bits_changed);
456 	}
457 }
458 #if IS_MODULE(CONFIG_LKDTM)
459 EXPORT_SYMBOL_GPL(native_write_cr4);
460 #endif
461 
cr4_update_irqsoff(unsigned long set,unsigned long clear)462 void cr4_update_irqsoff(unsigned long set, unsigned long clear)
463 {
464 	unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4);
465 
466 	lockdep_assert_irqs_disabled();
467 
468 	newval = (cr4 & ~clear) | set;
469 	if (newval != cr4) {
470 		this_cpu_write(cpu_tlbstate.cr4, newval);
471 		__write_cr4(newval);
472 	}
473 }
474 EXPORT_SYMBOL(cr4_update_irqsoff);
475 
476 /* Read the CR4 shadow. */
cr4_read_shadow(void)477 unsigned long cr4_read_shadow(void)
478 {
479 	return this_cpu_read(cpu_tlbstate.cr4);
480 }
481 EXPORT_SYMBOL_GPL(cr4_read_shadow);
482 
cr4_init(void)483 void cr4_init(void)
484 {
485 	unsigned long cr4 = __read_cr4();
486 
487 	if (boot_cpu_has(X86_FEATURE_PCID))
488 		cr4 |= X86_CR4_PCIDE;
489 	if (static_branch_likely(&cr_pinning))
490 		cr4 = (cr4 & ~cr4_pinned_mask) | cr4_pinned_bits;
491 
492 	__write_cr4(cr4);
493 
494 	/* Initialize cr4 shadow for this CPU. */
495 	this_cpu_write(cpu_tlbstate.cr4, cr4);
496 }
497 
498 /*
499  * Once CPU feature detection is finished (and boot params have been
500  * parsed), record any of the sensitive CR bits that are set, and
501  * enable CR pinning.
502  */
setup_cr_pinning(void)503 static void __init setup_cr_pinning(void)
504 {
505 	cr4_pinned_bits = this_cpu_read(cpu_tlbstate.cr4) & cr4_pinned_mask;
506 	static_key_enable(&cr_pinning.key);
507 }
508 
x86_nofsgsbase_setup(char * arg)509 static __init int x86_nofsgsbase_setup(char *arg)
510 {
511 	/* Require an exact match without trailing characters. */
512 	if (strlen(arg))
513 		return 0;
514 
515 	/* Do not emit a message if the feature is not present. */
516 	if (!boot_cpu_has(X86_FEATURE_FSGSBASE))
517 		return 1;
518 
519 	setup_clear_cpu_cap(X86_FEATURE_FSGSBASE);
520 	pr_info("FSGSBASE disabled via kernel command line\n");
521 	return 1;
522 }
523 __setup("nofsgsbase", x86_nofsgsbase_setup);
524 
525 /*
526  * Protection Keys are not available in 32-bit mode.
527  */
528 static bool pku_disabled;
529 
setup_pku(struct cpuinfo_x86 * c)530 static __always_inline void setup_pku(struct cpuinfo_x86 *c)
531 {
532 	if (c == &boot_cpu_data) {
533 		if (pku_disabled || !cpu_feature_enabled(X86_FEATURE_PKU))
534 			return;
535 		/*
536 		 * Setting CR4.PKE will cause the X86_FEATURE_OSPKE cpuid
537 		 * bit to be set.  Enforce it.
538 		 */
539 		setup_force_cpu_cap(X86_FEATURE_OSPKE);
540 
541 	} else if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) {
542 		return;
543 	}
544 
545 	cr4_set_bits(X86_CR4_PKE);
546 	/* Load the default PKRU value */
547 	pkru_write_default();
548 }
549 
550 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
setup_disable_pku(char * arg)551 static __init int setup_disable_pku(char *arg)
552 {
553 	/*
554 	 * Do not clear the X86_FEATURE_PKU bit.  All of the
555 	 * runtime checks are against OSPKE so clearing the
556 	 * bit does nothing.
557 	 *
558 	 * This way, we will see "pku" in cpuinfo, but not
559 	 * "ospke", which is exactly what we want.  It shows
560 	 * that the CPU has PKU, but the OS has not enabled it.
561 	 * This happens to be exactly how a system would look
562 	 * if we disabled the config option.
563 	 */
564 	pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
565 	pku_disabled = true;
566 	return 1;
567 }
568 __setup("nopku", setup_disable_pku);
569 #endif /* CONFIG_X86_64 */
570 
571 #ifdef CONFIG_X86_KERNEL_IBT
572 
ibt_save(void)573 __noendbr u64 ibt_save(void)
574 {
575 	u64 msr = 0;
576 
577 	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
578 		rdmsrl(MSR_IA32_S_CET, msr);
579 		wrmsrl(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN);
580 	}
581 
582 	return msr;
583 }
584 
ibt_restore(u64 save)585 __noendbr void ibt_restore(u64 save)
586 {
587 	u64 msr;
588 
589 	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
590 		rdmsrl(MSR_IA32_S_CET, msr);
591 		msr &= ~CET_ENDBR_EN;
592 		msr |= (save & CET_ENDBR_EN);
593 		wrmsrl(MSR_IA32_S_CET, msr);
594 	}
595 }
596 
597 #endif
598 
setup_cet(struct cpuinfo_x86 * c)599 static __always_inline void setup_cet(struct cpuinfo_x86 *c)
600 {
601 	u64 msr = CET_ENDBR_EN;
602 
603 	if (!HAS_KERNEL_IBT ||
604 	    !cpu_feature_enabled(X86_FEATURE_IBT))
605 		return;
606 
607 	wrmsrl(MSR_IA32_S_CET, msr);
608 	cr4_set_bits(X86_CR4_CET);
609 
610 	if (!ibt_selftest()) {
611 		pr_err("IBT selftest: Failed!\n");
612 		setup_clear_cpu_cap(X86_FEATURE_IBT);
613 		return;
614 	}
615 }
616 
cet_disable(void)617 __noendbr void cet_disable(void)
618 {
619 	if (cpu_feature_enabled(X86_FEATURE_IBT))
620 		wrmsrl(MSR_IA32_S_CET, 0);
621 }
622 
623 /*
624  * Some CPU features depend on higher CPUID levels, which may not always
625  * be available due to CPUID level capping or broken virtualization
626  * software.  Add those features to this table to auto-disable them.
627  */
628 struct cpuid_dependent_feature {
629 	u32 feature;
630 	u32 level;
631 };
632 
633 static const struct cpuid_dependent_feature
634 cpuid_dependent_features[] = {
635 	{ X86_FEATURE_MWAIT,		0x00000005 },
636 	{ X86_FEATURE_DCA,		0x00000009 },
637 	{ X86_FEATURE_XSAVE,		0x0000000d },
638 	{ 0, 0 }
639 };
640 
filter_cpuid_features(struct cpuinfo_x86 * c,bool warn)641 static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
642 {
643 	const struct cpuid_dependent_feature *df;
644 
645 	for (df = cpuid_dependent_features; df->feature; df++) {
646 
647 		if (!cpu_has(c, df->feature))
648 			continue;
649 		/*
650 		 * Note: cpuid_level is set to -1 if unavailable, but
651 		 * extended_extended_level is set to 0 if unavailable
652 		 * and the legitimate extended levels are all negative
653 		 * when signed; hence the weird messing around with
654 		 * signs here...
655 		 */
656 		if (!((s32)df->level < 0 ?
657 		     (u32)df->level > (u32)c->extended_cpuid_level :
658 		     (s32)df->level > (s32)c->cpuid_level))
659 			continue;
660 
661 		clear_cpu_cap(c, df->feature);
662 		if (!warn)
663 			continue;
664 
665 		pr_warn("CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
666 			x86_cap_flag(df->feature), df->level);
667 	}
668 }
669 
670 /*
671  * Naming convention should be: <Name> [(<Codename>)]
672  * This table only is used unless init_<vendor>() below doesn't set it;
673  * in particular, if CPUID levels 0x80000002..4 are supported, this
674  * isn't used
675  */
676 
677 /* Look up CPU names by table lookup. */
table_lookup_model(struct cpuinfo_x86 * c)678 static const char *table_lookup_model(struct cpuinfo_x86 *c)
679 {
680 #ifdef CONFIG_X86_32
681 	const struct legacy_cpu_model_info *info;
682 
683 	if (c->x86_model >= 16)
684 		return NULL;	/* Range check */
685 
686 	if (!this_cpu)
687 		return NULL;
688 
689 	info = this_cpu->legacy_models;
690 
691 	while (info->family) {
692 		if (info->family == c->x86)
693 			return info->model_names[c->x86_model];
694 		info++;
695 	}
696 #endif
697 	return NULL;		/* Not found */
698 }
699 
700 /* Aligned to unsigned long to avoid split lock in atomic bitmap ops */
701 __u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
702 __u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
703 
load_percpu_segment(int cpu)704 void load_percpu_segment(int cpu)
705 {
706 #ifdef CONFIG_X86_32
707 	loadsegment(fs, __KERNEL_PERCPU);
708 #else
709 	__loadsegment_simple(gs, 0);
710 	wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
711 #endif
712 }
713 
714 #ifdef CONFIG_X86_32
715 /* The 32-bit entry code needs to find cpu_entry_area. */
716 DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
717 #endif
718 
719 /* Load the original GDT from the per-cpu structure */
load_direct_gdt(int cpu)720 void load_direct_gdt(int cpu)
721 {
722 	struct desc_ptr gdt_descr;
723 
724 	gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
725 	gdt_descr.size = GDT_SIZE - 1;
726 	load_gdt(&gdt_descr);
727 }
728 EXPORT_SYMBOL_GPL(load_direct_gdt);
729 
730 /* Load a fixmap remapping of the per-cpu GDT */
load_fixmap_gdt(int cpu)731 void load_fixmap_gdt(int cpu)
732 {
733 	struct desc_ptr gdt_descr;
734 
735 	gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
736 	gdt_descr.size = GDT_SIZE - 1;
737 	load_gdt(&gdt_descr);
738 }
739 EXPORT_SYMBOL_GPL(load_fixmap_gdt);
740 
741 /*
742  * Current gdt points %fs at the "master" per-cpu area: after this,
743  * it's on the real one.
744  */
switch_to_new_gdt(int cpu)745 void switch_to_new_gdt(int cpu)
746 {
747 	/* Load the original GDT */
748 	load_direct_gdt(cpu);
749 	/* Reload the per-cpu base */
750 	load_percpu_segment(cpu);
751 }
752 
753 static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
754 
get_model_name(struct cpuinfo_x86 * c)755 static void get_model_name(struct cpuinfo_x86 *c)
756 {
757 	unsigned int *v;
758 	char *p, *q, *s;
759 
760 	if (c->extended_cpuid_level < 0x80000004)
761 		return;
762 
763 	v = (unsigned int *)c->x86_model_id;
764 	cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
765 	cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
766 	cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
767 	c->x86_model_id[48] = 0;
768 
769 	/* Trim whitespace */
770 	p = q = s = &c->x86_model_id[0];
771 
772 	while (*p == ' ')
773 		p++;
774 
775 	while (*p) {
776 		/* Note the last non-whitespace index */
777 		if (!isspace(*p))
778 			s = q;
779 
780 		*q++ = *p++;
781 	}
782 
783 	*(s + 1) = '\0';
784 }
785 
detect_num_cpu_cores(struct cpuinfo_x86 * c)786 void detect_num_cpu_cores(struct cpuinfo_x86 *c)
787 {
788 	unsigned int eax, ebx, ecx, edx;
789 
790 	c->x86_max_cores = 1;
791 	if (!IS_ENABLED(CONFIG_SMP) || c->cpuid_level < 4)
792 		return;
793 
794 	cpuid_count(4, 0, &eax, &ebx, &ecx, &edx);
795 	if (eax & 0x1f)
796 		c->x86_max_cores = (eax >> 26) + 1;
797 }
798 
cpu_detect_cache_sizes(struct cpuinfo_x86 * c)799 void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
800 {
801 	unsigned int n, dummy, ebx, ecx, edx, l2size;
802 
803 	n = c->extended_cpuid_level;
804 
805 	if (n >= 0x80000005) {
806 		cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
807 		c->x86_cache_size = (ecx>>24) + (edx>>24);
808 #ifdef CONFIG_X86_64
809 		/* On K8 L1 TLB is inclusive, so don't count it */
810 		c->x86_tlbsize = 0;
811 #endif
812 	}
813 
814 	if (n < 0x80000006)	/* Some chips just has a large L1. */
815 		return;
816 
817 	cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
818 	l2size = ecx >> 16;
819 
820 #ifdef CONFIG_X86_64
821 	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
822 #else
823 	/* do processor-specific cache resizing */
824 	if (this_cpu->legacy_cache_size)
825 		l2size = this_cpu->legacy_cache_size(c, l2size);
826 
827 	/* Allow user to override all this if necessary. */
828 	if (cachesize_override != -1)
829 		l2size = cachesize_override;
830 
831 	if (l2size == 0)
832 		return;		/* Again, no L2 cache is possible */
833 #endif
834 
835 	c->x86_cache_size = l2size;
836 }
837 
838 u16 __read_mostly tlb_lli_4k[NR_INFO];
839 u16 __read_mostly tlb_lli_2m[NR_INFO];
840 u16 __read_mostly tlb_lli_4m[NR_INFO];
841 u16 __read_mostly tlb_lld_4k[NR_INFO];
842 u16 __read_mostly tlb_lld_2m[NR_INFO];
843 u16 __read_mostly tlb_lld_4m[NR_INFO];
844 u16 __read_mostly tlb_lld_1g[NR_INFO];
845 
cpu_detect_tlb(struct cpuinfo_x86 * c)846 static void cpu_detect_tlb(struct cpuinfo_x86 *c)
847 {
848 	if (this_cpu->c_detect_tlb)
849 		this_cpu->c_detect_tlb(c);
850 
851 	pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
852 		tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
853 		tlb_lli_4m[ENTRIES]);
854 
855 	pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
856 		tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
857 		tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
858 }
859 
detect_ht_early(struct cpuinfo_x86 * c)860 int detect_ht_early(struct cpuinfo_x86 *c)
861 {
862 #ifdef CONFIG_SMP
863 	u32 eax, ebx, ecx, edx;
864 
865 	if (!cpu_has(c, X86_FEATURE_HT))
866 		return -1;
867 
868 	if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
869 		return -1;
870 
871 	if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
872 		return -1;
873 
874 	cpuid(1, &eax, &ebx, &ecx, &edx);
875 
876 	smp_num_siblings = (ebx & 0xff0000) >> 16;
877 	if (smp_num_siblings == 1)
878 		pr_info_once("CPU0: Hyper-Threading is disabled\n");
879 #endif
880 	return 0;
881 }
882 
detect_ht(struct cpuinfo_x86 * c)883 void detect_ht(struct cpuinfo_x86 *c)
884 {
885 #ifdef CONFIG_SMP
886 	int index_msb, core_bits;
887 
888 	if (detect_ht_early(c) < 0)
889 		return;
890 
891 	index_msb = get_count_order(smp_num_siblings);
892 	c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
893 
894 	smp_num_siblings = smp_num_siblings / c->x86_max_cores;
895 
896 	index_msb = get_count_order(smp_num_siblings);
897 
898 	core_bits = get_count_order(c->x86_max_cores);
899 
900 	c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
901 				       ((1 << core_bits) - 1);
902 #endif
903 }
904 
get_cpu_vendor(struct cpuinfo_x86 * c)905 static void get_cpu_vendor(struct cpuinfo_x86 *c)
906 {
907 	char *v = c->x86_vendor_id;
908 	int i;
909 
910 	for (i = 0; i < X86_VENDOR_NUM; i++) {
911 		if (!cpu_devs[i])
912 			break;
913 
914 		if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
915 		    (cpu_devs[i]->c_ident[1] &&
916 		     !strcmp(v, cpu_devs[i]->c_ident[1]))) {
917 
918 			this_cpu = cpu_devs[i];
919 			c->x86_vendor = this_cpu->c_x86_vendor;
920 			return;
921 		}
922 	}
923 
924 	pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
925 		    "CPU: Your system may be unstable.\n", v);
926 
927 	c->x86_vendor = X86_VENDOR_UNKNOWN;
928 	this_cpu = &default_cpu;
929 }
930 
cpu_detect(struct cpuinfo_x86 * c)931 void cpu_detect(struct cpuinfo_x86 *c)
932 {
933 	/* Get vendor name */
934 	cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
935 	      (unsigned int *)&c->x86_vendor_id[0],
936 	      (unsigned int *)&c->x86_vendor_id[8],
937 	      (unsigned int *)&c->x86_vendor_id[4]);
938 
939 	c->x86 = 4;
940 	/* Intel-defined flags: level 0x00000001 */
941 	if (c->cpuid_level >= 0x00000001) {
942 		u32 junk, tfms, cap0, misc;
943 
944 		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
945 		c->x86		= x86_family(tfms);
946 		c->x86_model	= x86_model(tfms);
947 		c->x86_stepping	= x86_stepping(tfms);
948 
949 		if (cap0 & (1<<19)) {
950 			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
951 			c->x86_cache_alignment = c->x86_clflush_size;
952 		}
953 	}
954 }
955 
apply_forced_caps(struct cpuinfo_x86 * c)956 static void apply_forced_caps(struct cpuinfo_x86 *c)
957 {
958 	int i;
959 
960 	for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
961 		c->x86_capability[i] &= ~cpu_caps_cleared[i];
962 		c->x86_capability[i] |= cpu_caps_set[i];
963 	}
964 }
965 
init_speculation_control(struct cpuinfo_x86 * c)966 static void init_speculation_control(struct cpuinfo_x86 *c)
967 {
968 	/*
969 	 * The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
970 	 * and they also have a different bit for STIBP support. Also,
971 	 * a hypervisor might have set the individual AMD bits even on
972 	 * Intel CPUs, for finer-grained selection of what's available.
973 	 */
974 	if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
975 		set_cpu_cap(c, X86_FEATURE_IBRS);
976 		set_cpu_cap(c, X86_FEATURE_IBPB);
977 		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
978 	}
979 
980 	if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
981 		set_cpu_cap(c, X86_FEATURE_STIBP);
982 
983 	if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
984 	    cpu_has(c, X86_FEATURE_VIRT_SSBD))
985 		set_cpu_cap(c, X86_FEATURE_SSBD);
986 
987 	if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
988 		set_cpu_cap(c, X86_FEATURE_IBRS);
989 		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
990 	}
991 
992 	if (cpu_has(c, X86_FEATURE_AMD_IBPB))
993 		set_cpu_cap(c, X86_FEATURE_IBPB);
994 
995 	if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
996 		set_cpu_cap(c, X86_FEATURE_STIBP);
997 		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
998 	}
999 
1000 	if (cpu_has(c, X86_FEATURE_AMD_SSBD)) {
1001 		set_cpu_cap(c, X86_FEATURE_SSBD);
1002 		set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
1003 		clear_cpu_cap(c, X86_FEATURE_VIRT_SSBD);
1004 	}
1005 }
1006 
get_cpu_cap(struct cpuinfo_x86 * c)1007 void get_cpu_cap(struct cpuinfo_x86 *c)
1008 {
1009 	u32 eax, ebx, ecx, edx;
1010 
1011 	/* Intel-defined flags: level 0x00000001 */
1012 	if (c->cpuid_level >= 0x00000001) {
1013 		cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
1014 
1015 		c->x86_capability[CPUID_1_ECX] = ecx;
1016 		c->x86_capability[CPUID_1_EDX] = edx;
1017 	}
1018 
1019 	/* Thermal and Power Management Leaf: level 0x00000006 (eax) */
1020 	if (c->cpuid_level >= 0x00000006)
1021 		c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);
1022 
1023 	/* Additional Intel-defined flags: level 0x00000007 */
1024 	if (c->cpuid_level >= 0x00000007) {
1025 		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
1026 		c->x86_capability[CPUID_7_0_EBX] = ebx;
1027 		c->x86_capability[CPUID_7_ECX] = ecx;
1028 		c->x86_capability[CPUID_7_EDX] = edx;
1029 
1030 		/* Check valid sub-leaf index before accessing it */
1031 		if (eax >= 1) {
1032 			cpuid_count(0x00000007, 1, &eax, &ebx, &ecx, &edx);
1033 			c->x86_capability[CPUID_7_1_EAX] = eax;
1034 		}
1035 	}
1036 
1037 	/* Extended state features: level 0x0000000d */
1038 	if (c->cpuid_level >= 0x0000000d) {
1039 		cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);
1040 
1041 		c->x86_capability[CPUID_D_1_EAX] = eax;
1042 	}
1043 
1044 	/* AMD-defined flags: level 0x80000001 */
1045 	eax = cpuid_eax(0x80000000);
1046 	c->extended_cpuid_level = eax;
1047 
1048 	if ((eax & 0xffff0000) == 0x80000000) {
1049 		if (eax >= 0x80000001) {
1050 			cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
1051 
1052 			c->x86_capability[CPUID_8000_0001_ECX] = ecx;
1053 			c->x86_capability[CPUID_8000_0001_EDX] = edx;
1054 		}
1055 	}
1056 
1057 	if (c->extended_cpuid_level >= 0x80000007) {
1058 		cpuid(0x80000007, &eax, &ebx, &ecx, &edx);
1059 
1060 		c->x86_capability[CPUID_8000_0007_EBX] = ebx;
1061 		c->x86_power = edx;
1062 	}
1063 
1064 	if (c->extended_cpuid_level >= 0x80000008) {
1065 		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1066 		c->x86_capability[CPUID_8000_0008_EBX] = ebx;
1067 	}
1068 
1069 	if (c->extended_cpuid_level >= 0x8000000a)
1070 		c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
1071 
1072 	if (c->extended_cpuid_level >= 0x8000001f)
1073 		c->x86_capability[CPUID_8000_001F_EAX] = cpuid_eax(0x8000001f);
1074 
1075 	init_scattered_cpuid_features(c);
1076 	init_speculation_control(c);
1077 
1078 	/*
1079 	 * Clear/Set all flags overridden by options, after probe.
1080 	 * This needs to happen each time we re-probe, which may happen
1081 	 * several times during CPU initialization.
1082 	 */
1083 	apply_forced_caps(c);
1084 }
1085 
get_cpu_address_sizes(struct cpuinfo_x86 * c)1086 void get_cpu_address_sizes(struct cpuinfo_x86 *c)
1087 {
1088 	u32 eax, ebx, ecx, edx;
1089 
1090 	if (c->extended_cpuid_level >= 0x80000008) {
1091 		cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
1092 
1093 		c->x86_virt_bits = (eax >> 8) & 0xff;
1094 		c->x86_phys_bits = eax & 0xff;
1095 	}
1096 #ifdef CONFIG_X86_32
1097 	else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
1098 		c->x86_phys_bits = 36;
1099 #endif
1100 	c->x86_cache_bits = c->x86_phys_bits;
1101 }
1102 
identify_cpu_without_cpuid(struct cpuinfo_x86 * c)1103 static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
1104 {
1105 #ifdef CONFIG_X86_32
1106 	int i;
1107 
1108 	/*
1109 	 * First of all, decide if this is a 486 or higher
1110 	 * It's a 486 if we can modify the AC flag
1111 	 */
1112 	if (flag_is_changeable_p(X86_EFLAGS_AC))
1113 		c->x86 = 4;
1114 	else
1115 		c->x86 = 3;
1116 
1117 	for (i = 0; i < X86_VENDOR_NUM; i++)
1118 		if (cpu_devs[i] && cpu_devs[i]->c_identify) {
1119 			c->x86_vendor_id[0] = 0;
1120 			cpu_devs[i]->c_identify(c);
1121 			if (c->x86_vendor_id[0]) {
1122 				get_cpu_vendor(c);
1123 				break;
1124 			}
1125 		}
1126 #endif
1127 }
1128 
1129 #define NO_SPECULATION		BIT(0)
1130 #define NO_MELTDOWN		BIT(1)
1131 #define NO_SSB			BIT(2)
1132 #define NO_L1TF			BIT(3)
1133 #define NO_MDS			BIT(4)
1134 #define MSBDS_ONLY		BIT(5)
1135 #define NO_SWAPGS		BIT(6)
1136 #define NO_ITLB_MULTIHIT	BIT(7)
1137 #define NO_SPECTRE_V2		BIT(8)
1138 #define NO_MMIO			BIT(9)
1139 #define NO_EIBRS_PBRSB		BIT(10)
1140 
1141 #define VULNWL(vendor, family, model, whitelist)	\
1142 	X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, whitelist)
1143 
1144 #define VULNWL_INTEL(model, whitelist)		\
1145 	VULNWL(INTEL, 6, INTEL_FAM6_##model, whitelist)
1146 
1147 #define VULNWL_AMD(family, whitelist)		\
1148 	VULNWL(AMD, family, X86_MODEL_ANY, whitelist)
1149 
1150 #define VULNWL_HYGON(family, whitelist)		\
1151 	VULNWL(HYGON, family, X86_MODEL_ANY, whitelist)
1152 
1153 static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = {
1154 	VULNWL(ANY,	4, X86_MODEL_ANY,	NO_SPECULATION),
1155 	VULNWL(CENTAUR,	5, X86_MODEL_ANY,	NO_SPECULATION),
1156 	VULNWL(INTEL,	5, X86_MODEL_ANY,	NO_SPECULATION),
1157 	VULNWL(NSC,	5, X86_MODEL_ANY,	NO_SPECULATION),
1158 	VULNWL(VORTEX,	5, X86_MODEL_ANY,	NO_SPECULATION),
1159 	VULNWL(VORTEX,	6, X86_MODEL_ANY,	NO_SPECULATION),
1160 
1161 	/* Intel Family 6 */
1162 	VULNWL_INTEL(TIGERLAKE,			NO_MMIO),
1163 	VULNWL_INTEL(TIGERLAKE_L,		NO_MMIO),
1164 	VULNWL_INTEL(ALDERLAKE,			NO_MMIO),
1165 	VULNWL_INTEL(ALDERLAKE_L,		NO_MMIO),
1166 
1167 	VULNWL_INTEL(ATOM_SALTWELL,		NO_SPECULATION | NO_ITLB_MULTIHIT),
1168 	VULNWL_INTEL(ATOM_SALTWELL_TABLET,	NO_SPECULATION | NO_ITLB_MULTIHIT),
1169 	VULNWL_INTEL(ATOM_SALTWELL_MID,		NO_SPECULATION | NO_ITLB_MULTIHIT),
1170 	VULNWL_INTEL(ATOM_BONNELL,		NO_SPECULATION | NO_ITLB_MULTIHIT),
1171 	VULNWL_INTEL(ATOM_BONNELL_MID,		NO_SPECULATION | NO_ITLB_MULTIHIT),
1172 
1173 	VULNWL_INTEL(ATOM_SILVERMONT,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1174 	VULNWL_INTEL(ATOM_SILVERMONT_D,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1175 	VULNWL_INTEL(ATOM_SILVERMONT_MID,	NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1176 	VULNWL_INTEL(ATOM_AIRMONT,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1177 	VULNWL_INTEL(XEON_PHI_KNL,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1178 	VULNWL_INTEL(XEON_PHI_KNM,		NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1179 
1180 	VULNWL_INTEL(CORE_YONAH,		NO_SSB),
1181 
1182 	VULNWL_INTEL(ATOM_AIRMONT_MID,		NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1183 	VULNWL_INTEL(ATOM_AIRMONT_NP,		NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
1184 
1185 	VULNWL_INTEL(ATOM_GOLDMONT,		NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1186 	VULNWL_INTEL(ATOM_GOLDMONT_D,		NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1187 	VULNWL_INTEL(ATOM_GOLDMONT_PLUS,	NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
1188 
1189 	/*
1190 	 * Technically, swapgs isn't serializing on AMD (despite it previously
1191 	 * being documented as such in the APM).  But according to AMD, %gs is
1192 	 * updated non-speculatively, and the issuing of %gs-relative memory
1193 	 * operands will be blocked until the %gs update completes, which is
1194 	 * good enough for our purposes.
1195 	 */
1196 
1197 	VULNWL_INTEL(ATOM_TREMONT,		NO_EIBRS_PBRSB),
1198 	VULNWL_INTEL(ATOM_TREMONT_L,		NO_EIBRS_PBRSB),
1199 	VULNWL_INTEL(ATOM_TREMONT_D,		NO_ITLB_MULTIHIT | NO_EIBRS_PBRSB),
1200 
1201 	/* AMD Family 0xf - 0x12 */
1202 	VULNWL_AMD(0x0f,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1203 	VULNWL_AMD(0x10,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1204 	VULNWL_AMD(0x11,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1205 	VULNWL_AMD(0x12,	NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1206 
1207 	/* FAMILY_ANY must be last, otherwise 0x0f - 0x12 matches won't work */
1208 	VULNWL_AMD(X86_FAMILY_ANY,	NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1209 	VULNWL_HYGON(X86_FAMILY_ANY,	NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1210 
1211 	/* Zhaoxin Family 7 */
1212 	VULNWL(CENTAUR,	7, X86_MODEL_ANY,	NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO),
1213 	VULNWL(ZHAOXIN,	7, X86_MODEL_ANY,	NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO),
1214 	{}
1215 };
1216 
1217 #define VULNBL(vendor, family, model, blacklist)	\
1218 	X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, blacklist)
1219 
1220 #define VULNBL_INTEL_STEPPINGS(model, steppings, issues)		   \
1221 	X86_MATCH_VENDOR_FAM_MODEL_STEPPINGS_FEATURE(INTEL, 6,		   \
1222 					    INTEL_FAM6_##model, steppings, \
1223 					    X86_FEATURE_ANY, issues)
1224 
1225 #define VULNBL_AMD(family, blacklist)		\
1226 	VULNBL(AMD, family, X86_MODEL_ANY, blacklist)
1227 
1228 #define VULNBL_HYGON(family, blacklist)		\
1229 	VULNBL(HYGON, family, X86_MODEL_ANY, blacklist)
1230 
1231 #define SRBDS		BIT(0)
1232 /* CPU is affected by X86_BUG_MMIO_STALE_DATA */
1233 #define MMIO		BIT(1)
1234 /* CPU is affected by Shared Buffers Data Sampling (SBDS), a variant of X86_BUG_MMIO_STALE_DATA */
1235 #define MMIO_SBDS	BIT(2)
1236 /* CPU is affected by RETbleed, speculating where you would not expect it */
1237 #define RETBLEED	BIT(3)
1238 
1239 static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = {
1240 	VULNBL_INTEL_STEPPINGS(IVYBRIDGE,	X86_STEPPING_ANY,		SRBDS),
1241 	VULNBL_INTEL_STEPPINGS(HASWELL,		X86_STEPPING_ANY,		SRBDS),
1242 	VULNBL_INTEL_STEPPINGS(HASWELL_L,	X86_STEPPING_ANY,		SRBDS),
1243 	VULNBL_INTEL_STEPPINGS(HASWELL_G,	X86_STEPPING_ANY,		SRBDS),
1244 	VULNBL_INTEL_STEPPINGS(HASWELL_X,	X86_STEPPING_ANY,		MMIO),
1245 	VULNBL_INTEL_STEPPINGS(BROADWELL_D,	X86_STEPPING_ANY,		MMIO),
1246 	VULNBL_INTEL_STEPPINGS(BROADWELL_G,	X86_STEPPING_ANY,		SRBDS),
1247 	VULNBL_INTEL_STEPPINGS(BROADWELL_X,	X86_STEPPING_ANY,		MMIO),
1248 	VULNBL_INTEL_STEPPINGS(BROADWELL,	X86_STEPPING_ANY,		SRBDS),
1249 	VULNBL_INTEL_STEPPINGS(SKYLAKE_L,	X86_STEPPING_ANY,		SRBDS | MMIO | RETBLEED),
1250 	VULNBL_INTEL_STEPPINGS(SKYLAKE_X,	X86_STEPPING_ANY,		MMIO | RETBLEED),
1251 	VULNBL_INTEL_STEPPINGS(SKYLAKE,		X86_STEPPING_ANY,		SRBDS | MMIO | RETBLEED),
1252 	VULNBL_INTEL_STEPPINGS(KABYLAKE_L,	X86_STEPPING_ANY,		SRBDS | MMIO | RETBLEED),
1253 	VULNBL_INTEL_STEPPINGS(KABYLAKE,	X86_STEPPING_ANY,		SRBDS | MMIO | RETBLEED),
1254 	VULNBL_INTEL_STEPPINGS(CANNONLAKE_L,	X86_STEPPING_ANY,		RETBLEED),
1255 	VULNBL_INTEL_STEPPINGS(ICELAKE_L,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RETBLEED),
1256 	VULNBL_INTEL_STEPPINGS(ICELAKE_D,	X86_STEPPING_ANY,		MMIO),
1257 	VULNBL_INTEL_STEPPINGS(ICELAKE_X,	X86_STEPPING_ANY,		MMIO),
1258 	VULNBL_INTEL_STEPPINGS(COMETLAKE,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RETBLEED),
1259 	VULNBL_INTEL_STEPPINGS(COMETLAKE_L,	X86_STEPPINGS(0x0, 0x0),	MMIO | RETBLEED),
1260 	VULNBL_INTEL_STEPPINGS(COMETLAKE_L,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RETBLEED),
1261 	VULNBL_INTEL_STEPPINGS(LAKEFIELD,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS | RETBLEED),
1262 	VULNBL_INTEL_STEPPINGS(ROCKETLAKE,	X86_STEPPING_ANY,		MMIO | RETBLEED),
1263 	VULNBL_INTEL_STEPPINGS(ATOM_TREMONT,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS),
1264 	VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_D,	X86_STEPPING_ANY,		MMIO),
1265 	VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_L,	X86_STEPPING_ANY,		MMIO | MMIO_SBDS),
1266 
1267 	VULNBL_AMD(0x15, RETBLEED),
1268 	VULNBL_AMD(0x16, RETBLEED),
1269 	VULNBL_AMD(0x17, RETBLEED),
1270 	VULNBL_HYGON(0x18, RETBLEED),
1271 	{}
1272 };
1273 
cpu_matches(const struct x86_cpu_id * table,unsigned long which)1274 static bool __init cpu_matches(const struct x86_cpu_id *table, unsigned long which)
1275 {
1276 	const struct x86_cpu_id *m = x86_match_cpu(table);
1277 
1278 	return m && !!(m->driver_data & which);
1279 }
1280 
x86_read_arch_cap_msr(void)1281 u64 x86_read_arch_cap_msr(void)
1282 {
1283 	u64 ia32_cap = 0;
1284 
1285 	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
1286 		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, ia32_cap);
1287 
1288 	return ia32_cap;
1289 }
1290 
arch_cap_mmio_immune(u64 ia32_cap)1291 static bool arch_cap_mmio_immune(u64 ia32_cap)
1292 {
1293 	return (ia32_cap & ARCH_CAP_FBSDP_NO &&
1294 		ia32_cap & ARCH_CAP_PSDP_NO &&
1295 		ia32_cap & ARCH_CAP_SBDR_SSDP_NO);
1296 }
1297 
cpu_set_bug_bits(struct cpuinfo_x86 * c)1298 static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
1299 {
1300 	u64 ia32_cap = x86_read_arch_cap_msr();
1301 
1302 	/* Set ITLB_MULTIHIT bug if cpu is not in the whitelist and not mitigated */
1303 	if (!cpu_matches(cpu_vuln_whitelist, NO_ITLB_MULTIHIT) &&
1304 	    !(ia32_cap & ARCH_CAP_PSCHANGE_MC_NO))
1305 		setup_force_cpu_bug(X86_BUG_ITLB_MULTIHIT);
1306 
1307 	if (cpu_matches(cpu_vuln_whitelist, NO_SPECULATION))
1308 		return;
1309 
1310 	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
1311 
1312 	if (!cpu_matches(cpu_vuln_whitelist, NO_SPECTRE_V2))
1313 		setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
1314 
1315 	if (!cpu_matches(cpu_vuln_whitelist, NO_SSB) &&
1316 	    !(ia32_cap & ARCH_CAP_SSB_NO) &&
1317 	   !cpu_has(c, X86_FEATURE_AMD_SSB_NO))
1318 		setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);
1319 
1320 	if (ia32_cap & ARCH_CAP_IBRS_ALL)
1321 		setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED);
1322 
1323 	if (!cpu_matches(cpu_vuln_whitelist, NO_MDS) &&
1324 	    !(ia32_cap & ARCH_CAP_MDS_NO)) {
1325 		setup_force_cpu_bug(X86_BUG_MDS);
1326 		if (cpu_matches(cpu_vuln_whitelist, MSBDS_ONLY))
1327 			setup_force_cpu_bug(X86_BUG_MSBDS_ONLY);
1328 	}
1329 
1330 	if (!cpu_matches(cpu_vuln_whitelist, NO_SWAPGS))
1331 		setup_force_cpu_bug(X86_BUG_SWAPGS);
1332 
1333 	/*
1334 	 * When the CPU is not mitigated for TAA (TAA_NO=0) set TAA bug when:
1335 	 *	- TSX is supported or
1336 	 *	- TSX_CTRL is present
1337 	 *
1338 	 * TSX_CTRL check is needed for cases when TSX could be disabled before
1339 	 * the kernel boot e.g. kexec.
1340 	 * TSX_CTRL check alone is not sufficient for cases when the microcode
1341 	 * update is not present or running as guest that don't get TSX_CTRL.
1342 	 */
1343 	if (!(ia32_cap & ARCH_CAP_TAA_NO) &&
1344 	    (cpu_has(c, X86_FEATURE_RTM) ||
1345 	     (ia32_cap & ARCH_CAP_TSX_CTRL_MSR)))
1346 		setup_force_cpu_bug(X86_BUG_TAA);
1347 
1348 	/*
1349 	 * SRBDS affects CPUs which support RDRAND or RDSEED and are listed
1350 	 * in the vulnerability blacklist.
1351 	 *
1352 	 * Some of the implications and mitigation of Shared Buffers Data
1353 	 * Sampling (SBDS) are similar to SRBDS. Give SBDS same treatment as
1354 	 * SRBDS.
1355 	 */
1356 	if ((cpu_has(c, X86_FEATURE_RDRAND) ||
1357 	     cpu_has(c, X86_FEATURE_RDSEED)) &&
1358 	    cpu_matches(cpu_vuln_blacklist, SRBDS | MMIO_SBDS))
1359 		    setup_force_cpu_bug(X86_BUG_SRBDS);
1360 
1361 	/*
1362 	 * Processor MMIO Stale Data bug enumeration
1363 	 *
1364 	 * Affected CPU list is generally enough to enumerate the vulnerability,
1365 	 * but for virtualization case check for ARCH_CAP MSR bits also, VMM may
1366 	 * not want the guest to enumerate the bug.
1367 	 *
1368 	 * Set X86_BUG_MMIO_UNKNOWN for CPUs that are neither in the blacklist,
1369 	 * nor in the whitelist and also don't enumerate MSR ARCH_CAP MMIO bits.
1370 	 */
1371 	if (!arch_cap_mmio_immune(ia32_cap)) {
1372 		if (cpu_matches(cpu_vuln_blacklist, MMIO))
1373 			setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA);
1374 		else if (!cpu_matches(cpu_vuln_whitelist, NO_MMIO))
1375 			setup_force_cpu_bug(X86_BUG_MMIO_UNKNOWN);
1376 	}
1377 
1378 	if (!cpu_has(c, X86_FEATURE_BTC_NO)) {
1379 		if (cpu_matches(cpu_vuln_blacklist, RETBLEED) || (ia32_cap & ARCH_CAP_RSBA))
1380 			setup_force_cpu_bug(X86_BUG_RETBLEED);
1381 	}
1382 
1383 	if (cpu_has(c, X86_FEATURE_IBRS_ENHANCED) &&
1384 	    !cpu_matches(cpu_vuln_whitelist, NO_EIBRS_PBRSB) &&
1385 	    !(ia32_cap & ARCH_CAP_PBRSB_NO))
1386 		setup_force_cpu_bug(X86_BUG_EIBRS_PBRSB);
1387 
1388 	if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN))
1389 		return;
1390 
1391 	/* Rogue Data Cache Load? No! */
1392 	if (ia32_cap & ARCH_CAP_RDCL_NO)
1393 		return;
1394 
1395 	setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
1396 
1397 	if (cpu_matches(cpu_vuln_whitelist, NO_L1TF))
1398 		return;
1399 
1400 	setup_force_cpu_bug(X86_BUG_L1TF);
1401 }
1402 
1403 /*
1404  * The NOPL instruction is supposed to exist on all CPUs of family >= 6;
1405  * unfortunately, that's not true in practice because of early VIA
1406  * chips and (more importantly) broken virtualizers that are not easy
1407  * to detect. In the latter case it doesn't even *fail* reliably, so
1408  * probing for it doesn't even work. Disable it completely on 32-bit
1409  * unless we can find a reliable way to detect all the broken cases.
1410  * Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
1411  */
detect_nopl(void)1412 static void detect_nopl(void)
1413 {
1414 #ifdef CONFIG_X86_32
1415 	setup_clear_cpu_cap(X86_FEATURE_NOPL);
1416 #else
1417 	setup_force_cpu_cap(X86_FEATURE_NOPL);
1418 #endif
1419 }
1420 
1421 /*
1422  * We parse cpu parameters early because fpu__init_system() is executed
1423  * before parse_early_param().
1424  */
cpu_parse_early_param(void)1425 static void __init cpu_parse_early_param(void)
1426 {
1427 	char arg[128];
1428 	char *argptr = arg, *opt;
1429 	int arglen, taint = 0;
1430 
1431 #ifdef CONFIG_X86_32
1432 	if (cmdline_find_option_bool(boot_command_line, "no387"))
1433 #ifdef CONFIG_MATH_EMULATION
1434 		setup_clear_cpu_cap(X86_FEATURE_FPU);
1435 #else
1436 		pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
1437 #endif
1438 
1439 	if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
1440 		setup_clear_cpu_cap(X86_FEATURE_FXSR);
1441 #endif
1442 
1443 	if (cmdline_find_option_bool(boot_command_line, "noxsave"))
1444 		setup_clear_cpu_cap(X86_FEATURE_XSAVE);
1445 
1446 	if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
1447 		setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
1448 
1449 	if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
1450 		setup_clear_cpu_cap(X86_FEATURE_XSAVES);
1451 
1452 	arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));
1453 	if (arglen <= 0)
1454 		return;
1455 
1456 	pr_info("Clearing CPUID bits:");
1457 
1458 	while (argptr) {
1459 		bool found __maybe_unused = false;
1460 		unsigned int bit;
1461 
1462 		opt = strsep(&argptr, ",");
1463 
1464 		/*
1465 		 * Handle naked numbers first for feature flags which don't
1466 		 * have names.
1467 		 */
1468 		if (!kstrtouint(opt, 10, &bit)) {
1469 			if (bit < NCAPINTS * 32) {
1470 
1471 #ifdef CONFIG_X86_FEATURE_NAMES
1472 				/* empty-string, i.e., ""-defined feature flags */
1473 				if (!x86_cap_flags[bit])
1474 					pr_cont(" " X86_CAP_FMT_NUM, x86_cap_flag_num(bit));
1475 				else
1476 #endif
1477 					pr_cont(" " X86_CAP_FMT, x86_cap_flag(bit));
1478 
1479 				setup_clear_cpu_cap(bit);
1480 				taint++;
1481 			}
1482 			/*
1483 			 * The assumption is that there are no feature names with only
1484 			 * numbers in the name thus go to the next argument.
1485 			 */
1486 			continue;
1487 		}
1488 
1489 #ifdef CONFIG_X86_FEATURE_NAMES
1490 		for (bit = 0; bit < 32 * NCAPINTS; bit++) {
1491 			if (!x86_cap_flag(bit))
1492 				continue;
1493 
1494 			if (strcmp(x86_cap_flag(bit), opt))
1495 				continue;
1496 
1497 			pr_cont(" %s", opt);
1498 			setup_clear_cpu_cap(bit);
1499 			taint++;
1500 			found = true;
1501 			break;
1502 		}
1503 
1504 		if (!found)
1505 			pr_cont(" (unknown: %s)", opt);
1506 #endif
1507 	}
1508 	pr_cont("\n");
1509 
1510 	if (taint)
1511 		add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1512 }
1513 
1514 /*
1515  * Do minimum CPU detection early.
1516  * Fields really needed: vendor, cpuid_level, family, model, mask,
1517  * cache alignment.
1518  * The others are not touched to avoid unwanted side effects.
1519  *
1520  * WARNING: this function is only called on the boot CPU.  Don't add code
1521  * here that is supposed to run on all CPUs.
1522  */
early_identify_cpu(struct cpuinfo_x86 * c)1523 static void __init early_identify_cpu(struct cpuinfo_x86 *c)
1524 {
1525 #ifdef CONFIG_X86_64
1526 	c->x86_clflush_size = 64;
1527 	c->x86_phys_bits = 36;
1528 	c->x86_virt_bits = 48;
1529 #else
1530 	c->x86_clflush_size = 32;
1531 	c->x86_phys_bits = 32;
1532 	c->x86_virt_bits = 32;
1533 #endif
1534 	c->x86_cache_alignment = c->x86_clflush_size;
1535 
1536 	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1537 	c->extended_cpuid_level = 0;
1538 
1539 	if (!have_cpuid_p())
1540 		identify_cpu_without_cpuid(c);
1541 
1542 	/* cyrix could have cpuid enabled via c_identify()*/
1543 	if (have_cpuid_p()) {
1544 		cpu_detect(c);
1545 		get_cpu_vendor(c);
1546 		get_cpu_cap(c);
1547 		get_cpu_address_sizes(c);
1548 		setup_force_cpu_cap(X86_FEATURE_CPUID);
1549 		cpu_parse_early_param();
1550 
1551 		if (this_cpu->c_early_init)
1552 			this_cpu->c_early_init(c);
1553 
1554 		c->cpu_index = 0;
1555 		filter_cpuid_features(c, false);
1556 
1557 		if (this_cpu->c_bsp_init)
1558 			this_cpu->c_bsp_init(c);
1559 	} else {
1560 		setup_clear_cpu_cap(X86_FEATURE_CPUID);
1561 	}
1562 
1563 	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
1564 
1565 	cpu_set_bug_bits(c);
1566 
1567 	sld_setup(c);
1568 
1569 	fpu__init_system(c);
1570 
1571 	init_sigframe_size();
1572 
1573 #ifdef CONFIG_X86_32
1574 	/*
1575 	 * Regardless of whether PCID is enumerated, the SDM says
1576 	 * that it can't be enabled in 32-bit mode.
1577 	 */
1578 	setup_clear_cpu_cap(X86_FEATURE_PCID);
1579 #endif
1580 
1581 	/*
1582 	 * Later in the boot process pgtable_l5_enabled() relies on
1583 	 * cpu_feature_enabled(X86_FEATURE_LA57). If 5-level paging is not
1584 	 * enabled by this point we need to clear the feature bit to avoid
1585 	 * false-positives at the later stage.
1586 	 *
1587 	 * pgtable_l5_enabled() can be false here for several reasons:
1588 	 *  - 5-level paging is disabled compile-time;
1589 	 *  - it's 32-bit kernel;
1590 	 *  - machine doesn't support 5-level paging;
1591 	 *  - user specified 'no5lvl' in kernel command line.
1592 	 */
1593 	if (!pgtable_l5_enabled())
1594 		setup_clear_cpu_cap(X86_FEATURE_LA57);
1595 
1596 	detect_nopl();
1597 }
1598 
early_cpu_init(void)1599 void __init early_cpu_init(void)
1600 {
1601 	const struct cpu_dev *const *cdev;
1602 	int count = 0;
1603 
1604 #ifdef CONFIG_PROCESSOR_SELECT
1605 	pr_info("KERNEL supported cpus:\n");
1606 #endif
1607 
1608 	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
1609 		const struct cpu_dev *cpudev = *cdev;
1610 
1611 		if (count >= X86_VENDOR_NUM)
1612 			break;
1613 		cpu_devs[count] = cpudev;
1614 		count++;
1615 
1616 #ifdef CONFIG_PROCESSOR_SELECT
1617 		{
1618 			unsigned int j;
1619 
1620 			for (j = 0; j < 2; j++) {
1621 				if (!cpudev->c_ident[j])
1622 					continue;
1623 				pr_info("  %s %s\n", cpudev->c_vendor,
1624 					cpudev->c_ident[j]);
1625 			}
1626 		}
1627 #endif
1628 	}
1629 	early_identify_cpu(&boot_cpu_data);
1630 }
1631 
detect_null_seg_behavior(void)1632 static bool detect_null_seg_behavior(void)
1633 {
1634 	/*
1635 	 * Empirically, writing zero to a segment selector on AMD does
1636 	 * not clear the base, whereas writing zero to a segment
1637 	 * selector on Intel does clear the base.  Intel's behavior
1638 	 * allows slightly faster context switches in the common case
1639 	 * where GS is unused by the prev and next threads.
1640 	 *
1641 	 * Since neither vendor documents this anywhere that I can see,
1642 	 * detect it directly instead of hard-coding the choice by
1643 	 * vendor.
1644 	 *
1645 	 * I've designated AMD's behavior as the "bug" because it's
1646 	 * counterintuitive and less friendly.
1647 	 */
1648 
1649 	unsigned long old_base, tmp;
1650 	rdmsrl(MSR_FS_BASE, old_base);
1651 	wrmsrl(MSR_FS_BASE, 1);
1652 	loadsegment(fs, 0);
1653 	rdmsrl(MSR_FS_BASE, tmp);
1654 	wrmsrl(MSR_FS_BASE, old_base);
1655 	return tmp == 0;
1656 }
1657 
check_null_seg_clears_base(struct cpuinfo_x86 * c)1658 void check_null_seg_clears_base(struct cpuinfo_x86 *c)
1659 {
1660 	/* BUG_NULL_SEG is only relevant with 64bit userspace */
1661 	if (!IS_ENABLED(CONFIG_X86_64))
1662 		return;
1663 
1664 	/* Zen3 CPUs advertise Null Selector Clears Base in CPUID. */
1665 	if (c->extended_cpuid_level >= 0x80000021 &&
1666 	    cpuid_eax(0x80000021) & BIT(6))
1667 		return;
1668 
1669 	/*
1670 	 * CPUID bit above wasn't set. If this kernel is still running
1671 	 * as a HV guest, then the HV has decided not to advertize
1672 	 * that CPUID bit for whatever reason.	For example, one
1673 	 * member of the migration pool might be vulnerable.  Which
1674 	 * means, the bug is present: set the BUG flag and return.
1675 	 */
1676 	if (cpu_has(c, X86_FEATURE_HYPERVISOR)) {
1677 		set_cpu_bug(c, X86_BUG_NULL_SEG);
1678 		return;
1679 	}
1680 
1681 	/*
1682 	 * Zen2 CPUs also have this behaviour, but no CPUID bit.
1683 	 * 0x18 is the respective family for Hygon.
1684 	 */
1685 	if ((c->x86 == 0x17 || c->x86 == 0x18) &&
1686 	    detect_null_seg_behavior())
1687 		return;
1688 
1689 	/* All the remaining ones are affected */
1690 	set_cpu_bug(c, X86_BUG_NULL_SEG);
1691 }
1692 
generic_identify(struct cpuinfo_x86 * c)1693 static void generic_identify(struct cpuinfo_x86 *c)
1694 {
1695 	c->extended_cpuid_level = 0;
1696 
1697 	if (!have_cpuid_p())
1698 		identify_cpu_without_cpuid(c);
1699 
1700 	/* cyrix could have cpuid enabled via c_identify()*/
1701 	if (!have_cpuid_p())
1702 		return;
1703 
1704 	cpu_detect(c);
1705 
1706 	get_cpu_vendor(c);
1707 
1708 	get_cpu_cap(c);
1709 
1710 	get_cpu_address_sizes(c);
1711 
1712 	if (c->cpuid_level >= 0x00000001) {
1713 		c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
1714 #ifdef CONFIG_X86_32
1715 # ifdef CONFIG_SMP
1716 		c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1717 # else
1718 		c->apicid = c->initial_apicid;
1719 # endif
1720 #endif
1721 		c->phys_proc_id = c->initial_apicid;
1722 	}
1723 
1724 	get_model_name(c); /* Default name */
1725 
1726 	/*
1727 	 * ESPFIX is a strange bug.  All real CPUs have it.  Paravirt
1728 	 * systems that run Linux at CPL > 0 may or may not have the
1729 	 * issue, but, even if they have the issue, there's absolutely
1730 	 * nothing we can do about it because we can't use the real IRET
1731 	 * instruction.
1732 	 *
1733 	 * NB: For the time being, only 32-bit kernels support
1734 	 * X86_BUG_ESPFIX as such.  64-bit kernels directly choose
1735 	 * whether to apply espfix using paravirt hooks.  If any
1736 	 * non-paravirt system ever shows up that does *not* have the
1737 	 * ESPFIX issue, we can change this.
1738 	 */
1739 #ifdef CONFIG_X86_32
1740 	set_cpu_bug(c, X86_BUG_ESPFIX);
1741 #endif
1742 }
1743 
1744 /*
1745  * Validate that ACPI/mptables have the same information about the
1746  * effective APIC id and update the package map.
1747  */
validate_apic_and_package_id(struct cpuinfo_x86 * c)1748 static void validate_apic_and_package_id(struct cpuinfo_x86 *c)
1749 {
1750 #ifdef CONFIG_SMP
1751 	unsigned int apicid, cpu = smp_processor_id();
1752 
1753 	apicid = apic->cpu_present_to_apicid(cpu);
1754 
1755 	if (apicid != c->apicid) {
1756 		pr_err(FW_BUG "CPU%u: APIC id mismatch. Firmware: %x APIC: %x\n",
1757 		       cpu, apicid, c->initial_apicid);
1758 	}
1759 	BUG_ON(topology_update_package_map(c->phys_proc_id, cpu));
1760 	BUG_ON(topology_update_die_map(c->cpu_die_id, cpu));
1761 #else
1762 	c->logical_proc_id = 0;
1763 #endif
1764 }
1765 
1766 /*
1767  * This does the hard work of actually picking apart the CPU stuff...
1768  */
identify_cpu(struct cpuinfo_x86 * c)1769 static void identify_cpu(struct cpuinfo_x86 *c)
1770 {
1771 	int i;
1772 
1773 	c->loops_per_jiffy = loops_per_jiffy;
1774 	c->x86_cache_size = 0;
1775 	c->x86_vendor = X86_VENDOR_UNKNOWN;
1776 	c->x86_model = c->x86_stepping = 0;	/* So far unknown... */
1777 	c->x86_vendor_id[0] = '\0'; /* Unset */
1778 	c->x86_model_id[0] = '\0';  /* Unset */
1779 	c->x86_max_cores = 1;
1780 	c->x86_coreid_bits = 0;
1781 	c->cu_id = 0xff;
1782 #ifdef CONFIG_X86_64
1783 	c->x86_clflush_size = 64;
1784 	c->x86_phys_bits = 36;
1785 	c->x86_virt_bits = 48;
1786 #else
1787 	c->cpuid_level = -1;	/* CPUID not detected */
1788 	c->x86_clflush_size = 32;
1789 	c->x86_phys_bits = 32;
1790 	c->x86_virt_bits = 32;
1791 #endif
1792 	c->x86_cache_alignment = c->x86_clflush_size;
1793 	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1794 #ifdef CONFIG_X86_VMX_FEATURE_NAMES
1795 	memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
1796 #endif
1797 
1798 	generic_identify(c);
1799 
1800 	if (this_cpu->c_identify)
1801 		this_cpu->c_identify(c);
1802 
1803 	/* Clear/Set all flags overridden by options, after probe */
1804 	apply_forced_caps(c);
1805 
1806 #ifdef CONFIG_X86_64
1807 	c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1808 #endif
1809 
1810 	/*
1811 	 * Vendor-specific initialization.  In this section we
1812 	 * canonicalize the feature flags, meaning if there are
1813 	 * features a certain CPU supports which CPUID doesn't
1814 	 * tell us, CPUID claiming incorrect flags, or other bugs,
1815 	 * we handle them here.
1816 	 *
1817 	 * At the end of this section, c->x86_capability better
1818 	 * indicate the features this CPU genuinely supports!
1819 	 */
1820 	if (this_cpu->c_init)
1821 		this_cpu->c_init(c);
1822 
1823 	/* Disable the PN if appropriate */
1824 	squash_the_stupid_serial_number(c);
1825 
1826 	/* Set up SMEP/SMAP/UMIP */
1827 	setup_smep(c);
1828 	setup_smap(c);
1829 	setup_umip(c);
1830 
1831 	/* Enable FSGSBASE instructions if available. */
1832 	if (cpu_has(c, X86_FEATURE_FSGSBASE)) {
1833 		cr4_set_bits(X86_CR4_FSGSBASE);
1834 		elf_hwcap2 |= HWCAP2_FSGSBASE;
1835 	}
1836 
1837 	/*
1838 	 * The vendor-specific functions might have changed features.
1839 	 * Now we do "generic changes."
1840 	 */
1841 
1842 	/* Filter out anything that depends on CPUID levels we don't have */
1843 	filter_cpuid_features(c, true);
1844 
1845 	/* If the model name is still unset, do table lookup. */
1846 	if (!c->x86_model_id[0]) {
1847 		const char *p;
1848 		p = table_lookup_model(c);
1849 		if (p)
1850 			strcpy(c->x86_model_id, p);
1851 		else
1852 			/* Last resort... */
1853 			sprintf(c->x86_model_id, "%02x/%02x",
1854 				c->x86, c->x86_model);
1855 	}
1856 
1857 #ifdef CONFIG_X86_64
1858 	detect_ht(c);
1859 #endif
1860 
1861 	x86_init_rdrand(c);
1862 	setup_pku(c);
1863 	setup_cet(c);
1864 
1865 	/*
1866 	 * Clear/Set all flags overridden by options, need do it
1867 	 * before following smp all cpus cap AND.
1868 	 */
1869 	apply_forced_caps(c);
1870 
1871 	/*
1872 	 * On SMP, boot_cpu_data holds the common feature set between
1873 	 * all CPUs; so make sure that we indicate which features are
1874 	 * common between the CPUs.  The first time this routine gets
1875 	 * executed, c == &boot_cpu_data.
1876 	 */
1877 	if (c != &boot_cpu_data) {
1878 		/* AND the already accumulated flags with these */
1879 		for (i = 0; i < NCAPINTS; i++)
1880 			boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1881 
1882 		/* OR, i.e. replicate the bug flags */
1883 		for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
1884 			c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
1885 	}
1886 
1887 	ppin_init(c);
1888 
1889 	/* Init Machine Check Exception if available. */
1890 	mcheck_cpu_init(c);
1891 
1892 	select_idle_routine(c);
1893 
1894 #ifdef CONFIG_NUMA
1895 	numa_add_cpu(smp_processor_id());
1896 #endif
1897 }
1898 
1899 /*
1900  * Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
1901  * on 32-bit kernels:
1902  */
1903 #ifdef CONFIG_X86_32
enable_sep_cpu(void)1904 void enable_sep_cpu(void)
1905 {
1906 	struct tss_struct *tss;
1907 	int cpu;
1908 
1909 	if (!boot_cpu_has(X86_FEATURE_SEP))
1910 		return;
1911 
1912 	cpu = get_cpu();
1913 	tss = &per_cpu(cpu_tss_rw, cpu);
1914 
1915 	/*
1916 	 * We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
1917 	 * see the big comment in struct x86_hw_tss's definition.
1918 	 */
1919 
1920 	tss->x86_tss.ss1 = __KERNEL_CS;
1921 	wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);
1922 	wrmsr(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1), 0);
1923 	wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0);
1924 
1925 	put_cpu();
1926 }
1927 #endif
1928 
identify_boot_cpu(void)1929 void __init identify_boot_cpu(void)
1930 {
1931 	identify_cpu(&boot_cpu_data);
1932 	if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
1933 		pr_info("CET detected: Indirect Branch Tracking enabled\n");
1934 #ifdef CONFIG_X86_32
1935 	sysenter_setup();
1936 	enable_sep_cpu();
1937 #endif
1938 	cpu_detect_tlb(&boot_cpu_data);
1939 	setup_cr_pinning();
1940 
1941 	tsx_init();
1942 }
1943 
identify_secondary_cpu(struct cpuinfo_x86 * c)1944 void identify_secondary_cpu(struct cpuinfo_x86 *c)
1945 {
1946 	BUG_ON(c == &boot_cpu_data);
1947 	identify_cpu(c);
1948 #ifdef CONFIG_X86_32
1949 	enable_sep_cpu();
1950 #endif
1951 	mtrr_ap_init();
1952 	validate_apic_and_package_id(c);
1953 	x86_spec_ctrl_setup_ap();
1954 	update_srbds_msr();
1955 
1956 	tsx_ap_init();
1957 }
1958 
print_cpu_info(struct cpuinfo_x86 * c)1959 void print_cpu_info(struct cpuinfo_x86 *c)
1960 {
1961 	const char *vendor = NULL;
1962 
1963 	if (c->x86_vendor < X86_VENDOR_NUM) {
1964 		vendor = this_cpu->c_vendor;
1965 	} else {
1966 		if (c->cpuid_level >= 0)
1967 			vendor = c->x86_vendor_id;
1968 	}
1969 
1970 	if (vendor && !strstr(c->x86_model_id, vendor))
1971 		pr_cont("%s ", vendor);
1972 
1973 	if (c->x86_model_id[0])
1974 		pr_cont("%s", c->x86_model_id);
1975 	else
1976 		pr_cont("%d86", c->x86);
1977 
1978 	pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
1979 
1980 	if (c->x86_stepping || c->cpuid_level >= 0)
1981 		pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
1982 	else
1983 		pr_cont(")\n");
1984 }
1985 
1986 /*
1987  * clearcpuid= was already parsed in cpu_parse_early_param().  This dummy
1988  * function prevents it from becoming an environment variable for init.
1989  */
setup_clearcpuid(char * arg)1990 static __init int setup_clearcpuid(char *arg)
1991 {
1992 	return 1;
1993 }
1994 __setup("clearcpuid=", setup_clearcpuid);
1995 
1996 #ifdef CONFIG_X86_64
1997 DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
1998 		     fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
1999 EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
2000 
2001 /*
2002  * The following percpu variables are hot.  Align current_task to
2003  * cacheline size such that they fall in the same cacheline.
2004  */
2005 DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
2006 	&init_task;
2007 EXPORT_PER_CPU_SYMBOL(current_task);
2008 
2009 DEFINE_PER_CPU(void *, hardirq_stack_ptr);
2010 DEFINE_PER_CPU(bool, hardirq_stack_inuse);
2011 
2012 DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
2013 EXPORT_PER_CPU_SYMBOL(__preempt_count);
2014 
2015 DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;
2016 
wrmsrl_cstar(unsigned long val)2017 static void wrmsrl_cstar(unsigned long val)
2018 {
2019 	/*
2020 	 * Intel CPUs do not support 32-bit SYSCALL. Writing to MSR_CSTAR
2021 	 * is so far ignored by the CPU, but raises a #VE trap in a TDX
2022 	 * guest. Avoid the pointless write on all Intel CPUs.
2023 	 */
2024 	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
2025 		wrmsrl(MSR_CSTAR, val);
2026 }
2027 
2028 /* May not be marked __init: used by software suspend */
syscall_init(void)2029 void syscall_init(void)
2030 {
2031 	wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
2032 	wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
2033 
2034 #ifdef CONFIG_IA32_EMULATION
2035 	wrmsrl_cstar((unsigned long)entry_SYSCALL_compat);
2036 	/*
2037 	 * This only works on Intel CPUs.
2038 	 * On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
2039 	 * This does not cause SYSENTER to jump to the wrong location, because
2040 	 * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
2041 	 */
2042 	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
2043 	wrmsrl_safe(MSR_IA32_SYSENTER_ESP,
2044 		    (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1));
2045 	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
2046 #else
2047 	wrmsrl_cstar((unsigned long)ignore_sysret);
2048 	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
2049 	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
2050 	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
2051 #endif
2052 
2053 	/*
2054 	 * Flags to clear on syscall; clear as much as possible
2055 	 * to minimize user space-kernel interference.
2056 	 */
2057 	wrmsrl(MSR_SYSCALL_MASK,
2058 	       X86_EFLAGS_CF|X86_EFLAGS_PF|X86_EFLAGS_AF|
2059 	       X86_EFLAGS_ZF|X86_EFLAGS_SF|X86_EFLAGS_TF|
2060 	       X86_EFLAGS_IF|X86_EFLAGS_DF|X86_EFLAGS_OF|
2061 	       X86_EFLAGS_IOPL|X86_EFLAGS_NT|X86_EFLAGS_RF|
2062 	       X86_EFLAGS_AC|X86_EFLAGS_ID);
2063 }
2064 
2065 #else	/* CONFIG_X86_64 */
2066 
2067 DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
2068 EXPORT_PER_CPU_SYMBOL(current_task);
2069 DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
2070 EXPORT_PER_CPU_SYMBOL(__preempt_count);
2071 
2072 /*
2073  * On x86_32, vm86 modifies tss.sp0, so sp0 isn't a reliable way to find
2074  * the top of the kernel stack.  Use an extra percpu variable to track the
2075  * top of the kernel stack directly.
2076  */
2077 DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
2078 	(unsigned long)&init_thread_union + THREAD_SIZE;
2079 EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);
2080 
2081 #ifdef CONFIG_STACKPROTECTOR
2082 DEFINE_PER_CPU(unsigned long, __stack_chk_guard);
2083 EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
2084 #endif
2085 
2086 #endif	/* CONFIG_X86_64 */
2087 
2088 /*
2089  * Clear all 6 debug registers:
2090  */
clear_all_debug_regs(void)2091 static void clear_all_debug_regs(void)
2092 {
2093 	int i;
2094 
2095 	for (i = 0; i < 8; i++) {
2096 		/* Ignore db4, db5 */
2097 		if ((i == 4) || (i == 5))
2098 			continue;
2099 
2100 		set_debugreg(0, i);
2101 	}
2102 }
2103 
2104 #ifdef CONFIG_KGDB
2105 /*
2106  * Restore debug regs if using kgdbwait and you have a kernel debugger
2107  * connection established.
2108  */
dbg_restore_debug_regs(void)2109 static void dbg_restore_debug_regs(void)
2110 {
2111 	if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
2112 		arch_kgdb_ops.correct_hw_break();
2113 }
2114 #else /* ! CONFIG_KGDB */
2115 #define dbg_restore_debug_regs()
2116 #endif /* ! CONFIG_KGDB */
2117 
wait_for_master_cpu(int cpu)2118 static void wait_for_master_cpu(int cpu)
2119 {
2120 #ifdef CONFIG_SMP
2121 	/*
2122 	 * wait for ACK from master CPU before continuing
2123 	 * with AP initialization
2124 	 */
2125 	WARN_ON(cpumask_test_and_set_cpu(cpu, cpu_initialized_mask));
2126 	while (!cpumask_test_cpu(cpu, cpu_callout_mask))
2127 		cpu_relax();
2128 #endif
2129 }
2130 
2131 #ifdef CONFIG_X86_64
setup_getcpu(int cpu)2132 static inline void setup_getcpu(int cpu)
2133 {
2134 	unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu));
2135 	struct desc_struct d = { };
2136 
2137 	if (boot_cpu_has(X86_FEATURE_RDTSCP) || boot_cpu_has(X86_FEATURE_RDPID))
2138 		wrmsr(MSR_TSC_AUX, cpudata, 0);
2139 
2140 	/* Store CPU and node number in limit. */
2141 	d.limit0 = cpudata;
2142 	d.limit1 = cpudata >> 16;
2143 
2144 	d.type = 5;		/* RO data, expand down, accessed */
2145 	d.dpl = 3;		/* Visible to user code */
2146 	d.s = 1;		/* Not a system segment */
2147 	d.p = 1;		/* Present */
2148 	d.d = 1;		/* 32-bit */
2149 
2150 	write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, &d, DESCTYPE_S);
2151 }
2152 
ucode_cpu_init(int cpu)2153 static inline void ucode_cpu_init(int cpu)
2154 {
2155 	if (cpu)
2156 		load_ucode_ap();
2157 }
2158 
tss_setup_ist(struct tss_struct * tss)2159 static inline void tss_setup_ist(struct tss_struct *tss)
2160 {
2161 	/* Set up the per-CPU TSS IST stacks */
2162 	tss->x86_tss.ist[IST_INDEX_DF] = __this_cpu_ist_top_va(DF);
2163 	tss->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI);
2164 	tss->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB);
2165 	tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE);
2166 	/* Only mapped when SEV-ES is active */
2167 	tss->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC);
2168 }
2169 
2170 #else /* CONFIG_X86_64 */
2171 
setup_getcpu(int cpu)2172 static inline void setup_getcpu(int cpu) { }
2173 
ucode_cpu_init(int cpu)2174 static inline void ucode_cpu_init(int cpu)
2175 {
2176 	show_ucode_info_early();
2177 }
2178 
tss_setup_ist(struct tss_struct * tss)2179 static inline void tss_setup_ist(struct tss_struct *tss) { }
2180 
2181 #endif /* !CONFIG_X86_64 */
2182 
tss_setup_io_bitmap(struct tss_struct * tss)2183 static inline void tss_setup_io_bitmap(struct tss_struct *tss)
2184 {
2185 	tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
2186 
2187 #ifdef CONFIG_X86_IOPL_IOPERM
2188 	tss->io_bitmap.prev_max = 0;
2189 	tss->io_bitmap.prev_sequence = 0;
2190 	memset(tss->io_bitmap.bitmap, 0xff, sizeof(tss->io_bitmap.bitmap));
2191 	/*
2192 	 * Invalidate the extra array entry past the end of the all
2193 	 * permission bitmap as required by the hardware.
2194 	 */
2195 	tss->io_bitmap.mapall[IO_BITMAP_LONGS] = ~0UL;
2196 #endif
2197 }
2198 
2199 /*
2200  * Setup everything needed to handle exceptions from the IDT, including the IST
2201  * exceptions which use paranoid_entry().
2202  */
cpu_init_exception_handling(void)2203 void cpu_init_exception_handling(void)
2204 {
2205 	struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
2206 	int cpu = raw_smp_processor_id();
2207 
2208 	/* paranoid_entry() gets the CPU number from the GDT */
2209 	setup_getcpu(cpu);
2210 
2211 	/* IST vectors need TSS to be set up. */
2212 	tss_setup_ist(tss);
2213 	tss_setup_io_bitmap(tss);
2214 	set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
2215 
2216 	load_TR_desc();
2217 
2218 	/* GHCB needs to be setup to handle #VC. */
2219 	setup_ghcb();
2220 
2221 	/* Finally load the IDT */
2222 	load_current_idt();
2223 }
2224 
2225 /*
2226  * cpu_init() initializes state that is per-CPU. Some data is already
2227  * initialized (naturally) in the bootstrap process, such as the GDT.  We
2228  * reload it nevertheless, this function acts as a 'CPU state barrier',
2229  * nothing should get across.
2230  */
cpu_init(void)2231 void cpu_init(void)
2232 {
2233 	struct task_struct *cur = current;
2234 	int cpu = raw_smp_processor_id();
2235 
2236 	wait_for_master_cpu(cpu);
2237 
2238 	ucode_cpu_init(cpu);
2239 
2240 #ifdef CONFIG_NUMA
2241 	if (this_cpu_read(numa_node) == 0 &&
2242 	    early_cpu_to_node(cpu) != NUMA_NO_NODE)
2243 		set_numa_node(early_cpu_to_node(cpu));
2244 #endif
2245 	pr_debug("Initializing CPU#%d\n", cpu);
2246 
2247 	if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) ||
2248 	    boot_cpu_has(X86_FEATURE_TSC) || boot_cpu_has(X86_FEATURE_DE))
2249 		cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
2250 
2251 	/*
2252 	 * Initialize the per-CPU GDT with the boot GDT,
2253 	 * and set up the GDT descriptor:
2254 	 */
2255 	switch_to_new_gdt(cpu);
2256 
2257 	if (IS_ENABLED(CONFIG_X86_64)) {
2258 		loadsegment(fs, 0);
2259 		memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
2260 		syscall_init();
2261 
2262 		wrmsrl(MSR_FS_BASE, 0);
2263 		wrmsrl(MSR_KERNEL_GS_BASE, 0);
2264 		barrier();
2265 
2266 		x2apic_setup();
2267 	}
2268 
2269 	mmgrab(&init_mm);
2270 	cur->active_mm = &init_mm;
2271 	BUG_ON(cur->mm);
2272 	initialize_tlbstate_and_flush();
2273 	enter_lazy_tlb(&init_mm, cur);
2274 
2275 	/*
2276 	 * sp0 points to the entry trampoline stack regardless of what task
2277 	 * is running.
2278 	 */
2279 	load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1));
2280 
2281 	load_mm_ldt(&init_mm);
2282 
2283 	clear_all_debug_regs();
2284 	dbg_restore_debug_regs();
2285 
2286 	doublefault_init_cpu_tss();
2287 
2288 	fpu__init_cpu();
2289 
2290 	if (is_uv_system())
2291 		uv_cpu_init();
2292 
2293 	load_fixmap_gdt(cpu);
2294 }
2295 
2296 #ifdef CONFIG_SMP
cpu_init_secondary(void)2297 void cpu_init_secondary(void)
2298 {
2299 	/*
2300 	 * Relies on the BP having set-up the IDT tables, which are loaded
2301 	 * on this CPU in cpu_init_exception_handling().
2302 	 */
2303 	cpu_init_exception_handling();
2304 	cpu_init();
2305 }
2306 #endif
2307 
2308 #ifdef CONFIG_MICROCODE_LATE_LOADING
2309 /*
2310  * The microcode loader calls this upon late microcode load to recheck features,
2311  * only when microcode has been updated. Caller holds microcode_mutex and CPU
2312  * hotplug lock.
2313  */
microcode_check(void)2314 void microcode_check(void)
2315 {
2316 	struct cpuinfo_x86 info;
2317 
2318 	perf_check_microcode();
2319 
2320 	/* Reload CPUID max function as it might've changed. */
2321 	info.cpuid_level = cpuid_eax(0);
2322 
2323 	/*
2324 	 * Copy all capability leafs to pick up the synthetic ones so that
2325 	 * memcmp() below doesn't fail on that. The ones coming from CPUID will
2326 	 * get overwritten in get_cpu_cap().
2327 	 */
2328 	memcpy(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability));
2329 
2330 	get_cpu_cap(&info);
2331 
2332 	if (!memcmp(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability)))
2333 		return;
2334 
2335 	pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
2336 	pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
2337 }
2338 #endif
2339 
2340 /*
2341  * Invoked from core CPU hotplug code after hotplug operations
2342  */
arch_smt_update(void)2343 void arch_smt_update(void)
2344 {
2345 	/* Handle the speculative execution misfeatures */
2346 	cpu_bugs_smt_update();
2347 	/* Check whether IPI broadcasting can be enabled */
2348 	apic_smt_update();
2349 }
2350