1 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
2 
3 #include <linux/kvm_host.h>
4 
5 #include "irq.h"
6 #include "mmu.h"
7 #include "kvm_cache_regs.h"
8 #include "x86.h"
9 #include "smm.h"
10 #include "cpuid.h"
11 #include "pmu.h"
12 
13 #include <linux/module.h>
14 #include <linux/mod_devicetable.h>
15 #include <linux/kernel.h>
16 #include <linux/vmalloc.h>
17 #include <linux/highmem.h>
18 #include <linux/amd-iommu.h>
19 #include <linux/sched.h>
20 #include <linux/trace_events.h>
21 #include <linux/slab.h>
22 #include <linux/hashtable.h>
23 #include <linux/objtool.h>
24 #include <linux/psp-sev.h>
25 #include <linux/file.h>
26 #include <linux/pagemap.h>
27 #include <linux/swap.h>
28 #include <linux/rwsem.h>
29 #include <linux/cc_platform.h>
30 #include <linux/smp.h>
31 
32 #include <asm/apic.h>
33 #include <asm/perf_event.h>
34 #include <asm/tlbflush.h>
35 #include <asm/desc.h>
36 #include <asm/debugreg.h>
37 #include <asm/kvm_para.h>
38 #include <asm/irq_remapping.h>
39 #include <asm/spec-ctrl.h>
40 #include <asm/cpu_device_id.h>
41 #include <asm/traps.h>
42 #include <asm/reboot.h>
43 #include <asm/fpu/api.h>
44 
45 #include <trace/events/ipi.h>
46 
47 #include "trace.h"
48 
49 #include "svm.h"
50 #include "svm_ops.h"
51 
52 #include "kvm_onhyperv.h"
53 #include "svm_onhyperv.h"
54 
55 MODULE_AUTHOR("Qumranet");
56 MODULE_LICENSE("GPL");
57 
58 #ifdef MODULE
59 static const struct x86_cpu_id svm_cpu_id[] = {
60 	X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL),
61 	{}
62 };
63 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
64 #endif
65 
66 #define SEG_TYPE_LDT 2
67 #define SEG_TYPE_BUSY_TSS16 3
68 
69 static bool erratum_383_found __read_mostly;
70 
71 u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
72 
73 /*
74  * Set osvw_len to higher value when updated Revision Guides
75  * are published and we know what the new status bits are
76  */
77 static uint64_t osvw_len = 4, osvw_status;
78 
79 static DEFINE_PER_CPU(u64, current_tsc_ratio);
80 
81 #define X2APIC_MSR(x)	(APIC_BASE_MSR + (x >> 4))
82 
83 static const struct svm_direct_access_msrs {
84 	u32 index;   /* Index of the MSR */
85 	bool always; /* True if intercept is initially cleared */
86 } direct_access_msrs[MAX_DIRECT_ACCESS_MSRS] = {
87 	{ .index = MSR_STAR,				.always = true  },
88 	{ .index = MSR_IA32_SYSENTER_CS,		.always = true  },
89 	{ .index = MSR_IA32_SYSENTER_EIP,		.always = false },
90 	{ .index = MSR_IA32_SYSENTER_ESP,		.always = false },
91 #ifdef CONFIG_X86_64
92 	{ .index = MSR_GS_BASE,				.always = true  },
93 	{ .index = MSR_FS_BASE,				.always = true  },
94 	{ .index = MSR_KERNEL_GS_BASE,			.always = true  },
95 	{ .index = MSR_LSTAR,				.always = true  },
96 	{ .index = MSR_CSTAR,				.always = true  },
97 	{ .index = MSR_SYSCALL_MASK,			.always = true  },
98 #endif
99 	{ .index = MSR_IA32_SPEC_CTRL,			.always = false },
100 	{ .index = MSR_IA32_PRED_CMD,			.always = false },
101 	{ .index = MSR_IA32_FLUSH_CMD,			.always = false },
102 	{ .index = MSR_IA32_LASTBRANCHFROMIP,		.always = false },
103 	{ .index = MSR_IA32_LASTBRANCHTOIP,		.always = false },
104 	{ .index = MSR_IA32_LASTINTFROMIP,		.always = false },
105 	{ .index = MSR_IA32_LASTINTTOIP,		.always = false },
106 	{ .index = MSR_EFER,				.always = false },
107 	{ .index = MSR_IA32_CR_PAT,			.always = false },
108 	{ .index = MSR_AMD64_SEV_ES_GHCB,		.always = true  },
109 	{ .index = MSR_TSC_AUX,				.always = false },
110 	{ .index = X2APIC_MSR(APIC_ID),			.always = false },
111 	{ .index = X2APIC_MSR(APIC_LVR),		.always = false },
112 	{ .index = X2APIC_MSR(APIC_TASKPRI),		.always = false },
113 	{ .index = X2APIC_MSR(APIC_ARBPRI),		.always = false },
114 	{ .index = X2APIC_MSR(APIC_PROCPRI),		.always = false },
115 	{ .index = X2APIC_MSR(APIC_EOI),		.always = false },
116 	{ .index = X2APIC_MSR(APIC_RRR),		.always = false },
117 	{ .index = X2APIC_MSR(APIC_LDR),		.always = false },
118 	{ .index = X2APIC_MSR(APIC_DFR),		.always = false },
119 	{ .index = X2APIC_MSR(APIC_SPIV),		.always = false },
120 	{ .index = X2APIC_MSR(APIC_ISR),		.always = false },
121 	{ .index = X2APIC_MSR(APIC_TMR),		.always = false },
122 	{ .index = X2APIC_MSR(APIC_IRR),		.always = false },
123 	{ .index = X2APIC_MSR(APIC_ESR),		.always = false },
124 	{ .index = X2APIC_MSR(APIC_ICR),		.always = false },
125 	{ .index = X2APIC_MSR(APIC_ICR2),		.always = false },
126 
127 	/*
128 	 * Note:
129 	 * AMD does not virtualize APIC TSC-deadline timer mode, but it is
130 	 * emulated by KVM. When setting APIC LVTT (0x832) register bit 18,
131 	 * the AVIC hardware would generate GP fault. Therefore, always
132 	 * intercept the MSR 0x832, and do not setup direct_access_msr.
133 	 */
134 	{ .index = X2APIC_MSR(APIC_LVTTHMR),		.always = false },
135 	{ .index = X2APIC_MSR(APIC_LVTPC),		.always = false },
136 	{ .index = X2APIC_MSR(APIC_LVT0),		.always = false },
137 	{ .index = X2APIC_MSR(APIC_LVT1),		.always = false },
138 	{ .index = X2APIC_MSR(APIC_LVTERR),		.always = false },
139 	{ .index = X2APIC_MSR(APIC_TMICT),		.always = false },
140 	{ .index = X2APIC_MSR(APIC_TMCCT),		.always = false },
141 	{ .index = X2APIC_MSR(APIC_TDCR),		.always = false },
142 	{ .index = MSR_INVALID,				.always = false },
143 };
144 
145 /*
146  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
147  * pause_filter_count: On processors that support Pause filtering(indicated
148  *	by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
149  *	count value. On VMRUN this value is loaded into an internal counter.
150  *	Each time a pause instruction is executed, this counter is decremented
151  *	until it reaches zero at which time a #VMEXIT is generated if pause
152  *	intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
153  *	Intercept Filtering for more details.
154  *	This also indicate if ple logic enabled.
155  *
156  * pause_filter_thresh: In addition, some processor families support advanced
157  *	pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
158  *	the amount of time a guest is allowed to execute in a pause loop.
159  *	In this mode, a 16-bit pause filter threshold field is added in the
160  *	VMCB. The threshold value is a cycle count that is used to reset the
161  *	pause counter. As with simple pause filtering, VMRUN loads the pause
162  *	count value from VMCB into an internal counter. Then, on each pause
163  *	instruction the hardware checks the elapsed number of cycles since
164  *	the most recent pause instruction against the pause filter threshold.
165  *	If the elapsed cycle count is greater than the pause filter threshold,
166  *	then the internal pause count is reloaded from the VMCB and execution
167  *	continues. If the elapsed cycle count is less than the pause filter
168  *	threshold, then the internal pause count is decremented. If the count
169  *	value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
170  *	triggered. If advanced pause filtering is supported and pause filter
171  *	threshold field is set to zero, the filter will operate in the simpler,
172  *	count only mode.
173  */
174 
175 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
176 module_param(pause_filter_thresh, ushort, 0444);
177 
178 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
179 module_param(pause_filter_count, ushort, 0444);
180 
181 /* Default doubles per-vcpu window every exit. */
182 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
183 module_param(pause_filter_count_grow, ushort, 0444);
184 
185 /* Default resets per-vcpu window every exit to pause_filter_count. */
186 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
187 module_param(pause_filter_count_shrink, ushort, 0444);
188 
189 /* Default is to compute the maximum so we can never overflow. */
190 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
191 module_param(pause_filter_count_max, ushort, 0444);
192 
193 /*
194  * Use nested page tables by default.  Note, NPT may get forced off by
195  * svm_hardware_setup() if it's unsupported by hardware or the host kernel.
196  */
197 bool npt_enabled = true;
198 module_param_named(npt, npt_enabled, bool, 0444);
199 
200 /* allow nested virtualization in KVM/SVM */
201 static int nested = true;
202 module_param(nested, int, S_IRUGO);
203 
204 /* enable/disable Next RIP Save */
205 int nrips = true;
206 module_param(nrips, int, 0444);
207 
208 /* enable/disable Virtual VMLOAD VMSAVE */
209 static int vls = true;
210 module_param(vls, int, 0444);
211 
212 /* enable/disable Virtual GIF */
213 int vgif = true;
214 module_param(vgif, int, 0444);
215 
216 /* enable/disable LBR virtualization */
217 static int lbrv = true;
218 module_param(lbrv, int, 0444);
219 
220 static int tsc_scaling = true;
221 module_param(tsc_scaling, int, 0444);
222 
223 /*
224  * enable / disable AVIC.  Because the defaults differ for APICv
225  * support between VMX and SVM we cannot use module_param_named.
226  */
227 static bool avic;
228 module_param(avic, bool, 0444);
229 
230 bool __read_mostly dump_invalid_vmcb;
231 module_param(dump_invalid_vmcb, bool, 0644);
232 
233 
234 bool intercept_smi = true;
235 module_param(intercept_smi, bool, 0444);
236 
237 bool vnmi = true;
238 module_param(vnmi, bool, 0444);
239 
240 static bool svm_gp_erratum_intercept = true;
241 
242 static u8 rsm_ins_bytes[] = "\x0f\xaa";
243 
244 static unsigned long iopm_base;
245 
246 DEFINE_PER_CPU(struct svm_cpu_data, svm_data);
247 
248 /*
249  * Only MSR_TSC_AUX is switched via the user return hook.  EFER is switched via
250  * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE.
251  *
252  * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to
253  * defer the restoration of TSC_AUX until the CPU returns to userspace.
254  */
255 static int tsc_aux_uret_slot __read_mostly = -1;
256 
257 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
258 
259 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
260 #define MSRS_RANGE_SIZE 2048
261 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
262 
svm_msrpm_offset(u32 msr)263 u32 svm_msrpm_offset(u32 msr)
264 {
265 	u32 offset;
266 	int i;
267 
268 	for (i = 0; i < NUM_MSR_MAPS; i++) {
269 		if (msr < msrpm_ranges[i] ||
270 		    msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
271 			continue;
272 
273 		offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
274 		offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
275 
276 		/* Now we have the u8 offset - but need the u32 offset */
277 		return offset / 4;
278 	}
279 
280 	/* MSR not in any range */
281 	return MSR_INVALID;
282 }
283 
284 static void svm_flush_tlb_current(struct kvm_vcpu *vcpu);
285 
get_npt_level(void)286 static int get_npt_level(void)
287 {
288 #ifdef CONFIG_X86_64
289 	return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL;
290 #else
291 	return PT32E_ROOT_LEVEL;
292 #endif
293 }
294 
svm_set_efer(struct kvm_vcpu * vcpu,u64 efer)295 int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
296 {
297 	struct vcpu_svm *svm = to_svm(vcpu);
298 	u64 old_efer = vcpu->arch.efer;
299 	vcpu->arch.efer = efer;
300 
301 	if (!npt_enabled) {
302 		/* Shadow paging assumes NX to be available.  */
303 		efer |= EFER_NX;
304 
305 		if (!(efer & EFER_LMA))
306 			efer &= ~EFER_LME;
307 	}
308 
309 	if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
310 		if (!(efer & EFER_SVME)) {
311 			svm_leave_nested(vcpu);
312 			svm_set_gif(svm, true);
313 			/* #GP intercept is still needed for vmware backdoor */
314 			if (!enable_vmware_backdoor)
315 				clr_exception_intercept(svm, GP_VECTOR);
316 
317 			/*
318 			 * Free the nested guest state, unless we are in SMM.
319 			 * In this case we will return to the nested guest
320 			 * as soon as we leave SMM.
321 			 */
322 			if (!is_smm(vcpu))
323 				svm_free_nested(svm);
324 
325 		} else {
326 			int ret = svm_allocate_nested(svm);
327 
328 			if (ret) {
329 				vcpu->arch.efer = old_efer;
330 				return ret;
331 			}
332 
333 			/*
334 			 * Never intercept #GP for SEV guests, KVM can't
335 			 * decrypt guest memory to workaround the erratum.
336 			 */
337 			if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm))
338 				set_exception_intercept(svm, GP_VECTOR);
339 		}
340 	}
341 
342 	svm->vmcb->save.efer = efer | EFER_SVME;
343 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
344 	return 0;
345 }
346 
svm_get_interrupt_shadow(struct kvm_vcpu * vcpu)347 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
348 {
349 	struct vcpu_svm *svm = to_svm(vcpu);
350 	u32 ret = 0;
351 
352 	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
353 		ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
354 	return ret;
355 }
356 
svm_set_interrupt_shadow(struct kvm_vcpu * vcpu,int mask)357 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
358 {
359 	struct vcpu_svm *svm = to_svm(vcpu);
360 
361 	if (mask == 0)
362 		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
363 	else
364 		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
365 
366 }
367 static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
368 					void *insn, int insn_len);
369 
__svm_skip_emulated_instruction(struct kvm_vcpu * vcpu,bool commit_side_effects)370 static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu,
371 					   bool commit_side_effects)
372 {
373 	struct vcpu_svm *svm = to_svm(vcpu);
374 	unsigned long old_rflags;
375 
376 	/*
377 	 * SEV-ES does not expose the next RIP. The RIP update is controlled by
378 	 * the type of exit and the #VC handler in the guest.
379 	 */
380 	if (sev_es_guest(vcpu->kvm))
381 		goto done;
382 
383 	if (nrips && svm->vmcb->control.next_rip != 0) {
384 		WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
385 		svm->next_rip = svm->vmcb->control.next_rip;
386 	}
387 
388 	if (!svm->next_rip) {
389 		/*
390 		 * FIXME: Drop this when kvm_emulate_instruction() does the
391 		 * right thing and treats "can't emulate" as outright failure
392 		 * for EMULTYPE_SKIP.
393 		 */
394 		if (!svm_can_emulate_instruction(vcpu, EMULTYPE_SKIP, NULL, 0))
395 			return 0;
396 
397 		if (unlikely(!commit_side_effects))
398 			old_rflags = svm->vmcb->save.rflags;
399 
400 		if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
401 			return 0;
402 
403 		if (unlikely(!commit_side_effects))
404 			svm->vmcb->save.rflags = old_rflags;
405 	} else {
406 		kvm_rip_write(vcpu, svm->next_rip);
407 	}
408 
409 done:
410 	if (likely(commit_side_effects))
411 		svm_set_interrupt_shadow(vcpu, 0);
412 
413 	return 1;
414 }
415 
svm_skip_emulated_instruction(struct kvm_vcpu * vcpu)416 static int svm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
417 {
418 	return __svm_skip_emulated_instruction(vcpu, true);
419 }
420 
svm_update_soft_interrupt_rip(struct kvm_vcpu * vcpu)421 static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu)
422 {
423 	unsigned long rip, old_rip = kvm_rip_read(vcpu);
424 	struct vcpu_svm *svm = to_svm(vcpu);
425 
426 	/*
427 	 * Due to architectural shortcomings, the CPU doesn't always provide
428 	 * NextRIP, e.g. if KVM intercepted an exception that occurred while
429 	 * the CPU was vectoring an INTO/INT3 in the guest.  Temporarily skip
430 	 * the instruction even if NextRIP is supported to acquire the next
431 	 * RIP so that it can be shoved into the NextRIP field, otherwise
432 	 * hardware will fail to advance guest RIP during event injection.
433 	 * Drop the exception/interrupt if emulation fails and effectively
434 	 * retry the instruction, it's the least awful option.  If NRIPS is
435 	 * in use, the skip must not commit any side effects such as clearing
436 	 * the interrupt shadow or RFLAGS.RF.
437 	 */
438 	if (!__svm_skip_emulated_instruction(vcpu, !nrips))
439 		return -EIO;
440 
441 	rip = kvm_rip_read(vcpu);
442 
443 	/*
444 	 * Save the injection information, even when using next_rip, as the
445 	 * VMCB's next_rip will be lost (cleared on VM-Exit) if the injection
446 	 * doesn't complete due to a VM-Exit occurring while the CPU is
447 	 * vectoring the event.   Decoding the instruction isn't guaranteed to
448 	 * work as there may be no backing instruction, e.g. if the event is
449 	 * being injected by L1 for L2, or if the guest is patching INT3 into
450 	 * a different instruction.
451 	 */
452 	svm->soft_int_injected = true;
453 	svm->soft_int_csbase = svm->vmcb->save.cs.base;
454 	svm->soft_int_old_rip = old_rip;
455 	svm->soft_int_next_rip = rip;
456 
457 	if (nrips)
458 		kvm_rip_write(vcpu, old_rip);
459 
460 	if (static_cpu_has(X86_FEATURE_NRIPS))
461 		svm->vmcb->control.next_rip = rip;
462 
463 	return 0;
464 }
465 
svm_inject_exception(struct kvm_vcpu * vcpu)466 static void svm_inject_exception(struct kvm_vcpu *vcpu)
467 {
468 	struct kvm_queued_exception *ex = &vcpu->arch.exception;
469 	struct vcpu_svm *svm = to_svm(vcpu);
470 
471 	kvm_deliver_exception_payload(vcpu, ex);
472 
473 	if (kvm_exception_is_soft(ex->vector) &&
474 	    svm_update_soft_interrupt_rip(vcpu))
475 		return;
476 
477 	svm->vmcb->control.event_inj = ex->vector
478 		| SVM_EVTINJ_VALID
479 		| (ex->has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
480 		| SVM_EVTINJ_TYPE_EXEPT;
481 	svm->vmcb->control.event_inj_err = ex->error_code;
482 }
483 
svm_init_erratum_383(void)484 static void svm_init_erratum_383(void)
485 {
486 	u32 low, high;
487 	int err;
488 	u64 val;
489 
490 	if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
491 		return;
492 
493 	/* Use _safe variants to not break nested virtualization */
494 	val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
495 	if (err)
496 		return;
497 
498 	val |= (1ULL << 47);
499 
500 	low  = lower_32_bits(val);
501 	high = upper_32_bits(val);
502 
503 	native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
504 
505 	erratum_383_found = true;
506 }
507 
svm_init_osvw(struct kvm_vcpu * vcpu)508 static void svm_init_osvw(struct kvm_vcpu *vcpu)
509 {
510 	/*
511 	 * Guests should see errata 400 and 415 as fixed (assuming that
512 	 * HLT and IO instructions are intercepted).
513 	 */
514 	vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
515 	vcpu->arch.osvw.status = osvw_status & ~(6ULL);
516 
517 	/*
518 	 * By increasing VCPU's osvw.length to 3 we are telling the guest that
519 	 * all osvw.status bits inside that length, including bit 0 (which is
520 	 * reserved for erratum 298), are valid. However, if host processor's
521 	 * osvw_len is 0 then osvw_status[0] carries no information. We need to
522 	 * be conservative here and therefore we tell the guest that erratum 298
523 	 * is present (because we really don't know).
524 	 */
525 	if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
526 		vcpu->arch.osvw.status |= 1;
527 }
528 
__kvm_is_svm_supported(void)529 static bool __kvm_is_svm_supported(void)
530 {
531 	int cpu = smp_processor_id();
532 	struct cpuinfo_x86 *c = &cpu_data(cpu);
533 
534 	u64 vm_cr;
535 
536 	if (c->x86_vendor != X86_VENDOR_AMD &&
537 	    c->x86_vendor != X86_VENDOR_HYGON) {
538 		pr_err("CPU %d isn't AMD or Hygon\n", cpu);
539 		return false;
540 	}
541 
542 	if (!cpu_has(c, X86_FEATURE_SVM)) {
543 		pr_err("SVM not supported by CPU %d\n", cpu);
544 		return false;
545 	}
546 
547 	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
548 		pr_info("KVM is unsupported when running as an SEV guest\n");
549 		return false;
550 	}
551 
552 	rdmsrl(MSR_VM_CR, vm_cr);
553 	if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE)) {
554 		pr_err("SVM disabled (by BIOS) in MSR_VM_CR on CPU %d\n", cpu);
555 		return false;
556 	}
557 
558 	return true;
559 }
560 
kvm_is_svm_supported(void)561 static bool kvm_is_svm_supported(void)
562 {
563 	bool supported;
564 
565 	migrate_disable();
566 	supported = __kvm_is_svm_supported();
567 	migrate_enable();
568 
569 	return supported;
570 }
571 
svm_check_processor_compat(void)572 static int svm_check_processor_compat(void)
573 {
574 	if (!__kvm_is_svm_supported())
575 		return -EIO;
576 
577 	return 0;
578 }
579 
__svm_write_tsc_multiplier(u64 multiplier)580 static void __svm_write_tsc_multiplier(u64 multiplier)
581 {
582 	if (multiplier == __this_cpu_read(current_tsc_ratio))
583 		return;
584 
585 	wrmsrl(MSR_AMD64_TSC_RATIO, multiplier);
586 	__this_cpu_write(current_tsc_ratio, multiplier);
587 }
588 
kvm_cpu_svm_disable(void)589 static inline void kvm_cpu_svm_disable(void)
590 {
591 	uint64_t efer;
592 
593 	wrmsrl(MSR_VM_HSAVE_PA, 0);
594 	rdmsrl(MSR_EFER, efer);
595 	if (efer & EFER_SVME) {
596 		/*
597 		 * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and
598 		 * NMI aren't blocked.
599 		 */
600 		stgi();
601 		wrmsrl(MSR_EFER, efer & ~EFER_SVME);
602 	}
603 }
604 
svm_emergency_disable(void)605 static void svm_emergency_disable(void)
606 {
607 	kvm_rebooting = true;
608 
609 	kvm_cpu_svm_disable();
610 }
611 
svm_hardware_disable(void)612 static void svm_hardware_disable(void)
613 {
614 	/* Make sure we clean up behind us */
615 	if (tsc_scaling)
616 		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
617 
618 	kvm_cpu_svm_disable();
619 
620 	amd_pmu_disable_virt();
621 }
622 
svm_hardware_enable(void)623 static int svm_hardware_enable(void)
624 {
625 
626 	struct svm_cpu_data *sd;
627 	uint64_t efer;
628 	int me = raw_smp_processor_id();
629 
630 	rdmsrl(MSR_EFER, efer);
631 	if (efer & EFER_SVME)
632 		return -EBUSY;
633 
634 	sd = per_cpu_ptr(&svm_data, me);
635 	sd->asid_generation = 1;
636 	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
637 	sd->next_asid = sd->max_asid + 1;
638 	sd->min_asid = max_sev_asid + 1;
639 
640 	wrmsrl(MSR_EFER, efer | EFER_SVME);
641 
642 	wrmsrl(MSR_VM_HSAVE_PA, sd->save_area_pa);
643 
644 	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
645 		/*
646 		 * Set the default value, even if we don't use TSC scaling
647 		 * to avoid having stale value in the msr
648 		 */
649 		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
650 	}
651 
652 
653 	/*
654 	 * Get OSVW bits.
655 	 *
656 	 * Note that it is possible to have a system with mixed processor
657 	 * revisions and therefore different OSVW bits. If bits are not the same
658 	 * on different processors then choose the worst case (i.e. if erratum
659 	 * is present on one processor and not on another then assume that the
660 	 * erratum is present everywhere).
661 	 */
662 	if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
663 		uint64_t len, status = 0;
664 		int err;
665 
666 		len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
667 		if (!err)
668 			status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
669 						      &err);
670 
671 		if (err)
672 			osvw_status = osvw_len = 0;
673 		else {
674 			if (len < osvw_len)
675 				osvw_len = len;
676 			osvw_status |= status;
677 			osvw_status &= (1ULL << osvw_len) - 1;
678 		}
679 	} else
680 		osvw_status = osvw_len = 0;
681 
682 	svm_init_erratum_383();
683 
684 	amd_pmu_enable_virt();
685 
686 	/*
687 	 * If TSC_AUX virtualization is supported, TSC_AUX becomes a swap type
688 	 * "B" field (see sev_es_prepare_switch_to_guest()) for SEV-ES guests.
689 	 * Since Linux does not change the value of TSC_AUX once set, prime the
690 	 * TSC_AUX field now to avoid a RDMSR on every vCPU run.
691 	 */
692 	if (boot_cpu_has(X86_FEATURE_V_TSC_AUX)) {
693 		struct sev_es_save_area *hostsa;
694 		u32 __maybe_unused msr_hi;
695 
696 		hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400);
697 
698 		rdmsr(MSR_TSC_AUX, hostsa->tsc_aux, msr_hi);
699 	}
700 
701 	return 0;
702 }
703 
svm_cpu_uninit(int cpu)704 static void svm_cpu_uninit(int cpu)
705 {
706 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
707 
708 	if (!sd->save_area)
709 		return;
710 
711 	kfree(sd->sev_vmcbs);
712 	__free_page(sd->save_area);
713 	sd->save_area_pa = 0;
714 	sd->save_area = NULL;
715 }
716 
svm_cpu_init(int cpu)717 static int svm_cpu_init(int cpu)
718 {
719 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
720 	int ret = -ENOMEM;
721 
722 	memset(sd, 0, sizeof(struct svm_cpu_data));
723 	sd->save_area = alloc_page(GFP_KERNEL | __GFP_ZERO);
724 	if (!sd->save_area)
725 		return ret;
726 
727 	ret = sev_cpu_init(sd);
728 	if (ret)
729 		goto free_save_area;
730 
731 	sd->save_area_pa = __sme_page_pa(sd->save_area);
732 	return 0;
733 
734 free_save_area:
735 	__free_page(sd->save_area);
736 	sd->save_area = NULL;
737 	return ret;
738 
739 }
740 
set_dr_intercepts(struct vcpu_svm * svm)741 static void set_dr_intercepts(struct vcpu_svm *svm)
742 {
743 	struct vmcb *vmcb = svm->vmcb01.ptr;
744 
745 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_READ);
746 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_READ);
747 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_READ);
748 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_READ);
749 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_READ);
750 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_READ);
751 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_READ);
752 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_WRITE);
753 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_WRITE);
754 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_WRITE);
755 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_WRITE);
756 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_WRITE);
757 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_WRITE);
758 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_WRITE);
759 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ);
760 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE);
761 
762 	recalc_intercepts(svm);
763 }
764 
clr_dr_intercepts(struct vcpu_svm * svm)765 static void clr_dr_intercepts(struct vcpu_svm *svm)
766 {
767 	struct vmcb *vmcb = svm->vmcb01.ptr;
768 
769 	vmcb->control.intercepts[INTERCEPT_DR] = 0;
770 
771 	recalc_intercepts(svm);
772 }
773 
direct_access_msr_slot(u32 msr)774 static int direct_access_msr_slot(u32 msr)
775 {
776 	u32 i;
777 
778 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
779 		if (direct_access_msrs[i].index == msr)
780 			return i;
781 
782 	return -ENOENT;
783 }
784 
set_shadow_msr_intercept(struct kvm_vcpu * vcpu,u32 msr,int read,int write)785 static void set_shadow_msr_intercept(struct kvm_vcpu *vcpu, u32 msr, int read,
786 				     int write)
787 {
788 	struct vcpu_svm *svm = to_svm(vcpu);
789 	int slot = direct_access_msr_slot(msr);
790 
791 	if (slot == -ENOENT)
792 		return;
793 
794 	/* Set the shadow bitmaps to the desired intercept states */
795 	if (read)
796 		set_bit(slot, svm->shadow_msr_intercept.read);
797 	else
798 		clear_bit(slot, svm->shadow_msr_intercept.read);
799 
800 	if (write)
801 		set_bit(slot, svm->shadow_msr_intercept.write);
802 	else
803 		clear_bit(slot, svm->shadow_msr_intercept.write);
804 }
805 
valid_msr_intercept(u32 index)806 static bool valid_msr_intercept(u32 index)
807 {
808 	return direct_access_msr_slot(index) != -ENOENT;
809 }
810 
msr_write_intercepted(struct kvm_vcpu * vcpu,u32 msr)811 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
812 {
813 	u8 bit_write;
814 	unsigned long tmp;
815 	u32 offset;
816 	u32 *msrpm;
817 
818 	/*
819 	 * For non-nested case:
820 	 * If the L01 MSR bitmap does not intercept the MSR, then we need to
821 	 * save it.
822 	 *
823 	 * For nested case:
824 	 * If the L02 MSR bitmap does not intercept the MSR, then we need to
825 	 * save it.
826 	 */
827 	msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
828 				      to_svm(vcpu)->msrpm;
829 
830 	offset    = svm_msrpm_offset(msr);
831 	bit_write = 2 * (msr & 0x0f) + 1;
832 	tmp       = msrpm[offset];
833 
834 	BUG_ON(offset == MSR_INVALID);
835 
836 	return test_bit(bit_write, &tmp);
837 }
838 
set_msr_interception_bitmap(struct kvm_vcpu * vcpu,u32 * msrpm,u32 msr,int read,int write)839 static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm,
840 					u32 msr, int read, int write)
841 {
842 	struct vcpu_svm *svm = to_svm(vcpu);
843 	u8 bit_read, bit_write;
844 	unsigned long tmp;
845 	u32 offset;
846 
847 	/*
848 	 * If this warning triggers extend the direct_access_msrs list at the
849 	 * beginning of the file
850 	 */
851 	WARN_ON(!valid_msr_intercept(msr));
852 
853 	/* Enforce non allowed MSRs to trap */
854 	if (read && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ))
855 		read = 0;
856 
857 	if (write && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE))
858 		write = 0;
859 
860 	offset    = svm_msrpm_offset(msr);
861 	bit_read  = 2 * (msr & 0x0f);
862 	bit_write = 2 * (msr & 0x0f) + 1;
863 	tmp       = msrpm[offset];
864 
865 	BUG_ON(offset == MSR_INVALID);
866 
867 	read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
868 	write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
869 
870 	msrpm[offset] = tmp;
871 
872 	svm_hv_vmcb_dirty_nested_enlightenments(vcpu);
873 	svm->nested.force_msr_bitmap_recalc = true;
874 }
875 
set_msr_interception(struct kvm_vcpu * vcpu,u32 * msrpm,u32 msr,int read,int write)876 void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr,
877 			  int read, int write)
878 {
879 	set_shadow_msr_intercept(vcpu, msr, read, write);
880 	set_msr_interception_bitmap(vcpu, msrpm, msr, read, write);
881 }
882 
svm_vcpu_alloc_msrpm(void)883 u32 *svm_vcpu_alloc_msrpm(void)
884 {
885 	unsigned int order = get_order(MSRPM_SIZE);
886 	struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, order);
887 	u32 *msrpm;
888 
889 	if (!pages)
890 		return NULL;
891 
892 	msrpm = page_address(pages);
893 	memset(msrpm, 0xff, PAGE_SIZE * (1 << order));
894 
895 	return msrpm;
896 }
897 
svm_vcpu_init_msrpm(struct kvm_vcpu * vcpu,u32 * msrpm)898 void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm)
899 {
900 	int i;
901 
902 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
903 		if (!direct_access_msrs[i].always)
904 			continue;
905 		set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1);
906 	}
907 }
908 
svm_set_x2apic_msr_interception(struct vcpu_svm * svm,bool intercept)909 void svm_set_x2apic_msr_interception(struct vcpu_svm *svm, bool intercept)
910 {
911 	int i;
912 
913 	if (intercept == svm->x2avic_msrs_intercepted)
914 		return;
915 
916 	if (!x2avic_enabled)
917 		return;
918 
919 	for (i = 0; i < MAX_DIRECT_ACCESS_MSRS; i++) {
920 		int index = direct_access_msrs[i].index;
921 
922 		if ((index < APIC_BASE_MSR) ||
923 		    (index > APIC_BASE_MSR + 0xff))
924 			continue;
925 		set_msr_interception(&svm->vcpu, svm->msrpm, index,
926 				     !intercept, !intercept);
927 	}
928 
929 	svm->x2avic_msrs_intercepted = intercept;
930 }
931 
svm_vcpu_free_msrpm(u32 * msrpm)932 void svm_vcpu_free_msrpm(u32 *msrpm)
933 {
934 	__free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE));
935 }
936 
svm_msr_filter_changed(struct kvm_vcpu * vcpu)937 static void svm_msr_filter_changed(struct kvm_vcpu *vcpu)
938 {
939 	struct vcpu_svm *svm = to_svm(vcpu);
940 	u32 i;
941 
942 	/*
943 	 * Set intercept permissions for all direct access MSRs again. They
944 	 * will automatically get filtered through the MSR filter, so we are
945 	 * back in sync after this.
946 	 */
947 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
948 		u32 msr = direct_access_msrs[i].index;
949 		u32 read = test_bit(i, svm->shadow_msr_intercept.read);
950 		u32 write = test_bit(i, svm->shadow_msr_intercept.write);
951 
952 		set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write);
953 	}
954 }
955 
add_msr_offset(u32 offset)956 static void add_msr_offset(u32 offset)
957 {
958 	int i;
959 
960 	for (i = 0; i < MSRPM_OFFSETS; ++i) {
961 
962 		/* Offset already in list? */
963 		if (msrpm_offsets[i] == offset)
964 			return;
965 
966 		/* Slot used by another offset? */
967 		if (msrpm_offsets[i] != MSR_INVALID)
968 			continue;
969 
970 		/* Add offset to list */
971 		msrpm_offsets[i] = offset;
972 
973 		return;
974 	}
975 
976 	/*
977 	 * If this BUG triggers the msrpm_offsets table has an overflow. Just
978 	 * increase MSRPM_OFFSETS in this case.
979 	 */
980 	BUG();
981 }
982 
init_msrpm_offsets(void)983 static void init_msrpm_offsets(void)
984 {
985 	int i;
986 
987 	memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
988 
989 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
990 		u32 offset;
991 
992 		offset = svm_msrpm_offset(direct_access_msrs[i].index);
993 		BUG_ON(offset == MSR_INVALID);
994 
995 		add_msr_offset(offset);
996 	}
997 }
998 
svm_copy_lbrs(struct vmcb * to_vmcb,struct vmcb * from_vmcb)999 void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb)
1000 {
1001 	to_vmcb->save.dbgctl		= from_vmcb->save.dbgctl;
1002 	to_vmcb->save.br_from		= from_vmcb->save.br_from;
1003 	to_vmcb->save.br_to		= from_vmcb->save.br_to;
1004 	to_vmcb->save.last_excp_from	= from_vmcb->save.last_excp_from;
1005 	to_vmcb->save.last_excp_to	= from_vmcb->save.last_excp_to;
1006 
1007 	vmcb_mark_dirty(to_vmcb, VMCB_LBR);
1008 }
1009 
svm_enable_lbrv(struct kvm_vcpu * vcpu)1010 static void svm_enable_lbrv(struct kvm_vcpu *vcpu)
1011 {
1012 	struct vcpu_svm *svm = to_svm(vcpu);
1013 
1014 	svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
1015 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
1016 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
1017 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
1018 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
1019 
1020 	/* Move the LBR msrs to the vmcb02 so that the guest can see them. */
1021 	if (is_guest_mode(vcpu))
1022 		svm_copy_lbrs(svm->vmcb, svm->vmcb01.ptr);
1023 }
1024 
svm_disable_lbrv(struct kvm_vcpu * vcpu)1025 static void svm_disable_lbrv(struct kvm_vcpu *vcpu)
1026 {
1027 	struct vcpu_svm *svm = to_svm(vcpu);
1028 
1029 	svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
1030 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
1031 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
1032 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
1033 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
1034 
1035 	/*
1036 	 * Move the LBR msrs back to the vmcb01 to avoid copying them
1037 	 * on nested guest entries.
1038 	 */
1039 	if (is_guest_mode(vcpu))
1040 		svm_copy_lbrs(svm->vmcb01.ptr, svm->vmcb);
1041 }
1042 
svm_get_lbr_vmcb(struct vcpu_svm * svm)1043 static struct vmcb *svm_get_lbr_vmcb(struct vcpu_svm *svm)
1044 {
1045 	/*
1046 	 * If LBR virtualization is disabled, the LBR MSRs are always kept in
1047 	 * vmcb01.  If LBR virtualization is enabled and L1 is running VMs of
1048 	 * its own, the MSRs are moved between vmcb01 and vmcb02 as needed.
1049 	 */
1050 	return svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK ? svm->vmcb :
1051 								   svm->vmcb01.ptr;
1052 }
1053 
svm_update_lbrv(struct kvm_vcpu * vcpu)1054 void svm_update_lbrv(struct kvm_vcpu *vcpu)
1055 {
1056 	struct vcpu_svm *svm = to_svm(vcpu);
1057 	bool current_enable_lbrv = svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK;
1058 	bool enable_lbrv = (svm_get_lbr_vmcb(svm)->save.dbgctl & DEBUGCTLMSR_LBR) ||
1059 			    (is_guest_mode(vcpu) && guest_can_use(vcpu, X86_FEATURE_LBRV) &&
1060 			    (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK));
1061 
1062 	if (enable_lbrv == current_enable_lbrv)
1063 		return;
1064 
1065 	if (enable_lbrv)
1066 		svm_enable_lbrv(vcpu);
1067 	else
1068 		svm_disable_lbrv(vcpu);
1069 }
1070 
disable_nmi_singlestep(struct vcpu_svm * svm)1071 void disable_nmi_singlestep(struct vcpu_svm *svm)
1072 {
1073 	svm->nmi_singlestep = false;
1074 
1075 	if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
1076 		/* Clear our flags if they were not set by the guest */
1077 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1078 			svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
1079 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1080 			svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
1081 	}
1082 }
1083 
grow_ple_window(struct kvm_vcpu * vcpu)1084 static void grow_ple_window(struct kvm_vcpu *vcpu)
1085 {
1086 	struct vcpu_svm *svm = to_svm(vcpu);
1087 	struct vmcb_control_area *control = &svm->vmcb->control;
1088 	int old = control->pause_filter_count;
1089 
1090 	if (kvm_pause_in_guest(vcpu->kvm))
1091 		return;
1092 
1093 	control->pause_filter_count = __grow_ple_window(old,
1094 							pause_filter_count,
1095 							pause_filter_count_grow,
1096 							pause_filter_count_max);
1097 
1098 	if (control->pause_filter_count != old) {
1099 		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1100 		trace_kvm_ple_window_update(vcpu->vcpu_id,
1101 					    control->pause_filter_count, old);
1102 	}
1103 }
1104 
shrink_ple_window(struct kvm_vcpu * vcpu)1105 static void shrink_ple_window(struct kvm_vcpu *vcpu)
1106 {
1107 	struct vcpu_svm *svm = to_svm(vcpu);
1108 	struct vmcb_control_area *control = &svm->vmcb->control;
1109 	int old = control->pause_filter_count;
1110 
1111 	if (kvm_pause_in_guest(vcpu->kvm))
1112 		return;
1113 
1114 	control->pause_filter_count =
1115 				__shrink_ple_window(old,
1116 						    pause_filter_count,
1117 						    pause_filter_count_shrink,
1118 						    pause_filter_count);
1119 	if (control->pause_filter_count != old) {
1120 		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1121 		trace_kvm_ple_window_update(vcpu->vcpu_id,
1122 					    control->pause_filter_count, old);
1123 	}
1124 }
1125 
svm_hardware_unsetup(void)1126 static void svm_hardware_unsetup(void)
1127 {
1128 	int cpu;
1129 
1130 	sev_hardware_unsetup();
1131 
1132 	for_each_possible_cpu(cpu)
1133 		svm_cpu_uninit(cpu);
1134 
1135 	__free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT),
1136 	get_order(IOPM_SIZE));
1137 	iopm_base = 0;
1138 }
1139 
init_seg(struct vmcb_seg * seg)1140 static void init_seg(struct vmcb_seg *seg)
1141 {
1142 	seg->selector = 0;
1143 	seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1144 		      SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1145 	seg->limit = 0xffff;
1146 	seg->base = 0;
1147 }
1148 
init_sys_seg(struct vmcb_seg * seg,uint32_t type)1149 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1150 {
1151 	seg->selector = 0;
1152 	seg->attrib = SVM_SELECTOR_P_MASK | type;
1153 	seg->limit = 0xffff;
1154 	seg->base = 0;
1155 }
1156 
svm_get_l2_tsc_offset(struct kvm_vcpu * vcpu)1157 static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu)
1158 {
1159 	struct vcpu_svm *svm = to_svm(vcpu);
1160 
1161 	return svm->nested.ctl.tsc_offset;
1162 }
1163 
svm_get_l2_tsc_multiplier(struct kvm_vcpu * vcpu)1164 static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)
1165 {
1166 	struct vcpu_svm *svm = to_svm(vcpu);
1167 
1168 	return svm->tsc_ratio_msr;
1169 }
1170 
svm_write_tsc_offset(struct kvm_vcpu * vcpu)1171 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu)
1172 {
1173 	struct vcpu_svm *svm = to_svm(vcpu);
1174 
1175 	svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset;
1176 	svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset;
1177 	vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1178 }
1179 
svm_write_tsc_multiplier(struct kvm_vcpu * vcpu)1180 void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu)
1181 {
1182 	preempt_disable();
1183 	if (to_svm(vcpu)->guest_state_loaded)
1184 		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1185 	preempt_enable();
1186 }
1187 
1188 /* Evaluate instruction intercepts that depend on guest CPUID features. */
svm_recalc_instruction_intercepts(struct kvm_vcpu * vcpu,struct vcpu_svm * svm)1189 static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu,
1190 					      struct vcpu_svm *svm)
1191 {
1192 	/*
1193 	 * Intercept INVPCID if shadow paging is enabled to sync/free shadow
1194 	 * roots, or if INVPCID is disabled in the guest to inject #UD.
1195 	 */
1196 	if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) {
1197 		if (!npt_enabled ||
1198 		    !guest_cpuid_has(&svm->vcpu, X86_FEATURE_INVPCID))
1199 			svm_set_intercept(svm, INTERCEPT_INVPCID);
1200 		else
1201 			svm_clr_intercept(svm, INTERCEPT_INVPCID);
1202 	}
1203 
1204 	if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) {
1205 		if (guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
1206 			svm_clr_intercept(svm, INTERCEPT_RDTSCP);
1207 		else
1208 			svm_set_intercept(svm, INTERCEPT_RDTSCP);
1209 	}
1210 }
1211 
init_vmcb_after_set_cpuid(struct kvm_vcpu * vcpu)1212 static inline void init_vmcb_after_set_cpuid(struct kvm_vcpu *vcpu)
1213 {
1214 	struct vcpu_svm *svm = to_svm(vcpu);
1215 
1216 	if (guest_cpuid_is_intel(vcpu)) {
1217 		/*
1218 		 * We must intercept SYSENTER_EIP and SYSENTER_ESP
1219 		 * accesses because the processor only stores 32 bits.
1220 		 * For the same reason we cannot use virtual VMLOAD/VMSAVE.
1221 		 */
1222 		svm_set_intercept(svm, INTERCEPT_VMLOAD);
1223 		svm_set_intercept(svm, INTERCEPT_VMSAVE);
1224 		svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1225 
1226 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0);
1227 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0);
1228 	} else {
1229 		/*
1230 		 * If hardware supports Virtual VMLOAD VMSAVE then enable it
1231 		 * in VMCB and clear intercepts to avoid #VMEXIT.
1232 		 */
1233 		if (vls) {
1234 			svm_clr_intercept(svm, INTERCEPT_VMLOAD);
1235 			svm_clr_intercept(svm, INTERCEPT_VMSAVE);
1236 			svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1237 		}
1238 		/* No need to intercept these MSRs */
1239 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1);
1240 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1);
1241 	}
1242 }
1243 
init_vmcb(struct kvm_vcpu * vcpu)1244 static void init_vmcb(struct kvm_vcpu *vcpu)
1245 {
1246 	struct vcpu_svm *svm = to_svm(vcpu);
1247 	struct vmcb *vmcb = svm->vmcb01.ptr;
1248 	struct vmcb_control_area *control = &vmcb->control;
1249 	struct vmcb_save_area *save = &vmcb->save;
1250 
1251 	svm_set_intercept(svm, INTERCEPT_CR0_READ);
1252 	svm_set_intercept(svm, INTERCEPT_CR3_READ);
1253 	svm_set_intercept(svm, INTERCEPT_CR4_READ);
1254 	svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1255 	svm_set_intercept(svm, INTERCEPT_CR3_WRITE);
1256 	svm_set_intercept(svm, INTERCEPT_CR4_WRITE);
1257 	if (!kvm_vcpu_apicv_active(vcpu))
1258 		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
1259 
1260 	set_dr_intercepts(svm);
1261 
1262 	set_exception_intercept(svm, PF_VECTOR);
1263 	set_exception_intercept(svm, UD_VECTOR);
1264 	set_exception_intercept(svm, MC_VECTOR);
1265 	set_exception_intercept(svm, AC_VECTOR);
1266 	set_exception_intercept(svm, DB_VECTOR);
1267 	/*
1268 	 * Guest access to VMware backdoor ports could legitimately
1269 	 * trigger #GP because of TSS I/O permission bitmap.
1270 	 * We intercept those #GP and allow access to them anyway
1271 	 * as VMware does.
1272 	 */
1273 	if (enable_vmware_backdoor)
1274 		set_exception_intercept(svm, GP_VECTOR);
1275 
1276 	svm_set_intercept(svm, INTERCEPT_INTR);
1277 	svm_set_intercept(svm, INTERCEPT_NMI);
1278 
1279 	if (intercept_smi)
1280 		svm_set_intercept(svm, INTERCEPT_SMI);
1281 
1282 	svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1283 	svm_set_intercept(svm, INTERCEPT_RDPMC);
1284 	svm_set_intercept(svm, INTERCEPT_CPUID);
1285 	svm_set_intercept(svm, INTERCEPT_INVD);
1286 	svm_set_intercept(svm, INTERCEPT_INVLPG);
1287 	svm_set_intercept(svm, INTERCEPT_INVLPGA);
1288 	svm_set_intercept(svm, INTERCEPT_IOIO_PROT);
1289 	svm_set_intercept(svm, INTERCEPT_MSR_PROT);
1290 	svm_set_intercept(svm, INTERCEPT_TASK_SWITCH);
1291 	svm_set_intercept(svm, INTERCEPT_SHUTDOWN);
1292 	svm_set_intercept(svm, INTERCEPT_VMRUN);
1293 	svm_set_intercept(svm, INTERCEPT_VMMCALL);
1294 	svm_set_intercept(svm, INTERCEPT_VMLOAD);
1295 	svm_set_intercept(svm, INTERCEPT_VMSAVE);
1296 	svm_set_intercept(svm, INTERCEPT_STGI);
1297 	svm_set_intercept(svm, INTERCEPT_CLGI);
1298 	svm_set_intercept(svm, INTERCEPT_SKINIT);
1299 	svm_set_intercept(svm, INTERCEPT_WBINVD);
1300 	svm_set_intercept(svm, INTERCEPT_XSETBV);
1301 	svm_set_intercept(svm, INTERCEPT_RDPRU);
1302 	svm_set_intercept(svm, INTERCEPT_RSM);
1303 
1304 	if (!kvm_mwait_in_guest(vcpu->kvm)) {
1305 		svm_set_intercept(svm, INTERCEPT_MONITOR);
1306 		svm_set_intercept(svm, INTERCEPT_MWAIT);
1307 	}
1308 
1309 	if (!kvm_hlt_in_guest(vcpu->kvm))
1310 		svm_set_intercept(svm, INTERCEPT_HLT);
1311 
1312 	control->iopm_base_pa = __sme_set(iopm_base);
1313 	control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1314 	control->int_ctl = V_INTR_MASKING_MASK;
1315 
1316 	init_seg(&save->es);
1317 	init_seg(&save->ss);
1318 	init_seg(&save->ds);
1319 	init_seg(&save->fs);
1320 	init_seg(&save->gs);
1321 
1322 	save->cs.selector = 0xf000;
1323 	save->cs.base = 0xffff0000;
1324 	/* Executable/Readable Code Segment */
1325 	save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1326 		SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1327 	save->cs.limit = 0xffff;
1328 
1329 	save->gdtr.base = 0;
1330 	save->gdtr.limit = 0xffff;
1331 	save->idtr.base = 0;
1332 	save->idtr.limit = 0xffff;
1333 
1334 	init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1335 	init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1336 
1337 	if (npt_enabled) {
1338 		/* Setup VMCB for Nested Paging */
1339 		control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1340 		svm_clr_intercept(svm, INTERCEPT_INVLPG);
1341 		clr_exception_intercept(svm, PF_VECTOR);
1342 		svm_clr_intercept(svm, INTERCEPT_CR3_READ);
1343 		svm_clr_intercept(svm, INTERCEPT_CR3_WRITE);
1344 		save->g_pat = vcpu->arch.pat;
1345 		save->cr3 = 0;
1346 	}
1347 	svm->current_vmcb->asid_generation = 0;
1348 	svm->asid = 0;
1349 
1350 	svm->nested.vmcb12_gpa = INVALID_GPA;
1351 	svm->nested.last_vmcb12_gpa = INVALID_GPA;
1352 
1353 	if (!kvm_pause_in_guest(vcpu->kvm)) {
1354 		control->pause_filter_count = pause_filter_count;
1355 		if (pause_filter_thresh)
1356 			control->pause_filter_thresh = pause_filter_thresh;
1357 		svm_set_intercept(svm, INTERCEPT_PAUSE);
1358 	} else {
1359 		svm_clr_intercept(svm, INTERCEPT_PAUSE);
1360 	}
1361 
1362 	svm_recalc_instruction_intercepts(vcpu, svm);
1363 
1364 	/*
1365 	 * If the host supports V_SPEC_CTRL then disable the interception
1366 	 * of MSR_IA32_SPEC_CTRL.
1367 	 */
1368 	if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
1369 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
1370 
1371 	if (kvm_vcpu_apicv_active(vcpu))
1372 		avic_init_vmcb(svm, vmcb);
1373 
1374 	if (vnmi)
1375 		svm->vmcb->control.int_ctl |= V_NMI_ENABLE_MASK;
1376 
1377 	if (vgif) {
1378 		svm_clr_intercept(svm, INTERCEPT_STGI);
1379 		svm_clr_intercept(svm, INTERCEPT_CLGI);
1380 		svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1381 	}
1382 
1383 	if (sev_guest(vcpu->kvm))
1384 		sev_init_vmcb(svm);
1385 
1386 	svm_hv_init_vmcb(vmcb);
1387 	init_vmcb_after_set_cpuid(vcpu);
1388 
1389 	vmcb_mark_all_dirty(vmcb);
1390 
1391 	enable_gif(svm);
1392 }
1393 
__svm_vcpu_reset(struct kvm_vcpu * vcpu)1394 static void __svm_vcpu_reset(struct kvm_vcpu *vcpu)
1395 {
1396 	struct vcpu_svm *svm = to_svm(vcpu);
1397 
1398 	svm_vcpu_init_msrpm(vcpu, svm->msrpm);
1399 
1400 	svm_init_osvw(vcpu);
1401 	vcpu->arch.microcode_version = 0x01000065;
1402 	svm->tsc_ratio_msr = kvm_caps.default_tsc_scaling_ratio;
1403 
1404 	svm->nmi_masked = false;
1405 	svm->awaiting_iret_completion = false;
1406 
1407 	if (sev_es_guest(vcpu->kvm))
1408 		sev_es_vcpu_reset(svm);
1409 }
1410 
svm_vcpu_reset(struct kvm_vcpu * vcpu,bool init_event)1411 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1412 {
1413 	struct vcpu_svm *svm = to_svm(vcpu);
1414 
1415 	svm->spec_ctrl = 0;
1416 	svm->virt_spec_ctrl = 0;
1417 
1418 	init_vmcb(vcpu);
1419 
1420 	if (!init_event)
1421 		__svm_vcpu_reset(vcpu);
1422 }
1423 
svm_switch_vmcb(struct vcpu_svm * svm,struct kvm_vmcb_info * target_vmcb)1424 void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb)
1425 {
1426 	svm->current_vmcb = target_vmcb;
1427 	svm->vmcb = target_vmcb->ptr;
1428 }
1429 
svm_vcpu_create(struct kvm_vcpu * vcpu)1430 static int svm_vcpu_create(struct kvm_vcpu *vcpu)
1431 {
1432 	struct vcpu_svm *svm;
1433 	struct page *vmcb01_page;
1434 	struct page *vmsa_page = NULL;
1435 	int err;
1436 
1437 	BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0);
1438 	svm = to_svm(vcpu);
1439 
1440 	err = -ENOMEM;
1441 	vmcb01_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
1442 	if (!vmcb01_page)
1443 		goto out;
1444 
1445 	if (sev_es_guest(vcpu->kvm)) {
1446 		/*
1447 		 * SEV-ES guests require a separate VMSA page used to contain
1448 		 * the encrypted register state of the guest.
1449 		 */
1450 		vmsa_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
1451 		if (!vmsa_page)
1452 			goto error_free_vmcb_page;
1453 
1454 		/*
1455 		 * SEV-ES guests maintain an encrypted version of their FPU
1456 		 * state which is restored and saved on VMRUN and VMEXIT.
1457 		 * Mark vcpu->arch.guest_fpu->fpstate as scratch so it won't
1458 		 * do xsave/xrstor on it.
1459 		 */
1460 		fpstate_set_confidential(&vcpu->arch.guest_fpu);
1461 	}
1462 
1463 	err = avic_init_vcpu(svm);
1464 	if (err)
1465 		goto error_free_vmsa_page;
1466 
1467 	svm->msrpm = svm_vcpu_alloc_msrpm();
1468 	if (!svm->msrpm) {
1469 		err = -ENOMEM;
1470 		goto error_free_vmsa_page;
1471 	}
1472 
1473 	svm->x2avic_msrs_intercepted = true;
1474 
1475 	svm->vmcb01.ptr = page_address(vmcb01_page);
1476 	svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT);
1477 	svm_switch_vmcb(svm, &svm->vmcb01);
1478 
1479 	if (vmsa_page)
1480 		svm->sev_es.vmsa = page_address(vmsa_page);
1481 
1482 	svm->guest_state_loaded = false;
1483 
1484 	return 0;
1485 
1486 error_free_vmsa_page:
1487 	if (vmsa_page)
1488 		__free_page(vmsa_page);
1489 error_free_vmcb_page:
1490 	__free_page(vmcb01_page);
1491 out:
1492 	return err;
1493 }
1494 
svm_clear_current_vmcb(struct vmcb * vmcb)1495 static void svm_clear_current_vmcb(struct vmcb *vmcb)
1496 {
1497 	int i;
1498 
1499 	for_each_online_cpu(i)
1500 		cmpxchg(per_cpu_ptr(&svm_data.current_vmcb, i), vmcb, NULL);
1501 }
1502 
svm_vcpu_free(struct kvm_vcpu * vcpu)1503 static void svm_vcpu_free(struct kvm_vcpu *vcpu)
1504 {
1505 	struct vcpu_svm *svm = to_svm(vcpu);
1506 
1507 	/*
1508 	 * The vmcb page can be recycled, causing a false negative in
1509 	 * svm_vcpu_load(). So, ensure that no logical CPU has this
1510 	 * vmcb page recorded as its current vmcb.
1511 	 */
1512 	svm_clear_current_vmcb(svm->vmcb);
1513 
1514 	svm_leave_nested(vcpu);
1515 	svm_free_nested(svm);
1516 
1517 	sev_free_vcpu(vcpu);
1518 
1519 	__free_page(pfn_to_page(__sme_clr(svm->vmcb01.pa) >> PAGE_SHIFT));
1520 	__free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE));
1521 }
1522 
svm_prepare_switch_to_guest(struct kvm_vcpu * vcpu)1523 static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
1524 {
1525 	struct vcpu_svm *svm = to_svm(vcpu);
1526 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
1527 
1528 	if (sev_es_guest(vcpu->kvm))
1529 		sev_es_unmap_ghcb(svm);
1530 
1531 	if (svm->guest_state_loaded)
1532 		return;
1533 
1534 	/*
1535 	 * Save additional host state that will be restored on VMEXIT (sev-es)
1536 	 * or subsequent vmload of host save area.
1537 	 */
1538 	vmsave(sd->save_area_pa);
1539 	if (sev_es_guest(vcpu->kvm)) {
1540 		struct sev_es_save_area *hostsa;
1541 		hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400);
1542 
1543 		sev_es_prepare_switch_to_guest(hostsa);
1544 	}
1545 
1546 	if (tsc_scaling)
1547 		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1548 
1549 	/*
1550 	 * TSC_AUX is always virtualized for SEV-ES guests when the feature is
1551 	 * available. The user return MSR support is not required in this case
1552 	 * because TSC_AUX is restored on #VMEXIT from the host save area
1553 	 * (which has been initialized in svm_hardware_enable()).
1554 	 */
1555 	if (likely(tsc_aux_uret_slot >= 0) &&
1556 	    (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) || !sev_es_guest(vcpu->kvm)))
1557 		kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);
1558 
1559 	svm->guest_state_loaded = true;
1560 }
1561 
svm_prepare_host_switch(struct kvm_vcpu * vcpu)1562 static void svm_prepare_host_switch(struct kvm_vcpu *vcpu)
1563 {
1564 	to_svm(vcpu)->guest_state_loaded = false;
1565 }
1566 
svm_vcpu_load(struct kvm_vcpu * vcpu,int cpu)1567 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1568 {
1569 	struct vcpu_svm *svm = to_svm(vcpu);
1570 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
1571 
1572 	if (sd->current_vmcb != svm->vmcb) {
1573 		sd->current_vmcb = svm->vmcb;
1574 
1575 		if (!cpu_feature_enabled(X86_FEATURE_IBPB_ON_VMEXIT))
1576 			indirect_branch_prediction_barrier();
1577 	}
1578 	if (kvm_vcpu_apicv_active(vcpu))
1579 		avic_vcpu_load(vcpu, cpu);
1580 }
1581 
svm_vcpu_put(struct kvm_vcpu * vcpu)1582 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1583 {
1584 	if (kvm_vcpu_apicv_active(vcpu))
1585 		avic_vcpu_put(vcpu);
1586 
1587 	svm_prepare_host_switch(vcpu);
1588 
1589 	++vcpu->stat.host_state_reload;
1590 }
1591 
svm_get_rflags(struct kvm_vcpu * vcpu)1592 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1593 {
1594 	struct vcpu_svm *svm = to_svm(vcpu);
1595 	unsigned long rflags = svm->vmcb->save.rflags;
1596 
1597 	if (svm->nmi_singlestep) {
1598 		/* Hide our flags if they were not set by the guest */
1599 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1600 			rflags &= ~X86_EFLAGS_TF;
1601 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1602 			rflags &= ~X86_EFLAGS_RF;
1603 	}
1604 	return rflags;
1605 }
1606 
svm_set_rflags(struct kvm_vcpu * vcpu,unsigned long rflags)1607 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1608 {
1609 	if (to_svm(vcpu)->nmi_singlestep)
1610 		rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
1611 
1612        /*
1613         * Any change of EFLAGS.VM is accompanied by a reload of SS
1614         * (caused by either a task switch or an inter-privilege IRET),
1615         * so we do not need to update the CPL here.
1616         */
1617 	to_svm(vcpu)->vmcb->save.rflags = rflags;
1618 }
1619 
svm_get_if_flag(struct kvm_vcpu * vcpu)1620 static bool svm_get_if_flag(struct kvm_vcpu *vcpu)
1621 {
1622 	struct vmcb *vmcb = to_svm(vcpu)->vmcb;
1623 
1624 	return sev_es_guest(vcpu->kvm)
1625 		? vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK
1626 		: kvm_get_rflags(vcpu) & X86_EFLAGS_IF;
1627 }
1628 
svm_cache_reg(struct kvm_vcpu * vcpu,enum kvm_reg reg)1629 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1630 {
1631 	kvm_register_mark_available(vcpu, reg);
1632 
1633 	switch (reg) {
1634 	case VCPU_EXREG_PDPTR:
1635 		/*
1636 		 * When !npt_enabled, mmu->pdptrs[] is already available since
1637 		 * it is always updated per SDM when moving to CRs.
1638 		 */
1639 		if (npt_enabled)
1640 			load_pdptrs(vcpu, kvm_read_cr3(vcpu));
1641 		break;
1642 	default:
1643 		KVM_BUG_ON(1, vcpu->kvm);
1644 	}
1645 }
1646 
svm_set_vintr(struct vcpu_svm * svm)1647 static void svm_set_vintr(struct vcpu_svm *svm)
1648 {
1649 	struct vmcb_control_area *control;
1650 
1651 	/*
1652 	 * The following fields are ignored when AVIC is enabled
1653 	 */
1654 	WARN_ON(kvm_vcpu_apicv_activated(&svm->vcpu));
1655 
1656 	svm_set_intercept(svm, INTERCEPT_VINTR);
1657 
1658 	/*
1659 	 * Recalculating intercepts may have cleared the VINTR intercept.  If
1660 	 * V_INTR_MASKING is enabled in vmcb12, then the effective RFLAGS.IF
1661 	 * for L1 physical interrupts is L1's RFLAGS.IF at the time of VMRUN.
1662 	 * Requesting an interrupt window if save.RFLAGS.IF=0 is pointless as
1663 	 * interrupts will never be unblocked while L2 is running.
1664 	 */
1665 	if (!svm_is_intercept(svm, INTERCEPT_VINTR))
1666 		return;
1667 
1668 	/*
1669 	 * This is just a dummy VINTR to actually cause a vmexit to happen.
1670 	 * Actual injection of virtual interrupts happens through EVENTINJ.
1671 	 */
1672 	control = &svm->vmcb->control;
1673 	control->int_vector = 0x0;
1674 	control->int_ctl &= ~V_INTR_PRIO_MASK;
1675 	control->int_ctl |= V_IRQ_MASK |
1676 		((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
1677 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1678 }
1679 
svm_clear_vintr(struct vcpu_svm * svm)1680 static void svm_clear_vintr(struct vcpu_svm *svm)
1681 {
1682 	svm_clr_intercept(svm, INTERCEPT_VINTR);
1683 
1684 	/* Drop int_ctl fields related to VINTR injection.  */
1685 	svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1686 	if (is_guest_mode(&svm->vcpu)) {
1687 		svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1688 
1689 		WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) !=
1690 			(svm->nested.ctl.int_ctl & V_TPR_MASK));
1691 
1692 		svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl &
1693 			V_IRQ_INJECTION_BITS_MASK;
1694 
1695 		svm->vmcb->control.int_vector = svm->nested.ctl.int_vector;
1696 	}
1697 
1698 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1699 }
1700 
svm_seg(struct kvm_vcpu * vcpu,int seg)1701 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1702 {
1703 	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1704 	struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save;
1705 
1706 	switch (seg) {
1707 	case VCPU_SREG_CS: return &save->cs;
1708 	case VCPU_SREG_DS: return &save->ds;
1709 	case VCPU_SREG_ES: return &save->es;
1710 	case VCPU_SREG_FS: return &save01->fs;
1711 	case VCPU_SREG_GS: return &save01->gs;
1712 	case VCPU_SREG_SS: return &save->ss;
1713 	case VCPU_SREG_TR: return &save01->tr;
1714 	case VCPU_SREG_LDTR: return &save01->ldtr;
1715 	}
1716 	BUG();
1717 	return NULL;
1718 }
1719 
svm_get_segment_base(struct kvm_vcpu * vcpu,int seg)1720 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1721 {
1722 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1723 
1724 	return s->base;
1725 }
1726 
svm_get_segment(struct kvm_vcpu * vcpu,struct kvm_segment * var,int seg)1727 static void svm_get_segment(struct kvm_vcpu *vcpu,
1728 			    struct kvm_segment *var, int seg)
1729 {
1730 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1731 
1732 	var->base = s->base;
1733 	var->limit = s->limit;
1734 	var->selector = s->selector;
1735 	var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1736 	var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1737 	var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1738 	var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1739 	var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1740 	var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1741 	var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1742 
1743 	/*
1744 	 * AMD CPUs circa 2014 track the G bit for all segments except CS.
1745 	 * However, the SVM spec states that the G bit is not observed by the
1746 	 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1747 	 * So let's synthesize a legal G bit for all segments, this helps
1748 	 * running KVM nested. It also helps cross-vendor migration, because
1749 	 * Intel's vmentry has a check on the 'G' bit.
1750 	 */
1751 	var->g = s->limit > 0xfffff;
1752 
1753 	/*
1754 	 * AMD's VMCB does not have an explicit unusable field, so emulate it
1755 	 * for cross vendor migration purposes by "not present"
1756 	 */
1757 	var->unusable = !var->present;
1758 
1759 	switch (seg) {
1760 	case VCPU_SREG_TR:
1761 		/*
1762 		 * Work around a bug where the busy flag in the tr selector
1763 		 * isn't exposed
1764 		 */
1765 		var->type |= 0x2;
1766 		break;
1767 	case VCPU_SREG_DS:
1768 	case VCPU_SREG_ES:
1769 	case VCPU_SREG_FS:
1770 	case VCPU_SREG_GS:
1771 		/*
1772 		 * The accessed bit must always be set in the segment
1773 		 * descriptor cache, although it can be cleared in the
1774 		 * descriptor, the cached bit always remains at 1. Since
1775 		 * Intel has a check on this, set it here to support
1776 		 * cross-vendor migration.
1777 		 */
1778 		if (!var->unusable)
1779 			var->type |= 0x1;
1780 		break;
1781 	case VCPU_SREG_SS:
1782 		/*
1783 		 * On AMD CPUs sometimes the DB bit in the segment
1784 		 * descriptor is left as 1, although the whole segment has
1785 		 * been made unusable. Clear it here to pass an Intel VMX
1786 		 * entry check when cross vendor migrating.
1787 		 */
1788 		if (var->unusable)
1789 			var->db = 0;
1790 		/* This is symmetric with svm_set_segment() */
1791 		var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1792 		break;
1793 	}
1794 }
1795 
svm_get_cpl(struct kvm_vcpu * vcpu)1796 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1797 {
1798 	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1799 
1800 	return save->cpl;
1801 }
1802 
svm_get_cs_db_l_bits(struct kvm_vcpu * vcpu,int * db,int * l)1803 static void svm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
1804 {
1805 	struct kvm_segment cs;
1806 
1807 	svm_get_segment(vcpu, &cs, VCPU_SREG_CS);
1808 	*db = cs.db;
1809 	*l = cs.l;
1810 }
1811 
svm_get_idt(struct kvm_vcpu * vcpu,struct desc_ptr * dt)1812 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1813 {
1814 	struct vcpu_svm *svm = to_svm(vcpu);
1815 
1816 	dt->size = svm->vmcb->save.idtr.limit;
1817 	dt->address = svm->vmcb->save.idtr.base;
1818 }
1819 
svm_set_idt(struct kvm_vcpu * vcpu,struct desc_ptr * dt)1820 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1821 {
1822 	struct vcpu_svm *svm = to_svm(vcpu);
1823 
1824 	svm->vmcb->save.idtr.limit = dt->size;
1825 	svm->vmcb->save.idtr.base = dt->address ;
1826 	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1827 }
1828 
svm_get_gdt(struct kvm_vcpu * vcpu,struct desc_ptr * dt)1829 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1830 {
1831 	struct vcpu_svm *svm = to_svm(vcpu);
1832 
1833 	dt->size = svm->vmcb->save.gdtr.limit;
1834 	dt->address = svm->vmcb->save.gdtr.base;
1835 }
1836 
svm_set_gdt(struct kvm_vcpu * vcpu,struct desc_ptr * dt)1837 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1838 {
1839 	struct vcpu_svm *svm = to_svm(vcpu);
1840 
1841 	svm->vmcb->save.gdtr.limit = dt->size;
1842 	svm->vmcb->save.gdtr.base = dt->address ;
1843 	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1844 }
1845 
sev_post_set_cr3(struct kvm_vcpu * vcpu,unsigned long cr3)1846 static void sev_post_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1847 {
1848 	struct vcpu_svm *svm = to_svm(vcpu);
1849 
1850 	/*
1851 	 * For guests that don't set guest_state_protected, the cr3 update is
1852 	 * handled via kvm_mmu_load() while entering the guest. For guests
1853 	 * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to
1854 	 * VMCB save area now, since the save area will become the initial
1855 	 * contents of the VMSA, and future VMCB save area updates won't be
1856 	 * seen.
1857 	 */
1858 	if (sev_es_guest(vcpu->kvm)) {
1859 		svm->vmcb->save.cr3 = cr3;
1860 		vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1861 	}
1862 }
1863 
svm_is_valid_cr0(struct kvm_vcpu * vcpu,unsigned long cr0)1864 static bool svm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1865 {
1866 	return true;
1867 }
1868 
svm_set_cr0(struct kvm_vcpu * vcpu,unsigned long cr0)1869 void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1870 {
1871 	struct vcpu_svm *svm = to_svm(vcpu);
1872 	u64 hcr0 = cr0;
1873 	bool old_paging = is_paging(vcpu);
1874 
1875 #ifdef CONFIG_X86_64
1876 	if (vcpu->arch.efer & EFER_LME) {
1877 		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1878 			vcpu->arch.efer |= EFER_LMA;
1879 			if (!vcpu->arch.guest_state_protected)
1880 				svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1881 		}
1882 
1883 		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1884 			vcpu->arch.efer &= ~EFER_LMA;
1885 			if (!vcpu->arch.guest_state_protected)
1886 				svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1887 		}
1888 	}
1889 #endif
1890 	vcpu->arch.cr0 = cr0;
1891 
1892 	if (!npt_enabled) {
1893 		hcr0 |= X86_CR0_PG | X86_CR0_WP;
1894 		if (old_paging != is_paging(vcpu))
1895 			svm_set_cr4(vcpu, kvm_read_cr4(vcpu));
1896 	}
1897 
1898 	/*
1899 	 * re-enable caching here because the QEMU bios
1900 	 * does not do it - this results in some delay at
1901 	 * reboot
1902 	 */
1903 	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1904 		hcr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1905 
1906 	svm->vmcb->save.cr0 = hcr0;
1907 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1908 
1909 	/*
1910 	 * SEV-ES guests must always keep the CR intercepts cleared. CR
1911 	 * tracking is done using the CR write traps.
1912 	 */
1913 	if (sev_es_guest(vcpu->kvm))
1914 		return;
1915 
1916 	if (hcr0 == cr0) {
1917 		/* Selective CR0 write remains on.  */
1918 		svm_clr_intercept(svm, INTERCEPT_CR0_READ);
1919 		svm_clr_intercept(svm, INTERCEPT_CR0_WRITE);
1920 	} else {
1921 		svm_set_intercept(svm, INTERCEPT_CR0_READ);
1922 		svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1923 	}
1924 }
1925 
svm_is_valid_cr4(struct kvm_vcpu * vcpu,unsigned long cr4)1926 static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1927 {
1928 	return true;
1929 }
1930 
svm_set_cr4(struct kvm_vcpu * vcpu,unsigned long cr4)1931 void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1932 {
1933 	unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1934 	unsigned long old_cr4 = vcpu->arch.cr4;
1935 
1936 	if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1937 		svm_flush_tlb_current(vcpu);
1938 
1939 	vcpu->arch.cr4 = cr4;
1940 	if (!npt_enabled) {
1941 		cr4 |= X86_CR4_PAE;
1942 
1943 		if (!is_paging(vcpu))
1944 			cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
1945 	}
1946 	cr4 |= host_cr4_mce;
1947 	to_svm(vcpu)->vmcb->save.cr4 = cr4;
1948 	vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1949 
1950 	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
1951 		kvm_update_cpuid_runtime(vcpu);
1952 }
1953 
svm_set_segment(struct kvm_vcpu * vcpu,struct kvm_segment * var,int seg)1954 static void svm_set_segment(struct kvm_vcpu *vcpu,
1955 			    struct kvm_segment *var, int seg)
1956 {
1957 	struct vcpu_svm *svm = to_svm(vcpu);
1958 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1959 
1960 	s->base = var->base;
1961 	s->limit = var->limit;
1962 	s->selector = var->selector;
1963 	s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1964 	s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1965 	s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1966 	s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
1967 	s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1968 	s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1969 	s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1970 	s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1971 
1972 	/*
1973 	 * This is always accurate, except if SYSRET returned to a segment
1974 	 * with SS.DPL != 3.  Intel does not have this quirk, and always
1975 	 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1976 	 * would entail passing the CPL to userspace and back.
1977 	 */
1978 	if (seg == VCPU_SREG_SS)
1979 		/* This is symmetric with svm_get_segment() */
1980 		svm->vmcb->save.cpl = (var->dpl & 3);
1981 
1982 	vmcb_mark_dirty(svm->vmcb, VMCB_SEG);
1983 }
1984 
svm_update_exception_bitmap(struct kvm_vcpu * vcpu)1985 static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)
1986 {
1987 	struct vcpu_svm *svm = to_svm(vcpu);
1988 
1989 	clr_exception_intercept(svm, BP_VECTOR);
1990 
1991 	if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1992 		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1993 			set_exception_intercept(svm, BP_VECTOR);
1994 	}
1995 }
1996 
new_asid(struct vcpu_svm * svm,struct svm_cpu_data * sd)1997 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1998 {
1999 	if (sd->next_asid > sd->max_asid) {
2000 		++sd->asid_generation;
2001 		sd->next_asid = sd->min_asid;
2002 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
2003 		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
2004 	}
2005 
2006 	svm->current_vmcb->asid_generation = sd->asid_generation;
2007 	svm->asid = sd->next_asid++;
2008 }
2009 
svm_set_dr6(struct vcpu_svm * svm,unsigned long value)2010 static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value)
2011 {
2012 	struct vmcb *vmcb = svm->vmcb;
2013 
2014 	if (svm->vcpu.arch.guest_state_protected)
2015 		return;
2016 
2017 	if (unlikely(value != vmcb->save.dr6)) {
2018 		vmcb->save.dr6 = value;
2019 		vmcb_mark_dirty(vmcb, VMCB_DR);
2020 	}
2021 }
2022 
svm_sync_dirty_debug_regs(struct kvm_vcpu * vcpu)2023 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
2024 {
2025 	struct vcpu_svm *svm = to_svm(vcpu);
2026 
2027 	if (WARN_ON_ONCE(sev_es_guest(vcpu->kvm)))
2028 		return;
2029 
2030 	get_debugreg(vcpu->arch.db[0], 0);
2031 	get_debugreg(vcpu->arch.db[1], 1);
2032 	get_debugreg(vcpu->arch.db[2], 2);
2033 	get_debugreg(vcpu->arch.db[3], 3);
2034 	/*
2035 	 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here,
2036 	 * because db_interception might need it.  We can do it before vmentry.
2037 	 */
2038 	vcpu->arch.dr6 = svm->vmcb->save.dr6;
2039 	vcpu->arch.dr7 = svm->vmcb->save.dr7;
2040 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2041 	set_dr_intercepts(svm);
2042 }
2043 
svm_set_dr7(struct kvm_vcpu * vcpu,unsigned long value)2044 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
2045 {
2046 	struct vcpu_svm *svm = to_svm(vcpu);
2047 
2048 	if (vcpu->arch.guest_state_protected)
2049 		return;
2050 
2051 	svm->vmcb->save.dr7 = value;
2052 	vmcb_mark_dirty(svm->vmcb, VMCB_DR);
2053 }
2054 
pf_interception(struct kvm_vcpu * vcpu)2055 static int pf_interception(struct kvm_vcpu *vcpu)
2056 {
2057 	struct vcpu_svm *svm = to_svm(vcpu);
2058 
2059 	u64 fault_address = svm->vmcb->control.exit_info_2;
2060 	u64 error_code = svm->vmcb->control.exit_info_1;
2061 
2062 	return kvm_handle_page_fault(vcpu, error_code, fault_address,
2063 			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2064 			svm->vmcb->control.insn_bytes : NULL,
2065 			svm->vmcb->control.insn_len);
2066 }
2067 
npf_interception(struct kvm_vcpu * vcpu)2068 static int npf_interception(struct kvm_vcpu *vcpu)
2069 {
2070 	struct vcpu_svm *svm = to_svm(vcpu);
2071 
2072 	u64 fault_address = svm->vmcb->control.exit_info_2;
2073 	u64 error_code = svm->vmcb->control.exit_info_1;
2074 
2075 	trace_kvm_page_fault(vcpu, fault_address, error_code);
2076 	return kvm_mmu_page_fault(vcpu, fault_address, error_code,
2077 			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2078 			svm->vmcb->control.insn_bytes : NULL,
2079 			svm->vmcb->control.insn_len);
2080 }
2081 
db_interception(struct kvm_vcpu * vcpu)2082 static int db_interception(struct kvm_vcpu *vcpu)
2083 {
2084 	struct kvm_run *kvm_run = vcpu->run;
2085 	struct vcpu_svm *svm = to_svm(vcpu);
2086 
2087 	if (!(vcpu->guest_debug &
2088 	      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
2089 		!svm->nmi_singlestep) {
2090 		u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW;
2091 		kvm_queue_exception_p(vcpu, DB_VECTOR, payload);
2092 		return 1;
2093 	}
2094 
2095 	if (svm->nmi_singlestep) {
2096 		disable_nmi_singlestep(svm);
2097 		/* Make sure we check for pending NMIs upon entry */
2098 		kvm_make_request(KVM_REQ_EVENT, vcpu);
2099 	}
2100 
2101 	if (vcpu->guest_debug &
2102 	    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2103 		kvm_run->exit_reason = KVM_EXIT_DEBUG;
2104 		kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6;
2105 		kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7;
2106 		kvm_run->debug.arch.pc =
2107 			svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2108 		kvm_run->debug.arch.exception = DB_VECTOR;
2109 		return 0;
2110 	}
2111 
2112 	return 1;
2113 }
2114 
bp_interception(struct kvm_vcpu * vcpu)2115 static int bp_interception(struct kvm_vcpu *vcpu)
2116 {
2117 	struct vcpu_svm *svm = to_svm(vcpu);
2118 	struct kvm_run *kvm_run = vcpu->run;
2119 
2120 	kvm_run->exit_reason = KVM_EXIT_DEBUG;
2121 	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2122 	kvm_run->debug.arch.exception = BP_VECTOR;
2123 	return 0;
2124 }
2125 
ud_interception(struct kvm_vcpu * vcpu)2126 static int ud_interception(struct kvm_vcpu *vcpu)
2127 {
2128 	return handle_ud(vcpu);
2129 }
2130 
ac_interception(struct kvm_vcpu * vcpu)2131 static int ac_interception(struct kvm_vcpu *vcpu)
2132 {
2133 	kvm_queue_exception_e(vcpu, AC_VECTOR, 0);
2134 	return 1;
2135 }
2136 
is_erratum_383(void)2137 static bool is_erratum_383(void)
2138 {
2139 	int err, i;
2140 	u64 value;
2141 
2142 	if (!erratum_383_found)
2143 		return false;
2144 
2145 	value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2146 	if (err)
2147 		return false;
2148 
2149 	/* Bit 62 may or may not be set for this mce */
2150 	value &= ~(1ULL << 62);
2151 
2152 	if (value != 0xb600000000010015ULL)
2153 		return false;
2154 
2155 	/* Clear MCi_STATUS registers */
2156 	for (i = 0; i < 6; ++i)
2157 		native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2158 
2159 	value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2160 	if (!err) {
2161 		u32 low, high;
2162 
2163 		value &= ~(1ULL << 2);
2164 		low    = lower_32_bits(value);
2165 		high   = upper_32_bits(value);
2166 
2167 		native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2168 	}
2169 
2170 	/* Flush tlb to evict multi-match entries */
2171 	__flush_tlb_all();
2172 
2173 	return true;
2174 }
2175 
svm_handle_mce(struct kvm_vcpu * vcpu)2176 static void svm_handle_mce(struct kvm_vcpu *vcpu)
2177 {
2178 	if (is_erratum_383()) {
2179 		/*
2180 		 * Erratum 383 triggered. Guest state is corrupt so kill the
2181 		 * guest.
2182 		 */
2183 		pr_err("Guest triggered AMD Erratum 383\n");
2184 
2185 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2186 
2187 		return;
2188 	}
2189 
2190 	/*
2191 	 * On an #MC intercept the MCE handler is not called automatically in
2192 	 * the host. So do it by hand here.
2193 	 */
2194 	kvm_machine_check();
2195 }
2196 
mc_interception(struct kvm_vcpu * vcpu)2197 static int mc_interception(struct kvm_vcpu *vcpu)
2198 {
2199 	return 1;
2200 }
2201 
shutdown_interception(struct kvm_vcpu * vcpu)2202 static int shutdown_interception(struct kvm_vcpu *vcpu)
2203 {
2204 	struct kvm_run *kvm_run = vcpu->run;
2205 	struct vcpu_svm *svm = to_svm(vcpu);
2206 
2207 	/*
2208 	 * The VM save area has already been encrypted so it
2209 	 * cannot be reinitialized - just terminate.
2210 	 */
2211 	if (sev_es_guest(vcpu->kvm))
2212 		return -EINVAL;
2213 
2214 	/*
2215 	 * VMCB is undefined after a SHUTDOWN intercept.  INIT the vCPU to put
2216 	 * the VMCB in a known good state.  Unfortuately, KVM doesn't have
2217 	 * KVM_MP_STATE_SHUTDOWN and can't add it without potentially breaking
2218 	 * userspace.  At a platform view, INIT is acceptable behavior as
2219 	 * there exist bare metal platforms that automatically INIT the CPU
2220 	 * in response to shutdown.
2221 	 */
2222 	clear_page(svm->vmcb);
2223 	kvm_vcpu_reset(vcpu, true);
2224 
2225 	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2226 	return 0;
2227 }
2228 
io_interception(struct kvm_vcpu * vcpu)2229 static int io_interception(struct kvm_vcpu *vcpu)
2230 {
2231 	struct vcpu_svm *svm = to_svm(vcpu);
2232 	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2233 	int size, in, string;
2234 	unsigned port;
2235 
2236 	++vcpu->stat.io_exits;
2237 	string = (io_info & SVM_IOIO_STR_MASK) != 0;
2238 	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2239 	port = io_info >> 16;
2240 	size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2241 
2242 	if (string) {
2243 		if (sev_es_guest(vcpu->kvm))
2244 			return sev_es_string_io(svm, size, port, in);
2245 		else
2246 			return kvm_emulate_instruction(vcpu, 0);
2247 	}
2248 
2249 	svm->next_rip = svm->vmcb->control.exit_info_2;
2250 
2251 	return kvm_fast_pio(vcpu, size, port, in);
2252 }
2253 
nmi_interception(struct kvm_vcpu * vcpu)2254 static int nmi_interception(struct kvm_vcpu *vcpu)
2255 {
2256 	return 1;
2257 }
2258 
smi_interception(struct kvm_vcpu * vcpu)2259 static int smi_interception(struct kvm_vcpu *vcpu)
2260 {
2261 	return 1;
2262 }
2263 
intr_interception(struct kvm_vcpu * vcpu)2264 static int intr_interception(struct kvm_vcpu *vcpu)
2265 {
2266 	++vcpu->stat.irq_exits;
2267 	return 1;
2268 }
2269 
vmload_vmsave_interception(struct kvm_vcpu * vcpu,bool vmload)2270 static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload)
2271 {
2272 	struct vcpu_svm *svm = to_svm(vcpu);
2273 	struct vmcb *vmcb12;
2274 	struct kvm_host_map map;
2275 	int ret;
2276 
2277 	if (nested_svm_check_permissions(vcpu))
2278 		return 1;
2279 
2280 	ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
2281 	if (ret) {
2282 		if (ret == -EINVAL)
2283 			kvm_inject_gp(vcpu, 0);
2284 		return 1;
2285 	}
2286 
2287 	vmcb12 = map.hva;
2288 
2289 	ret = kvm_skip_emulated_instruction(vcpu);
2290 
2291 	if (vmload) {
2292 		svm_copy_vmloadsave_state(svm->vmcb, vmcb12);
2293 		svm->sysenter_eip_hi = 0;
2294 		svm->sysenter_esp_hi = 0;
2295 	} else {
2296 		svm_copy_vmloadsave_state(vmcb12, svm->vmcb);
2297 	}
2298 
2299 	kvm_vcpu_unmap(vcpu, &map, true);
2300 
2301 	return ret;
2302 }
2303 
vmload_interception(struct kvm_vcpu * vcpu)2304 static int vmload_interception(struct kvm_vcpu *vcpu)
2305 {
2306 	return vmload_vmsave_interception(vcpu, true);
2307 }
2308 
vmsave_interception(struct kvm_vcpu * vcpu)2309 static int vmsave_interception(struct kvm_vcpu *vcpu)
2310 {
2311 	return vmload_vmsave_interception(vcpu, false);
2312 }
2313 
vmrun_interception(struct kvm_vcpu * vcpu)2314 static int vmrun_interception(struct kvm_vcpu *vcpu)
2315 {
2316 	if (nested_svm_check_permissions(vcpu))
2317 		return 1;
2318 
2319 	return nested_svm_vmrun(vcpu);
2320 }
2321 
2322 enum {
2323 	NONE_SVM_INSTR,
2324 	SVM_INSTR_VMRUN,
2325 	SVM_INSTR_VMLOAD,
2326 	SVM_INSTR_VMSAVE,
2327 };
2328 
2329 /* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */
svm_instr_opcode(struct kvm_vcpu * vcpu)2330 static int svm_instr_opcode(struct kvm_vcpu *vcpu)
2331 {
2332 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
2333 
2334 	if (ctxt->b != 0x1 || ctxt->opcode_len != 2)
2335 		return NONE_SVM_INSTR;
2336 
2337 	switch (ctxt->modrm) {
2338 	case 0xd8: /* VMRUN */
2339 		return SVM_INSTR_VMRUN;
2340 	case 0xda: /* VMLOAD */
2341 		return SVM_INSTR_VMLOAD;
2342 	case 0xdb: /* VMSAVE */
2343 		return SVM_INSTR_VMSAVE;
2344 	default:
2345 		break;
2346 	}
2347 
2348 	return NONE_SVM_INSTR;
2349 }
2350 
emulate_svm_instr(struct kvm_vcpu * vcpu,int opcode)2351 static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode)
2352 {
2353 	const int guest_mode_exit_codes[] = {
2354 		[SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN,
2355 		[SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD,
2356 		[SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE,
2357 	};
2358 	int (*const svm_instr_handlers[])(struct kvm_vcpu *vcpu) = {
2359 		[SVM_INSTR_VMRUN] = vmrun_interception,
2360 		[SVM_INSTR_VMLOAD] = vmload_interception,
2361 		[SVM_INSTR_VMSAVE] = vmsave_interception,
2362 	};
2363 	struct vcpu_svm *svm = to_svm(vcpu);
2364 	int ret;
2365 
2366 	if (is_guest_mode(vcpu)) {
2367 		/* Returns '1' or -errno on failure, '0' on success. */
2368 		ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]);
2369 		if (ret)
2370 			return ret;
2371 		return 1;
2372 	}
2373 	return svm_instr_handlers[opcode](vcpu);
2374 }
2375 
2376 /*
2377  * #GP handling code. Note that #GP can be triggered under the following two
2378  * cases:
2379  *   1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on
2380  *      some AMD CPUs when EAX of these instructions are in the reserved memory
2381  *      regions (e.g. SMM memory on host).
2382  *   2) VMware backdoor
2383  */
gp_interception(struct kvm_vcpu * vcpu)2384 static int gp_interception(struct kvm_vcpu *vcpu)
2385 {
2386 	struct vcpu_svm *svm = to_svm(vcpu);
2387 	u32 error_code = svm->vmcb->control.exit_info_1;
2388 	int opcode;
2389 
2390 	/* Both #GP cases have zero error_code */
2391 	if (error_code)
2392 		goto reinject;
2393 
2394 	/* Decode the instruction for usage later */
2395 	if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK)
2396 		goto reinject;
2397 
2398 	opcode = svm_instr_opcode(vcpu);
2399 
2400 	if (opcode == NONE_SVM_INSTR) {
2401 		if (!enable_vmware_backdoor)
2402 			goto reinject;
2403 
2404 		/*
2405 		 * VMware backdoor emulation on #GP interception only handles
2406 		 * IN{S}, OUT{S}, and RDPMC.
2407 		 */
2408 		if (!is_guest_mode(vcpu))
2409 			return kvm_emulate_instruction(vcpu,
2410 				EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE);
2411 	} else {
2412 		/* All SVM instructions expect page aligned RAX */
2413 		if (svm->vmcb->save.rax & ~PAGE_MASK)
2414 			goto reinject;
2415 
2416 		return emulate_svm_instr(vcpu, opcode);
2417 	}
2418 
2419 reinject:
2420 	kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2421 	return 1;
2422 }
2423 
svm_set_gif(struct vcpu_svm * svm,bool value)2424 void svm_set_gif(struct vcpu_svm *svm, bool value)
2425 {
2426 	if (value) {
2427 		/*
2428 		 * If VGIF is enabled, the STGI intercept is only added to
2429 		 * detect the opening of the SMI/NMI window; remove it now.
2430 		 * Likewise, clear the VINTR intercept, we will set it
2431 		 * again while processing KVM_REQ_EVENT if needed.
2432 		 */
2433 		if (vgif)
2434 			svm_clr_intercept(svm, INTERCEPT_STGI);
2435 		if (svm_is_intercept(svm, INTERCEPT_VINTR))
2436 			svm_clear_vintr(svm);
2437 
2438 		enable_gif(svm);
2439 		if (svm->vcpu.arch.smi_pending ||
2440 		    svm->vcpu.arch.nmi_pending ||
2441 		    kvm_cpu_has_injectable_intr(&svm->vcpu) ||
2442 		    kvm_apic_has_pending_init_or_sipi(&svm->vcpu))
2443 			kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2444 	} else {
2445 		disable_gif(svm);
2446 
2447 		/*
2448 		 * After a CLGI no interrupts should come.  But if vGIF is
2449 		 * in use, we still rely on the VINTR intercept (rather than
2450 		 * STGI) to detect an open interrupt window.
2451 		*/
2452 		if (!vgif)
2453 			svm_clear_vintr(svm);
2454 	}
2455 }
2456 
stgi_interception(struct kvm_vcpu * vcpu)2457 static int stgi_interception(struct kvm_vcpu *vcpu)
2458 {
2459 	int ret;
2460 
2461 	if (nested_svm_check_permissions(vcpu))
2462 		return 1;
2463 
2464 	ret = kvm_skip_emulated_instruction(vcpu);
2465 	svm_set_gif(to_svm(vcpu), true);
2466 	return ret;
2467 }
2468 
clgi_interception(struct kvm_vcpu * vcpu)2469 static int clgi_interception(struct kvm_vcpu *vcpu)
2470 {
2471 	int ret;
2472 
2473 	if (nested_svm_check_permissions(vcpu))
2474 		return 1;
2475 
2476 	ret = kvm_skip_emulated_instruction(vcpu);
2477 	svm_set_gif(to_svm(vcpu), false);
2478 	return ret;
2479 }
2480 
invlpga_interception(struct kvm_vcpu * vcpu)2481 static int invlpga_interception(struct kvm_vcpu *vcpu)
2482 {
2483 	gva_t gva = kvm_rax_read(vcpu);
2484 	u32 asid = kvm_rcx_read(vcpu);
2485 
2486 	/* FIXME: Handle an address size prefix. */
2487 	if (!is_long_mode(vcpu))
2488 		gva = (u32)gva;
2489 
2490 	trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva);
2491 
2492 	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2493 	kvm_mmu_invlpg(vcpu, gva);
2494 
2495 	return kvm_skip_emulated_instruction(vcpu);
2496 }
2497 
skinit_interception(struct kvm_vcpu * vcpu)2498 static int skinit_interception(struct kvm_vcpu *vcpu)
2499 {
2500 	trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu));
2501 
2502 	kvm_queue_exception(vcpu, UD_VECTOR);
2503 	return 1;
2504 }
2505 
task_switch_interception(struct kvm_vcpu * vcpu)2506 static int task_switch_interception(struct kvm_vcpu *vcpu)
2507 {
2508 	struct vcpu_svm *svm = to_svm(vcpu);
2509 	u16 tss_selector;
2510 	int reason;
2511 	int int_type = svm->vmcb->control.exit_int_info &
2512 		SVM_EXITINTINFO_TYPE_MASK;
2513 	int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2514 	uint32_t type =
2515 		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2516 	uint32_t idt_v =
2517 		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2518 	bool has_error_code = false;
2519 	u32 error_code = 0;
2520 
2521 	tss_selector = (u16)svm->vmcb->control.exit_info_1;
2522 
2523 	if (svm->vmcb->control.exit_info_2 &
2524 	    (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2525 		reason = TASK_SWITCH_IRET;
2526 	else if (svm->vmcb->control.exit_info_2 &
2527 		 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2528 		reason = TASK_SWITCH_JMP;
2529 	else if (idt_v)
2530 		reason = TASK_SWITCH_GATE;
2531 	else
2532 		reason = TASK_SWITCH_CALL;
2533 
2534 	if (reason == TASK_SWITCH_GATE) {
2535 		switch (type) {
2536 		case SVM_EXITINTINFO_TYPE_NMI:
2537 			vcpu->arch.nmi_injected = false;
2538 			break;
2539 		case SVM_EXITINTINFO_TYPE_EXEPT:
2540 			if (svm->vmcb->control.exit_info_2 &
2541 			    (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2542 				has_error_code = true;
2543 				error_code =
2544 					(u32)svm->vmcb->control.exit_info_2;
2545 			}
2546 			kvm_clear_exception_queue(vcpu);
2547 			break;
2548 		case SVM_EXITINTINFO_TYPE_INTR:
2549 		case SVM_EXITINTINFO_TYPE_SOFT:
2550 			kvm_clear_interrupt_queue(vcpu);
2551 			break;
2552 		default:
2553 			break;
2554 		}
2555 	}
2556 
2557 	if (reason != TASK_SWITCH_GATE ||
2558 	    int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2559 	    (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2560 	     (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
2561 		if (!svm_skip_emulated_instruction(vcpu))
2562 			return 0;
2563 	}
2564 
2565 	if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2566 		int_vec = -1;
2567 
2568 	return kvm_task_switch(vcpu, tss_selector, int_vec, reason,
2569 			       has_error_code, error_code);
2570 }
2571 
svm_clr_iret_intercept(struct vcpu_svm * svm)2572 static void svm_clr_iret_intercept(struct vcpu_svm *svm)
2573 {
2574 	if (!sev_es_guest(svm->vcpu.kvm))
2575 		svm_clr_intercept(svm, INTERCEPT_IRET);
2576 }
2577 
svm_set_iret_intercept(struct vcpu_svm * svm)2578 static void svm_set_iret_intercept(struct vcpu_svm *svm)
2579 {
2580 	if (!sev_es_guest(svm->vcpu.kvm))
2581 		svm_set_intercept(svm, INTERCEPT_IRET);
2582 }
2583 
iret_interception(struct kvm_vcpu * vcpu)2584 static int iret_interception(struct kvm_vcpu *vcpu)
2585 {
2586 	struct vcpu_svm *svm = to_svm(vcpu);
2587 
2588 	WARN_ON_ONCE(sev_es_guest(vcpu->kvm));
2589 
2590 	++vcpu->stat.nmi_window_exits;
2591 	svm->awaiting_iret_completion = true;
2592 
2593 	svm_clr_iret_intercept(svm);
2594 	svm->nmi_iret_rip = kvm_rip_read(vcpu);
2595 
2596 	kvm_make_request(KVM_REQ_EVENT, vcpu);
2597 	return 1;
2598 }
2599 
invlpg_interception(struct kvm_vcpu * vcpu)2600 static int invlpg_interception(struct kvm_vcpu *vcpu)
2601 {
2602 	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2603 		return kvm_emulate_instruction(vcpu, 0);
2604 
2605 	kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1);
2606 	return kvm_skip_emulated_instruction(vcpu);
2607 }
2608 
emulate_on_interception(struct kvm_vcpu * vcpu)2609 static int emulate_on_interception(struct kvm_vcpu *vcpu)
2610 {
2611 	return kvm_emulate_instruction(vcpu, 0);
2612 }
2613 
rsm_interception(struct kvm_vcpu * vcpu)2614 static int rsm_interception(struct kvm_vcpu *vcpu)
2615 {
2616 	return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2);
2617 }
2618 
check_selective_cr0_intercepted(struct kvm_vcpu * vcpu,unsigned long val)2619 static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu,
2620 					    unsigned long val)
2621 {
2622 	struct vcpu_svm *svm = to_svm(vcpu);
2623 	unsigned long cr0 = vcpu->arch.cr0;
2624 	bool ret = false;
2625 
2626 	if (!is_guest_mode(vcpu) ||
2627 	    (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0))))
2628 		return false;
2629 
2630 	cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2631 	val &= ~SVM_CR0_SELECTIVE_MASK;
2632 
2633 	if (cr0 ^ val) {
2634 		svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2635 		ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2636 	}
2637 
2638 	return ret;
2639 }
2640 
2641 #define CR_VALID (1ULL << 63)
2642 
cr_interception(struct kvm_vcpu * vcpu)2643 static int cr_interception(struct kvm_vcpu *vcpu)
2644 {
2645 	struct vcpu_svm *svm = to_svm(vcpu);
2646 	int reg, cr;
2647 	unsigned long val;
2648 	int err;
2649 
2650 	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2651 		return emulate_on_interception(vcpu);
2652 
2653 	if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2654 		return emulate_on_interception(vcpu);
2655 
2656 	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2657 	if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2658 		cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2659 	else
2660 		cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2661 
2662 	err = 0;
2663 	if (cr >= 16) { /* mov to cr */
2664 		cr -= 16;
2665 		val = kvm_register_read(vcpu, reg);
2666 		trace_kvm_cr_write(cr, val);
2667 		switch (cr) {
2668 		case 0:
2669 			if (!check_selective_cr0_intercepted(vcpu, val))
2670 				err = kvm_set_cr0(vcpu, val);
2671 			else
2672 				return 1;
2673 
2674 			break;
2675 		case 3:
2676 			err = kvm_set_cr3(vcpu, val);
2677 			break;
2678 		case 4:
2679 			err = kvm_set_cr4(vcpu, val);
2680 			break;
2681 		case 8:
2682 			err = kvm_set_cr8(vcpu, val);
2683 			break;
2684 		default:
2685 			WARN(1, "unhandled write to CR%d", cr);
2686 			kvm_queue_exception(vcpu, UD_VECTOR);
2687 			return 1;
2688 		}
2689 	} else { /* mov from cr */
2690 		switch (cr) {
2691 		case 0:
2692 			val = kvm_read_cr0(vcpu);
2693 			break;
2694 		case 2:
2695 			val = vcpu->arch.cr2;
2696 			break;
2697 		case 3:
2698 			val = kvm_read_cr3(vcpu);
2699 			break;
2700 		case 4:
2701 			val = kvm_read_cr4(vcpu);
2702 			break;
2703 		case 8:
2704 			val = kvm_get_cr8(vcpu);
2705 			break;
2706 		default:
2707 			WARN(1, "unhandled read from CR%d", cr);
2708 			kvm_queue_exception(vcpu, UD_VECTOR);
2709 			return 1;
2710 		}
2711 		kvm_register_write(vcpu, reg, val);
2712 		trace_kvm_cr_read(cr, val);
2713 	}
2714 	return kvm_complete_insn_gp(vcpu, err);
2715 }
2716 
cr_trap(struct kvm_vcpu * vcpu)2717 static int cr_trap(struct kvm_vcpu *vcpu)
2718 {
2719 	struct vcpu_svm *svm = to_svm(vcpu);
2720 	unsigned long old_value, new_value;
2721 	unsigned int cr;
2722 	int ret = 0;
2723 
2724 	new_value = (unsigned long)svm->vmcb->control.exit_info_1;
2725 
2726 	cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP;
2727 	switch (cr) {
2728 	case 0:
2729 		old_value = kvm_read_cr0(vcpu);
2730 		svm_set_cr0(vcpu, new_value);
2731 
2732 		kvm_post_set_cr0(vcpu, old_value, new_value);
2733 		break;
2734 	case 4:
2735 		old_value = kvm_read_cr4(vcpu);
2736 		svm_set_cr4(vcpu, new_value);
2737 
2738 		kvm_post_set_cr4(vcpu, old_value, new_value);
2739 		break;
2740 	case 8:
2741 		ret = kvm_set_cr8(vcpu, new_value);
2742 		break;
2743 	default:
2744 		WARN(1, "unhandled CR%d write trap", cr);
2745 		kvm_queue_exception(vcpu, UD_VECTOR);
2746 		return 1;
2747 	}
2748 
2749 	return kvm_complete_insn_gp(vcpu, ret);
2750 }
2751 
dr_interception(struct kvm_vcpu * vcpu)2752 static int dr_interception(struct kvm_vcpu *vcpu)
2753 {
2754 	struct vcpu_svm *svm = to_svm(vcpu);
2755 	int reg, dr;
2756 	unsigned long val;
2757 	int err = 0;
2758 
2759 	/*
2760 	 * SEV-ES intercepts DR7 only to disable guest debugging and the guest issues a VMGEXIT
2761 	 * for DR7 write only. KVM cannot change DR7 (always swapped as type 'A') so return early.
2762 	 */
2763 	if (sev_es_guest(vcpu->kvm))
2764 		return 1;
2765 
2766 	if (vcpu->guest_debug == 0) {
2767 		/*
2768 		 * No more DR vmexits; force a reload of the debug registers
2769 		 * and reenter on this instruction.  The next vmexit will
2770 		 * retrieve the full state of the debug registers.
2771 		 */
2772 		clr_dr_intercepts(svm);
2773 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2774 		return 1;
2775 	}
2776 
2777 	if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2778 		return emulate_on_interception(vcpu);
2779 
2780 	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2781 	dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2782 	if (dr >= 16) { /* mov to DRn  */
2783 		dr -= 16;
2784 		val = kvm_register_read(vcpu, reg);
2785 		err = kvm_set_dr(vcpu, dr, val);
2786 	} else {
2787 		kvm_get_dr(vcpu, dr, &val);
2788 		kvm_register_write(vcpu, reg, val);
2789 	}
2790 
2791 	return kvm_complete_insn_gp(vcpu, err);
2792 }
2793 
cr8_write_interception(struct kvm_vcpu * vcpu)2794 static int cr8_write_interception(struct kvm_vcpu *vcpu)
2795 {
2796 	int r;
2797 
2798 	u8 cr8_prev = kvm_get_cr8(vcpu);
2799 	/* instruction emulation calls kvm_set_cr8() */
2800 	r = cr_interception(vcpu);
2801 	if (lapic_in_kernel(vcpu))
2802 		return r;
2803 	if (cr8_prev <= kvm_get_cr8(vcpu))
2804 		return r;
2805 	vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
2806 	return 0;
2807 }
2808 
efer_trap(struct kvm_vcpu * vcpu)2809 static int efer_trap(struct kvm_vcpu *vcpu)
2810 {
2811 	struct msr_data msr_info;
2812 	int ret;
2813 
2814 	/*
2815 	 * Clear the EFER_SVME bit from EFER. The SVM code always sets this
2816 	 * bit in svm_set_efer(), but __kvm_valid_efer() checks it against
2817 	 * whether the guest has X86_FEATURE_SVM - this avoids a failure if
2818 	 * the guest doesn't have X86_FEATURE_SVM.
2819 	 */
2820 	msr_info.host_initiated = false;
2821 	msr_info.index = MSR_EFER;
2822 	msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME;
2823 	ret = kvm_set_msr_common(vcpu, &msr_info);
2824 
2825 	return kvm_complete_insn_gp(vcpu, ret);
2826 }
2827 
svm_get_msr_feature(struct kvm_msr_entry * msr)2828 static int svm_get_msr_feature(struct kvm_msr_entry *msr)
2829 {
2830 	msr->data = 0;
2831 
2832 	switch (msr->index) {
2833 	case MSR_AMD64_DE_CFG:
2834 		if (cpu_feature_enabled(X86_FEATURE_LFENCE_RDTSC))
2835 			msr->data |= MSR_AMD64_DE_CFG_LFENCE_SERIALIZE;
2836 		break;
2837 	default:
2838 		return KVM_MSR_RET_INVALID;
2839 	}
2840 
2841 	return 0;
2842 }
2843 
svm_get_msr(struct kvm_vcpu * vcpu,struct msr_data * msr_info)2844 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2845 {
2846 	struct vcpu_svm *svm = to_svm(vcpu);
2847 
2848 	switch (msr_info->index) {
2849 	case MSR_AMD64_TSC_RATIO:
2850 		if (!msr_info->host_initiated &&
2851 		    !guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR))
2852 			return 1;
2853 		msr_info->data = svm->tsc_ratio_msr;
2854 		break;
2855 	case MSR_STAR:
2856 		msr_info->data = svm->vmcb01.ptr->save.star;
2857 		break;
2858 #ifdef CONFIG_X86_64
2859 	case MSR_LSTAR:
2860 		msr_info->data = svm->vmcb01.ptr->save.lstar;
2861 		break;
2862 	case MSR_CSTAR:
2863 		msr_info->data = svm->vmcb01.ptr->save.cstar;
2864 		break;
2865 	case MSR_KERNEL_GS_BASE:
2866 		msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base;
2867 		break;
2868 	case MSR_SYSCALL_MASK:
2869 		msr_info->data = svm->vmcb01.ptr->save.sfmask;
2870 		break;
2871 #endif
2872 	case MSR_IA32_SYSENTER_CS:
2873 		msr_info->data = svm->vmcb01.ptr->save.sysenter_cs;
2874 		break;
2875 	case MSR_IA32_SYSENTER_EIP:
2876 		msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip;
2877 		if (guest_cpuid_is_intel(vcpu))
2878 			msr_info->data |= (u64)svm->sysenter_eip_hi << 32;
2879 		break;
2880 	case MSR_IA32_SYSENTER_ESP:
2881 		msr_info->data = svm->vmcb01.ptr->save.sysenter_esp;
2882 		if (guest_cpuid_is_intel(vcpu))
2883 			msr_info->data |= (u64)svm->sysenter_esp_hi << 32;
2884 		break;
2885 	case MSR_TSC_AUX:
2886 		msr_info->data = svm->tsc_aux;
2887 		break;
2888 	case MSR_IA32_DEBUGCTLMSR:
2889 		msr_info->data = svm_get_lbr_vmcb(svm)->save.dbgctl;
2890 		break;
2891 	case MSR_IA32_LASTBRANCHFROMIP:
2892 		msr_info->data = svm_get_lbr_vmcb(svm)->save.br_from;
2893 		break;
2894 	case MSR_IA32_LASTBRANCHTOIP:
2895 		msr_info->data = svm_get_lbr_vmcb(svm)->save.br_to;
2896 		break;
2897 	case MSR_IA32_LASTINTFROMIP:
2898 		msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_from;
2899 		break;
2900 	case MSR_IA32_LASTINTTOIP:
2901 		msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_to;
2902 		break;
2903 	case MSR_VM_HSAVE_PA:
2904 		msr_info->data = svm->nested.hsave_msr;
2905 		break;
2906 	case MSR_VM_CR:
2907 		msr_info->data = svm->nested.vm_cr_msr;
2908 		break;
2909 	case MSR_IA32_SPEC_CTRL:
2910 		if (!msr_info->host_initiated &&
2911 		    !guest_has_spec_ctrl_msr(vcpu))
2912 			return 1;
2913 
2914 		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
2915 			msr_info->data = svm->vmcb->save.spec_ctrl;
2916 		else
2917 			msr_info->data = svm->spec_ctrl;
2918 		break;
2919 	case MSR_AMD64_VIRT_SPEC_CTRL:
2920 		if (!msr_info->host_initiated &&
2921 		    !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2922 			return 1;
2923 
2924 		msr_info->data = svm->virt_spec_ctrl;
2925 		break;
2926 	case MSR_F15H_IC_CFG: {
2927 
2928 		int family, model;
2929 
2930 		family = guest_cpuid_family(vcpu);
2931 		model  = guest_cpuid_model(vcpu);
2932 
2933 		if (family < 0 || model < 0)
2934 			return kvm_get_msr_common(vcpu, msr_info);
2935 
2936 		msr_info->data = 0;
2937 
2938 		if (family == 0x15 &&
2939 		    (model >= 0x2 && model < 0x20))
2940 			msr_info->data = 0x1E;
2941 		}
2942 		break;
2943 	case MSR_AMD64_DE_CFG:
2944 		msr_info->data = svm->msr_decfg;
2945 		break;
2946 	default:
2947 		return kvm_get_msr_common(vcpu, msr_info);
2948 	}
2949 	return 0;
2950 }
2951 
svm_complete_emulated_msr(struct kvm_vcpu * vcpu,int err)2952 static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
2953 {
2954 	struct vcpu_svm *svm = to_svm(vcpu);
2955 	if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->sev_es.ghcb))
2956 		return kvm_complete_insn_gp(vcpu, err);
2957 
2958 	ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 1);
2959 	ghcb_set_sw_exit_info_2(svm->sev_es.ghcb,
2960 				X86_TRAP_GP |
2961 				SVM_EVTINJ_TYPE_EXEPT |
2962 				SVM_EVTINJ_VALID);
2963 	return 1;
2964 }
2965 
svm_set_vm_cr(struct kvm_vcpu * vcpu,u64 data)2966 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
2967 {
2968 	struct vcpu_svm *svm = to_svm(vcpu);
2969 	int svm_dis, chg_mask;
2970 
2971 	if (data & ~SVM_VM_CR_VALID_MASK)
2972 		return 1;
2973 
2974 	chg_mask = SVM_VM_CR_VALID_MASK;
2975 
2976 	if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
2977 		chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
2978 
2979 	svm->nested.vm_cr_msr &= ~chg_mask;
2980 	svm->nested.vm_cr_msr |= (data & chg_mask);
2981 
2982 	svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
2983 
2984 	/* check for svm_disable while efer.svme is set */
2985 	if (svm_dis && (vcpu->arch.efer & EFER_SVME))
2986 		return 1;
2987 
2988 	return 0;
2989 }
2990 
svm_set_msr(struct kvm_vcpu * vcpu,struct msr_data * msr)2991 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
2992 {
2993 	struct vcpu_svm *svm = to_svm(vcpu);
2994 	int ret = 0;
2995 
2996 	u32 ecx = msr->index;
2997 	u64 data = msr->data;
2998 	switch (ecx) {
2999 	case MSR_AMD64_TSC_RATIO:
3000 
3001 		if (!guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR)) {
3002 
3003 			if (!msr->host_initiated)
3004 				return 1;
3005 			/*
3006 			 * In case TSC scaling is not enabled, always
3007 			 * leave this MSR at the default value.
3008 			 *
3009 			 * Due to bug in qemu 6.2.0, it would try to set
3010 			 * this msr to 0 if tsc scaling is not enabled.
3011 			 * Ignore this value as well.
3012 			 */
3013 			if (data != 0 && data != svm->tsc_ratio_msr)
3014 				return 1;
3015 			break;
3016 		}
3017 
3018 		if (data & SVM_TSC_RATIO_RSVD)
3019 			return 1;
3020 
3021 		svm->tsc_ratio_msr = data;
3022 
3023 		if (guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR) &&
3024 		    is_guest_mode(vcpu))
3025 			nested_svm_update_tsc_ratio_msr(vcpu);
3026 
3027 		break;
3028 	case MSR_IA32_CR_PAT:
3029 		ret = kvm_set_msr_common(vcpu, msr);
3030 		if (ret)
3031 			break;
3032 
3033 		svm->vmcb01.ptr->save.g_pat = data;
3034 		if (is_guest_mode(vcpu))
3035 			nested_vmcb02_compute_g_pat(svm);
3036 		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
3037 		break;
3038 	case MSR_IA32_SPEC_CTRL:
3039 		if (!msr->host_initiated &&
3040 		    !guest_has_spec_ctrl_msr(vcpu))
3041 			return 1;
3042 
3043 		if (kvm_spec_ctrl_test_value(data))
3044 			return 1;
3045 
3046 		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
3047 			svm->vmcb->save.spec_ctrl = data;
3048 		else
3049 			svm->spec_ctrl = data;
3050 		if (!data)
3051 			break;
3052 
3053 		/*
3054 		 * For non-nested:
3055 		 * When it's written (to non-zero) for the first time, pass
3056 		 * it through.
3057 		 *
3058 		 * For nested:
3059 		 * The handling of the MSR bitmap for L2 guests is done in
3060 		 * nested_svm_vmrun_msrpm.
3061 		 * We update the L1 MSR bit as well since it will end up
3062 		 * touching the MSR anyway now.
3063 		 */
3064 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
3065 		break;
3066 	case MSR_AMD64_VIRT_SPEC_CTRL:
3067 		if (!msr->host_initiated &&
3068 		    !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
3069 			return 1;
3070 
3071 		if (data & ~SPEC_CTRL_SSBD)
3072 			return 1;
3073 
3074 		svm->virt_spec_ctrl = data;
3075 		break;
3076 	case MSR_STAR:
3077 		svm->vmcb01.ptr->save.star = data;
3078 		break;
3079 #ifdef CONFIG_X86_64
3080 	case MSR_LSTAR:
3081 		svm->vmcb01.ptr->save.lstar = data;
3082 		break;
3083 	case MSR_CSTAR:
3084 		svm->vmcb01.ptr->save.cstar = data;
3085 		break;
3086 	case MSR_KERNEL_GS_BASE:
3087 		svm->vmcb01.ptr->save.kernel_gs_base = data;
3088 		break;
3089 	case MSR_SYSCALL_MASK:
3090 		svm->vmcb01.ptr->save.sfmask = data;
3091 		break;
3092 #endif
3093 	case MSR_IA32_SYSENTER_CS:
3094 		svm->vmcb01.ptr->save.sysenter_cs = data;
3095 		break;
3096 	case MSR_IA32_SYSENTER_EIP:
3097 		svm->vmcb01.ptr->save.sysenter_eip = (u32)data;
3098 		/*
3099 		 * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs
3100 		 * when we spoof an Intel vendor ID (for cross vendor migration).
3101 		 * In this case we use this intercept to track the high
3102 		 * 32 bit part of these msrs to support Intel's
3103 		 * implementation of SYSENTER/SYSEXIT.
3104 		 */
3105 		svm->sysenter_eip_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
3106 		break;
3107 	case MSR_IA32_SYSENTER_ESP:
3108 		svm->vmcb01.ptr->save.sysenter_esp = (u32)data;
3109 		svm->sysenter_esp_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
3110 		break;
3111 	case MSR_TSC_AUX:
3112 		/*
3113 		 * TSC_AUX is always virtualized for SEV-ES guests when the
3114 		 * feature is available. The user return MSR support is not
3115 		 * required in this case because TSC_AUX is restored on #VMEXIT
3116 		 * from the host save area (which has been initialized in
3117 		 * svm_hardware_enable()).
3118 		 */
3119 		if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) && sev_es_guest(vcpu->kvm))
3120 			break;
3121 
3122 		/*
3123 		 * TSC_AUX is usually changed only during boot and never read
3124 		 * directly.  Intercept TSC_AUX instead of exposing it to the
3125 		 * guest via direct_access_msrs, and switch it via user return.
3126 		 */
3127 		preempt_disable();
3128 		ret = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull);
3129 		preempt_enable();
3130 		if (ret)
3131 			break;
3132 
3133 		svm->tsc_aux = data;
3134 		break;
3135 	case MSR_IA32_DEBUGCTLMSR:
3136 		if (!lbrv) {
3137 			kvm_pr_unimpl_wrmsr(vcpu, ecx, data);
3138 			break;
3139 		}
3140 		if (data & DEBUGCTL_RESERVED_BITS)
3141 			return 1;
3142 
3143 		svm_get_lbr_vmcb(svm)->save.dbgctl = data;
3144 		svm_update_lbrv(vcpu);
3145 		break;
3146 	case MSR_VM_HSAVE_PA:
3147 		/*
3148 		 * Old kernels did not validate the value written to
3149 		 * MSR_VM_HSAVE_PA.  Allow KVM_SET_MSR to set an invalid
3150 		 * value to allow live migrating buggy or malicious guests
3151 		 * originating from those kernels.
3152 		 */
3153 		if (!msr->host_initiated && !page_address_valid(vcpu, data))
3154 			return 1;
3155 
3156 		svm->nested.hsave_msr = data & PAGE_MASK;
3157 		break;
3158 	case MSR_VM_CR:
3159 		return svm_set_vm_cr(vcpu, data);
3160 	case MSR_VM_IGNNE:
3161 		kvm_pr_unimpl_wrmsr(vcpu, ecx, data);
3162 		break;
3163 	case MSR_AMD64_DE_CFG: {
3164 		struct kvm_msr_entry msr_entry;
3165 
3166 		msr_entry.index = msr->index;
3167 		if (svm_get_msr_feature(&msr_entry))
3168 			return 1;
3169 
3170 		/* Check the supported bits */
3171 		if (data & ~msr_entry.data)
3172 			return 1;
3173 
3174 		/* Don't allow the guest to change a bit, #GP */
3175 		if (!msr->host_initiated && (data ^ msr_entry.data))
3176 			return 1;
3177 
3178 		svm->msr_decfg = data;
3179 		break;
3180 	}
3181 	default:
3182 		return kvm_set_msr_common(vcpu, msr);
3183 	}
3184 	return ret;
3185 }
3186 
msr_interception(struct kvm_vcpu * vcpu)3187 static int msr_interception(struct kvm_vcpu *vcpu)
3188 {
3189 	if (to_svm(vcpu)->vmcb->control.exit_info_1)
3190 		return kvm_emulate_wrmsr(vcpu);
3191 	else
3192 		return kvm_emulate_rdmsr(vcpu);
3193 }
3194 
interrupt_window_interception(struct kvm_vcpu * vcpu)3195 static int interrupt_window_interception(struct kvm_vcpu *vcpu)
3196 {
3197 	kvm_make_request(KVM_REQ_EVENT, vcpu);
3198 	svm_clear_vintr(to_svm(vcpu));
3199 
3200 	/*
3201 	 * If not running nested, for AVIC, the only reason to end up here is ExtINTs.
3202 	 * In this case AVIC was temporarily disabled for
3203 	 * requesting the IRQ window and we have to re-enable it.
3204 	 *
3205 	 * If running nested, still remove the VM wide AVIC inhibit to
3206 	 * support case in which the interrupt window was requested when the
3207 	 * vCPU was not running nested.
3208 
3209 	 * All vCPUs which run still run nested, will remain to have their
3210 	 * AVIC still inhibited due to per-cpu AVIC inhibition.
3211 	 */
3212 	kvm_clear_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
3213 
3214 	++vcpu->stat.irq_window_exits;
3215 	return 1;
3216 }
3217 
pause_interception(struct kvm_vcpu * vcpu)3218 static int pause_interception(struct kvm_vcpu *vcpu)
3219 {
3220 	bool in_kernel;
3221 	/*
3222 	 * CPL is not made available for an SEV-ES guest, therefore
3223 	 * vcpu->arch.preempted_in_kernel can never be true.  Just
3224 	 * set in_kernel to false as well.
3225 	 */
3226 	in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0;
3227 
3228 	grow_ple_window(vcpu);
3229 
3230 	kvm_vcpu_on_spin(vcpu, in_kernel);
3231 	return kvm_skip_emulated_instruction(vcpu);
3232 }
3233 
invpcid_interception(struct kvm_vcpu * vcpu)3234 static int invpcid_interception(struct kvm_vcpu *vcpu)
3235 {
3236 	struct vcpu_svm *svm = to_svm(vcpu);
3237 	unsigned long type;
3238 	gva_t gva;
3239 
3240 	if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
3241 		kvm_queue_exception(vcpu, UD_VECTOR);
3242 		return 1;
3243 	}
3244 
3245 	/*
3246 	 * For an INVPCID intercept:
3247 	 * EXITINFO1 provides the linear address of the memory operand.
3248 	 * EXITINFO2 provides the contents of the register operand.
3249 	 */
3250 	type = svm->vmcb->control.exit_info_2;
3251 	gva = svm->vmcb->control.exit_info_1;
3252 
3253 	return kvm_handle_invpcid(vcpu, type, gva);
3254 }
3255 
3256 static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = {
3257 	[SVM_EXIT_READ_CR0]			= cr_interception,
3258 	[SVM_EXIT_READ_CR3]			= cr_interception,
3259 	[SVM_EXIT_READ_CR4]			= cr_interception,
3260 	[SVM_EXIT_READ_CR8]			= cr_interception,
3261 	[SVM_EXIT_CR0_SEL_WRITE]		= cr_interception,
3262 	[SVM_EXIT_WRITE_CR0]			= cr_interception,
3263 	[SVM_EXIT_WRITE_CR3]			= cr_interception,
3264 	[SVM_EXIT_WRITE_CR4]			= cr_interception,
3265 	[SVM_EXIT_WRITE_CR8]			= cr8_write_interception,
3266 	[SVM_EXIT_READ_DR0]			= dr_interception,
3267 	[SVM_EXIT_READ_DR1]			= dr_interception,
3268 	[SVM_EXIT_READ_DR2]			= dr_interception,
3269 	[SVM_EXIT_READ_DR3]			= dr_interception,
3270 	[SVM_EXIT_READ_DR4]			= dr_interception,
3271 	[SVM_EXIT_READ_DR5]			= dr_interception,
3272 	[SVM_EXIT_READ_DR6]			= dr_interception,
3273 	[SVM_EXIT_READ_DR7]			= dr_interception,
3274 	[SVM_EXIT_WRITE_DR0]			= dr_interception,
3275 	[SVM_EXIT_WRITE_DR1]			= dr_interception,
3276 	[SVM_EXIT_WRITE_DR2]			= dr_interception,
3277 	[SVM_EXIT_WRITE_DR3]			= dr_interception,
3278 	[SVM_EXIT_WRITE_DR4]			= dr_interception,
3279 	[SVM_EXIT_WRITE_DR5]			= dr_interception,
3280 	[SVM_EXIT_WRITE_DR6]			= dr_interception,
3281 	[SVM_EXIT_WRITE_DR7]			= dr_interception,
3282 	[SVM_EXIT_EXCP_BASE + DB_VECTOR]	= db_interception,
3283 	[SVM_EXIT_EXCP_BASE + BP_VECTOR]	= bp_interception,
3284 	[SVM_EXIT_EXCP_BASE + UD_VECTOR]	= ud_interception,
3285 	[SVM_EXIT_EXCP_BASE + PF_VECTOR]	= pf_interception,
3286 	[SVM_EXIT_EXCP_BASE + MC_VECTOR]	= mc_interception,
3287 	[SVM_EXIT_EXCP_BASE + AC_VECTOR]	= ac_interception,
3288 	[SVM_EXIT_EXCP_BASE + GP_VECTOR]	= gp_interception,
3289 	[SVM_EXIT_INTR]				= intr_interception,
3290 	[SVM_EXIT_NMI]				= nmi_interception,
3291 	[SVM_EXIT_SMI]				= smi_interception,
3292 	[SVM_EXIT_VINTR]			= interrupt_window_interception,
3293 	[SVM_EXIT_RDPMC]			= kvm_emulate_rdpmc,
3294 	[SVM_EXIT_CPUID]			= kvm_emulate_cpuid,
3295 	[SVM_EXIT_IRET]                         = iret_interception,
3296 	[SVM_EXIT_INVD]                         = kvm_emulate_invd,
3297 	[SVM_EXIT_PAUSE]			= pause_interception,
3298 	[SVM_EXIT_HLT]				= kvm_emulate_halt,
3299 	[SVM_EXIT_INVLPG]			= invlpg_interception,
3300 	[SVM_EXIT_INVLPGA]			= invlpga_interception,
3301 	[SVM_EXIT_IOIO]				= io_interception,
3302 	[SVM_EXIT_MSR]				= msr_interception,
3303 	[SVM_EXIT_TASK_SWITCH]			= task_switch_interception,
3304 	[SVM_EXIT_SHUTDOWN]			= shutdown_interception,
3305 	[SVM_EXIT_VMRUN]			= vmrun_interception,
3306 	[SVM_EXIT_VMMCALL]			= kvm_emulate_hypercall,
3307 	[SVM_EXIT_VMLOAD]			= vmload_interception,
3308 	[SVM_EXIT_VMSAVE]			= vmsave_interception,
3309 	[SVM_EXIT_STGI]				= stgi_interception,
3310 	[SVM_EXIT_CLGI]				= clgi_interception,
3311 	[SVM_EXIT_SKINIT]			= skinit_interception,
3312 	[SVM_EXIT_RDTSCP]			= kvm_handle_invalid_op,
3313 	[SVM_EXIT_WBINVD]                       = kvm_emulate_wbinvd,
3314 	[SVM_EXIT_MONITOR]			= kvm_emulate_monitor,
3315 	[SVM_EXIT_MWAIT]			= kvm_emulate_mwait,
3316 	[SVM_EXIT_XSETBV]			= kvm_emulate_xsetbv,
3317 	[SVM_EXIT_RDPRU]			= kvm_handle_invalid_op,
3318 	[SVM_EXIT_EFER_WRITE_TRAP]		= efer_trap,
3319 	[SVM_EXIT_CR0_WRITE_TRAP]		= cr_trap,
3320 	[SVM_EXIT_CR4_WRITE_TRAP]		= cr_trap,
3321 	[SVM_EXIT_CR8_WRITE_TRAP]		= cr_trap,
3322 	[SVM_EXIT_INVPCID]                      = invpcid_interception,
3323 	[SVM_EXIT_NPF]				= npf_interception,
3324 	[SVM_EXIT_RSM]                          = rsm_interception,
3325 	[SVM_EXIT_AVIC_INCOMPLETE_IPI]		= avic_incomplete_ipi_interception,
3326 	[SVM_EXIT_AVIC_UNACCELERATED_ACCESS]	= avic_unaccelerated_access_interception,
3327 	[SVM_EXIT_VMGEXIT]			= sev_handle_vmgexit,
3328 };
3329 
dump_vmcb(struct kvm_vcpu * vcpu)3330 static void dump_vmcb(struct kvm_vcpu *vcpu)
3331 {
3332 	struct vcpu_svm *svm = to_svm(vcpu);
3333 	struct vmcb_control_area *control = &svm->vmcb->control;
3334 	struct vmcb_save_area *save = &svm->vmcb->save;
3335 	struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save;
3336 
3337 	if (!dump_invalid_vmcb) {
3338 		pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
3339 		return;
3340 	}
3341 
3342 	pr_err("VMCB %p, last attempted VMRUN on CPU %d\n",
3343 	       svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu);
3344 	pr_err("VMCB Control Area:\n");
3345 	pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff);
3346 	pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16);
3347 	pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff);
3348 	pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16);
3349 	pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]);
3350 	pr_err("%-20s%08x %08x\n", "intercepts:",
3351               control->intercepts[INTERCEPT_WORD3],
3352 	       control->intercepts[INTERCEPT_WORD4]);
3353 	pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3354 	pr_err("%-20s%d\n", "pause filter threshold:",
3355 	       control->pause_filter_thresh);
3356 	pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3357 	pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3358 	pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3359 	pr_err("%-20s%d\n", "asid:", control->asid);
3360 	pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3361 	pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3362 	pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3363 	pr_err("%-20s%08x\n", "int_state:", control->int_state);
3364 	pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3365 	pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3366 	pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3367 	pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3368 	pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3369 	pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3370 	pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3371 	pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
3372 	pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa);
3373 	pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3374 	pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3375 	pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
3376 	pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3377 	pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
3378 	pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
3379 	pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
3380 	pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa);
3381 	pr_err("VMCB State Save Area:\n");
3382 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3383 	       "es:",
3384 	       save->es.selector, save->es.attrib,
3385 	       save->es.limit, save->es.base);
3386 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3387 	       "cs:",
3388 	       save->cs.selector, save->cs.attrib,
3389 	       save->cs.limit, save->cs.base);
3390 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3391 	       "ss:",
3392 	       save->ss.selector, save->ss.attrib,
3393 	       save->ss.limit, save->ss.base);
3394 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3395 	       "ds:",
3396 	       save->ds.selector, save->ds.attrib,
3397 	       save->ds.limit, save->ds.base);
3398 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3399 	       "fs:",
3400 	       save01->fs.selector, save01->fs.attrib,
3401 	       save01->fs.limit, save01->fs.base);
3402 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3403 	       "gs:",
3404 	       save01->gs.selector, save01->gs.attrib,
3405 	       save01->gs.limit, save01->gs.base);
3406 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3407 	       "gdtr:",
3408 	       save->gdtr.selector, save->gdtr.attrib,
3409 	       save->gdtr.limit, save->gdtr.base);
3410 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3411 	       "ldtr:",
3412 	       save01->ldtr.selector, save01->ldtr.attrib,
3413 	       save01->ldtr.limit, save01->ldtr.base);
3414 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3415 	       "idtr:",
3416 	       save->idtr.selector, save->idtr.attrib,
3417 	       save->idtr.limit, save->idtr.base);
3418 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3419 	       "tr:",
3420 	       save01->tr.selector, save01->tr.attrib,
3421 	       save01->tr.limit, save01->tr.base);
3422 	pr_err("vmpl: %d   cpl:  %d               efer:          %016llx\n",
3423 	       save->vmpl, save->cpl, save->efer);
3424 	pr_err("%-15s %016llx %-13s %016llx\n",
3425 	       "cr0:", save->cr0, "cr2:", save->cr2);
3426 	pr_err("%-15s %016llx %-13s %016llx\n",
3427 	       "cr3:", save->cr3, "cr4:", save->cr4);
3428 	pr_err("%-15s %016llx %-13s %016llx\n",
3429 	       "dr6:", save->dr6, "dr7:", save->dr7);
3430 	pr_err("%-15s %016llx %-13s %016llx\n",
3431 	       "rip:", save->rip, "rflags:", save->rflags);
3432 	pr_err("%-15s %016llx %-13s %016llx\n",
3433 	       "rsp:", save->rsp, "rax:", save->rax);
3434 	pr_err("%-15s %016llx %-13s %016llx\n",
3435 	       "star:", save01->star, "lstar:", save01->lstar);
3436 	pr_err("%-15s %016llx %-13s %016llx\n",
3437 	       "cstar:", save01->cstar, "sfmask:", save01->sfmask);
3438 	pr_err("%-15s %016llx %-13s %016llx\n",
3439 	       "kernel_gs_base:", save01->kernel_gs_base,
3440 	       "sysenter_cs:", save01->sysenter_cs);
3441 	pr_err("%-15s %016llx %-13s %016llx\n",
3442 	       "sysenter_esp:", save01->sysenter_esp,
3443 	       "sysenter_eip:", save01->sysenter_eip);
3444 	pr_err("%-15s %016llx %-13s %016llx\n",
3445 	       "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3446 	pr_err("%-15s %016llx %-13s %016llx\n",
3447 	       "br_from:", save->br_from, "br_to:", save->br_to);
3448 	pr_err("%-15s %016llx %-13s %016llx\n",
3449 	       "excp_from:", save->last_excp_from,
3450 	       "excp_to:", save->last_excp_to);
3451 }
3452 
svm_check_exit_valid(u64 exit_code)3453 static bool svm_check_exit_valid(u64 exit_code)
3454 {
3455 	return (exit_code < ARRAY_SIZE(svm_exit_handlers) &&
3456 		svm_exit_handlers[exit_code]);
3457 }
3458 
svm_handle_invalid_exit(struct kvm_vcpu * vcpu,u64 exit_code)3459 static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code)
3460 {
3461 	vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%llx\n", exit_code);
3462 	dump_vmcb(vcpu);
3463 	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3464 	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
3465 	vcpu->run->internal.ndata = 2;
3466 	vcpu->run->internal.data[0] = exit_code;
3467 	vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
3468 	return 0;
3469 }
3470 
svm_invoke_exit_handler(struct kvm_vcpu * vcpu,u64 exit_code)3471 int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code)
3472 {
3473 	if (!svm_check_exit_valid(exit_code))
3474 		return svm_handle_invalid_exit(vcpu, exit_code);
3475 
3476 #ifdef CONFIG_RETPOLINE
3477 	if (exit_code == SVM_EXIT_MSR)
3478 		return msr_interception(vcpu);
3479 	else if (exit_code == SVM_EXIT_VINTR)
3480 		return interrupt_window_interception(vcpu);
3481 	else if (exit_code == SVM_EXIT_INTR)
3482 		return intr_interception(vcpu);
3483 	else if (exit_code == SVM_EXIT_HLT)
3484 		return kvm_emulate_halt(vcpu);
3485 	else if (exit_code == SVM_EXIT_NPF)
3486 		return npf_interception(vcpu);
3487 #endif
3488 	return svm_exit_handlers[exit_code](vcpu);
3489 }
3490 
svm_get_exit_info(struct kvm_vcpu * vcpu,u32 * reason,u64 * info1,u64 * info2,u32 * intr_info,u32 * error_code)3491 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
3492 			      u64 *info1, u64 *info2,
3493 			      u32 *intr_info, u32 *error_code)
3494 {
3495 	struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3496 
3497 	*reason = control->exit_code;
3498 	*info1 = control->exit_info_1;
3499 	*info2 = control->exit_info_2;
3500 	*intr_info = control->exit_int_info;
3501 	if ((*intr_info & SVM_EXITINTINFO_VALID) &&
3502 	    (*intr_info & SVM_EXITINTINFO_VALID_ERR))
3503 		*error_code = control->exit_int_info_err;
3504 	else
3505 		*error_code = 0;
3506 }
3507 
svm_handle_exit(struct kvm_vcpu * vcpu,fastpath_t exit_fastpath)3508 static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
3509 {
3510 	struct vcpu_svm *svm = to_svm(vcpu);
3511 	struct kvm_run *kvm_run = vcpu->run;
3512 	u32 exit_code = svm->vmcb->control.exit_code;
3513 
3514 	/* SEV-ES guests must use the CR write traps to track CR registers. */
3515 	if (!sev_es_guest(vcpu->kvm)) {
3516 		if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
3517 			vcpu->arch.cr0 = svm->vmcb->save.cr0;
3518 		if (npt_enabled)
3519 			vcpu->arch.cr3 = svm->vmcb->save.cr3;
3520 	}
3521 
3522 	if (is_guest_mode(vcpu)) {
3523 		int vmexit;
3524 
3525 		trace_kvm_nested_vmexit(vcpu, KVM_ISA_SVM);
3526 
3527 		vmexit = nested_svm_exit_special(svm);
3528 
3529 		if (vmexit == NESTED_EXIT_CONTINUE)
3530 			vmexit = nested_svm_exit_handled(svm);
3531 
3532 		if (vmexit == NESTED_EXIT_DONE)
3533 			return 1;
3534 	}
3535 
3536 	if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3537 		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3538 		kvm_run->fail_entry.hardware_entry_failure_reason
3539 			= svm->vmcb->control.exit_code;
3540 		kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
3541 		dump_vmcb(vcpu);
3542 		return 0;
3543 	}
3544 
3545 	if (exit_fastpath != EXIT_FASTPATH_NONE)
3546 		return 1;
3547 
3548 	return svm_invoke_exit_handler(vcpu, exit_code);
3549 }
3550 
pre_svm_run(struct kvm_vcpu * vcpu)3551 static void pre_svm_run(struct kvm_vcpu *vcpu)
3552 {
3553 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
3554 	struct vcpu_svm *svm = to_svm(vcpu);
3555 
3556 	/*
3557 	 * If the previous vmrun of the vmcb occurred on a different physical
3558 	 * cpu, then mark the vmcb dirty and assign a new asid.  Hardware's
3559 	 * vmcb clean bits are per logical CPU, as are KVM's asid assignments.
3560 	 */
3561 	if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) {
3562 		svm->current_vmcb->asid_generation = 0;
3563 		vmcb_mark_all_dirty(svm->vmcb);
3564 		svm->current_vmcb->cpu = vcpu->cpu;
3565         }
3566 
3567 	if (sev_guest(vcpu->kvm))
3568 		return pre_sev_run(svm, vcpu->cpu);
3569 
3570 	/* FIXME: handle wraparound of asid_generation */
3571 	if (svm->current_vmcb->asid_generation != sd->asid_generation)
3572 		new_asid(svm, sd);
3573 }
3574 
svm_inject_nmi(struct kvm_vcpu * vcpu)3575 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3576 {
3577 	struct vcpu_svm *svm = to_svm(vcpu);
3578 
3579 	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3580 
3581 	if (svm->nmi_l1_to_l2)
3582 		return;
3583 
3584 	svm->nmi_masked = true;
3585 	svm_set_iret_intercept(svm);
3586 	++vcpu->stat.nmi_injections;
3587 }
3588 
svm_is_vnmi_pending(struct kvm_vcpu * vcpu)3589 static bool svm_is_vnmi_pending(struct kvm_vcpu *vcpu)
3590 {
3591 	struct vcpu_svm *svm = to_svm(vcpu);
3592 
3593 	if (!is_vnmi_enabled(svm))
3594 		return false;
3595 
3596 	return !!(svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK);
3597 }
3598 
svm_set_vnmi_pending(struct kvm_vcpu * vcpu)3599 static bool svm_set_vnmi_pending(struct kvm_vcpu *vcpu)
3600 {
3601 	struct vcpu_svm *svm = to_svm(vcpu);
3602 
3603 	if (!is_vnmi_enabled(svm))
3604 		return false;
3605 
3606 	if (svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK)
3607 		return false;
3608 
3609 	svm->vmcb->control.int_ctl |= V_NMI_PENDING_MASK;
3610 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
3611 
3612 	/*
3613 	 * Because the pending NMI is serviced by hardware, KVM can't know when
3614 	 * the NMI is "injected", but for all intents and purposes, passing the
3615 	 * NMI off to hardware counts as injection.
3616 	 */
3617 	++vcpu->stat.nmi_injections;
3618 
3619 	return true;
3620 }
3621 
svm_inject_irq(struct kvm_vcpu * vcpu,bool reinjected)3622 static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
3623 {
3624 	struct vcpu_svm *svm = to_svm(vcpu);
3625 	u32 type;
3626 
3627 	if (vcpu->arch.interrupt.soft) {
3628 		if (svm_update_soft_interrupt_rip(vcpu))
3629 			return;
3630 
3631 		type = SVM_EVTINJ_TYPE_SOFT;
3632 	} else {
3633 		type = SVM_EVTINJ_TYPE_INTR;
3634 	}
3635 
3636 	trace_kvm_inj_virq(vcpu->arch.interrupt.nr,
3637 			   vcpu->arch.interrupt.soft, reinjected);
3638 	++vcpu->stat.irq_injections;
3639 
3640 	svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3641 				       SVM_EVTINJ_VALID | type;
3642 }
3643 
svm_complete_interrupt_delivery(struct kvm_vcpu * vcpu,int delivery_mode,int trig_mode,int vector)3644 void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode,
3645 				     int trig_mode, int vector)
3646 {
3647 	/*
3648 	 * apic->apicv_active must be read after vcpu->mode.
3649 	 * Pairs with smp_store_release in vcpu_enter_guest.
3650 	 */
3651 	bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE);
3652 
3653 	/* Note, this is called iff the local APIC is in-kernel. */
3654 	if (!READ_ONCE(vcpu->arch.apic->apicv_active)) {
3655 		/* Process the interrupt via kvm_check_and_inject_events(). */
3656 		kvm_make_request(KVM_REQ_EVENT, vcpu);
3657 		kvm_vcpu_kick(vcpu);
3658 		return;
3659 	}
3660 
3661 	trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector);
3662 	if (in_guest_mode) {
3663 		/*
3664 		 * Signal the doorbell to tell hardware to inject the IRQ.  If
3665 		 * the vCPU exits the guest before the doorbell chimes, hardware
3666 		 * will automatically process AVIC interrupts at the next VMRUN.
3667 		 */
3668 		avic_ring_doorbell(vcpu);
3669 	} else {
3670 		/*
3671 		 * Wake the vCPU if it was blocking.  KVM will then detect the
3672 		 * pending IRQ when checking if the vCPU has a wake event.
3673 		 */
3674 		kvm_vcpu_wake_up(vcpu);
3675 	}
3676 }
3677 
svm_deliver_interrupt(struct kvm_lapic * apic,int delivery_mode,int trig_mode,int vector)3678 static void svm_deliver_interrupt(struct kvm_lapic *apic,  int delivery_mode,
3679 				  int trig_mode, int vector)
3680 {
3681 	kvm_lapic_set_irr(vector, apic);
3682 
3683 	/*
3684 	 * Pairs with the smp_mb_*() after setting vcpu->guest_mode in
3685 	 * vcpu_enter_guest() to ensure the write to the vIRR is ordered before
3686 	 * the read of guest_mode.  This guarantees that either VMRUN will see
3687 	 * and process the new vIRR entry, or that svm_complete_interrupt_delivery
3688 	 * will signal the doorbell if the CPU has already entered the guest.
3689 	 */
3690 	smp_mb__after_atomic();
3691 	svm_complete_interrupt_delivery(apic->vcpu, delivery_mode, trig_mode, vector);
3692 }
3693 
svm_update_cr8_intercept(struct kvm_vcpu * vcpu,int tpr,int irr)3694 static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3695 {
3696 	struct vcpu_svm *svm = to_svm(vcpu);
3697 
3698 	/*
3699 	 * SEV-ES guests must always keep the CR intercepts cleared. CR
3700 	 * tracking is done using the CR write traps.
3701 	 */
3702 	if (sev_es_guest(vcpu->kvm))
3703 		return;
3704 
3705 	if (nested_svm_virtualize_tpr(vcpu))
3706 		return;
3707 
3708 	svm_clr_intercept(svm, INTERCEPT_CR8_WRITE);
3709 
3710 	if (irr == -1)
3711 		return;
3712 
3713 	if (tpr >= irr)
3714 		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
3715 }
3716 
svm_get_nmi_mask(struct kvm_vcpu * vcpu)3717 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3718 {
3719 	struct vcpu_svm *svm = to_svm(vcpu);
3720 
3721 	if (is_vnmi_enabled(svm))
3722 		return svm->vmcb->control.int_ctl & V_NMI_BLOCKING_MASK;
3723 	else
3724 		return svm->nmi_masked;
3725 }
3726 
svm_set_nmi_mask(struct kvm_vcpu * vcpu,bool masked)3727 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3728 {
3729 	struct vcpu_svm *svm = to_svm(vcpu);
3730 
3731 	if (is_vnmi_enabled(svm)) {
3732 		if (masked)
3733 			svm->vmcb->control.int_ctl |= V_NMI_BLOCKING_MASK;
3734 		else
3735 			svm->vmcb->control.int_ctl &= ~V_NMI_BLOCKING_MASK;
3736 
3737 	} else {
3738 		svm->nmi_masked = masked;
3739 		if (masked)
3740 			svm_set_iret_intercept(svm);
3741 		else
3742 			svm_clr_iret_intercept(svm);
3743 	}
3744 }
3745 
svm_nmi_blocked(struct kvm_vcpu * vcpu)3746 bool svm_nmi_blocked(struct kvm_vcpu *vcpu)
3747 {
3748 	struct vcpu_svm *svm = to_svm(vcpu);
3749 	struct vmcb *vmcb = svm->vmcb;
3750 
3751 	if (!gif_set(svm))
3752 		return true;
3753 
3754 	if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3755 		return false;
3756 
3757 	if (svm_get_nmi_mask(vcpu))
3758 		return true;
3759 
3760 	return vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK;
3761 }
3762 
svm_nmi_allowed(struct kvm_vcpu * vcpu,bool for_injection)3763 static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3764 {
3765 	struct vcpu_svm *svm = to_svm(vcpu);
3766 	if (svm->nested.nested_run_pending)
3767 		return -EBUSY;
3768 
3769 	if (svm_nmi_blocked(vcpu))
3770 		return 0;
3771 
3772 	/* An NMI must not be injected into L2 if it's supposed to VM-Exit.  */
3773 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3774 		return -EBUSY;
3775 	return 1;
3776 }
3777 
svm_interrupt_blocked(struct kvm_vcpu * vcpu)3778 bool svm_interrupt_blocked(struct kvm_vcpu *vcpu)
3779 {
3780 	struct vcpu_svm *svm = to_svm(vcpu);
3781 	struct vmcb *vmcb = svm->vmcb;
3782 
3783 	if (!gif_set(svm))
3784 		return true;
3785 
3786 	if (is_guest_mode(vcpu)) {
3787 		/* As long as interrupts are being delivered...  */
3788 		if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK)
3789 		    ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF)
3790 		    : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
3791 			return true;
3792 
3793 		/* ... vmexits aren't blocked by the interrupt shadow  */
3794 		if (nested_exit_on_intr(svm))
3795 			return false;
3796 	} else {
3797 		if (!svm_get_if_flag(vcpu))
3798 			return true;
3799 	}
3800 
3801 	return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK);
3802 }
3803 
svm_interrupt_allowed(struct kvm_vcpu * vcpu,bool for_injection)3804 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3805 {
3806 	struct vcpu_svm *svm = to_svm(vcpu);
3807 
3808 	if (svm->nested.nested_run_pending)
3809 		return -EBUSY;
3810 
3811 	if (svm_interrupt_blocked(vcpu))
3812 		return 0;
3813 
3814 	/*
3815 	 * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
3816 	 * e.g. if the IRQ arrived asynchronously after checking nested events.
3817 	 */
3818 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm))
3819 		return -EBUSY;
3820 
3821 	return 1;
3822 }
3823 
svm_enable_irq_window(struct kvm_vcpu * vcpu)3824 static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
3825 {
3826 	struct vcpu_svm *svm = to_svm(vcpu);
3827 
3828 	/*
3829 	 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3830 	 * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3831 	 * get that intercept, this function will be called again though and
3832 	 * we'll get the vintr intercept. However, if the vGIF feature is
3833 	 * enabled, the STGI interception will not occur. Enable the irq
3834 	 * window under the assumption that the hardware will set the GIF.
3835 	 */
3836 	if (vgif || gif_set(svm)) {
3837 		/*
3838 		 * IRQ window is not needed when AVIC is enabled,
3839 		 * unless we have pending ExtINT since it cannot be injected
3840 		 * via AVIC. In such case, KVM needs to temporarily disable AVIC,
3841 		 * and fallback to injecting IRQ via V_IRQ.
3842 		 *
3843 		 * If running nested, AVIC is already locally inhibited
3844 		 * on this vCPU, therefore there is no need to request
3845 		 * the VM wide AVIC inhibition.
3846 		 */
3847 		if (!is_guest_mode(vcpu))
3848 			kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
3849 
3850 		svm_set_vintr(svm);
3851 	}
3852 }
3853 
svm_enable_nmi_window(struct kvm_vcpu * vcpu)3854 static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
3855 {
3856 	struct vcpu_svm *svm = to_svm(vcpu);
3857 
3858 	/*
3859 	 * KVM should never request an NMI window when vNMI is enabled, as KVM
3860 	 * allows at most one to-be-injected NMI and one pending NMI, i.e. if
3861 	 * two NMIs arrive simultaneously, KVM will inject one and set
3862 	 * V_NMI_PENDING for the other.  WARN, but continue with the standard
3863 	 * single-step approach to try and salvage the pending NMI.
3864 	 */
3865 	WARN_ON_ONCE(is_vnmi_enabled(svm));
3866 
3867 	if (svm_get_nmi_mask(vcpu) && !svm->awaiting_iret_completion)
3868 		return; /* IRET will cause a vm exit */
3869 
3870 	/*
3871 	 * SEV-ES guests are responsible for signaling when a vCPU is ready to
3872 	 * receive a new NMI, as SEV-ES guests can't be single-stepped, i.e.
3873 	 * KVM can't intercept and single-step IRET to detect when NMIs are
3874 	 * unblocked (architecturally speaking).  See SVM_VMGEXIT_NMI_COMPLETE.
3875 	 *
3876 	 * Note, GIF is guaranteed to be '1' for SEV-ES guests as hardware
3877 	 * ignores SEV-ES guest writes to EFER.SVME *and* CLGI/STGI are not
3878 	 * supported NAEs in the GHCB protocol.
3879 	 */
3880 	if (sev_es_guest(vcpu->kvm))
3881 		return;
3882 
3883 	if (!gif_set(svm)) {
3884 		if (vgif)
3885 			svm_set_intercept(svm, INTERCEPT_STGI);
3886 		return; /* STGI will cause a vm exit */
3887 	}
3888 
3889 	/*
3890 	 * Something prevents NMI from been injected. Single step over possible
3891 	 * problem (IRET or exception injection or interrupt shadow)
3892 	 */
3893 	svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
3894 	svm->nmi_singlestep = true;
3895 	svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3896 }
3897 
svm_flush_tlb_asid(struct kvm_vcpu * vcpu)3898 static void svm_flush_tlb_asid(struct kvm_vcpu *vcpu)
3899 {
3900 	struct vcpu_svm *svm = to_svm(vcpu);
3901 
3902 	/*
3903 	 * Unlike VMX, SVM doesn't provide a way to flush only NPT TLB entries.
3904 	 * A TLB flush for the current ASID flushes both "host" and "guest" TLB
3905 	 * entries, and thus is a superset of Hyper-V's fine grained flushing.
3906 	 */
3907 	kvm_hv_vcpu_purge_flush_tlb(vcpu);
3908 
3909 	/*
3910 	 * Flush only the current ASID even if the TLB flush was invoked via
3911 	 * kvm_flush_remote_tlbs().  Although flushing remote TLBs requires all
3912 	 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and
3913 	 * unconditionally does a TLB flush on both nested VM-Enter and nested
3914 	 * VM-Exit (via kvm_mmu_reset_context()).
3915 	 */
3916 	if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3917 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3918 	else
3919 		svm->current_vmcb->asid_generation--;
3920 }
3921 
svm_flush_tlb_current(struct kvm_vcpu * vcpu)3922 static void svm_flush_tlb_current(struct kvm_vcpu *vcpu)
3923 {
3924 	hpa_t root_tdp = vcpu->arch.mmu->root.hpa;
3925 
3926 	/*
3927 	 * When running on Hyper-V with EnlightenedNptTlb enabled, explicitly
3928 	 * flush the NPT mappings via hypercall as flushing the ASID only
3929 	 * affects virtual to physical mappings, it does not invalidate guest
3930 	 * physical to host physical mappings.
3931 	 */
3932 	if (svm_hv_is_enlightened_tlb_enabled(vcpu) && VALID_PAGE(root_tdp))
3933 		hyperv_flush_guest_mapping(root_tdp);
3934 
3935 	svm_flush_tlb_asid(vcpu);
3936 }
3937 
svm_flush_tlb_all(struct kvm_vcpu * vcpu)3938 static void svm_flush_tlb_all(struct kvm_vcpu *vcpu)
3939 {
3940 	/*
3941 	 * When running on Hyper-V with EnlightenedNptTlb enabled, remote TLB
3942 	 * flushes should be routed to hv_flush_remote_tlbs() without requesting
3943 	 * a "regular" remote flush.  Reaching this point means either there's
3944 	 * a KVM bug or a prior hv_flush_remote_tlbs() call failed, both of
3945 	 * which might be fatal to the guest.  Yell, but try to recover.
3946 	 */
3947 	if (WARN_ON_ONCE(svm_hv_is_enlightened_tlb_enabled(vcpu)))
3948 		hv_flush_remote_tlbs(vcpu->kvm);
3949 
3950 	svm_flush_tlb_asid(vcpu);
3951 }
3952 
svm_flush_tlb_gva(struct kvm_vcpu * vcpu,gva_t gva)3953 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
3954 {
3955 	struct vcpu_svm *svm = to_svm(vcpu);
3956 
3957 	invlpga(gva, svm->vmcb->control.asid);
3958 }
3959 
sync_cr8_to_lapic(struct kvm_vcpu * vcpu)3960 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3961 {
3962 	struct vcpu_svm *svm = to_svm(vcpu);
3963 
3964 	if (nested_svm_virtualize_tpr(vcpu))
3965 		return;
3966 
3967 	if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) {
3968 		int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3969 		kvm_set_cr8(vcpu, cr8);
3970 	}
3971 }
3972 
sync_lapic_to_cr8(struct kvm_vcpu * vcpu)3973 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
3974 {
3975 	struct vcpu_svm *svm = to_svm(vcpu);
3976 	u64 cr8;
3977 
3978 	if (nested_svm_virtualize_tpr(vcpu) ||
3979 	    kvm_vcpu_apicv_active(vcpu))
3980 		return;
3981 
3982 	cr8 = kvm_get_cr8(vcpu);
3983 	svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
3984 	svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
3985 }
3986 
svm_complete_soft_interrupt(struct kvm_vcpu * vcpu,u8 vector,int type)3987 static void svm_complete_soft_interrupt(struct kvm_vcpu *vcpu, u8 vector,
3988 					int type)
3989 {
3990 	bool is_exception = (type == SVM_EXITINTINFO_TYPE_EXEPT);
3991 	bool is_soft = (type == SVM_EXITINTINFO_TYPE_SOFT);
3992 	struct vcpu_svm *svm = to_svm(vcpu);
3993 
3994 	/*
3995 	 * If NRIPS is enabled, KVM must snapshot the pre-VMRUN next_rip that's
3996 	 * associated with the original soft exception/interrupt.  next_rip is
3997 	 * cleared on all exits that can occur while vectoring an event, so KVM
3998 	 * needs to manually set next_rip for re-injection.  Unlike the !nrips
3999 	 * case below, this needs to be done if and only if KVM is re-injecting
4000 	 * the same event, i.e. if the event is a soft exception/interrupt,
4001 	 * otherwise next_rip is unused on VMRUN.
4002 	 */
4003 	if (nrips && (is_soft || (is_exception && kvm_exception_is_soft(vector))) &&
4004 	    kvm_is_linear_rip(vcpu, svm->soft_int_old_rip + svm->soft_int_csbase))
4005 		svm->vmcb->control.next_rip = svm->soft_int_next_rip;
4006 	/*
4007 	 * If NRIPS isn't enabled, KVM must manually advance RIP prior to
4008 	 * injecting the soft exception/interrupt.  That advancement needs to
4009 	 * be unwound if vectoring didn't complete.  Note, the new event may
4010 	 * not be the injected event, e.g. if KVM injected an INTn, the INTn
4011 	 * hit a #NP in the guest, and the #NP encountered a #PF, the #NP will
4012 	 * be the reported vectored event, but RIP still needs to be unwound.
4013 	 */
4014 	else if (!nrips && (is_soft || is_exception) &&
4015 		 kvm_is_linear_rip(vcpu, svm->soft_int_next_rip + svm->soft_int_csbase))
4016 		kvm_rip_write(vcpu, svm->soft_int_old_rip);
4017 }
4018 
svm_complete_interrupts(struct kvm_vcpu * vcpu)4019 static void svm_complete_interrupts(struct kvm_vcpu *vcpu)
4020 {
4021 	struct vcpu_svm *svm = to_svm(vcpu);
4022 	u8 vector;
4023 	int type;
4024 	u32 exitintinfo = svm->vmcb->control.exit_int_info;
4025 	bool nmi_l1_to_l2 = svm->nmi_l1_to_l2;
4026 	bool soft_int_injected = svm->soft_int_injected;
4027 
4028 	svm->nmi_l1_to_l2 = false;
4029 	svm->soft_int_injected = false;
4030 
4031 	/*
4032 	 * If we've made progress since setting awaiting_iret_completion, we've
4033 	 * executed an IRET and can allow NMI injection.
4034 	 */
4035 	if (svm->awaiting_iret_completion &&
4036 	    kvm_rip_read(vcpu) != svm->nmi_iret_rip) {
4037 		svm->awaiting_iret_completion = false;
4038 		svm->nmi_masked = false;
4039 		kvm_make_request(KVM_REQ_EVENT, vcpu);
4040 	}
4041 
4042 	vcpu->arch.nmi_injected = false;
4043 	kvm_clear_exception_queue(vcpu);
4044 	kvm_clear_interrupt_queue(vcpu);
4045 
4046 	if (!(exitintinfo & SVM_EXITINTINFO_VALID))
4047 		return;
4048 
4049 	kvm_make_request(KVM_REQ_EVENT, vcpu);
4050 
4051 	vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
4052 	type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
4053 
4054 	if (soft_int_injected)
4055 		svm_complete_soft_interrupt(vcpu, vector, type);
4056 
4057 	switch (type) {
4058 	case SVM_EXITINTINFO_TYPE_NMI:
4059 		vcpu->arch.nmi_injected = true;
4060 		svm->nmi_l1_to_l2 = nmi_l1_to_l2;
4061 		break;
4062 	case SVM_EXITINTINFO_TYPE_EXEPT:
4063 		/*
4064 		 * Never re-inject a #VC exception.
4065 		 */
4066 		if (vector == X86_TRAP_VC)
4067 			break;
4068 
4069 		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
4070 			u32 err = svm->vmcb->control.exit_int_info_err;
4071 			kvm_requeue_exception_e(vcpu, vector, err);
4072 
4073 		} else
4074 			kvm_requeue_exception(vcpu, vector);
4075 		break;
4076 	case SVM_EXITINTINFO_TYPE_INTR:
4077 		kvm_queue_interrupt(vcpu, vector, false);
4078 		break;
4079 	case SVM_EXITINTINFO_TYPE_SOFT:
4080 		kvm_queue_interrupt(vcpu, vector, true);
4081 		break;
4082 	default:
4083 		break;
4084 	}
4085 
4086 }
4087 
svm_cancel_injection(struct kvm_vcpu * vcpu)4088 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
4089 {
4090 	struct vcpu_svm *svm = to_svm(vcpu);
4091 	struct vmcb_control_area *control = &svm->vmcb->control;
4092 
4093 	control->exit_int_info = control->event_inj;
4094 	control->exit_int_info_err = control->event_inj_err;
4095 	control->event_inj = 0;
4096 	svm_complete_interrupts(vcpu);
4097 }
4098 
svm_vcpu_pre_run(struct kvm_vcpu * vcpu)4099 static int svm_vcpu_pre_run(struct kvm_vcpu *vcpu)
4100 {
4101 	return 1;
4102 }
4103 
svm_exit_handlers_fastpath(struct kvm_vcpu * vcpu)4104 static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
4105 {
4106 	if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR &&
4107 	    to_svm(vcpu)->vmcb->control.exit_info_1)
4108 		return handle_fastpath_set_msr_irqoff(vcpu);
4109 
4110 	return EXIT_FASTPATH_NONE;
4111 }
4112 
svm_vcpu_enter_exit(struct kvm_vcpu * vcpu,bool spec_ctrl_intercepted)4113 static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted)
4114 {
4115 	struct vcpu_svm *svm = to_svm(vcpu);
4116 
4117 	guest_state_enter_irqoff();
4118 
4119 	amd_clear_divider();
4120 
4121 	if (sev_es_guest(vcpu->kvm))
4122 		__svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted);
4123 	else
4124 		__svm_vcpu_run(svm, spec_ctrl_intercepted);
4125 
4126 	guest_state_exit_irqoff();
4127 }
4128 
svm_vcpu_run(struct kvm_vcpu * vcpu)4129 static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)
4130 {
4131 	struct vcpu_svm *svm = to_svm(vcpu);
4132 	bool spec_ctrl_intercepted = msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL);
4133 
4134 	trace_kvm_entry(vcpu);
4135 
4136 	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4137 	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4138 	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4139 
4140 	/*
4141 	 * Disable singlestep if we're injecting an interrupt/exception.
4142 	 * We don't want our modified rflags to be pushed on the stack where
4143 	 * we might not be able to easily reset them if we disabled NMI
4144 	 * singlestep later.
4145 	 */
4146 	if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
4147 		/*
4148 		 * Event injection happens before external interrupts cause a
4149 		 * vmexit and interrupts are disabled here, so smp_send_reschedule
4150 		 * is enough to force an immediate vmexit.
4151 		 */
4152 		disable_nmi_singlestep(svm);
4153 		smp_send_reschedule(vcpu->cpu);
4154 	}
4155 
4156 	pre_svm_run(vcpu);
4157 
4158 	sync_lapic_to_cr8(vcpu);
4159 
4160 	if (unlikely(svm->asid != svm->vmcb->control.asid)) {
4161 		svm->vmcb->control.asid = svm->asid;
4162 		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
4163 	}
4164 	svm->vmcb->save.cr2 = vcpu->arch.cr2;
4165 
4166 	svm_hv_update_vp_id(svm->vmcb, vcpu);
4167 
4168 	/*
4169 	 * Run with all-zero DR6 unless needed, so that we can get the exact cause
4170 	 * of a #DB.
4171 	 */
4172 	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))
4173 		svm_set_dr6(svm, vcpu->arch.dr6);
4174 	else
4175 		svm_set_dr6(svm, DR6_ACTIVE_LOW);
4176 
4177 	clgi();
4178 	kvm_load_guest_xsave_state(vcpu);
4179 
4180 	kvm_wait_lapic_expire(vcpu);
4181 
4182 	/*
4183 	 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
4184 	 * it's non-zero. Since vmentry is serialising on affected CPUs, there
4185 	 * is no need to worry about the conditional branch over the wrmsr
4186 	 * being speculatively taken.
4187 	 */
4188 	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
4189 		x86_spec_ctrl_set_guest(svm->virt_spec_ctrl);
4190 
4191 	svm_vcpu_enter_exit(vcpu, spec_ctrl_intercepted);
4192 
4193 	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
4194 		x86_spec_ctrl_restore_host(svm->virt_spec_ctrl);
4195 
4196 	if (!sev_es_guest(vcpu->kvm)) {
4197 		vcpu->arch.cr2 = svm->vmcb->save.cr2;
4198 		vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
4199 		vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
4200 		vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
4201 	}
4202 	vcpu->arch.regs_dirty = 0;
4203 
4204 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4205 		kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
4206 
4207 	kvm_load_host_xsave_state(vcpu);
4208 	stgi();
4209 
4210 	/* Any pending NMI will happen here */
4211 
4212 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4213 		kvm_after_interrupt(vcpu);
4214 
4215 	sync_cr8_to_lapic(vcpu);
4216 
4217 	svm->next_rip = 0;
4218 	if (is_guest_mode(vcpu)) {
4219 		nested_sync_control_from_vmcb02(svm);
4220 
4221 		/* Track VMRUNs that have made past consistency checking */
4222 		if (svm->nested.nested_run_pending &&
4223 		    svm->vmcb->control.exit_code != SVM_EXIT_ERR)
4224                         ++vcpu->stat.nested_run;
4225 
4226 		svm->nested.nested_run_pending = 0;
4227 	}
4228 
4229 	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
4230 	vmcb_mark_all_clean(svm->vmcb);
4231 
4232 	/* if exit due to PF check for async PF */
4233 	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
4234 		vcpu->arch.apf.host_apf_flags =
4235 			kvm_read_and_reset_apf_flags();
4236 
4237 	vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET;
4238 
4239 	/*
4240 	 * We need to handle MC intercepts here before the vcpu has a chance to
4241 	 * change the physical cpu
4242 	 */
4243 	if (unlikely(svm->vmcb->control.exit_code ==
4244 		     SVM_EXIT_EXCP_BASE + MC_VECTOR))
4245 		svm_handle_mce(vcpu);
4246 
4247 	trace_kvm_exit(vcpu, KVM_ISA_SVM);
4248 
4249 	svm_complete_interrupts(vcpu);
4250 
4251 	if (is_guest_mode(vcpu))
4252 		return EXIT_FASTPATH_NONE;
4253 
4254 	return svm_exit_handlers_fastpath(vcpu);
4255 }
4256 
svm_load_mmu_pgd(struct kvm_vcpu * vcpu,hpa_t root_hpa,int root_level)4257 static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
4258 			     int root_level)
4259 {
4260 	struct vcpu_svm *svm = to_svm(vcpu);
4261 	unsigned long cr3;
4262 
4263 	if (npt_enabled) {
4264 		svm->vmcb->control.nested_cr3 = __sme_set(root_hpa);
4265 		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
4266 
4267 		hv_track_root_tdp(vcpu, root_hpa);
4268 
4269 		cr3 = vcpu->arch.cr3;
4270 	} else if (root_level >= PT64_ROOT_4LEVEL) {
4271 		cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu);
4272 	} else {
4273 		/* PCID in the guest should be impossible with a 32-bit MMU. */
4274 		WARN_ON_ONCE(kvm_get_active_pcid(vcpu));
4275 		cr3 = root_hpa;
4276 	}
4277 
4278 	svm->vmcb->save.cr3 = cr3;
4279 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
4280 }
4281 
4282 static void
svm_patch_hypercall(struct kvm_vcpu * vcpu,unsigned char * hypercall)4283 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4284 {
4285 	/*
4286 	 * Patch in the VMMCALL instruction:
4287 	 */
4288 	hypercall[0] = 0x0f;
4289 	hypercall[1] = 0x01;
4290 	hypercall[2] = 0xd9;
4291 }
4292 
4293 /*
4294  * The kvm parameter can be NULL (module initialization, or invocation before
4295  * VM creation). Be sure to check the kvm parameter before using it.
4296  */
svm_has_emulated_msr(struct kvm * kvm,u32 index)4297 static bool svm_has_emulated_msr(struct kvm *kvm, u32 index)
4298 {
4299 	switch (index) {
4300 	case MSR_IA32_MCG_EXT_CTL:
4301 	case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR:
4302 		return false;
4303 	case MSR_IA32_SMBASE:
4304 		if (!IS_ENABLED(CONFIG_KVM_SMM))
4305 			return false;
4306 		/* SEV-ES guests do not support SMM, so report false */
4307 		if (kvm && sev_es_guest(kvm))
4308 			return false;
4309 		break;
4310 	default:
4311 		break;
4312 	}
4313 
4314 	return true;
4315 }
4316 
svm_vcpu_after_set_cpuid(struct kvm_vcpu * vcpu)4317 static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
4318 {
4319 	struct vcpu_svm *svm = to_svm(vcpu);
4320 
4321 	/*
4322 	 * SVM doesn't provide a way to disable just XSAVES in the guest, KVM
4323 	 * can only disable all variants of by disallowing CR4.OSXSAVE from
4324 	 * being set.  As a result, if the host has XSAVE and XSAVES, and the
4325 	 * guest has XSAVE enabled, the guest can execute XSAVES without
4326 	 * faulting.  Treat XSAVES as enabled in this case regardless of
4327 	 * whether it's advertised to the guest so that KVM context switches
4328 	 * XSS on VM-Enter/VM-Exit.  Failure to do so would effectively give
4329 	 * the guest read/write access to the host's XSS.
4330 	 */
4331 	if (boot_cpu_has(X86_FEATURE_XSAVE) &&
4332 	    boot_cpu_has(X86_FEATURE_XSAVES) &&
4333 	    guest_cpuid_has(vcpu, X86_FEATURE_XSAVE))
4334 		kvm_governed_feature_set(vcpu, X86_FEATURE_XSAVES);
4335 
4336 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_NRIPS);
4337 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_TSCRATEMSR);
4338 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_LBRV);
4339 
4340 	/*
4341 	 * Intercept VMLOAD if the vCPU mode is Intel in order to emulate that
4342 	 * VMLOAD drops bits 63:32 of SYSENTER (ignoring the fact that exposing
4343 	 * SVM on Intel is bonkers and extremely unlikely to work).
4344 	 */
4345 	if (!guest_cpuid_is_intel(vcpu))
4346 		kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_V_VMSAVE_VMLOAD);
4347 
4348 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_PAUSEFILTER);
4349 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_PFTHRESHOLD);
4350 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_VGIF);
4351 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_VNMI);
4352 
4353 	svm_recalc_instruction_intercepts(vcpu, svm);
4354 
4355 	if (boot_cpu_has(X86_FEATURE_IBPB))
4356 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0,
4357 				     !!guest_has_pred_cmd_msr(vcpu));
4358 
4359 	if (boot_cpu_has(X86_FEATURE_FLUSH_L1D))
4360 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_FLUSH_CMD, 0,
4361 				     !!guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D));
4362 
4363 	if (sev_guest(vcpu->kvm))
4364 		sev_vcpu_after_set_cpuid(svm);
4365 
4366 	init_vmcb_after_set_cpuid(vcpu);
4367 }
4368 
svm_has_wbinvd_exit(void)4369 static bool svm_has_wbinvd_exit(void)
4370 {
4371 	return true;
4372 }
4373 
4374 #define PRE_EX(exit)  { .exit_code = (exit), \
4375 			.stage = X86_ICPT_PRE_EXCEPT, }
4376 #define POST_EX(exit) { .exit_code = (exit), \
4377 			.stage = X86_ICPT_POST_EXCEPT, }
4378 #define POST_MEM(exit) { .exit_code = (exit), \
4379 			.stage = X86_ICPT_POST_MEMACCESS, }
4380 
4381 static const struct __x86_intercept {
4382 	u32 exit_code;
4383 	enum x86_intercept_stage stage;
4384 } x86_intercept_map[] = {
4385 	[x86_intercept_cr_read]		= POST_EX(SVM_EXIT_READ_CR0),
4386 	[x86_intercept_cr_write]	= POST_EX(SVM_EXIT_WRITE_CR0),
4387 	[x86_intercept_clts]		= POST_EX(SVM_EXIT_WRITE_CR0),
4388 	[x86_intercept_lmsw]		= POST_EX(SVM_EXIT_WRITE_CR0),
4389 	[x86_intercept_smsw]		= POST_EX(SVM_EXIT_READ_CR0),
4390 	[x86_intercept_dr_read]		= POST_EX(SVM_EXIT_READ_DR0),
4391 	[x86_intercept_dr_write]	= POST_EX(SVM_EXIT_WRITE_DR0),
4392 	[x86_intercept_sldt]		= POST_EX(SVM_EXIT_LDTR_READ),
4393 	[x86_intercept_str]		= POST_EX(SVM_EXIT_TR_READ),
4394 	[x86_intercept_lldt]		= POST_EX(SVM_EXIT_LDTR_WRITE),
4395 	[x86_intercept_ltr]		= POST_EX(SVM_EXIT_TR_WRITE),
4396 	[x86_intercept_sgdt]		= POST_EX(SVM_EXIT_GDTR_READ),
4397 	[x86_intercept_sidt]		= POST_EX(SVM_EXIT_IDTR_READ),
4398 	[x86_intercept_lgdt]		= POST_EX(SVM_EXIT_GDTR_WRITE),
4399 	[x86_intercept_lidt]		= POST_EX(SVM_EXIT_IDTR_WRITE),
4400 	[x86_intercept_vmrun]		= POST_EX(SVM_EXIT_VMRUN),
4401 	[x86_intercept_vmmcall]		= POST_EX(SVM_EXIT_VMMCALL),
4402 	[x86_intercept_vmload]		= POST_EX(SVM_EXIT_VMLOAD),
4403 	[x86_intercept_vmsave]		= POST_EX(SVM_EXIT_VMSAVE),
4404 	[x86_intercept_stgi]		= POST_EX(SVM_EXIT_STGI),
4405 	[x86_intercept_clgi]		= POST_EX(SVM_EXIT_CLGI),
4406 	[x86_intercept_skinit]		= POST_EX(SVM_EXIT_SKINIT),
4407 	[x86_intercept_invlpga]		= POST_EX(SVM_EXIT_INVLPGA),
4408 	[x86_intercept_rdtscp]		= POST_EX(SVM_EXIT_RDTSCP),
4409 	[x86_intercept_monitor]		= POST_MEM(SVM_EXIT_MONITOR),
4410 	[x86_intercept_mwait]		= POST_EX(SVM_EXIT_MWAIT),
4411 	[x86_intercept_invlpg]		= POST_EX(SVM_EXIT_INVLPG),
4412 	[x86_intercept_invd]		= POST_EX(SVM_EXIT_INVD),
4413 	[x86_intercept_wbinvd]		= POST_EX(SVM_EXIT_WBINVD),
4414 	[x86_intercept_wrmsr]		= POST_EX(SVM_EXIT_MSR),
4415 	[x86_intercept_rdtsc]		= POST_EX(SVM_EXIT_RDTSC),
4416 	[x86_intercept_rdmsr]		= POST_EX(SVM_EXIT_MSR),
4417 	[x86_intercept_rdpmc]		= POST_EX(SVM_EXIT_RDPMC),
4418 	[x86_intercept_cpuid]		= PRE_EX(SVM_EXIT_CPUID),
4419 	[x86_intercept_rsm]		= PRE_EX(SVM_EXIT_RSM),
4420 	[x86_intercept_pause]		= PRE_EX(SVM_EXIT_PAUSE),
4421 	[x86_intercept_pushf]		= PRE_EX(SVM_EXIT_PUSHF),
4422 	[x86_intercept_popf]		= PRE_EX(SVM_EXIT_POPF),
4423 	[x86_intercept_intn]		= PRE_EX(SVM_EXIT_SWINT),
4424 	[x86_intercept_iret]		= PRE_EX(SVM_EXIT_IRET),
4425 	[x86_intercept_icebp]		= PRE_EX(SVM_EXIT_ICEBP),
4426 	[x86_intercept_hlt]		= POST_EX(SVM_EXIT_HLT),
4427 	[x86_intercept_in]		= POST_EX(SVM_EXIT_IOIO),
4428 	[x86_intercept_ins]		= POST_EX(SVM_EXIT_IOIO),
4429 	[x86_intercept_out]		= POST_EX(SVM_EXIT_IOIO),
4430 	[x86_intercept_outs]		= POST_EX(SVM_EXIT_IOIO),
4431 	[x86_intercept_xsetbv]		= PRE_EX(SVM_EXIT_XSETBV),
4432 };
4433 
4434 #undef PRE_EX
4435 #undef POST_EX
4436 #undef POST_MEM
4437 
svm_check_intercept(struct kvm_vcpu * vcpu,struct x86_instruction_info * info,enum x86_intercept_stage stage,struct x86_exception * exception)4438 static int svm_check_intercept(struct kvm_vcpu *vcpu,
4439 			       struct x86_instruction_info *info,
4440 			       enum x86_intercept_stage stage,
4441 			       struct x86_exception *exception)
4442 {
4443 	struct vcpu_svm *svm = to_svm(vcpu);
4444 	int vmexit, ret = X86EMUL_CONTINUE;
4445 	struct __x86_intercept icpt_info;
4446 	struct vmcb *vmcb = svm->vmcb;
4447 
4448 	if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4449 		goto out;
4450 
4451 	icpt_info = x86_intercept_map[info->intercept];
4452 
4453 	if (stage != icpt_info.stage)
4454 		goto out;
4455 
4456 	switch (icpt_info.exit_code) {
4457 	case SVM_EXIT_READ_CR0:
4458 		if (info->intercept == x86_intercept_cr_read)
4459 			icpt_info.exit_code += info->modrm_reg;
4460 		break;
4461 	case SVM_EXIT_WRITE_CR0: {
4462 		unsigned long cr0, val;
4463 
4464 		if (info->intercept == x86_intercept_cr_write)
4465 			icpt_info.exit_code += info->modrm_reg;
4466 
4467 		if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
4468 		    info->intercept == x86_intercept_clts)
4469 			break;
4470 
4471 		if (!(vmcb12_is_intercept(&svm->nested.ctl,
4472 					INTERCEPT_SELECTIVE_CR0)))
4473 			break;
4474 
4475 		cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4476 		val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
4477 
4478 		if (info->intercept == x86_intercept_lmsw) {
4479 			cr0 &= 0xfUL;
4480 			val &= 0xfUL;
4481 			/* lmsw can't clear PE - catch this here */
4482 			if (cr0 & X86_CR0_PE)
4483 				val |= X86_CR0_PE;
4484 		}
4485 
4486 		if (cr0 ^ val)
4487 			icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4488 
4489 		break;
4490 	}
4491 	case SVM_EXIT_READ_DR0:
4492 	case SVM_EXIT_WRITE_DR0:
4493 		icpt_info.exit_code += info->modrm_reg;
4494 		break;
4495 	case SVM_EXIT_MSR:
4496 		if (info->intercept == x86_intercept_wrmsr)
4497 			vmcb->control.exit_info_1 = 1;
4498 		else
4499 			vmcb->control.exit_info_1 = 0;
4500 		break;
4501 	case SVM_EXIT_PAUSE:
4502 		/*
4503 		 * We get this for NOP only, but pause
4504 		 * is rep not, check this here
4505 		 */
4506 		if (info->rep_prefix != REPE_PREFIX)
4507 			goto out;
4508 		break;
4509 	case SVM_EXIT_IOIO: {
4510 		u64 exit_info;
4511 		u32 bytes;
4512 
4513 		if (info->intercept == x86_intercept_in ||
4514 		    info->intercept == x86_intercept_ins) {
4515 			exit_info = ((info->src_val & 0xffff) << 16) |
4516 				SVM_IOIO_TYPE_MASK;
4517 			bytes = info->dst_bytes;
4518 		} else {
4519 			exit_info = (info->dst_val & 0xffff) << 16;
4520 			bytes = info->src_bytes;
4521 		}
4522 
4523 		if (info->intercept == x86_intercept_outs ||
4524 		    info->intercept == x86_intercept_ins)
4525 			exit_info |= SVM_IOIO_STR_MASK;
4526 
4527 		if (info->rep_prefix)
4528 			exit_info |= SVM_IOIO_REP_MASK;
4529 
4530 		bytes = min(bytes, 4u);
4531 
4532 		exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4533 
4534 		exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4535 
4536 		vmcb->control.exit_info_1 = exit_info;
4537 		vmcb->control.exit_info_2 = info->next_rip;
4538 
4539 		break;
4540 	}
4541 	default:
4542 		break;
4543 	}
4544 
4545 	/* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4546 	if (static_cpu_has(X86_FEATURE_NRIPS))
4547 		vmcb->control.next_rip  = info->next_rip;
4548 	vmcb->control.exit_code = icpt_info.exit_code;
4549 	vmexit = nested_svm_exit_handled(svm);
4550 
4551 	ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4552 					   : X86EMUL_CONTINUE;
4553 
4554 out:
4555 	return ret;
4556 }
4557 
svm_handle_exit_irqoff(struct kvm_vcpu * vcpu)4558 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
4559 {
4560 	if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_INTR)
4561 		vcpu->arch.at_instruction_boundary = true;
4562 }
4563 
svm_sched_in(struct kvm_vcpu * vcpu,int cpu)4564 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
4565 {
4566 	if (!kvm_pause_in_guest(vcpu->kvm))
4567 		shrink_ple_window(vcpu);
4568 }
4569 
svm_setup_mce(struct kvm_vcpu * vcpu)4570 static void svm_setup_mce(struct kvm_vcpu *vcpu)
4571 {
4572 	/* [63:9] are reserved. */
4573 	vcpu->arch.mcg_cap &= 0x1ff;
4574 }
4575 
4576 #ifdef CONFIG_KVM_SMM
svm_smi_blocked(struct kvm_vcpu * vcpu)4577 bool svm_smi_blocked(struct kvm_vcpu *vcpu)
4578 {
4579 	struct vcpu_svm *svm = to_svm(vcpu);
4580 
4581 	/* Per APM Vol.2 15.22.2 "Response to SMI" */
4582 	if (!gif_set(svm))
4583 		return true;
4584 
4585 	return is_smm(vcpu);
4586 }
4587 
svm_smi_allowed(struct kvm_vcpu * vcpu,bool for_injection)4588 static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
4589 {
4590 	struct vcpu_svm *svm = to_svm(vcpu);
4591 	if (svm->nested.nested_run_pending)
4592 		return -EBUSY;
4593 
4594 	if (svm_smi_blocked(vcpu))
4595 		return 0;
4596 
4597 	/* An SMI must not be injected into L2 if it's supposed to VM-Exit.  */
4598 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm))
4599 		return -EBUSY;
4600 
4601 	return 1;
4602 }
4603 
svm_enter_smm(struct kvm_vcpu * vcpu,union kvm_smram * smram)4604 static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram)
4605 {
4606 	struct vcpu_svm *svm = to_svm(vcpu);
4607 	struct kvm_host_map map_save;
4608 	int ret;
4609 
4610 	if (!is_guest_mode(vcpu))
4611 		return 0;
4612 
4613 	/*
4614 	 * 32-bit SMRAM format doesn't preserve EFER and SVM state.  Userspace is
4615 	 * responsible for ensuring nested SVM and SMIs are mutually exclusive.
4616 	 */
4617 
4618 	if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
4619 		return 1;
4620 
4621 	smram->smram64.svm_guest_flag = 1;
4622 	smram->smram64.svm_guest_vmcb_gpa = svm->nested.vmcb12_gpa;
4623 
4624 	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4625 	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4626 	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4627 
4628 	ret = nested_svm_simple_vmexit(svm, SVM_EXIT_SW);
4629 	if (ret)
4630 		return ret;
4631 
4632 	/*
4633 	 * KVM uses VMCB01 to store L1 host state while L2 runs but
4634 	 * VMCB01 is going to be used during SMM and thus the state will
4635 	 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save
4636 	 * area pointed to by MSR_VM_HSAVE_PA. APM guarantees that the
4637 	 * format of the area is identical to guest save area offsetted
4638 	 * by 0x400 (matches the offset of 'struct vmcb_save_area'
4639 	 * within 'struct vmcb'). Note: HSAVE area may also be used by
4640 	 * L1 hypervisor to save additional host context (e.g. KVM does
4641 	 * that, see svm_prepare_switch_to_guest()) which must be
4642 	 * preserved.
4643 	 */
4644 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save))
4645 		return 1;
4646 
4647 	BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400);
4648 
4649 	svm_copy_vmrun_state(map_save.hva + 0x400,
4650 			     &svm->vmcb01.ptr->save);
4651 
4652 	kvm_vcpu_unmap(vcpu, &map_save, true);
4653 	return 0;
4654 }
4655 
svm_leave_smm(struct kvm_vcpu * vcpu,const union kvm_smram * smram)4656 static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram)
4657 {
4658 	struct vcpu_svm *svm = to_svm(vcpu);
4659 	struct kvm_host_map map, map_save;
4660 	struct vmcb *vmcb12;
4661 	int ret;
4662 
4663 	const struct kvm_smram_state_64 *smram64 = &smram->smram64;
4664 
4665 	if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
4666 		return 0;
4667 
4668 	/* Non-zero if SMI arrived while vCPU was in guest mode. */
4669 	if (!smram64->svm_guest_flag)
4670 		return 0;
4671 
4672 	if (!guest_cpuid_has(vcpu, X86_FEATURE_SVM))
4673 		return 1;
4674 
4675 	if (!(smram64->efer & EFER_SVME))
4676 		return 1;
4677 
4678 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(smram64->svm_guest_vmcb_gpa), &map))
4679 		return 1;
4680 
4681 	ret = 1;
4682 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save))
4683 		goto unmap_map;
4684 
4685 	if (svm_allocate_nested(svm))
4686 		goto unmap_save;
4687 
4688 	/*
4689 	 * Restore L1 host state from L1 HSAVE area as VMCB01 was
4690 	 * used during SMM (see svm_enter_smm())
4691 	 */
4692 
4693 	svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400);
4694 
4695 	/*
4696 	 * Enter the nested guest now
4697 	 */
4698 
4699 	vmcb_mark_all_dirty(svm->vmcb01.ptr);
4700 
4701 	vmcb12 = map.hva;
4702 	nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
4703 	nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
4704 	ret = enter_svm_guest_mode(vcpu, smram64->svm_guest_vmcb_gpa, vmcb12, false);
4705 
4706 	if (ret)
4707 		goto unmap_save;
4708 
4709 	svm->nested.nested_run_pending = 1;
4710 
4711 unmap_save:
4712 	kvm_vcpu_unmap(vcpu, &map_save, true);
4713 unmap_map:
4714 	kvm_vcpu_unmap(vcpu, &map, true);
4715 	return ret;
4716 }
4717 
svm_enable_smi_window(struct kvm_vcpu * vcpu)4718 static void svm_enable_smi_window(struct kvm_vcpu *vcpu)
4719 {
4720 	struct vcpu_svm *svm = to_svm(vcpu);
4721 
4722 	if (!gif_set(svm)) {
4723 		if (vgif)
4724 			svm_set_intercept(svm, INTERCEPT_STGI);
4725 		/* STGI will cause a vm exit */
4726 	} else {
4727 		/* We must be in SMM; RSM will cause a vmexit anyway.  */
4728 	}
4729 }
4730 #endif
4731 
svm_can_emulate_instruction(struct kvm_vcpu * vcpu,int emul_type,void * insn,int insn_len)4732 static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
4733 					void *insn, int insn_len)
4734 {
4735 	bool smep, smap, is_user;
4736 	u64 error_code;
4737 
4738 	/* Emulation is always possible when KVM has access to all guest state. */
4739 	if (!sev_guest(vcpu->kvm))
4740 		return true;
4741 
4742 	/* #UD and #GP should never be intercepted for SEV guests. */
4743 	WARN_ON_ONCE(emul_type & (EMULTYPE_TRAP_UD |
4744 				  EMULTYPE_TRAP_UD_FORCED |
4745 				  EMULTYPE_VMWARE_GP));
4746 
4747 	/*
4748 	 * Emulation is impossible for SEV-ES guests as KVM doesn't have access
4749 	 * to guest register state.
4750 	 */
4751 	if (sev_es_guest(vcpu->kvm))
4752 		return false;
4753 
4754 	/*
4755 	 * Emulation is possible if the instruction is already decoded, e.g.
4756 	 * when completing I/O after returning from userspace.
4757 	 */
4758 	if (emul_type & EMULTYPE_NO_DECODE)
4759 		return true;
4760 
4761 	/*
4762 	 * Emulation is possible for SEV guests if and only if a prefilled
4763 	 * buffer containing the bytes of the intercepted instruction is
4764 	 * available. SEV guest memory is encrypted with a guest specific key
4765 	 * and cannot be decrypted by KVM, i.e. KVM would read cyphertext and
4766 	 * decode garbage.
4767 	 *
4768 	 * If KVM is NOT trying to simply skip an instruction, inject #UD if
4769 	 * KVM reached this point without an instruction buffer.  In practice,
4770 	 * this path should never be hit by a well-behaved guest, e.g. KVM
4771 	 * doesn't intercept #UD or #GP for SEV guests, but this path is still
4772 	 * theoretically reachable, e.g. via unaccelerated fault-like AVIC
4773 	 * access, and needs to be handled by KVM to avoid putting the guest
4774 	 * into an infinite loop.   Injecting #UD is somewhat arbitrary, but
4775 	 * its the least awful option given lack of insight into the guest.
4776 	 *
4777 	 * If KVM is trying to skip an instruction, simply resume the guest.
4778 	 * If a #NPF occurs while the guest is vectoring an INT3/INTO, then KVM
4779 	 * will attempt to re-inject the INT3/INTO and skip the instruction.
4780 	 * In that scenario, retrying the INT3/INTO and hoping the guest will
4781 	 * make forward progress is the only option that has a chance of
4782 	 * success (and in practice it will work the vast majority of the time).
4783 	 */
4784 	if (unlikely(!insn)) {
4785 		if (!(emul_type & EMULTYPE_SKIP))
4786 			kvm_queue_exception(vcpu, UD_VECTOR);
4787 		return false;
4788 	}
4789 
4790 	/*
4791 	 * Emulate for SEV guests if the insn buffer is not empty.  The buffer
4792 	 * will be empty if the DecodeAssist microcode cannot fetch bytes for
4793 	 * the faulting instruction because the code fetch itself faulted, e.g.
4794 	 * the guest attempted to fetch from emulated MMIO or a guest page
4795 	 * table used to translate CS:RIP resides in emulated MMIO.
4796 	 */
4797 	if (likely(insn_len))
4798 		return true;
4799 
4800 	/*
4801 	 * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
4802 	 *
4803 	 * Errata:
4804 	 * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is
4805 	 * possible that CPU microcode implementing DecodeAssist will fail to
4806 	 * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly
4807 	 * be '0'.  This happens because microcode reads CS:RIP using a _data_
4808 	 * loap uop with CPL=0 privileges.  If the load hits a SMAP #PF, ucode
4809 	 * gives up and does not fill the instruction bytes buffer.
4810 	 *
4811 	 * As above, KVM reaches this point iff the VM is an SEV guest, the CPU
4812 	 * supports DecodeAssist, a #NPF was raised, KVM's page fault handler
4813 	 * triggered emulation (e.g. for MMIO), and the CPU returned 0 in the
4814 	 * GuestIntrBytes field of the VMCB.
4815 	 *
4816 	 * This does _not_ mean that the erratum has been encountered, as the
4817 	 * DecodeAssist will also fail if the load for CS:RIP hits a legitimate
4818 	 * #PF, e.g. if the guest attempt to execute from emulated MMIO and
4819 	 * encountered a reserved/not-present #PF.
4820 	 *
4821 	 * To hit the erratum, the following conditions must be true:
4822 	 *    1. CR4.SMAP=1 (obviously).
4823 	 *    2. CR4.SMEP=0 || CPL=3.  If SMEP=1 and CPL<3, the erratum cannot
4824 	 *       have been hit as the guest would have encountered a SMEP
4825 	 *       violation #PF, not a #NPF.
4826 	 *    3. The #NPF is not due to a code fetch, in which case failure to
4827 	 *       retrieve the instruction bytes is legitimate (see abvoe).
4828 	 *
4829 	 * In addition, don't apply the erratum workaround if the #NPF occurred
4830 	 * while translating guest page tables (see below).
4831 	 */
4832 	error_code = to_svm(vcpu)->vmcb->control.exit_info_1;
4833 	if (error_code & (PFERR_GUEST_PAGE_MASK | PFERR_FETCH_MASK))
4834 		goto resume_guest;
4835 
4836 	smep = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMEP);
4837 	smap = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMAP);
4838 	is_user = svm_get_cpl(vcpu) == 3;
4839 	if (smap && (!smep || is_user)) {
4840 		pr_err_ratelimited("SEV Guest triggered AMD Erratum 1096\n");
4841 
4842 		/*
4843 		 * If the fault occurred in userspace, arbitrarily inject #GP
4844 		 * to avoid killing the guest and to hopefully avoid confusing
4845 		 * the guest kernel too much, e.g. injecting #PF would not be
4846 		 * coherent with respect to the guest's page tables.  Request
4847 		 * triple fault if the fault occurred in the kernel as there's
4848 		 * no fault that KVM can inject without confusing the guest.
4849 		 * In practice, the triple fault is moot as no sane SEV kernel
4850 		 * will execute from user memory while also running with SMAP=1.
4851 		 */
4852 		if (is_user)
4853 			kvm_inject_gp(vcpu, 0);
4854 		else
4855 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4856 	}
4857 
4858 resume_guest:
4859 	/*
4860 	 * If the erratum was not hit, simply resume the guest and let it fault
4861 	 * again.  While awful, e.g. the vCPU may get stuck in an infinite loop
4862 	 * if the fault is at CPL=0, it's the lesser of all evils.  Exiting to
4863 	 * userspace will kill the guest, and letting the emulator read garbage
4864 	 * will yield random behavior and potentially corrupt the guest.
4865 	 *
4866 	 * Simply resuming the guest is technically not a violation of the SEV
4867 	 * architecture.  AMD's APM states that all code fetches and page table
4868 	 * accesses for SEV guest are encrypted, regardless of the C-Bit.  The
4869 	 * APM also states that encrypted accesses to MMIO are "ignored", but
4870 	 * doesn't explicitly define "ignored", i.e. doing nothing and letting
4871 	 * the guest spin is technically "ignoring" the access.
4872 	 */
4873 	return false;
4874 }
4875 
svm_apic_init_signal_blocked(struct kvm_vcpu * vcpu)4876 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
4877 {
4878 	struct vcpu_svm *svm = to_svm(vcpu);
4879 
4880 	return !gif_set(svm);
4881 }
4882 
svm_vcpu_deliver_sipi_vector(struct kvm_vcpu * vcpu,u8 vector)4883 static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
4884 {
4885 	if (!sev_es_guest(vcpu->kvm))
4886 		return kvm_vcpu_deliver_sipi_vector(vcpu, vector);
4887 
4888 	sev_vcpu_deliver_sipi_vector(vcpu, vector);
4889 }
4890 
svm_vm_destroy(struct kvm * kvm)4891 static void svm_vm_destroy(struct kvm *kvm)
4892 {
4893 	avic_vm_destroy(kvm);
4894 	sev_vm_destroy(kvm);
4895 }
4896 
svm_vm_init(struct kvm * kvm)4897 static int svm_vm_init(struct kvm *kvm)
4898 {
4899 	if (!pause_filter_count || !pause_filter_thresh)
4900 		kvm->arch.pause_in_guest = true;
4901 
4902 	if (enable_apicv) {
4903 		int ret = avic_vm_init(kvm);
4904 		if (ret)
4905 			return ret;
4906 	}
4907 
4908 	return 0;
4909 }
4910 
4911 static struct kvm_x86_ops svm_x86_ops __initdata = {
4912 	.name = KBUILD_MODNAME,
4913 
4914 	.check_processor_compatibility = svm_check_processor_compat,
4915 
4916 	.hardware_unsetup = svm_hardware_unsetup,
4917 	.hardware_enable = svm_hardware_enable,
4918 	.hardware_disable = svm_hardware_disable,
4919 	.has_emulated_msr = svm_has_emulated_msr,
4920 
4921 	.vcpu_create = svm_vcpu_create,
4922 	.vcpu_free = svm_vcpu_free,
4923 	.vcpu_reset = svm_vcpu_reset,
4924 
4925 	.vm_size = sizeof(struct kvm_svm),
4926 	.vm_init = svm_vm_init,
4927 	.vm_destroy = svm_vm_destroy,
4928 
4929 	.prepare_switch_to_guest = svm_prepare_switch_to_guest,
4930 	.vcpu_load = svm_vcpu_load,
4931 	.vcpu_put = svm_vcpu_put,
4932 	.vcpu_blocking = avic_vcpu_blocking,
4933 	.vcpu_unblocking = avic_vcpu_unblocking,
4934 
4935 	.update_exception_bitmap = svm_update_exception_bitmap,
4936 	.get_msr_feature = svm_get_msr_feature,
4937 	.get_msr = svm_get_msr,
4938 	.set_msr = svm_set_msr,
4939 	.get_segment_base = svm_get_segment_base,
4940 	.get_segment = svm_get_segment,
4941 	.set_segment = svm_set_segment,
4942 	.get_cpl = svm_get_cpl,
4943 	.get_cs_db_l_bits = svm_get_cs_db_l_bits,
4944 	.is_valid_cr0 = svm_is_valid_cr0,
4945 	.set_cr0 = svm_set_cr0,
4946 	.post_set_cr3 = sev_post_set_cr3,
4947 	.is_valid_cr4 = svm_is_valid_cr4,
4948 	.set_cr4 = svm_set_cr4,
4949 	.set_efer = svm_set_efer,
4950 	.get_idt = svm_get_idt,
4951 	.set_idt = svm_set_idt,
4952 	.get_gdt = svm_get_gdt,
4953 	.set_gdt = svm_set_gdt,
4954 	.set_dr7 = svm_set_dr7,
4955 	.sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
4956 	.cache_reg = svm_cache_reg,
4957 	.get_rflags = svm_get_rflags,
4958 	.set_rflags = svm_set_rflags,
4959 	.get_if_flag = svm_get_if_flag,
4960 
4961 	.flush_tlb_all = svm_flush_tlb_all,
4962 	.flush_tlb_current = svm_flush_tlb_current,
4963 	.flush_tlb_gva = svm_flush_tlb_gva,
4964 	.flush_tlb_guest = svm_flush_tlb_asid,
4965 
4966 	.vcpu_pre_run = svm_vcpu_pre_run,
4967 	.vcpu_run = svm_vcpu_run,
4968 	.handle_exit = svm_handle_exit,
4969 	.skip_emulated_instruction = svm_skip_emulated_instruction,
4970 	.update_emulated_instruction = NULL,
4971 	.set_interrupt_shadow = svm_set_interrupt_shadow,
4972 	.get_interrupt_shadow = svm_get_interrupt_shadow,
4973 	.patch_hypercall = svm_patch_hypercall,
4974 	.inject_irq = svm_inject_irq,
4975 	.inject_nmi = svm_inject_nmi,
4976 	.is_vnmi_pending = svm_is_vnmi_pending,
4977 	.set_vnmi_pending = svm_set_vnmi_pending,
4978 	.inject_exception = svm_inject_exception,
4979 	.cancel_injection = svm_cancel_injection,
4980 	.interrupt_allowed = svm_interrupt_allowed,
4981 	.nmi_allowed = svm_nmi_allowed,
4982 	.get_nmi_mask = svm_get_nmi_mask,
4983 	.set_nmi_mask = svm_set_nmi_mask,
4984 	.enable_nmi_window = svm_enable_nmi_window,
4985 	.enable_irq_window = svm_enable_irq_window,
4986 	.update_cr8_intercept = svm_update_cr8_intercept,
4987 	.set_virtual_apic_mode = avic_refresh_virtual_apic_mode,
4988 	.refresh_apicv_exec_ctrl = avic_refresh_apicv_exec_ctrl,
4989 	.apicv_post_state_restore = avic_apicv_post_state_restore,
4990 	.required_apicv_inhibits = AVIC_REQUIRED_APICV_INHIBITS,
4991 
4992 	.get_exit_info = svm_get_exit_info,
4993 
4994 	.vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid,
4995 
4996 	.has_wbinvd_exit = svm_has_wbinvd_exit,
4997 
4998 	.get_l2_tsc_offset = svm_get_l2_tsc_offset,
4999 	.get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier,
5000 	.write_tsc_offset = svm_write_tsc_offset,
5001 	.write_tsc_multiplier = svm_write_tsc_multiplier,
5002 
5003 	.load_mmu_pgd = svm_load_mmu_pgd,
5004 
5005 	.check_intercept = svm_check_intercept,
5006 	.handle_exit_irqoff = svm_handle_exit_irqoff,
5007 
5008 	.request_immediate_exit = __kvm_request_immediate_exit,
5009 
5010 	.sched_in = svm_sched_in,
5011 
5012 	.nested_ops = &svm_nested_ops,
5013 
5014 	.deliver_interrupt = svm_deliver_interrupt,
5015 	.pi_update_irte = avic_pi_update_irte,
5016 	.setup_mce = svm_setup_mce,
5017 
5018 #ifdef CONFIG_KVM_SMM
5019 	.smi_allowed = svm_smi_allowed,
5020 	.enter_smm = svm_enter_smm,
5021 	.leave_smm = svm_leave_smm,
5022 	.enable_smi_window = svm_enable_smi_window,
5023 #endif
5024 
5025 	.mem_enc_ioctl = sev_mem_enc_ioctl,
5026 	.mem_enc_register_region = sev_mem_enc_register_region,
5027 	.mem_enc_unregister_region = sev_mem_enc_unregister_region,
5028 	.guest_memory_reclaimed = sev_guest_memory_reclaimed,
5029 
5030 	.vm_copy_enc_context_from = sev_vm_copy_enc_context_from,
5031 	.vm_move_enc_context_from = sev_vm_move_enc_context_from,
5032 
5033 	.can_emulate_instruction = svm_can_emulate_instruction,
5034 
5035 	.apic_init_signal_blocked = svm_apic_init_signal_blocked,
5036 
5037 	.msr_filter_changed = svm_msr_filter_changed,
5038 	.complete_emulated_msr = svm_complete_emulated_msr,
5039 
5040 	.vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector,
5041 	.vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons,
5042 };
5043 
5044 /*
5045  * The default MMIO mask is a single bit (excluding the present bit),
5046  * which could conflict with the memory encryption bit. Check for
5047  * memory encryption support and override the default MMIO mask if
5048  * memory encryption is enabled.
5049  */
svm_adjust_mmio_mask(void)5050 static __init void svm_adjust_mmio_mask(void)
5051 {
5052 	unsigned int enc_bit, mask_bit;
5053 	u64 msr, mask;
5054 
5055 	/* If there is no memory encryption support, use existing mask */
5056 	if (cpuid_eax(0x80000000) < 0x8000001f)
5057 		return;
5058 
5059 	/* If memory encryption is not enabled, use existing mask */
5060 	rdmsrl(MSR_AMD64_SYSCFG, msr);
5061 	if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
5062 		return;
5063 
5064 	enc_bit = cpuid_ebx(0x8000001f) & 0x3f;
5065 	mask_bit = boot_cpu_data.x86_phys_bits;
5066 
5067 	/* Increment the mask bit if it is the same as the encryption bit */
5068 	if (enc_bit == mask_bit)
5069 		mask_bit++;
5070 
5071 	/*
5072 	 * If the mask bit location is below 52, then some bits above the
5073 	 * physical addressing limit will always be reserved, so use the
5074 	 * rsvd_bits() function to generate the mask. This mask, along with
5075 	 * the present bit, will be used to generate a page fault with
5076 	 * PFER.RSV = 1.
5077 	 *
5078 	 * If the mask bit location is 52 (or above), then clear the mask.
5079 	 */
5080 	mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0;
5081 
5082 	kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK);
5083 }
5084 
svm_set_cpu_caps(void)5085 static __init void svm_set_cpu_caps(void)
5086 {
5087 	kvm_set_cpu_caps();
5088 
5089 	kvm_caps.supported_perf_cap = 0;
5090 	kvm_caps.supported_xss = 0;
5091 
5092 	/* CPUID 0x80000001 and 0x8000000A (SVM features) */
5093 	if (nested) {
5094 		kvm_cpu_cap_set(X86_FEATURE_SVM);
5095 		kvm_cpu_cap_set(X86_FEATURE_VMCBCLEAN);
5096 
5097 		if (nrips)
5098 			kvm_cpu_cap_set(X86_FEATURE_NRIPS);
5099 
5100 		if (npt_enabled)
5101 			kvm_cpu_cap_set(X86_FEATURE_NPT);
5102 
5103 		if (tsc_scaling)
5104 			kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR);
5105 
5106 		if (vls)
5107 			kvm_cpu_cap_set(X86_FEATURE_V_VMSAVE_VMLOAD);
5108 		if (lbrv)
5109 			kvm_cpu_cap_set(X86_FEATURE_LBRV);
5110 
5111 		if (boot_cpu_has(X86_FEATURE_PAUSEFILTER))
5112 			kvm_cpu_cap_set(X86_FEATURE_PAUSEFILTER);
5113 
5114 		if (boot_cpu_has(X86_FEATURE_PFTHRESHOLD))
5115 			kvm_cpu_cap_set(X86_FEATURE_PFTHRESHOLD);
5116 
5117 		if (vgif)
5118 			kvm_cpu_cap_set(X86_FEATURE_VGIF);
5119 
5120 		if (vnmi)
5121 			kvm_cpu_cap_set(X86_FEATURE_VNMI);
5122 
5123 		/* Nested VM can receive #VMEXIT instead of triggering #GP */
5124 		kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK);
5125 	}
5126 
5127 	/* CPUID 0x80000008 */
5128 	if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
5129 	    boot_cpu_has(X86_FEATURE_AMD_SSBD))
5130 		kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
5131 
5132 	if (enable_pmu) {
5133 		/*
5134 		 * Enumerate support for PERFCTR_CORE if and only if KVM has
5135 		 * access to enough counters to virtualize "core" support,
5136 		 * otherwise limit vPMU support to the legacy number of counters.
5137 		 */
5138 		if (kvm_pmu_cap.num_counters_gp < AMD64_NUM_COUNTERS_CORE)
5139 			kvm_pmu_cap.num_counters_gp = min(AMD64_NUM_COUNTERS,
5140 							  kvm_pmu_cap.num_counters_gp);
5141 		else
5142 			kvm_cpu_cap_check_and_set(X86_FEATURE_PERFCTR_CORE);
5143 
5144 		if (kvm_pmu_cap.version != 2 ||
5145 		    !kvm_cpu_cap_has(X86_FEATURE_PERFCTR_CORE))
5146 			kvm_cpu_cap_clear(X86_FEATURE_PERFMON_V2);
5147 	}
5148 
5149 	/* CPUID 0x8000001F (SME/SEV features) */
5150 	sev_set_cpu_caps();
5151 }
5152 
svm_hardware_setup(void)5153 static __init int svm_hardware_setup(void)
5154 {
5155 	int cpu;
5156 	struct page *iopm_pages;
5157 	void *iopm_va;
5158 	int r;
5159 	unsigned int order = get_order(IOPM_SIZE);
5160 
5161 	/*
5162 	 * NX is required for shadow paging and for NPT if the NX huge pages
5163 	 * mitigation is enabled.
5164 	 */
5165 	if (!boot_cpu_has(X86_FEATURE_NX)) {
5166 		pr_err_ratelimited("NX (Execute Disable) not supported\n");
5167 		return -EOPNOTSUPP;
5168 	}
5169 	kvm_enable_efer_bits(EFER_NX);
5170 
5171 	iopm_pages = alloc_pages(GFP_KERNEL, order);
5172 
5173 	if (!iopm_pages)
5174 		return -ENOMEM;
5175 
5176 	iopm_va = page_address(iopm_pages);
5177 	memset(iopm_va, 0xff, PAGE_SIZE * (1 << order));
5178 	iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
5179 
5180 	init_msrpm_offsets();
5181 
5182 	kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS |
5183 				     XFEATURE_MASK_BNDCSR);
5184 
5185 	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
5186 		kvm_enable_efer_bits(EFER_FFXSR);
5187 
5188 	if (tsc_scaling) {
5189 		if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
5190 			tsc_scaling = false;
5191 		} else {
5192 			pr_info("TSC scaling supported\n");
5193 			kvm_caps.has_tsc_control = true;
5194 		}
5195 	}
5196 	kvm_caps.max_tsc_scaling_ratio = SVM_TSC_RATIO_MAX;
5197 	kvm_caps.tsc_scaling_ratio_frac_bits = 32;
5198 
5199 	tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX);
5200 
5201 	if (boot_cpu_has(X86_FEATURE_AUTOIBRS))
5202 		kvm_enable_efer_bits(EFER_AUTOIBRS);
5203 
5204 	/* Check for pause filtering support */
5205 	if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
5206 		pause_filter_count = 0;
5207 		pause_filter_thresh = 0;
5208 	} else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
5209 		pause_filter_thresh = 0;
5210 	}
5211 
5212 	if (nested) {
5213 		pr_info("Nested Virtualization enabled\n");
5214 		kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
5215 	}
5216 
5217 	/*
5218 	 * KVM's MMU doesn't support using 2-level paging for itself, and thus
5219 	 * NPT isn't supported if the host is using 2-level paging since host
5220 	 * CR4 is unchanged on VMRUN.
5221 	 */
5222 	if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE))
5223 		npt_enabled = false;
5224 
5225 	if (!boot_cpu_has(X86_FEATURE_NPT))
5226 		npt_enabled = false;
5227 
5228 	/* Force VM NPT level equal to the host's paging level */
5229 	kvm_configure_mmu(npt_enabled, get_npt_level(),
5230 			  get_npt_level(), PG_LEVEL_1G);
5231 	pr_info("Nested Paging %sabled\n", npt_enabled ? "en" : "dis");
5232 
5233 	/* Setup shadow_me_value and shadow_me_mask */
5234 	kvm_mmu_set_me_spte_mask(sme_me_mask, sme_me_mask);
5235 
5236 	svm_adjust_mmio_mask();
5237 
5238 	nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS);
5239 
5240 	/*
5241 	 * Note, SEV setup consumes npt_enabled and enable_mmio_caching (which
5242 	 * may be modified by svm_adjust_mmio_mask()), as well as nrips.
5243 	 */
5244 	sev_hardware_setup();
5245 
5246 	svm_hv_hardware_setup();
5247 
5248 	for_each_possible_cpu(cpu) {
5249 		r = svm_cpu_init(cpu);
5250 		if (r)
5251 			goto err;
5252 	}
5253 
5254 	enable_apicv = avic = avic && avic_hardware_setup();
5255 
5256 	if (!enable_apicv) {
5257 		svm_x86_ops.vcpu_blocking = NULL;
5258 		svm_x86_ops.vcpu_unblocking = NULL;
5259 		svm_x86_ops.vcpu_get_apicv_inhibit_reasons = NULL;
5260 	} else if (!x2avic_enabled) {
5261 		svm_x86_ops.allow_apicv_in_x2apic_without_x2apic_virtualization = true;
5262 	}
5263 
5264 	if (vls) {
5265 		if (!npt_enabled ||
5266 		    !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
5267 		    !IS_ENABLED(CONFIG_X86_64)) {
5268 			vls = false;
5269 		} else {
5270 			pr_info("Virtual VMLOAD VMSAVE supported\n");
5271 		}
5272 	}
5273 
5274 	if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK))
5275 		svm_gp_erratum_intercept = false;
5276 
5277 	if (vgif) {
5278 		if (!boot_cpu_has(X86_FEATURE_VGIF))
5279 			vgif = false;
5280 		else
5281 			pr_info("Virtual GIF supported\n");
5282 	}
5283 
5284 	vnmi = vgif && vnmi && boot_cpu_has(X86_FEATURE_VNMI);
5285 	if (vnmi)
5286 		pr_info("Virtual NMI enabled\n");
5287 
5288 	if (!vnmi) {
5289 		svm_x86_ops.is_vnmi_pending = NULL;
5290 		svm_x86_ops.set_vnmi_pending = NULL;
5291 	}
5292 
5293 
5294 	if (lbrv) {
5295 		if (!boot_cpu_has(X86_FEATURE_LBRV))
5296 			lbrv = false;
5297 		else
5298 			pr_info("LBR virtualization supported\n");
5299 	}
5300 
5301 	if (!enable_pmu)
5302 		pr_info("PMU virtualization is disabled\n");
5303 
5304 	svm_set_cpu_caps();
5305 
5306 	/*
5307 	 * It seems that on AMD processors PTE's accessed bit is
5308 	 * being set by the CPU hardware before the NPF vmexit.
5309 	 * This is not expected behaviour and our tests fail because
5310 	 * of it.
5311 	 * A workaround here is to disable support for
5312 	 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled.
5313 	 * In this case userspace can know if there is support using
5314 	 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle
5315 	 * it
5316 	 * If future AMD CPU models change the behaviour described above,
5317 	 * this variable can be changed accordingly
5318 	 */
5319 	allow_smaller_maxphyaddr = !npt_enabled;
5320 
5321 	return 0;
5322 
5323 err:
5324 	svm_hardware_unsetup();
5325 	return r;
5326 }
5327 
5328 
5329 static struct kvm_x86_init_ops svm_init_ops __initdata = {
5330 	.hardware_setup = svm_hardware_setup,
5331 
5332 	.runtime_ops = &svm_x86_ops,
5333 	.pmu_ops = &amd_pmu_ops,
5334 };
5335 
__svm_exit(void)5336 static void __svm_exit(void)
5337 {
5338 	kvm_x86_vendor_exit();
5339 
5340 	cpu_emergency_unregister_virt_callback(svm_emergency_disable);
5341 }
5342 
svm_init(void)5343 static int __init svm_init(void)
5344 {
5345 	int r;
5346 
5347 	__unused_size_checks();
5348 
5349 	if (!kvm_is_svm_supported())
5350 		return -EOPNOTSUPP;
5351 
5352 	r = kvm_x86_vendor_init(&svm_init_ops);
5353 	if (r)
5354 		return r;
5355 
5356 	cpu_emergency_register_virt_callback(svm_emergency_disable);
5357 
5358 	/*
5359 	 * Common KVM initialization _must_ come last, after this, /dev/kvm is
5360 	 * exposed to userspace!
5361 	 */
5362 	r = kvm_init(sizeof(struct vcpu_svm), __alignof__(struct vcpu_svm),
5363 		     THIS_MODULE);
5364 	if (r)
5365 		goto err_kvm_init;
5366 
5367 	return 0;
5368 
5369 err_kvm_init:
5370 	__svm_exit();
5371 	return r;
5372 }
5373 
svm_exit(void)5374 static void __exit svm_exit(void)
5375 {
5376 	kvm_exit();
5377 	__svm_exit();
5378 }
5379 
5380 module_init(svm_init)
5381 module_exit(svm_exit)
5382