1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Kernel-based Virtual Machine driver for Linux
4  *
5  * derived from drivers/kvm/kvm_main.c
6  *
7  * Copyright (C) 2006 Qumranet, Inc.
8  * Copyright (C) 2008 Qumranet, Inc.
9  * Copyright IBM Corporation, 2008
10  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
11  *
12  * Authors:
13  *   Avi Kivity   <avi@qumranet.com>
14  *   Yaniv Kamay  <yaniv@qumranet.com>
15  *   Amit Shah    <amit.shah@qumranet.com>
16  *   Ben-Ami Yassour <benami@il.ibm.com>
17  */
18 
19 #include <linux/kvm_host.h>
20 #include "irq.h"
21 #include "ioapic.h"
22 #include "mmu.h"
23 #include "i8254.h"
24 #include "tss.h"
25 #include "kvm_cache_regs.h"
26 #include "kvm_emulate.h"
27 #include "x86.h"
28 #include "cpuid.h"
29 #include "pmu.h"
30 #include "hyperv.h"
31 #include "lapic.h"
32 #include "xen.h"
33 
34 #include <linux/clocksource.h>
35 #include <linux/interrupt.h>
36 #include <linux/kvm.h>
37 #include <linux/fs.h>
38 #include <linux/vmalloc.h>
39 #include <linux/export.h>
40 #include <linux/moduleparam.h>
41 #include <linux/mman.h>
42 #include <linux/highmem.h>
43 #include <linux/iommu.h>
44 #include <linux/cpufreq.h>
45 #include <linux/user-return-notifier.h>
46 #include <linux/srcu.h>
47 #include <linux/slab.h>
48 #include <linux/perf_event.h>
49 #include <linux/uaccess.h>
50 #include <linux/hash.h>
51 #include <linux/pci.h>
52 #include <linux/timekeeper_internal.h>
53 #include <linux/pvclock_gtod.h>
54 #include <linux/kvm_irqfd.h>
55 #include <linux/irqbypass.h>
56 #include <linux/sched/stat.h>
57 #include <linux/sched/isolation.h>
58 #include <linux/mem_encrypt.h>
59 #include <linux/entry-kvm.h>
60 #include <linux/suspend.h>
61 
62 #include <trace/events/kvm.h>
63 
64 #include <asm/debugreg.h>
65 #include <asm/msr.h>
66 #include <asm/desc.h>
67 #include <asm/mce.h>
68 #include <asm/pkru.h>
69 #include <linux/kernel_stat.h>
70 #include <asm/fpu/api.h>
71 #include <asm/fpu/xcr.h>
72 #include <asm/fpu/xstate.h>
73 #include <asm/pvclock.h>
74 #include <asm/div64.h>
75 #include <asm/irq_remapping.h>
76 #include <asm/mshyperv.h>
77 #include <asm/hypervisor.h>
78 #include <asm/tlbflush.h>
79 #include <asm/intel_pt.h>
80 #include <asm/emulate_prefix.h>
81 #include <asm/sgx.h>
82 #include <clocksource/hyperv_timer.h>
83 
84 #define CREATE_TRACE_POINTS
85 #include "trace.h"
86 
87 #define MAX_IO_MSRS 256
88 #define KVM_MAX_MCE_BANKS 32
89 
90 struct kvm_caps kvm_caps __read_mostly = {
91 	.supported_mce_cap = MCG_CTL_P | MCG_SER_P,
92 };
93 EXPORT_SYMBOL_GPL(kvm_caps);
94 
95 #define  ERR_PTR_USR(e)  ((void __user *)ERR_PTR(e))
96 
97 #define emul_to_vcpu(ctxt) \
98 	((struct kvm_vcpu *)(ctxt)->vcpu)
99 
100 /* EFER defaults:
101  * - enable syscall per default because its emulated by KVM
102  * - enable LME and LMA per default on 64 bit KVM
103  */
104 #ifdef CONFIG_X86_64
105 static
106 u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
107 #else
108 static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
109 #endif
110 
111 static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS;
112 
113 #define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE)
114 
115 #define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE
116 
117 #define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
118                                     KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
119 
120 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
121 static void process_nmi(struct kvm_vcpu *vcpu);
122 static void process_smi(struct kvm_vcpu *vcpu);
123 static void enter_smm(struct kvm_vcpu *vcpu);
124 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
125 static void store_regs(struct kvm_vcpu *vcpu);
126 static int sync_regs(struct kvm_vcpu *vcpu);
127 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu);
128 
129 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
130 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
131 
132 struct kvm_x86_ops kvm_x86_ops __read_mostly;
133 
134 #define KVM_X86_OP(func)					     \
135 	DEFINE_STATIC_CALL_NULL(kvm_x86_##func,			     \
136 				*(((struct kvm_x86_ops *)0)->func));
137 #define KVM_X86_OP_OPTIONAL KVM_X86_OP
138 #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP
139 #include <asm/kvm-x86-ops.h>
140 EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
141 EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
142 
143 static bool __read_mostly ignore_msrs = 0;
144 module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
145 
146 bool __read_mostly report_ignored_msrs = true;
147 module_param(report_ignored_msrs, bool, S_IRUGO | S_IWUSR);
148 EXPORT_SYMBOL_GPL(report_ignored_msrs);
149 
150 unsigned int min_timer_period_us = 200;
151 module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);
152 
153 static bool __read_mostly kvmclock_periodic_sync = true;
154 module_param(kvmclock_periodic_sync, bool, S_IRUGO);
155 
156 /* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
157 static u32 __read_mostly tsc_tolerance_ppm = 250;
158 module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);
159 
160 /*
161  * lapic timer advance (tscdeadline mode only) in nanoseconds.  '-1' enables
162  * adaptive tuning starting from default advancement of 1000ns.  '0' disables
163  * advancement entirely.  Any other value is used as-is and disables adaptive
164  * tuning, i.e. allows privileged userspace to set an exact advancement time.
165  */
166 static int __read_mostly lapic_timer_advance_ns = -1;
167 module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR);
168 
169 static bool __read_mostly vector_hashing = true;
170 module_param(vector_hashing, bool, S_IRUGO);
171 
172 bool __read_mostly enable_vmware_backdoor = false;
173 module_param(enable_vmware_backdoor, bool, S_IRUGO);
174 EXPORT_SYMBOL_GPL(enable_vmware_backdoor);
175 
176 /*
177  * Flags to manipulate forced emulation behavior (any non-zero value will
178  * enable forced emulation).
179  */
180 #define KVM_FEP_CLEAR_RFLAGS_RF	BIT(1)
181 static int __read_mostly force_emulation_prefix;
182 module_param(force_emulation_prefix, int, 0644);
183 
184 int __read_mostly pi_inject_timer = -1;
185 module_param(pi_inject_timer, bint, S_IRUGO | S_IWUSR);
186 
187 /* Enable/disable PMU virtualization */
188 bool __read_mostly enable_pmu = true;
189 EXPORT_SYMBOL_GPL(enable_pmu);
190 module_param(enable_pmu, bool, 0444);
191 
192 bool __read_mostly eager_page_split = true;
193 module_param(eager_page_split, bool, 0644);
194 
195 /*
196  * Restoring the host value for MSRs that are only consumed when running in
197  * usermode, e.g. SYSCALL MSRs and TSC_AUX, can be deferred until the CPU
198  * returns to userspace, i.e. the kernel can run with the guest's value.
199  */
200 #define KVM_MAX_NR_USER_RETURN_MSRS 16
201 
202 struct kvm_user_return_msrs {
203 	struct user_return_notifier urn;
204 	bool registered;
205 	struct kvm_user_return_msr_values {
206 		u64 host;
207 		u64 curr;
208 	} values[KVM_MAX_NR_USER_RETURN_MSRS];
209 };
210 
211 u32 __read_mostly kvm_nr_uret_msrs;
212 EXPORT_SYMBOL_GPL(kvm_nr_uret_msrs);
213 static u32 __read_mostly kvm_uret_msrs_list[KVM_MAX_NR_USER_RETURN_MSRS];
214 static struct kvm_user_return_msrs __percpu *user_return_msrs;
215 
216 #define KVM_SUPPORTED_XCR0     (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \
217 				| XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \
218 				| XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \
219 				| XFEATURE_MASK_PKRU | XFEATURE_MASK_XTILE)
220 
221 u64 __read_mostly host_efer;
222 EXPORT_SYMBOL_GPL(host_efer);
223 
224 bool __read_mostly allow_smaller_maxphyaddr = 0;
225 EXPORT_SYMBOL_GPL(allow_smaller_maxphyaddr);
226 
227 bool __read_mostly enable_apicv = true;
228 EXPORT_SYMBOL_GPL(enable_apicv);
229 
230 u64 __read_mostly host_xss;
231 EXPORT_SYMBOL_GPL(host_xss);
232 
233 const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
234 	KVM_GENERIC_VM_STATS(),
235 	STATS_DESC_COUNTER(VM, mmu_shadow_zapped),
236 	STATS_DESC_COUNTER(VM, mmu_pte_write),
237 	STATS_DESC_COUNTER(VM, mmu_pde_zapped),
238 	STATS_DESC_COUNTER(VM, mmu_flooded),
239 	STATS_DESC_COUNTER(VM, mmu_recycled),
240 	STATS_DESC_COUNTER(VM, mmu_cache_miss),
241 	STATS_DESC_ICOUNTER(VM, mmu_unsync),
242 	STATS_DESC_ICOUNTER(VM, pages_4k),
243 	STATS_DESC_ICOUNTER(VM, pages_2m),
244 	STATS_DESC_ICOUNTER(VM, pages_1g),
245 	STATS_DESC_ICOUNTER(VM, nx_lpage_splits),
246 	STATS_DESC_PCOUNTER(VM, max_mmu_rmap_size),
247 	STATS_DESC_PCOUNTER(VM, max_mmu_page_hash_collisions)
248 };
249 
250 const struct kvm_stats_header kvm_vm_stats_header = {
251 	.name_size = KVM_STATS_NAME_SIZE,
252 	.num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
253 	.id_offset = sizeof(struct kvm_stats_header),
254 	.desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
255 	.data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
256 		       sizeof(kvm_vm_stats_desc),
257 };
258 
259 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
260 	KVM_GENERIC_VCPU_STATS(),
261 	STATS_DESC_COUNTER(VCPU, pf_taken),
262 	STATS_DESC_COUNTER(VCPU, pf_fixed),
263 	STATS_DESC_COUNTER(VCPU, pf_emulate),
264 	STATS_DESC_COUNTER(VCPU, pf_spurious),
265 	STATS_DESC_COUNTER(VCPU, pf_fast),
266 	STATS_DESC_COUNTER(VCPU, pf_mmio_spte_created),
267 	STATS_DESC_COUNTER(VCPU, pf_guest),
268 	STATS_DESC_COUNTER(VCPU, tlb_flush),
269 	STATS_DESC_COUNTER(VCPU, invlpg),
270 	STATS_DESC_COUNTER(VCPU, exits),
271 	STATS_DESC_COUNTER(VCPU, io_exits),
272 	STATS_DESC_COUNTER(VCPU, mmio_exits),
273 	STATS_DESC_COUNTER(VCPU, signal_exits),
274 	STATS_DESC_COUNTER(VCPU, irq_window_exits),
275 	STATS_DESC_COUNTER(VCPU, nmi_window_exits),
276 	STATS_DESC_COUNTER(VCPU, l1d_flush),
277 	STATS_DESC_COUNTER(VCPU, halt_exits),
278 	STATS_DESC_COUNTER(VCPU, request_irq_exits),
279 	STATS_DESC_COUNTER(VCPU, irq_exits),
280 	STATS_DESC_COUNTER(VCPU, host_state_reload),
281 	STATS_DESC_COUNTER(VCPU, fpu_reload),
282 	STATS_DESC_COUNTER(VCPU, insn_emulation),
283 	STATS_DESC_COUNTER(VCPU, insn_emulation_fail),
284 	STATS_DESC_COUNTER(VCPU, hypercalls),
285 	STATS_DESC_COUNTER(VCPU, irq_injections),
286 	STATS_DESC_COUNTER(VCPU, nmi_injections),
287 	STATS_DESC_COUNTER(VCPU, req_event),
288 	STATS_DESC_COUNTER(VCPU, nested_run),
289 	STATS_DESC_COUNTER(VCPU, directed_yield_attempted),
290 	STATS_DESC_COUNTER(VCPU, directed_yield_successful),
291 	STATS_DESC_COUNTER(VCPU, preemption_reported),
292 	STATS_DESC_COUNTER(VCPU, preemption_other),
293 	STATS_DESC_IBOOLEAN(VCPU, guest_mode),
294 	STATS_DESC_COUNTER(VCPU, notify_window_exits),
295 };
296 
297 const struct kvm_stats_header kvm_vcpu_stats_header = {
298 	.name_size = KVM_STATS_NAME_SIZE,
299 	.num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
300 	.id_offset = sizeof(struct kvm_stats_header),
301 	.desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
302 	.data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
303 		       sizeof(kvm_vcpu_stats_desc),
304 };
305 
306 u64 __read_mostly host_xcr0;
307 
308 static struct kmem_cache *x86_emulator_cache;
309 
310 /*
311  * When called, it means the previous get/set msr reached an invalid msr.
312  * Return true if we want to ignore/silent this failed msr access.
313  */
kvm_msr_ignored_check(u32 msr,u64 data,bool write)314 static bool kvm_msr_ignored_check(u32 msr, u64 data, bool write)
315 {
316 	const char *op = write ? "wrmsr" : "rdmsr";
317 
318 	if (ignore_msrs) {
319 		if (report_ignored_msrs)
320 			kvm_pr_unimpl("ignored %s: 0x%x data 0x%llx\n",
321 				      op, msr, data);
322 		/* Mask the error */
323 		return true;
324 	} else {
325 		kvm_debug_ratelimited("unhandled %s: 0x%x data 0x%llx\n",
326 				      op, msr, data);
327 		return false;
328 	}
329 }
330 
kvm_alloc_emulator_cache(void)331 static struct kmem_cache *kvm_alloc_emulator_cache(void)
332 {
333 	unsigned int useroffset = offsetof(struct x86_emulate_ctxt, src);
334 	unsigned int size = sizeof(struct x86_emulate_ctxt);
335 
336 	return kmem_cache_create_usercopy("x86_emulator", size,
337 					  __alignof__(struct x86_emulate_ctxt),
338 					  SLAB_ACCOUNT, useroffset,
339 					  size - useroffset, NULL);
340 }
341 
342 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
343 
kvm_async_pf_hash_reset(struct kvm_vcpu * vcpu)344 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
345 {
346 	int i;
347 	for (i = 0; i < ASYNC_PF_PER_VCPU; i++)
348 		vcpu->arch.apf.gfns[i] = ~0;
349 }
350 
kvm_on_user_return(struct user_return_notifier * urn)351 static void kvm_on_user_return(struct user_return_notifier *urn)
352 {
353 	unsigned slot;
354 	struct kvm_user_return_msrs *msrs
355 		= container_of(urn, struct kvm_user_return_msrs, urn);
356 	struct kvm_user_return_msr_values *values;
357 	unsigned long flags;
358 
359 	/*
360 	 * Disabling irqs at this point since the following code could be
361 	 * interrupted and executed through kvm_arch_hardware_disable()
362 	 */
363 	local_irq_save(flags);
364 	if (msrs->registered) {
365 		msrs->registered = false;
366 		user_return_notifier_unregister(urn);
367 	}
368 	local_irq_restore(flags);
369 	for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) {
370 		values = &msrs->values[slot];
371 		if (values->host != values->curr) {
372 			wrmsrl(kvm_uret_msrs_list[slot], values->host);
373 			values->curr = values->host;
374 		}
375 	}
376 }
377 
kvm_probe_user_return_msr(u32 msr)378 static int kvm_probe_user_return_msr(u32 msr)
379 {
380 	u64 val;
381 	int ret;
382 
383 	preempt_disable();
384 	ret = rdmsrl_safe(msr, &val);
385 	if (ret)
386 		goto out;
387 	ret = wrmsrl_safe(msr, val);
388 out:
389 	preempt_enable();
390 	return ret;
391 }
392 
kvm_add_user_return_msr(u32 msr)393 int kvm_add_user_return_msr(u32 msr)
394 {
395 	BUG_ON(kvm_nr_uret_msrs >= KVM_MAX_NR_USER_RETURN_MSRS);
396 
397 	if (kvm_probe_user_return_msr(msr))
398 		return -1;
399 
400 	kvm_uret_msrs_list[kvm_nr_uret_msrs] = msr;
401 	return kvm_nr_uret_msrs++;
402 }
403 EXPORT_SYMBOL_GPL(kvm_add_user_return_msr);
404 
kvm_find_user_return_msr(u32 msr)405 int kvm_find_user_return_msr(u32 msr)
406 {
407 	int i;
408 
409 	for (i = 0; i < kvm_nr_uret_msrs; ++i) {
410 		if (kvm_uret_msrs_list[i] == msr)
411 			return i;
412 	}
413 	return -1;
414 }
415 EXPORT_SYMBOL_GPL(kvm_find_user_return_msr);
416 
kvm_user_return_msr_cpu_online(void)417 static void kvm_user_return_msr_cpu_online(void)
418 {
419 	unsigned int cpu = smp_processor_id();
420 	struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
421 	u64 value;
422 	int i;
423 
424 	for (i = 0; i < kvm_nr_uret_msrs; ++i) {
425 		rdmsrl_safe(kvm_uret_msrs_list[i], &value);
426 		msrs->values[i].host = value;
427 		msrs->values[i].curr = value;
428 	}
429 }
430 
kvm_set_user_return_msr(unsigned slot,u64 value,u64 mask)431 int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask)
432 {
433 	unsigned int cpu = smp_processor_id();
434 	struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
435 	int err;
436 
437 	value = (value & mask) | (msrs->values[slot].host & ~mask);
438 	if (value == msrs->values[slot].curr)
439 		return 0;
440 	err = wrmsrl_safe(kvm_uret_msrs_list[slot], value);
441 	if (err)
442 		return 1;
443 
444 	msrs->values[slot].curr = value;
445 	if (!msrs->registered) {
446 		msrs->urn.on_user_return = kvm_on_user_return;
447 		user_return_notifier_register(&msrs->urn);
448 		msrs->registered = true;
449 	}
450 	return 0;
451 }
452 EXPORT_SYMBOL_GPL(kvm_set_user_return_msr);
453 
drop_user_return_notifiers(void)454 static void drop_user_return_notifiers(void)
455 {
456 	unsigned int cpu = smp_processor_id();
457 	struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
458 
459 	if (msrs->registered)
460 		kvm_on_user_return(&msrs->urn);
461 }
462 
kvm_get_apic_base(struct kvm_vcpu * vcpu)463 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
464 {
465 	return vcpu->arch.apic_base;
466 }
467 EXPORT_SYMBOL_GPL(kvm_get_apic_base);
468 
kvm_get_apic_mode(struct kvm_vcpu * vcpu)469 enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
470 {
471 	return kvm_apic_mode(kvm_get_apic_base(vcpu));
472 }
473 EXPORT_SYMBOL_GPL(kvm_get_apic_mode);
474 
kvm_set_apic_base(struct kvm_vcpu * vcpu,struct msr_data * msr_info)475 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
476 {
477 	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
478 	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
479 	u64 reserved_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu) | 0x2ff |
480 		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
481 
482 	if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
483 		return 1;
484 	if (!msr_info->host_initiated) {
485 		if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
486 			return 1;
487 		if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
488 			return 1;
489 	}
490 
491 	kvm_lapic_set_base(vcpu, msr_info->data);
492 	kvm_recalculate_apic_map(vcpu->kvm);
493 	return 0;
494 }
495 EXPORT_SYMBOL_GPL(kvm_set_apic_base);
496 
497 /*
498  * Handle a fault on a hardware virtualization (VMX or SVM) instruction.
499  *
500  * Hardware virtualization extension instructions may fault if a reboot turns
501  * off virtualization while processes are running.  Usually after catching the
502  * fault we just panic; during reboot instead the instruction is ignored.
503  */
kvm_spurious_fault(void)504 noinstr void kvm_spurious_fault(void)
505 {
506 	/* Fault while not rebooting.  We want the trace. */
507 	BUG_ON(!kvm_rebooting);
508 }
509 EXPORT_SYMBOL_GPL(kvm_spurious_fault);
510 
511 #define EXCPT_BENIGN		0
512 #define EXCPT_CONTRIBUTORY	1
513 #define EXCPT_PF		2
514 
exception_class(int vector)515 static int exception_class(int vector)
516 {
517 	switch (vector) {
518 	case PF_VECTOR:
519 		return EXCPT_PF;
520 	case DE_VECTOR:
521 	case TS_VECTOR:
522 	case NP_VECTOR:
523 	case SS_VECTOR:
524 	case GP_VECTOR:
525 		return EXCPT_CONTRIBUTORY;
526 	default:
527 		break;
528 	}
529 	return EXCPT_BENIGN;
530 }
531 
532 #define EXCPT_FAULT		0
533 #define EXCPT_TRAP		1
534 #define EXCPT_ABORT		2
535 #define EXCPT_INTERRUPT		3
536 #define EXCPT_DB		4
537 
exception_type(int vector)538 static int exception_type(int vector)
539 {
540 	unsigned int mask;
541 
542 	if (WARN_ON(vector > 31 || vector == NMI_VECTOR))
543 		return EXCPT_INTERRUPT;
544 
545 	mask = 1 << vector;
546 
547 	/*
548 	 * #DBs can be trap-like or fault-like, the caller must check other CPU
549 	 * state, e.g. DR6, to determine whether a #DB is a trap or fault.
550 	 */
551 	if (mask & (1 << DB_VECTOR))
552 		return EXCPT_DB;
553 
554 	if (mask & ((1 << BP_VECTOR) | (1 << OF_VECTOR)))
555 		return EXCPT_TRAP;
556 
557 	if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR)))
558 		return EXCPT_ABORT;
559 
560 	/* Reserved exceptions will result in fault */
561 	return EXCPT_FAULT;
562 }
563 
kvm_deliver_exception_payload(struct kvm_vcpu * vcpu,struct kvm_queued_exception * ex)564 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu,
565 				   struct kvm_queued_exception *ex)
566 {
567 	if (!ex->has_payload)
568 		return;
569 
570 	switch (ex->vector) {
571 	case DB_VECTOR:
572 		/*
573 		 * "Certain debug exceptions may clear bit 0-3.  The
574 		 * remaining contents of the DR6 register are never
575 		 * cleared by the processor".
576 		 */
577 		vcpu->arch.dr6 &= ~DR_TRAP_BITS;
578 		/*
579 		 * In order to reflect the #DB exception payload in guest
580 		 * dr6, three components need to be considered: active low
581 		 * bit, FIXED_1 bits and active high bits (e.g. DR6_BD,
582 		 * DR6_BS and DR6_BT)
583 		 * DR6_ACTIVE_LOW contains the FIXED_1 and active low bits.
584 		 * In the target guest dr6:
585 		 * FIXED_1 bits should always be set.
586 		 * Active low bits should be cleared if 1-setting in payload.
587 		 * Active high bits should be set if 1-setting in payload.
588 		 *
589 		 * Note, the payload is compatible with the pending debug
590 		 * exceptions/exit qualification under VMX, that active_low bits
591 		 * are active high in payload.
592 		 * So they need to be flipped for DR6.
593 		 */
594 		vcpu->arch.dr6 |= DR6_ACTIVE_LOW;
595 		vcpu->arch.dr6 |= ex->payload;
596 		vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW;
597 
598 		/*
599 		 * The #DB payload is defined as compatible with the 'pending
600 		 * debug exceptions' field under VMX, not DR6. While bit 12 is
601 		 * defined in the 'pending debug exceptions' field (enabled
602 		 * breakpoint), it is reserved and must be zero in DR6.
603 		 */
604 		vcpu->arch.dr6 &= ~BIT(12);
605 		break;
606 	case PF_VECTOR:
607 		vcpu->arch.cr2 = ex->payload;
608 		break;
609 	}
610 
611 	ex->has_payload = false;
612 	ex->payload = 0;
613 }
614 EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload);
615 
kvm_queue_exception_vmexit(struct kvm_vcpu * vcpu,unsigned int vector,bool has_error_code,u32 error_code,bool has_payload,unsigned long payload)616 static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vector,
617 				       bool has_error_code, u32 error_code,
618 				       bool has_payload, unsigned long payload)
619 {
620 	struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit;
621 
622 	ex->vector = vector;
623 	ex->injected = false;
624 	ex->pending = true;
625 	ex->has_error_code = has_error_code;
626 	ex->error_code = error_code;
627 	ex->has_payload = has_payload;
628 	ex->payload = payload;
629 }
630 
631 /* Forcibly leave the nested mode in cases like a vCPU reset */
kvm_leave_nested(struct kvm_vcpu * vcpu)632 static void kvm_leave_nested(struct kvm_vcpu *vcpu)
633 {
634 	kvm_x86_ops.nested_ops->leave_nested(vcpu);
635 }
636 
kvm_multiple_exception(struct kvm_vcpu * vcpu,unsigned nr,bool has_error,u32 error_code,bool has_payload,unsigned long payload,bool reinject)637 static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
638 		unsigned nr, bool has_error, u32 error_code,
639 	        bool has_payload, unsigned long payload, bool reinject)
640 {
641 	u32 prev_nr;
642 	int class1, class2;
643 
644 	kvm_make_request(KVM_REQ_EVENT, vcpu);
645 
646 	/*
647 	 * If the exception is destined for L2 and isn't being reinjected,
648 	 * morph it to a VM-Exit if L1 wants to intercept the exception.  A
649 	 * previously injected exception is not checked because it was checked
650 	 * when it was original queued, and re-checking is incorrect if _L1_
651 	 * injected the exception, in which case it's exempt from interception.
652 	 */
653 	if (!reinject && is_guest_mode(vcpu) &&
654 	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {
655 		kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code,
656 					   has_payload, payload);
657 		return;
658 	}
659 
660 	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
661 	queue:
662 		if (reinject) {
663 			/*
664 			 * On VM-Entry, an exception can be pending if and only
665 			 * if event injection was blocked by nested_run_pending.
666 			 * In that case, however, vcpu_enter_guest() requests an
667 			 * immediate exit, and the guest shouldn't proceed far
668 			 * enough to need reinjection.
669 			 */
670 			WARN_ON_ONCE(kvm_is_exception_pending(vcpu));
671 			vcpu->arch.exception.injected = true;
672 			if (WARN_ON_ONCE(has_payload)) {
673 				/*
674 				 * A reinjected event has already
675 				 * delivered its payload.
676 				 */
677 				has_payload = false;
678 				payload = 0;
679 			}
680 		} else {
681 			vcpu->arch.exception.pending = true;
682 			vcpu->arch.exception.injected = false;
683 		}
684 		vcpu->arch.exception.has_error_code = has_error;
685 		vcpu->arch.exception.vector = nr;
686 		vcpu->arch.exception.error_code = error_code;
687 		vcpu->arch.exception.has_payload = has_payload;
688 		vcpu->arch.exception.payload = payload;
689 		if (!is_guest_mode(vcpu))
690 			kvm_deliver_exception_payload(vcpu,
691 						      &vcpu->arch.exception);
692 		return;
693 	}
694 
695 	/* to check exception */
696 	prev_nr = vcpu->arch.exception.vector;
697 	if (prev_nr == DF_VECTOR) {
698 		/* triple fault -> shutdown */
699 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
700 		return;
701 	}
702 	class1 = exception_class(prev_nr);
703 	class2 = exception_class(nr);
704 	if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY) ||
705 	    (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
706 		/*
707 		 * Synthesize #DF.  Clear the previously injected or pending
708 		 * exception so as not to incorrectly trigger shutdown.
709 		 */
710 		vcpu->arch.exception.injected = false;
711 		vcpu->arch.exception.pending = false;
712 
713 		kvm_queue_exception_e(vcpu, DF_VECTOR, 0);
714 	} else {
715 		/* replace previous exception with a new one in a hope
716 		   that instruction re-execution will regenerate lost
717 		   exception */
718 		goto queue;
719 	}
720 }
721 
kvm_queue_exception(struct kvm_vcpu * vcpu,unsigned nr)722 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
723 {
724 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
725 }
726 EXPORT_SYMBOL_GPL(kvm_queue_exception);
727 
kvm_requeue_exception(struct kvm_vcpu * vcpu,unsigned nr)728 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
729 {
730 	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
731 }
732 EXPORT_SYMBOL_GPL(kvm_requeue_exception);
733 
kvm_queue_exception_p(struct kvm_vcpu * vcpu,unsigned nr,unsigned long payload)734 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
735 			   unsigned long payload)
736 {
737 	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
738 }
739 EXPORT_SYMBOL_GPL(kvm_queue_exception_p);
740 
kvm_queue_exception_e_p(struct kvm_vcpu * vcpu,unsigned nr,u32 error_code,unsigned long payload)741 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
742 				    u32 error_code, unsigned long payload)
743 {
744 	kvm_multiple_exception(vcpu, nr, true, error_code,
745 			       true, payload, false);
746 }
747 
kvm_complete_insn_gp(struct kvm_vcpu * vcpu,int err)748 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
749 {
750 	if (err)
751 		kvm_inject_gp(vcpu, 0);
752 	else
753 		return kvm_skip_emulated_instruction(vcpu);
754 
755 	return 1;
756 }
757 EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
758 
complete_emulated_insn_gp(struct kvm_vcpu * vcpu,int err)759 static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err)
760 {
761 	if (err) {
762 		kvm_inject_gp(vcpu, 0);
763 		return 1;
764 	}
765 
766 	return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP |
767 				       EMULTYPE_COMPLETE_USER_EXIT);
768 }
769 
kvm_inject_page_fault(struct kvm_vcpu * vcpu,struct x86_exception * fault)770 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
771 {
772 	++vcpu->stat.pf_guest;
773 
774 	/*
775 	 * Async #PF in L2 is always forwarded to L1 as a VM-Exit regardless of
776 	 * whether or not L1 wants to intercept "regular" #PF.
777 	 */
778 	if (is_guest_mode(vcpu) && fault->async_page_fault)
779 		kvm_queue_exception_vmexit(vcpu, PF_VECTOR,
780 					   true, fault->error_code,
781 					   true, fault->address);
782 	else
783 		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
784 					fault->address);
785 }
786 EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
787 
kvm_inject_emulated_page_fault(struct kvm_vcpu * vcpu,struct x86_exception * fault)788 void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
789 				    struct x86_exception *fault)
790 {
791 	struct kvm_mmu *fault_mmu;
792 	WARN_ON_ONCE(fault->vector != PF_VECTOR);
793 
794 	fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
795 					       vcpu->arch.walk_mmu;
796 
797 	/*
798 	 * Invalidate the TLB entry for the faulting address, if it exists,
799 	 * else the access will fault indefinitely (and to emulate hardware).
800 	 */
801 	if ((fault->error_code & PFERR_PRESENT_MASK) &&
802 	    !(fault->error_code & PFERR_RSVD_MASK))
803 		kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address,
804 				       fault_mmu->root.hpa);
805 
806 	fault_mmu->inject_page_fault(vcpu, fault);
807 }
808 EXPORT_SYMBOL_GPL(kvm_inject_emulated_page_fault);
809 
kvm_inject_nmi(struct kvm_vcpu * vcpu)810 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
811 {
812 	atomic_inc(&vcpu->arch.nmi_queued);
813 	kvm_make_request(KVM_REQ_NMI, vcpu);
814 }
815 EXPORT_SYMBOL_GPL(kvm_inject_nmi);
816 
kvm_queue_exception_e(struct kvm_vcpu * vcpu,unsigned nr,u32 error_code)817 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
818 {
819 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
820 }
821 EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
822 
kvm_requeue_exception_e(struct kvm_vcpu * vcpu,unsigned nr,u32 error_code)823 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
824 {
825 	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
826 }
827 EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);
828 
829 /*
830  * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
831  * a #GP and return false.
832  */
kvm_require_cpl(struct kvm_vcpu * vcpu,int required_cpl)833 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
834 {
835 	if (static_call(kvm_x86_get_cpl)(vcpu) <= required_cpl)
836 		return true;
837 	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
838 	return false;
839 }
840 EXPORT_SYMBOL_GPL(kvm_require_cpl);
841 
kvm_require_dr(struct kvm_vcpu * vcpu,int dr)842 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
843 {
844 	if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))
845 		return true;
846 
847 	kvm_queue_exception(vcpu, UD_VECTOR);
848 	return false;
849 }
850 EXPORT_SYMBOL_GPL(kvm_require_dr);
851 
pdptr_rsvd_bits(struct kvm_vcpu * vcpu)852 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
853 {
854 	return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);
855 }
856 
857 /*
858  * Load the pae pdptrs.  Return 1 if they are all valid, 0 otherwise.
859  */
load_pdptrs(struct kvm_vcpu * vcpu,unsigned long cr3)860 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
861 {
862 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
863 	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
864 	gpa_t real_gpa;
865 	int i;
866 	int ret;
867 	u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
868 
869 	/*
870 	 * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated
871 	 * to an L1 GPA.
872 	 */
873 	real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn),
874 				     PFERR_USER_MASK | PFERR_WRITE_MASK, NULL);
875 	if (real_gpa == INVALID_GPA)
876 		return 0;
877 
878 	/* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */
879 	ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte,
880 				       cr3 & GENMASK(11, 5), sizeof(pdpte));
881 	if (ret < 0)
882 		return 0;
883 
884 	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
885 		if ((pdpte[i] & PT_PRESENT_MASK) &&
886 		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
887 			return 0;
888 		}
889 	}
890 
891 	/*
892 	 * Marking VCPU_EXREG_PDPTR dirty doesn't work for !tdp_enabled.
893 	 * Shadow page roots need to be reconstructed instead.
894 	 */
895 	if (!tdp_enabled && memcmp(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)))
896 		kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT);
897 
898 	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
899 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
900 	kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
901 	vcpu->arch.pdptrs_from_userspace = false;
902 
903 	return 1;
904 }
905 EXPORT_SYMBOL_GPL(load_pdptrs);
906 
kvm_post_set_cr0(struct kvm_vcpu * vcpu,unsigned long old_cr0,unsigned long cr0)907 void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0)
908 {
909 	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
910 		kvm_clear_async_pf_completion_queue(vcpu);
911 		kvm_async_pf_hash_reset(vcpu);
912 
913 		/*
914 		 * Clearing CR0.PG is defined to flush the TLB from the guest's
915 		 * perspective.
916 		 */
917 		if (!(cr0 & X86_CR0_PG))
918 			kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
919 	}
920 
921 	if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS)
922 		kvm_mmu_reset_context(vcpu);
923 
924 	if (((cr0 ^ old_cr0) & X86_CR0_CD) &&
925 	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
926 	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
927 		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);
928 }
929 EXPORT_SYMBOL_GPL(kvm_post_set_cr0);
930 
kvm_set_cr0(struct kvm_vcpu * vcpu,unsigned long cr0)931 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
932 {
933 	unsigned long old_cr0 = kvm_read_cr0(vcpu);
934 
935 	cr0 |= X86_CR0_ET;
936 
937 #ifdef CONFIG_X86_64
938 	if (cr0 & 0xffffffff00000000UL)
939 		return 1;
940 #endif
941 
942 	cr0 &= ~CR0_RESERVED_BITS;
943 
944 	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
945 		return 1;
946 
947 	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
948 		return 1;
949 
950 #ifdef CONFIG_X86_64
951 	if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&
952 	    (cr0 & X86_CR0_PG)) {
953 		int cs_db, cs_l;
954 
955 		if (!is_pae(vcpu))
956 			return 1;
957 		static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
958 		if (cs_l)
959 			return 1;
960 	}
961 #endif
962 	if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&
963 	    is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) &&
964 	    !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
965 		return 1;
966 
967 	if (!(cr0 & X86_CR0_PG) &&
968 	    (is_64_bit_mode(vcpu) || kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE)))
969 		return 1;
970 
971 	static_call(kvm_x86_set_cr0)(vcpu, cr0);
972 
973 	kvm_post_set_cr0(vcpu, old_cr0, cr0);
974 
975 	return 0;
976 }
977 EXPORT_SYMBOL_GPL(kvm_set_cr0);
978 
kvm_lmsw(struct kvm_vcpu * vcpu,unsigned long msw)979 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
980 {
981 	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
982 }
983 EXPORT_SYMBOL_GPL(kvm_lmsw);
984 
kvm_load_guest_xsave_state(struct kvm_vcpu * vcpu)985 void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)
986 {
987 	if (vcpu->arch.guest_state_protected)
988 		return;
989 
990 	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {
991 
992 		if (vcpu->arch.xcr0 != host_xcr0)
993 			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
994 
995 		if (vcpu->arch.xsaves_enabled &&
996 		    vcpu->arch.ia32_xss != host_xss)
997 			wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);
998 	}
999 
1000 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
1001 	if (static_cpu_has(X86_FEATURE_PKU) &&
1002 	    vcpu->arch.pkru != vcpu->arch.host_pkru &&
1003 	    ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||
1004 	     kvm_read_cr4_bits(vcpu, X86_CR4_PKE)))
1005 		write_pkru(vcpu->arch.pkru);
1006 #endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */
1007 }
1008 EXPORT_SYMBOL_GPL(kvm_load_guest_xsave_state);
1009 
kvm_load_host_xsave_state(struct kvm_vcpu * vcpu)1010 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)
1011 {
1012 	if (vcpu->arch.guest_state_protected)
1013 		return;
1014 
1015 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
1016 	if (static_cpu_has(X86_FEATURE_PKU) &&
1017 	    ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||
1018 	     kvm_read_cr4_bits(vcpu, X86_CR4_PKE))) {
1019 		vcpu->arch.pkru = rdpkru();
1020 		if (vcpu->arch.pkru != vcpu->arch.host_pkru)
1021 			write_pkru(vcpu->arch.host_pkru);
1022 	}
1023 #endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */
1024 
1025 	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {
1026 
1027 		if (vcpu->arch.xcr0 != host_xcr0)
1028 			xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
1029 
1030 		if (vcpu->arch.xsaves_enabled &&
1031 		    vcpu->arch.ia32_xss != host_xss)
1032 			wrmsrl(MSR_IA32_XSS, host_xss);
1033 	}
1034 
1035 }
1036 EXPORT_SYMBOL_GPL(kvm_load_host_xsave_state);
1037 
1038 #ifdef CONFIG_X86_64
kvm_guest_supported_xfd(struct kvm_vcpu * vcpu)1039 static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu)
1040 {
1041 	return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC;
1042 }
1043 #endif
1044 
__kvm_set_xcr(struct kvm_vcpu * vcpu,u32 index,u64 xcr)1045 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
1046 {
1047 	u64 xcr0 = xcr;
1048 	u64 old_xcr0 = vcpu->arch.xcr0;
1049 	u64 valid_bits;
1050 
1051 	/* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
1052 	if (index != XCR_XFEATURE_ENABLED_MASK)
1053 		return 1;
1054 	if (!(xcr0 & XFEATURE_MASK_FP))
1055 		return 1;
1056 	if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE))
1057 		return 1;
1058 
1059 	/*
1060 	 * Do not allow the guest to set bits that we do not support
1061 	 * saving.  However, xcr0 bit 0 is always set, even if the
1062 	 * emulated CPU does not support XSAVE (see kvm_vcpu_reset()).
1063 	 */
1064 	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
1065 	if (xcr0 & ~valid_bits)
1066 		return 1;
1067 
1068 	if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) !=
1069 	    (!(xcr0 & XFEATURE_MASK_BNDCSR)))
1070 		return 1;
1071 
1072 	if (xcr0 & XFEATURE_MASK_AVX512) {
1073 		if (!(xcr0 & XFEATURE_MASK_YMM))
1074 			return 1;
1075 		if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512)
1076 			return 1;
1077 	}
1078 
1079 	if ((xcr0 & XFEATURE_MASK_XTILE) &&
1080 	    ((xcr0 & XFEATURE_MASK_XTILE) != XFEATURE_MASK_XTILE))
1081 		return 1;
1082 
1083 	vcpu->arch.xcr0 = xcr0;
1084 
1085 	if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND)
1086 		kvm_update_cpuid_runtime(vcpu);
1087 	return 0;
1088 }
1089 
kvm_emulate_xsetbv(struct kvm_vcpu * vcpu)1090 int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu)
1091 {
1092 	/* Note, #UD due to CR4.OSXSAVE=0 has priority over the intercept. */
1093 	if (static_call(kvm_x86_get_cpl)(vcpu) != 0 ||
1094 	    __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) {
1095 		kvm_inject_gp(vcpu, 0);
1096 		return 1;
1097 	}
1098 
1099 	return kvm_skip_emulated_instruction(vcpu);
1100 }
1101 EXPORT_SYMBOL_GPL(kvm_emulate_xsetbv);
1102 
__kvm_is_valid_cr4(struct kvm_vcpu * vcpu,unsigned long cr4)1103 bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1104 {
1105 	if (cr4 & cr4_reserved_bits)
1106 		return false;
1107 
1108 	if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)
1109 		return false;
1110 
1111 	return true;
1112 }
1113 EXPORT_SYMBOL_GPL(__kvm_is_valid_cr4);
1114 
kvm_is_valid_cr4(struct kvm_vcpu * vcpu,unsigned long cr4)1115 static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1116 {
1117 	return __kvm_is_valid_cr4(vcpu, cr4) &&
1118 	       static_call(kvm_x86_is_valid_cr4)(vcpu, cr4);
1119 }
1120 
kvm_post_set_cr4(struct kvm_vcpu * vcpu,unsigned long old_cr4,unsigned long cr4)1121 void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4)
1122 {
1123 	if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS)
1124 		kvm_mmu_reset_context(vcpu);
1125 
1126 	/*
1127 	 * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB
1128 	 * according to the SDM; however, stale prev_roots could be reused
1129 	 * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we
1130 	 * free them all.  This is *not* a superset of KVM_REQ_TLB_FLUSH_GUEST
1131 	 * or KVM_REQ_TLB_FLUSH_CURRENT, because the hardware TLB is not flushed,
1132 	 * so fall through.
1133 	 */
1134 	if (!tdp_enabled &&
1135 	    (cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE))
1136 		kvm_mmu_unload(vcpu);
1137 
1138 	/*
1139 	 * The TLB has to be flushed for all PCIDs if any of the following
1140 	 * (architecturally required) changes happen:
1141 	 * - CR4.PCIDE is changed from 1 to 0
1142 	 * - CR4.PGE is toggled
1143 	 *
1144 	 * This is a superset of KVM_REQ_TLB_FLUSH_CURRENT.
1145 	 */
1146 	if (((cr4 ^ old_cr4) & X86_CR4_PGE) ||
1147 	    (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
1148 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
1149 
1150 	/*
1151 	 * The TLB has to be flushed for the current PCID if any of the
1152 	 * following (architecturally required) changes happen:
1153 	 * - CR4.SMEP is changed from 0 to 1
1154 	 * - CR4.PAE is toggled
1155 	 */
1156 	else if (((cr4 ^ old_cr4) & X86_CR4_PAE) ||
1157 		 ((cr4 & X86_CR4_SMEP) && !(old_cr4 & X86_CR4_SMEP)))
1158 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
1159 
1160 }
1161 EXPORT_SYMBOL_GPL(kvm_post_set_cr4);
1162 
kvm_set_cr4(struct kvm_vcpu * vcpu,unsigned long cr4)1163 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1164 {
1165 	unsigned long old_cr4 = kvm_read_cr4(vcpu);
1166 
1167 	if (!kvm_is_valid_cr4(vcpu, cr4))
1168 		return 1;
1169 
1170 	if (is_long_mode(vcpu)) {
1171 		if (!(cr4 & X86_CR4_PAE))
1172 			return 1;
1173 		if ((cr4 ^ old_cr4) & X86_CR4_LA57)
1174 			return 1;
1175 	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
1176 		   && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS)
1177 		   && !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
1178 		return 1;
1179 
1180 	if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
1181 		if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))
1182 			return 1;
1183 
1184 		/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
1185 		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
1186 			return 1;
1187 	}
1188 
1189 	static_call(kvm_x86_set_cr4)(vcpu, cr4);
1190 
1191 	kvm_post_set_cr4(vcpu, old_cr4, cr4);
1192 
1193 	return 0;
1194 }
1195 EXPORT_SYMBOL_GPL(kvm_set_cr4);
1196 
kvm_invalidate_pcid(struct kvm_vcpu * vcpu,unsigned long pcid)1197 static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)
1198 {
1199 	struct kvm_mmu *mmu = vcpu->arch.mmu;
1200 	unsigned long roots_to_free = 0;
1201 	int i;
1202 
1203 	/*
1204 	 * MOV CR3 and INVPCID are usually not intercepted when using TDP, but
1205 	 * this is reachable when running EPT=1 and unrestricted_guest=0,  and
1206 	 * also via the emulator.  KVM's TDP page tables are not in the scope of
1207 	 * the invalidation, but the guest's TLB entries need to be flushed as
1208 	 * the CPU may have cached entries in its TLB for the target PCID.
1209 	 */
1210 	if (unlikely(tdp_enabled)) {
1211 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
1212 		return;
1213 	}
1214 
1215 	/*
1216 	 * If neither the current CR3 nor any of the prev_roots use the given
1217 	 * PCID, then nothing needs to be done here because a resync will
1218 	 * happen anyway before switching to any other CR3.
1219 	 */
1220 	if (kvm_get_active_pcid(vcpu) == pcid) {
1221 		kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
1222 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
1223 	}
1224 
1225 	/*
1226 	 * If PCID is disabled, there is no need to free prev_roots even if the
1227 	 * PCIDs for them are also 0, because MOV to CR3 always flushes the TLB
1228 	 * with PCIDE=0.
1229 	 */
1230 	if (!kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
1231 		return;
1232 
1233 	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
1234 		if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid)
1235 			roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
1236 
1237 	kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free);
1238 }
1239 
kvm_set_cr3(struct kvm_vcpu * vcpu,unsigned long cr3)1240 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1241 {
1242 	bool skip_tlb_flush = false;
1243 	unsigned long pcid = 0;
1244 #ifdef CONFIG_X86_64
1245 	bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);
1246 
1247 	if (pcid_enabled) {
1248 		skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
1249 		cr3 &= ~X86_CR3_PCID_NOFLUSH;
1250 		pcid = cr3 & X86_CR3_PCID_MASK;
1251 	}
1252 #endif
1253 
1254 	/* PDPTRs are always reloaded for PAE paging. */
1255 	if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu))
1256 		goto handle_tlb_flush;
1257 
1258 	/*
1259 	 * Do not condition the GPA check on long mode, this helper is used to
1260 	 * stuff CR3, e.g. for RSM emulation, and there is no guarantee that
1261 	 * the current vCPU mode is accurate.
1262 	 */
1263 	if (kvm_vcpu_is_illegal_gpa(vcpu, cr3))
1264 		return 1;
1265 
1266 	if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3))
1267 		return 1;
1268 
1269 	if (cr3 != kvm_read_cr3(vcpu))
1270 		kvm_mmu_new_pgd(vcpu, cr3);
1271 
1272 	vcpu->arch.cr3 = cr3;
1273 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
1274 	/* Do not call post_set_cr3, we do not get here for confidential guests.  */
1275 
1276 handle_tlb_flush:
1277 	/*
1278 	 * A load of CR3 that flushes the TLB flushes only the current PCID,
1279 	 * even if PCID is disabled, in which case PCID=0 is flushed.  It's a
1280 	 * moot point in the end because _disabling_ PCID will flush all PCIDs,
1281 	 * and it's impossible to use a non-zero PCID when PCID is disabled,
1282 	 * i.e. only PCID=0 can be relevant.
1283 	 */
1284 	if (!skip_tlb_flush)
1285 		kvm_invalidate_pcid(vcpu, pcid);
1286 
1287 	return 0;
1288 }
1289 EXPORT_SYMBOL_GPL(kvm_set_cr3);
1290 
kvm_set_cr8(struct kvm_vcpu * vcpu,unsigned long cr8)1291 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
1292 {
1293 	if (cr8 & CR8_RESERVED_BITS)
1294 		return 1;
1295 	if (lapic_in_kernel(vcpu))
1296 		kvm_lapic_set_tpr(vcpu, cr8);
1297 	else
1298 		vcpu->arch.cr8 = cr8;
1299 	return 0;
1300 }
1301 EXPORT_SYMBOL_GPL(kvm_set_cr8);
1302 
kvm_get_cr8(struct kvm_vcpu * vcpu)1303 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
1304 {
1305 	if (lapic_in_kernel(vcpu))
1306 		return kvm_lapic_get_cr8(vcpu);
1307 	else
1308 		return vcpu->arch.cr8;
1309 }
1310 EXPORT_SYMBOL_GPL(kvm_get_cr8);
1311 
kvm_update_dr0123(struct kvm_vcpu * vcpu)1312 static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
1313 {
1314 	int i;
1315 
1316 	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
1317 		for (i = 0; i < KVM_NR_DB_REGS; i++)
1318 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
1319 	}
1320 }
1321 
kvm_update_dr7(struct kvm_vcpu * vcpu)1322 void kvm_update_dr7(struct kvm_vcpu *vcpu)
1323 {
1324 	unsigned long dr7;
1325 
1326 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1327 		dr7 = vcpu->arch.guest_debug_dr7;
1328 	else
1329 		dr7 = vcpu->arch.dr7;
1330 	static_call(kvm_x86_set_dr7)(vcpu, dr7);
1331 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
1332 	if (dr7 & DR7_BP_EN_MASK)
1333 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
1334 }
1335 EXPORT_SYMBOL_GPL(kvm_update_dr7);
1336 
kvm_dr6_fixed(struct kvm_vcpu * vcpu)1337 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
1338 {
1339 	u64 fixed = DR6_FIXED_1;
1340 
1341 	if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
1342 		fixed |= DR6_RTM;
1343 
1344 	if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))
1345 		fixed |= DR6_BUS_LOCK;
1346 	return fixed;
1347 }
1348 
kvm_set_dr(struct kvm_vcpu * vcpu,int dr,unsigned long val)1349 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1350 {
1351 	size_t size = ARRAY_SIZE(vcpu->arch.db);
1352 
1353 	switch (dr) {
1354 	case 0 ... 3:
1355 		vcpu->arch.db[array_index_nospec(dr, size)] = val;
1356 		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
1357 			vcpu->arch.eff_db[dr] = val;
1358 		break;
1359 	case 4:
1360 	case 6:
1361 		if (!kvm_dr6_valid(val))
1362 			return 1; /* #GP */
1363 		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
1364 		break;
1365 	case 5:
1366 	default: /* 7 */
1367 		if (!kvm_dr7_valid(val))
1368 			return 1; /* #GP */
1369 		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
1370 		kvm_update_dr7(vcpu);
1371 		break;
1372 	}
1373 
1374 	return 0;
1375 }
1376 EXPORT_SYMBOL_GPL(kvm_set_dr);
1377 
kvm_get_dr(struct kvm_vcpu * vcpu,int dr,unsigned long * val)1378 void kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
1379 {
1380 	size_t size = ARRAY_SIZE(vcpu->arch.db);
1381 
1382 	switch (dr) {
1383 	case 0 ... 3:
1384 		*val = vcpu->arch.db[array_index_nospec(dr, size)];
1385 		break;
1386 	case 4:
1387 	case 6:
1388 		*val = vcpu->arch.dr6;
1389 		break;
1390 	case 5:
1391 	default: /* 7 */
1392 		*val = vcpu->arch.dr7;
1393 		break;
1394 	}
1395 }
1396 EXPORT_SYMBOL_GPL(kvm_get_dr);
1397 
kvm_emulate_rdpmc(struct kvm_vcpu * vcpu)1398 int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)
1399 {
1400 	u32 ecx = kvm_rcx_read(vcpu);
1401 	u64 data;
1402 
1403 	if (kvm_pmu_rdpmc(vcpu, ecx, &data)) {
1404 		kvm_inject_gp(vcpu, 0);
1405 		return 1;
1406 	}
1407 
1408 	kvm_rax_write(vcpu, (u32)data);
1409 	kvm_rdx_write(vcpu, data >> 32);
1410 	return kvm_skip_emulated_instruction(vcpu);
1411 }
1412 EXPORT_SYMBOL_GPL(kvm_emulate_rdpmc);
1413 
1414 /*
1415  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1416  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1417  *
1418  * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features)
1419  * extract the supported MSRs from the related const lists.
1420  * msrs_to_save is selected from the msrs_to_save_all to reflect the
1421  * capabilities of the host cpu. This capabilities test skips MSRs that are
1422  * kvm-specific. Those are put in emulated_msrs_all; filtering of emulated_msrs
1423  * may depend on host virtualization features rather than host cpu features.
1424  */
1425 
1426 static const u32 msrs_to_save_all[] = {
1427 	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
1428 	MSR_STAR,
1429 #ifdef CONFIG_X86_64
1430 	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1431 #endif
1432 	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
1433 	MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
1434 	MSR_IA32_SPEC_CTRL,
1435 	MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
1436 	MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
1437 	MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
1438 	MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
1439 	MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
1440 	MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
1441 	MSR_IA32_UMWAIT_CONTROL,
1442 
1443 	MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
1444 	MSR_ARCH_PERFMON_FIXED_CTR0 + 2,
1445 	MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
1446 	MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
1447 	MSR_IA32_PEBS_ENABLE, MSR_IA32_DS_AREA, MSR_PEBS_DATA_CFG,
1448 
1449 	/* This part of MSRs should match KVM_INTEL_PMC_MAX_GENERIC. */
1450 	MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
1451 	MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3,
1452 	MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5,
1453 	MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7,
1454 	MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1,
1455 	MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
1456 	MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
1457 	MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
1458 
1459 	MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3,
1460 	MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3,
1461 
1462 	/* This part of MSRs should match KVM_AMD_PMC_MAX_GENERIC. */
1463 	MSR_F15H_PERF_CTL0, MSR_F15H_PERF_CTL1, MSR_F15H_PERF_CTL2,
1464 	MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5,
1465 	MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2,
1466 	MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5,
1467 
1468 	MSR_IA32_XFD, MSR_IA32_XFD_ERR,
1469 };
1470 
1471 static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_all)];
1472 static unsigned num_msrs_to_save;
1473 
1474 static const u32 emulated_msrs_all[] = {
1475 	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
1476 	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
1477 	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
1478 	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
1479 	HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
1480 	HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
1481 	HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
1482 	HV_X64_MSR_RESET,
1483 	HV_X64_MSR_VP_INDEX,
1484 	HV_X64_MSR_VP_RUNTIME,
1485 	HV_X64_MSR_SCONTROL,
1486 	HV_X64_MSR_STIMER0_CONFIG,
1487 	HV_X64_MSR_VP_ASSIST_PAGE,
1488 	HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
1489 	HV_X64_MSR_TSC_EMULATION_STATUS,
1490 	HV_X64_MSR_SYNDBG_OPTIONS,
1491 	HV_X64_MSR_SYNDBG_CONTROL, HV_X64_MSR_SYNDBG_STATUS,
1492 	HV_X64_MSR_SYNDBG_SEND_BUFFER, HV_X64_MSR_SYNDBG_RECV_BUFFER,
1493 	HV_X64_MSR_SYNDBG_PENDING_BUFFER,
1494 
1495 	MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
1496 	MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK,
1497 
1498 	MSR_IA32_TSC_ADJUST,
1499 	MSR_IA32_TSC_DEADLINE,
1500 	MSR_IA32_ARCH_CAPABILITIES,
1501 	MSR_IA32_PERF_CAPABILITIES,
1502 	MSR_IA32_MISC_ENABLE,
1503 	MSR_IA32_MCG_STATUS,
1504 	MSR_IA32_MCG_CTL,
1505 	MSR_IA32_MCG_EXT_CTL,
1506 	MSR_IA32_SMBASE,
1507 	MSR_SMI_COUNT,
1508 	MSR_PLATFORM_INFO,
1509 	MSR_MISC_FEATURES_ENABLES,
1510 	MSR_AMD64_VIRT_SPEC_CTRL,
1511 	MSR_AMD64_TSC_RATIO,
1512 	MSR_IA32_POWER_CTL,
1513 	MSR_IA32_UCODE_REV,
1514 
1515 	/*
1516 	 * The following list leaves out MSRs whose values are determined
1517 	 * by arch/x86/kvm/vmx/nested.c based on CPUID or other MSRs.
1518 	 * We always support the "true" VMX control MSRs, even if the host
1519 	 * processor does not, so I am putting these registers here rather
1520 	 * than in msrs_to_save_all.
1521 	 */
1522 	MSR_IA32_VMX_BASIC,
1523 	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
1524 	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
1525 	MSR_IA32_VMX_TRUE_EXIT_CTLS,
1526 	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
1527 	MSR_IA32_VMX_MISC,
1528 	MSR_IA32_VMX_CR0_FIXED0,
1529 	MSR_IA32_VMX_CR4_FIXED0,
1530 	MSR_IA32_VMX_VMCS_ENUM,
1531 	MSR_IA32_VMX_PROCBASED_CTLS2,
1532 	MSR_IA32_VMX_EPT_VPID_CAP,
1533 	MSR_IA32_VMX_VMFUNC,
1534 
1535 	MSR_K7_HWCR,
1536 	MSR_KVM_POLL_CONTROL,
1537 };
1538 
1539 static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)];
1540 static unsigned num_emulated_msrs;
1541 
1542 /*
1543  * List of msr numbers which are used to expose MSR-based features that
1544  * can be used by a hypervisor to validate requested CPU features.
1545  */
1546 static const u32 msr_based_features_all[] = {
1547 	MSR_IA32_VMX_BASIC,
1548 	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
1549 	MSR_IA32_VMX_PINBASED_CTLS,
1550 	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
1551 	MSR_IA32_VMX_PROCBASED_CTLS,
1552 	MSR_IA32_VMX_TRUE_EXIT_CTLS,
1553 	MSR_IA32_VMX_EXIT_CTLS,
1554 	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
1555 	MSR_IA32_VMX_ENTRY_CTLS,
1556 	MSR_IA32_VMX_MISC,
1557 	MSR_IA32_VMX_CR0_FIXED0,
1558 	MSR_IA32_VMX_CR0_FIXED1,
1559 	MSR_IA32_VMX_CR4_FIXED0,
1560 	MSR_IA32_VMX_CR4_FIXED1,
1561 	MSR_IA32_VMX_VMCS_ENUM,
1562 	MSR_IA32_VMX_PROCBASED_CTLS2,
1563 	MSR_IA32_VMX_EPT_VPID_CAP,
1564 	MSR_IA32_VMX_VMFUNC,
1565 
1566 	MSR_AMD64_DE_CFG,
1567 	MSR_IA32_UCODE_REV,
1568 	MSR_IA32_ARCH_CAPABILITIES,
1569 	MSR_IA32_PERF_CAPABILITIES,
1570 };
1571 
1572 static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all)];
1573 static unsigned int num_msr_based_features;
1574 
1575 /*
1576  * Some IA32_ARCH_CAPABILITIES bits have dependencies on MSRs that KVM
1577  * does not yet virtualize. These include:
1578  *   10 - MISC_PACKAGE_CTRLS
1579  *   11 - ENERGY_FILTERING_CTL
1580  *   12 - DOITM
1581  *   18 - FB_CLEAR_CTRL
1582  *   21 - XAPIC_DISABLE_STATUS
1583  *   23 - OVERCLOCKING_STATUS
1584  */
1585 
1586 #define KVM_SUPPORTED_ARCH_CAP \
1587 	(ARCH_CAP_RDCL_NO | ARCH_CAP_IBRS_ALL | ARCH_CAP_RSBA | \
1588 	 ARCH_CAP_SKIP_VMENTRY_L1DFLUSH | ARCH_CAP_SSB_NO | ARCH_CAP_MDS_NO | \
1589 	 ARCH_CAP_PSCHANGE_MC_NO | ARCH_CAP_TSX_CTRL_MSR | ARCH_CAP_TAA_NO | \
1590 	 ARCH_CAP_SBDR_SSDP_NO | ARCH_CAP_FBSDP_NO | ARCH_CAP_PSDP_NO | \
1591 	 ARCH_CAP_FB_CLEAR | ARCH_CAP_RRSBA | ARCH_CAP_PBRSB_NO)
1592 
kvm_get_arch_capabilities(void)1593 static u64 kvm_get_arch_capabilities(void)
1594 {
1595 	u64 data = 0;
1596 
1597 	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) {
1598 		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, data);
1599 		data &= KVM_SUPPORTED_ARCH_CAP;
1600 	}
1601 
1602 	/*
1603 	 * If nx_huge_pages is enabled, KVM's shadow paging will ensure that
1604 	 * the nested hypervisor runs with NX huge pages.  If it is not,
1605 	 * L1 is anyway vulnerable to ITLB_MULTIHIT exploits from other
1606 	 * L1 guests, so it need not worry about its own (L2) guests.
1607 	 */
1608 	data |= ARCH_CAP_PSCHANGE_MC_NO;
1609 
1610 	/*
1611 	 * If we're doing cache flushes (either "always" or "cond")
1612 	 * we will do one whenever the guest does a vmlaunch/vmresume.
1613 	 * If an outer hypervisor is doing the cache flush for us
1614 	 * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that
1615 	 * capability to the guest too, and if EPT is disabled we're not
1616 	 * vulnerable.  Overall, only VMENTER_L1D_FLUSH_NEVER will
1617 	 * require a nested hypervisor to do a flush of its own.
1618 	 */
1619 	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
1620 		data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;
1621 
1622 	if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
1623 		data |= ARCH_CAP_RDCL_NO;
1624 	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1625 		data |= ARCH_CAP_SSB_NO;
1626 	if (!boot_cpu_has_bug(X86_BUG_MDS))
1627 		data |= ARCH_CAP_MDS_NO;
1628 
1629 	if (!boot_cpu_has(X86_FEATURE_RTM)) {
1630 		/*
1631 		 * If RTM=0 because the kernel has disabled TSX, the host might
1632 		 * have TAA_NO or TSX_CTRL.  Clear TAA_NO (the guest sees RTM=0
1633 		 * and therefore knows that there cannot be TAA) but keep
1634 		 * TSX_CTRL: some buggy userspaces leave it set on tsx=on hosts,
1635 		 * and we want to allow migrating those guests to tsx=off hosts.
1636 		 */
1637 		data &= ~ARCH_CAP_TAA_NO;
1638 	} else if (!boot_cpu_has_bug(X86_BUG_TAA)) {
1639 		data |= ARCH_CAP_TAA_NO;
1640 	} else {
1641 		/*
1642 		 * Nothing to do here; we emulate TSX_CTRL if present on the
1643 		 * host so the guest can choose between disabling TSX or
1644 		 * using VERW to clear CPU buffers.
1645 		 */
1646 	}
1647 
1648 	return data;
1649 }
1650 
kvm_get_msr_feature(struct kvm_msr_entry * msr)1651 static int kvm_get_msr_feature(struct kvm_msr_entry *msr)
1652 {
1653 	switch (msr->index) {
1654 	case MSR_IA32_ARCH_CAPABILITIES:
1655 		msr->data = kvm_get_arch_capabilities();
1656 		break;
1657 	case MSR_IA32_UCODE_REV:
1658 		rdmsrl_safe(msr->index, &msr->data);
1659 		break;
1660 	default:
1661 		return static_call(kvm_x86_get_msr_feature)(msr);
1662 	}
1663 	return 0;
1664 }
1665 
do_get_msr_feature(struct kvm_vcpu * vcpu,unsigned index,u64 * data)1666 static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1667 {
1668 	struct kvm_msr_entry msr;
1669 	int r;
1670 
1671 	msr.index = index;
1672 	r = kvm_get_msr_feature(&msr);
1673 
1674 	if (r == KVM_MSR_RET_INVALID) {
1675 		/* Unconditionally clear the output for simplicity */
1676 		*data = 0;
1677 		if (kvm_msr_ignored_check(index, 0, false))
1678 			r = 0;
1679 	}
1680 
1681 	if (r)
1682 		return r;
1683 
1684 	*data = msr.data;
1685 
1686 	return 0;
1687 }
1688 
__kvm_valid_efer(struct kvm_vcpu * vcpu,u64 efer)1689 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1690 {
1691 	if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
1692 		return false;
1693 
1694 	if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
1695 		return false;
1696 
1697 	if (efer & (EFER_LME | EFER_LMA) &&
1698 	    !guest_cpuid_has(vcpu, X86_FEATURE_LM))
1699 		return false;
1700 
1701 	if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
1702 		return false;
1703 
1704 	return true;
1705 
1706 }
kvm_valid_efer(struct kvm_vcpu * vcpu,u64 efer)1707 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1708 {
1709 	if (efer & efer_reserved_bits)
1710 		return false;
1711 
1712 	return __kvm_valid_efer(vcpu, efer);
1713 }
1714 EXPORT_SYMBOL_GPL(kvm_valid_efer);
1715 
set_efer(struct kvm_vcpu * vcpu,struct msr_data * msr_info)1716 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1717 {
1718 	u64 old_efer = vcpu->arch.efer;
1719 	u64 efer = msr_info->data;
1720 	int r;
1721 
1722 	if (efer & efer_reserved_bits)
1723 		return 1;
1724 
1725 	if (!msr_info->host_initiated) {
1726 		if (!__kvm_valid_efer(vcpu, efer))
1727 			return 1;
1728 
1729 		if (is_paging(vcpu) &&
1730 		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
1731 			return 1;
1732 	}
1733 
1734 	efer &= ~EFER_LMA;
1735 	efer |= vcpu->arch.efer & EFER_LMA;
1736 
1737 	r = static_call(kvm_x86_set_efer)(vcpu, efer);
1738 	if (r) {
1739 		WARN_ON(r > 0);
1740 		return r;
1741 	}
1742 
1743 	if ((efer ^ old_efer) & KVM_MMU_EFER_ROLE_BITS)
1744 		kvm_mmu_reset_context(vcpu);
1745 
1746 	return 0;
1747 }
1748 
kvm_enable_efer_bits(u64 mask)1749 void kvm_enable_efer_bits(u64 mask)
1750 {
1751        efer_reserved_bits &= ~mask;
1752 }
1753 EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
1754 
kvm_msr_allowed(struct kvm_vcpu * vcpu,u32 index,u32 type)1755 bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)
1756 {
1757 	struct kvm_x86_msr_filter *msr_filter;
1758 	struct msr_bitmap_range *ranges;
1759 	struct kvm *kvm = vcpu->kvm;
1760 	bool allowed;
1761 	int idx;
1762 	u32 i;
1763 
1764 	/* x2APIC MSRs do not support filtering. */
1765 	if (index >= 0x800 && index <= 0x8ff)
1766 		return true;
1767 
1768 	idx = srcu_read_lock(&kvm->srcu);
1769 
1770 	msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu);
1771 	if (!msr_filter) {
1772 		allowed = true;
1773 		goto out;
1774 	}
1775 
1776 	allowed = msr_filter->default_allow;
1777 	ranges = msr_filter->ranges;
1778 
1779 	for (i = 0; i < msr_filter->count; i++) {
1780 		u32 start = ranges[i].base;
1781 		u32 end = start + ranges[i].nmsrs;
1782 		u32 flags = ranges[i].flags;
1783 		unsigned long *bitmap = ranges[i].bitmap;
1784 
1785 		if ((index >= start) && (index < end) && (flags & type)) {
1786 			allowed = !!test_bit(index - start, bitmap);
1787 			break;
1788 		}
1789 	}
1790 
1791 out:
1792 	srcu_read_unlock(&kvm->srcu, idx);
1793 
1794 	return allowed;
1795 }
1796 EXPORT_SYMBOL_GPL(kvm_msr_allowed);
1797 
1798 /*
1799  * Write @data into the MSR specified by @index.  Select MSR specific fault
1800  * checks are bypassed if @host_initiated is %true.
1801  * Returns 0 on success, non-0 otherwise.
1802  * Assumes vcpu_load() was already called.
1803  */
__kvm_set_msr(struct kvm_vcpu * vcpu,u32 index,u64 data,bool host_initiated)1804 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
1805 			 bool host_initiated)
1806 {
1807 	struct msr_data msr;
1808 
1809 	switch (index) {
1810 	case MSR_FS_BASE:
1811 	case MSR_GS_BASE:
1812 	case MSR_KERNEL_GS_BASE:
1813 	case MSR_CSTAR:
1814 	case MSR_LSTAR:
1815 		if (is_noncanonical_address(data, vcpu))
1816 			return 1;
1817 		break;
1818 	case MSR_IA32_SYSENTER_EIP:
1819 	case MSR_IA32_SYSENTER_ESP:
1820 		/*
1821 		 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
1822 		 * non-canonical address is written on Intel but not on
1823 		 * AMD (which ignores the top 32-bits, because it does
1824 		 * not implement 64-bit SYSENTER).
1825 		 *
1826 		 * 64-bit code should hence be able to write a non-canonical
1827 		 * value on AMD.  Making the address canonical ensures that
1828 		 * vmentry does not fail on Intel after writing a non-canonical
1829 		 * value, and that something deterministic happens if the guest
1830 		 * invokes 64-bit SYSENTER.
1831 		 */
1832 		data = __canonical_address(data, vcpu_virt_addr_bits(vcpu));
1833 		break;
1834 	case MSR_TSC_AUX:
1835 		if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
1836 			return 1;
1837 
1838 		if (!host_initiated &&
1839 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&
1840 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))
1841 			return 1;
1842 
1843 		/*
1844 		 * Per Intel's SDM, bits 63:32 are reserved, but AMD's APM has
1845 		 * incomplete and conflicting architectural behavior.  Current
1846 		 * AMD CPUs completely ignore bits 63:32, i.e. they aren't
1847 		 * reserved and always read as zeros.  Enforce Intel's reserved
1848 		 * bits check if and only if the guest CPU is Intel, and clear
1849 		 * the bits in all other cases.  This ensures cross-vendor
1850 		 * migration will provide consistent behavior for the guest.
1851 		 */
1852 		if (guest_cpuid_is_intel(vcpu) && (data >> 32) != 0)
1853 			return 1;
1854 
1855 		data = (u32)data;
1856 		break;
1857 	}
1858 
1859 	msr.data = data;
1860 	msr.index = index;
1861 	msr.host_initiated = host_initiated;
1862 
1863 	return static_call(kvm_x86_set_msr)(vcpu, &msr);
1864 }
1865 
kvm_set_msr_ignored_check(struct kvm_vcpu * vcpu,u32 index,u64 data,bool host_initiated)1866 static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,
1867 				     u32 index, u64 data, bool host_initiated)
1868 {
1869 	int ret = __kvm_set_msr(vcpu, index, data, host_initiated);
1870 
1871 	if (ret == KVM_MSR_RET_INVALID)
1872 		if (kvm_msr_ignored_check(index, data, true))
1873 			ret = 0;
1874 
1875 	return ret;
1876 }
1877 
1878 /*
1879  * Read the MSR specified by @index into @data.  Select MSR specific fault
1880  * checks are bypassed if @host_initiated is %true.
1881  * Returns 0 on success, non-0 otherwise.
1882  * Assumes vcpu_load() was already called.
1883  */
__kvm_get_msr(struct kvm_vcpu * vcpu,u32 index,u64 * data,bool host_initiated)1884 int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
1885 		  bool host_initiated)
1886 {
1887 	struct msr_data msr;
1888 	int ret;
1889 
1890 	switch (index) {
1891 	case MSR_TSC_AUX:
1892 		if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
1893 			return 1;
1894 
1895 		if (!host_initiated &&
1896 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&
1897 		    !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))
1898 			return 1;
1899 		break;
1900 	}
1901 
1902 	msr.index = index;
1903 	msr.host_initiated = host_initiated;
1904 
1905 	ret = static_call(kvm_x86_get_msr)(vcpu, &msr);
1906 	if (!ret)
1907 		*data = msr.data;
1908 	return ret;
1909 }
1910 
kvm_get_msr_ignored_check(struct kvm_vcpu * vcpu,u32 index,u64 * data,bool host_initiated)1911 static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,
1912 				     u32 index, u64 *data, bool host_initiated)
1913 {
1914 	int ret = __kvm_get_msr(vcpu, index, data, host_initiated);
1915 
1916 	if (ret == KVM_MSR_RET_INVALID) {
1917 		/* Unconditionally clear *data for simplicity */
1918 		*data = 0;
1919 		if (kvm_msr_ignored_check(index, 0, false))
1920 			ret = 0;
1921 	}
1922 
1923 	return ret;
1924 }
1925 
kvm_get_msr_with_filter(struct kvm_vcpu * vcpu,u32 index,u64 * data)1926 static int kvm_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1927 {
1928 	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))
1929 		return KVM_MSR_RET_FILTERED;
1930 	return kvm_get_msr_ignored_check(vcpu, index, data, false);
1931 }
1932 
kvm_set_msr_with_filter(struct kvm_vcpu * vcpu,u32 index,u64 data)1933 static int kvm_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 data)
1934 {
1935 	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))
1936 		return KVM_MSR_RET_FILTERED;
1937 	return kvm_set_msr_ignored_check(vcpu, index, data, false);
1938 }
1939 
kvm_get_msr(struct kvm_vcpu * vcpu,u32 index,u64 * data)1940 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1941 {
1942 	return kvm_get_msr_ignored_check(vcpu, index, data, false);
1943 }
1944 EXPORT_SYMBOL_GPL(kvm_get_msr);
1945 
kvm_set_msr(struct kvm_vcpu * vcpu,u32 index,u64 data)1946 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
1947 {
1948 	return kvm_set_msr_ignored_check(vcpu, index, data, false);
1949 }
1950 EXPORT_SYMBOL_GPL(kvm_set_msr);
1951 
complete_userspace_rdmsr(struct kvm_vcpu * vcpu)1952 static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu)
1953 {
1954 	if (!vcpu->run->msr.error) {
1955 		kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);
1956 		kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
1957 	}
1958 }
1959 
complete_emulated_msr_access(struct kvm_vcpu * vcpu)1960 static int complete_emulated_msr_access(struct kvm_vcpu *vcpu)
1961 {
1962 	return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error);
1963 }
1964 
complete_emulated_rdmsr(struct kvm_vcpu * vcpu)1965 static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
1966 {
1967 	complete_userspace_rdmsr(vcpu);
1968 	return complete_emulated_msr_access(vcpu);
1969 }
1970 
complete_fast_msr_access(struct kvm_vcpu * vcpu)1971 static int complete_fast_msr_access(struct kvm_vcpu *vcpu)
1972 {
1973 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error);
1974 }
1975 
complete_fast_rdmsr(struct kvm_vcpu * vcpu)1976 static int complete_fast_rdmsr(struct kvm_vcpu *vcpu)
1977 {
1978 	complete_userspace_rdmsr(vcpu);
1979 	return complete_fast_msr_access(vcpu);
1980 }
1981 
kvm_msr_reason(int r)1982 static u64 kvm_msr_reason(int r)
1983 {
1984 	switch (r) {
1985 	case KVM_MSR_RET_INVALID:
1986 		return KVM_MSR_EXIT_REASON_UNKNOWN;
1987 	case KVM_MSR_RET_FILTERED:
1988 		return KVM_MSR_EXIT_REASON_FILTER;
1989 	default:
1990 		return KVM_MSR_EXIT_REASON_INVAL;
1991 	}
1992 }
1993 
kvm_msr_user_space(struct kvm_vcpu * vcpu,u32 index,u32 exit_reason,u64 data,int (* completion)(struct kvm_vcpu * vcpu),int r)1994 static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,
1995 			      u32 exit_reason, u64 data,
1996 			      int (*completion)(struct kvm_vcpu *vcpu),
1997 			      int r)
1998 {
1999 	u64 msr_reason = kvm_msr_reason(r);
2000 
2001 	/* Check if the user wanted to know about this MSR fault */
2002 	if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))
2003 		return 0;
2004 
2005 	vcpu->run->exit_reason = exit_reason;
2006 	vcpu->run->msr.error = 0;
2007 	memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));
2008 	vcpu->run->msr.reason = msr_reason;
2009 	vcpu->run->msr.index = index;
2010 	vcpu->run->msr.data = data;
2011 	vcpu->arch.complete_userspace_io = completion;
2012 
2013 	return 1;
2014 }
2015 
kvm_emulate_rdmsr(struct kvm_vcpu * vcpu)2016 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
2017 {
2018 	u32 ecx = kvm_rcx_read(vcpu);
2019 	u64 data;
2020 	int r;
2021 
2022 	r = kvm_get_msr_with_filter(vcpu, ecx, &data);
2023 
2024 	if (!r) {
2025 		trace_kvm_msr_read(ecx, data);
2026 
2027 		kvm_rax_write(vcpu, data & -1u);
2028 		kvm_rdx_write(vcpu, (data >> 32) & -1u);
2029 	} else {
2030 		/* MSR read failed? See if we should ask user space */
2031 		if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_RDMSR, 0,
2032 				       complete_fast_rdmsr, r))
2033 			return 0;
2034 		trace_kvm_msr_read_ex(ecx);
2035 	}
2036 
2037 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
2038 }
2039 EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
2040 
kvm_emulate_wrmsr(struct kvm_vcpu * vcpu)2041 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
2042 {
2043 	u32 ecx = kvm_rcx_read(vcpu);
2044 	u64 data = kvm_read_edx_eax(vcpu);
2045 	int r;
2046 
2047 	r = kvm_set_msr_with_filter(vcpu, ecx, data);
2048 
2049 	if (!r) {
2050 		trace_kvm_msr_write(ecx, data);
2051 	} else {
2052 		/* MSR write failed? See if we should ask user space */
2053 		if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_WRMSR, data,
2054 				       complete_fast_msr_access, r))
2055 			return 0;
2056 		/* Signal all other negative errors to userspace */
2057 		if (r < 0)
2058 			return r;
2059 		trace_kvm_msr_write_ex(ecx, data);
2060 	}
2061 
2062 	return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
2063 }
2064 EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
2065 
kvm_emulate_as_nop(struct kvm_vcpu * vcpu)2066 int kvm_emulate_as_nop(struct kvm_vcpu *vcpu)
2067 {
2068 	return kvm_skip_emulated_instruction(vcpu);
2069 }
2070 EXPORT_SYMBOL_GPL(kvm_emulate_as_nop);
2071 
kvm_emulate_invd(struct kvm_vcpu * vcpu)2072 int kvm_emulate_invd(struct kvm_vcpu *vcpu)
2073 {
2074 	/* Treat an INVD instruction as a NOP and just skip it. */
2075 	return kvm_emulate_as_nop(vcpu);
2076 }
2077 EXPORT_SYMBOL_GPL(kvm_emulate_invd);
2078 
kvm_handle_invalid_op(struct kvm_vcpu * vcpu)2079 int kvm_handle_invalid_op(struct kvm_vcpu *vcpu)
2080 {
2081 	kvm_queue_exception(vcpu, UD_VECTOR);
2082 	return 1;
2083 }
2084 EXPORT_SYMBOL_GPL(kvm_handle_invalid_op);
2085 
2086 
kvm_emulate_monitor_mwait(struct kvm_vcpu * vcpu,const char * insn)2087 static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn)
2088 {
2089 	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS) &&
2090 	    !guest_cpuid_has(vcpu, X86_FEATURE_MWAIT))
2091 		return kvm_handle_invalid_op(vcpu);
2092 
2093 	pr_warn_once("kvm: %s instruction emulated as NOP!\n", insn);
2094 	return kvm_emulate_as_nop(vcpu);
2095 }
kvm_emulate_mwait(struct kvm_vcpu * vcpu)2096 int kvm_emulate_mwait(struct kvm_vcpu *vcpu)
2097 {
2098 	return kvm_emulate_monitor_mwait(vcpu, "MWAIT");
2099 }
2100 EXPORT_SYMBOL_GPL(kvm_emulate_mwait);
2101 
kvm_emulate_monitor(struct kvm_vcpu * vcpu)2102 int kvm_emulate_monitor(struct kvm_vcpu *vcpu)
2103 {
2104 	return kvm_emulate_monitor_mwait(vcpu, "MONITOR");
2105 }
2106 EXPORT_SYMBOL_GPL(kvm_emulate_monitor);
2107 
kvm_vcpu_exit_request(struct kvm_vcpu * vcpu)2108 static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
2109 {
2110 	xfer_to_guest_mode_prepare();
2111 	return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||
2112 		xfer_to_guest_mode_work_pending();
2113 }
2114 
2115 /*
2116  * The fast path for frequent and performance sensitive wrmsr emulation,
2117  * i.e. the sending of IPI, sending IPI early in the VM-Exit flow reduces
2118  * the latency of virtual IPI by avoiding the expensive bits of transitioning
2119  * from guest to host, e.g. reacquiring KVM's SRCU lock. In contrast to the
2120  * other cases which must be called after interrupts are enabled on the host.
2121  */
handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu * vcpu,u64 data)2122 static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)
2123 {
2124 	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))
2125 		return 1;
2126 
2127 	if (((data & APIC_SHORT_MASK) == APIC_DEST_NOSHORT) &&
2128 	    ((data & APIC_DEST_MASK) == APIC_DEST_PHYSICAL) &&
2129 	    ((data & APIC_MODE_MASK) == APIC_DM_FIXED) &&
2130 	    ((u32)(data >> 32) != X2APIC_BROADCAST))
2131 		return kvm_x2apic_icr_write(vcpu->arch.apic, data);
2132 
2133 	return 1;
2134 }
2135 
handle_fastpath_set_tscdeadline(struct kvm_vcpu * vcpu,u64 data)2136 static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data)
2137 {
2138 	if (!kvm_can_use_hv_timer(vcpu))
2139 		return 1;
2140 
2141 	kvm_set_lapic_tscdeadline_msr(vcpu, data);
2142 	return 0;
2143 }
2144 
handle_fastpath_set_msr_irqoff(struct kvm_vcpu * vcpu)2145 fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)
2146 {
2147 	u32 msr = kvm_rcx_read(vcpu);
2148 	u64 data;
2149 	fastpath_t ret = EXIT_FASTPATH_NONE;
2150 
2151 	switch (msr) {
2152 	case APIC_BASE_MSR + (APIC_ICR >> 4):
2153 		data = kvm_read_edx_eax(vcpu);
2154 		if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) {
2155 			kvm_skip_emulated_instruction(vcpu);
2156 			ret = EXIT_FASTPATH_EXIT_HANDLED;
2157 		}
2158 		break;
2159 	case MSR_IA32_TSC_DEADLINE:
2160 		data = kvm_read_edx_eax(vcpu);
2161 		if (!handle_fastpath_set_tscdeadline(vcpu, data)) {
2162 			kvm_skip_emulated_instruction(vcpu);
2163 			ret = EXIT_FASTPATH_REENTER_GUEST;
2164 		}
2165 		break;
2166 	default:
2167 		break;
2168 	}
2169 
2170 	if (ret != EXIT_FASTPATH_NONE)
2171 		trace_kvm_msr_write(msr, data);
2172 
2173 	return ret;
2174 }
2175 EXPORT_SYMBOL_GPL(handle_fastpath_set_msr_irqoff);
2176 
2177 /*
2178  * Adapt set_msr() to msr_io()'s calling convention
2179  */
do_get_msr(struct kvm_vcpu * vcpu,unsigned index,u64 * data)2180 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2181 {
2182 	return kvm_get_msr_ignored_check(vcpu, index, data, true);
2183 }
2184 
do_set_msr(struct kvm_vcpu * vcpu,unsigned index,u64 * data)2185 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2186 {
2187 	return kvm_set_msr_ignored_check(vcpu, index, *data, true);
2188 }
2189 
2190 #ifdef CONFIG_X86_64
2191 struct pvclock_clock {
2192 	int vclock_mode;
2193 	u64 cycle_last;
2194 	u64 mask;
2195 	u32 mult;
2196 	u32 shift;
2197 	u64 base_cycles;
2198 	u64 offset;
2199 };
2200 
2201 struct pvclock_gtod_data {
2202 	seqcount_t	seq;
2203 
2204 	struct pvclock_clock clock; /* extract of a clocksource struct */
2205 	struct pvclock_clock raw_clock; /* extract of a clocksource struct */
2206 
2207 	ktime_t		offs_boot;
2208 	u64		wall_time_sec;
2209 };
2210 
2211 static struct pvclock_gtod_data pvclock_gtod_data;
2212 
update_pvclock_gtod(struct timekeeper * tk)2213 static void update_pvclock_gtod(struct timekeeper *tk)
2214 {
2215 	struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
2216 
2217 	write_seqcount_begin(&vdata->seq);
2218 
2219 	/* copy pvclock gtod data */
2220 	vdata->clock.vclock_mode	= tk->tkr_mono.clock->vdso_clock_mode;
2221 	vdata->clock.cycle_last		= tk->tkr_mono.cycle_last;
2222 	vdata->clock.mask		= tk->tkr_mono.mask;
2223 	vdata->clock.mult		= tk->tkr_mono.mult;
2224 	vdata->clock.shift		= tk->tkr_mono.shift;
2225 	vdata->clock.base_cycles	= tk->tkr_mono.xtime_nsec;
2226 	vdata->clock.offset		= tk->tkr_mono.base;
2227 
2228 	vdata->raw_clock.vclock_mode	= tk->tkr_raw.clock->vdso_clock_mode;
2229 	vdata->raw_clock.cycle_last	= tk->tkr_raw.cycle_last;
2230 	vdata->raw_clock.mask		= tk->tkr_raw.mask;
2231 	vdata->raw_clock.mult		= tk->tkr_raw.mult;
2232 	vdata->raw_clock.shift		= tk->tkr_raw.shift;
2233 	vdata->raw_clock.base_cycles	= tk->tkr_raw.xtime_nsec;
2234 	vdata->raw_clock.offset		= tk->tkr_raw.base;
2235 
2236 	vdata->wall_time_sec            = tk->xtime_sec;
2237 
2238 	vdata->offs_boot		= tk->offs_boot;
2239 
2240 	write_seqcount_end(&vdata->seq);
2241 }
2242 
get_kvmclock_base_ns(void)2243 static s64 get_kvmclock_base_ns(void)
2244 {
2245 	/* Count up from boot time, but with the frequency of the raw clock.  */
2246 	return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot));
2247 }
2248 #else
get_kvmclock_base_ns(void)2249 static s64 get_kvmclock_base_ns(void)
2250 {
2251 	/* Master clock not used, so we can just use CLOCK_BOOTTIME.  */
2252 	return ktime_get_boottime_ns();
2253 }
2254 #endif
2255 
kvm_write_wall_clock(struct kvm * kvm,gpa_t wall_clock,int sec_hi_ofs)2256 static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs)
2257 {
2258 	int version;
2259 	int r;
2260 	struct pvclock_wall_clock wc;
2261 	u32 wc_sec_hi;
2262 	u64 wall_nsec;
2263 
2264 	if (!wall_clock)
2265 		return;
2266 
2267 	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
2268 	if (r)
2269 		return;
2270 
2271 	if (version & 1)
2272 		++version;  /* first time write, random junk */
2273 
2274 	++version;
2275 
2276 	if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
2277 		return;
2278 
2279 	/*
2280 	 * The guest calculates current wall clock time by adding
2281 	 * system time (updated by kvm_guest_time_update below) to the
2282 	 * wall clock specified here.  We do the reverse here.
2283 	 */
2284 	wall_nsec = ktime_get_real_ns() - get_kvmclock_ns(kvm);
2285 
2286 	wc.nsec = do_div(wall_nsec, 1000000000);
2287 	wc.sec = (u32)wall_nsec; /* overflow in 2106 guest time */
2288 	wc.version = version;
2289 
2290 	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
2291 
2292 	if (sec_hi_ofs) {
2293 		wc_sec_hi = wall_nsec >> 32;
2294 		kvm_write_guest(kvm, wall_clock + sec_hi_ofs,
2295 				&wc_sec_hi, sizeof(wc_sec_hi));
2296 	}
2297 
2298 	version++;
2299 	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
2300 }
2301 
kvm_write_system_time(struct kvm_vcpu * vcpu,gpa_t system_time,bool old_msr,bool host_initiated)2302 static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time,
2303 				  bool old_msr, bool host_initiated)
2304 {
2305 	struct kvm_arch *ka = &vcpu->kvm->arch;
2306 
2307 	if (vcpu->vcpu_id == 0 && !host_initiated) {
2308 		if (ka->boot_vcpu_runs_old_kvmclock != old_msr)
2309 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2310 
2311 		ka->boot_vcpu_runs_old_kvmclock = old_msr;
2312 	}
2313 
2314 	vcpu->arch.time = system_time;
2315 	kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2316 
2317 	/* we verify if the enable bit is set... */
2318 	if (system_time & 1) {
2319 		kvm_gpc_activate(vcpu->kvm, &vcpu->arch.pv_time, vcpu,
2320 				 KVM_HOST_USES_PFN, system_time & ~1ULL,
2321 				 sizeof(struct pvclock_vcpu_time_info));
2322 	} else {
2323 		kvm_gpc_deactivate(vcpu->kvm, &vcpu->arch.pv_time);
2324 	}
2325 
2326 	return;
2327 }
2328 
div_frac(uint32_t dividend,uint32_t divisor)2329 static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
2330 {
2331 	do_shl32_div32(dividend, divisor);
2332 	return dividend;
2333 }
2334 
kvm_get_time_scale(uint64_t scaled_hz,uint64_t base_hz,s8 * pshift,u32 * pmultiplier)2335 static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
2336 			       s8 *pshift, u32 *pmultiplier)
2337 {
2338 	uint64_t scaled64;
2339 	int32_t  shift = 0;
2340 	uint64_t tps64;
2341 	uint32_t tps32;
2342 
2343 	tps64 = base_hz;
2344 	scaled64 = scaled_hz;
2345 	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
2346 		tps64 >>= 1;
2347 		shift--;
2348 	}
2349 
2350 	tps32 = (uint32_t)tps64;
2351 	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
2352 		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
2353 			scaled64 >>= 1;
2354 		else
2355 			tps32 <<= 1;
2356 		shift++;
2357 	}
2358 
2359 	*pshift = shift;
2360 	*pmultiplier = div_frac(scaled64, tps32);
2361 }
2362 
2363 #ifdef CONFIG_X86_64
2364 static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
2365 #endif
2366 
2367 static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
2368 static unsigned long max_tsc_khz;
2369 
adjust_tsc_khz(u32 khz,s32 ppm)2370 static u32 adjust_tsc_khz(u32 khz, s32 ppm)
2371 {
2372 	u64 v = (u64)khz * (1000000 + ppm);
2373 	do_div(v, 1000000);
2374 	return v;
2375 }
2376 
2377 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
2378 
set_tsc_khz(struct kvm_vcpu * vcpu,u32 user_tsc_khz,bool scale)2379 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
2380 {
2381 	u64 ratio;
2382 
2383 	/* Guest TSC same frequency as host TSC? */
2384 	if (!scale) {
2385 		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
2386 		return 0;
2387 	}
2388 
2389 	/* TSC scaling supported? */
2390 	if (!kvm_caps.has_tsc_control) {
2391 		if (user_tsc_khz > tsc_khz) {
2392 			vcpu->arch.tsc_catchup = 1;
2393 			vcpu->arch.tsc_always_catchup = 1;
2394 			return 0;
2395 		} else {
2396 			pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
2397 			return -1;
2398 		}
2399 	}
2400 
2401 	/* TSC scaling required  - calculate ratio */
2402 	ratio = mul_u64_u32_div(1ULL << kvm_caps.tsc_scaling_ratio_frac_bits,
2403 				user_tsc_khz, tsc_khz);
2404 
2405 	if (ratio == 0 || ratio >= kvm_caps.max_tsc_scaling_ratio) {
2406 		pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
2407 			            user_tsc_khz);
2408 		return -1;
2409 	}
2410 
2411 	kvm_vcpu_write_tsc_multiplier(vcpu, ratio);
2412 	return 0;
2413 }
2414 
kvm_set_tsc_khz(struct kvm_vcpu * vcpu,u32 user_tsc_khz)2415 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
2416 {
2417 	u32 thresh_lo, thresh_hi;
2418 	int use_scaling = 0;
2419 
2420 	/* tsc_khz can be zero if TSC calibration fails */
2421 	if (user_tsc_khz == 0) {
2422 		/* set tsc_scaling_ratio to a safe value */
2423 		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
2424 		return -1;
2425 	}
2426 
2427 	/* Compute a scale to convert nanoseconds in TSC cycles */
2428 	kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
2429 			   &vcpu->arch.virtual_tsc_shift,
2430 			   &vcpu->arch.virtual_tsc_mult);
2431 	vcpu->arch.virtual_tsc_khz = user_tsc_khz;
2432 
2433 	/*
2434 	 * Compute the variation in TSC rate which is acceptable
2435 	 * within the range of tolerance and decide if the
2436 	 * rate being applied is within that bounds of the hardware
2437 	 * rate.  If so, no scaling or compensation need be done.
2438 	 */
2439 	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
2440 	thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
2441 	if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) {
2442 		pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", user_tsc_khz, thresh_lo, thresh_hi);
2443 		use_scaling = 1;
2444 	}
2445 	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
2446 }
2447 
compute_guest_tsc(struct kvm_vcpu * vcpu,s64 kernel_ns)2448 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
2449 {
2450 	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
2451 				      vcpu->arch.virtual_tsc_mult,
2452 				      vcpu->arch.virtual_tsc_shift);
2453 	tsc += vcpu->arch.this_tsc_write;
2454 	return tsc;
2455 }
2456 
2457 #ifdef CONFIG_X86_64
gtod_is_based_on_tsc(int mode)2458 static inline int gtod_is_based_on_tsc(int mode)
2459 {
2460 	return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK;
2461 }
2462 #endif
2463 
kvm_track_tsc_matching(struct kvm_vcpu * vcpu)2464 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
2465 {
2466 #ifdef CONFIG_X86_64
2467 	bool vcpus_matched;
2468 	struct kvm_arch *ka = &vcpu->kvm->arch;
2469 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2470 
2471 	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
2472 			 atomic_read(&vcpu->kvm->online_vcpus));
2473 
2474 	/*
2475 	 * Once the masterclock is enabled, always perform request in
2476 	 * order to update it.
2477 	 *
2478 	 * In order to enable masterclock, the host clocksource must be TSC
2479 	 * and the vcpus need to have matched TSCs.  When that happens,
2480 	 * perform request to enable masterclock.
2481 	 */
2482 	if (ka->use_master_clock ||
2483 	    (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched))
2484 		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2485 
2486 	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
2487 			    atomic_read(&vcpu->kvm->online_vcpus),
2488 		            ka->use_master_clock, gtod->clock.vclock_mode);
2489 #endif
2490 }
2491 
2492 /*
2493  * Multiply tsc by a fixed point number represented by ratio.
2494  *
2495  * The most significant 64-N bits (mult) of ratio represent the
2496  * integral part of the fixed point number; the remaining N bits
2497  * (frac) represent the fractional part, ie. ratio represents a fixed
2498  * point number (mult + frac * 2^(-N)).
2499  *
2500  * N equals to kvm_caps.tsc_scaling_ratio_frac_bits.
2501  */
__scale_tsc(u64 ratio,u64 tsc)2502 static inline u64 __scale_tsc(u64 ratio, u64 tsc)
2503 {
2504 	return mul_u64_u64_shr(tsc, ratio, kvm_caps.tsc_scaling_ratio_frac_bits);
2505 }
2506 
kvm_scale_tsc(u64 tsc,u64 ratio)2507 u64 kvm_scale_tsc(u64 tsc, u64 ratio)
2508 {
2509 	u64 _tsc = tsc;
2510 
2511 	if (ratio != kvm_caps.default_tsc_scaling_ratio)
2512 		_tsc = __scale_tsc(ratio, tsc);
2513 
2514 	return _tsc;
2515 }
2516 EXPORT_SYMBOL_GPL(kvm_scale_tsc);
2517 
kvm_compute_l1_tsc_offset(struct kvm_vcpu * vcpu,u64 target_tsc)2518 static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
2519 {
2520 	u64 tsc;
2521 
2522 	tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);
2523 
2524 	return target_tsc - tsc;
2525 }
2526 
kvm_read_l1_tsc(struct kvm_vcpu * vcpu,u64 host_tsc)2527 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
2528 {
2529 	return vcpu->arch.l1_tsc_offset +
2530 		kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);
2531 }
2532 EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);
2533 
kvm_calc_nested_tsc_offset(u64 l1_offset,u64 l2_offset,u64 l2_multiplier)2534 u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier)
2535 {
2536 	u64 nested_offset;
2537 
2538 	if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio)
2539 		nested_offset = l1_offset;
2540 	else
2541 		nested_offset = mul_s64_u64_shr((s64) l1_offset, l2_multiplier,
2542 						kvm_caps.tsc_scaling_ratio_frac_bits);
2543 
2544 	nested_offset += l2_offset;
2545 	return nested_offset;
2546 }
2547 EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_offset);
2548 
kvm_calc_nested_tsc_multiplier(u64 l1_multiplier,u64 l2_multiplier)2549 u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier)
2550 {
2551 	if (l2_multiplier != kvm_caps.default_tsc_scaling_ratio)
2552 		return mul_u64_u64_shr(l1_multiplier, l2_multiplier,
2553 				       kvm_caps.tsc_scaling_ratio_frac_bits);
2554 
2555 	return l1_multiplier;
2556 }
2557 EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_multiplier);
2558 
kvm_vcpu_write_tsc_offset(struct kvm_vcpu * vcpu,u64 l1_offset)2559 static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)
2560 {
2561 	trace_kvm_write_tsc_offset(vcpu->vcpu_id,
2562 				   vcpu->arch.l1_tsc_offset,
2563 				   l1_offset);
2564 
2565 	vcpu->arch.l1_tsc_offset = l1_offset;
2566 
2567 	/*
2568 	 * If we are here because L1 chose not to trap WRMSR to TSC then
2569 	 * according to the spec this should set L1's TSC (as opposed to
2570 	 * setting L1's offset for L2).
2571 	 */
2572 	if (is_guest_mode(vcpu))
2573 		vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(
2574 			l1_offset,
2575 			static_call(kvm_x86_get_l2_tsc_offset)(vcpu),
2576 			static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));
2577 	else
2578 		vcpu->arch.tsc_offset = l1_offset;
2579 
2580 	static_call(kvm_x86_write_tsc_offset)(vcpu, vcpu->arch.tsc_offset);
2581 }
2582 
kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu * vcpu,u64 l1_multiplier)2583 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier)
2584 {
2585 	vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier;
2586 
2587 	/* Userspace is changing the multiplier while L2 is active */
2588 	if (is_guest_mode(vcpu))
2589 		vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(
2590 			l1_multiplier,
2591 			static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));
2592 	else
2593 		vcpu->arch.tsc_scaling_ratio = l1_multiplier;
2594 
2595 	if (kvm_caps.has_tsc_control)
2596 		static_call(kvm_x86_write_tsc_multiplier)(
2597 			vcpu, vcpu->arch.tsc_scaling_ratio);
2598 }
2599 
kvm_check_tsc_unstable(void)2600 static inline bool kvm_check_tsc_unstable(void)
2601 {
2602 #ifdef CONFIG_X86_64
2603 	/*
2604 	 * TSC is marked unstable when we're running on Hyper-V,
2605 	 * 'TSC page' clocksource is good.
2606 	 */
2607 	if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK)
2608 		return false;
2609 #endif
2610 	return check_tsc_unstable();
2611 }
2612 
2613 /*
2614  * Infers attempts to synchronize the guest's tsc from host writes. Sets the
2615  * offset for the vcpu and tracks the TSC matching generation that the vcpu
2616  * participates in.
2617  */
__kvm_synchronize_tsc(struct kvm_vcpu * vcpu,u64 offset,u64 tsc,u64 ns,bool matched)2618 static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
2619 				  u64 ns, bool matched)
2620 {
2621 	struct kvm *kvm = vcpu->kvm;
2622 
2623 	lockdep_assert_held(&kvm->arch.tsc_write_lock);
2624 
2625 	/*
2626 	 * We also track th most recent recorded KHZ, write and time to
2627 	 * allow the matching interval to be extended at each write.
2628 	 */
2629 	kvm->arch.last_tsc_nsec = ns;
2630 	kvm->arch.last_tsc_write = tsc;
2631 	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
2632 	kvm->arch.last_tsc_offset = offset;
2633 
2634 	vcpu->arch.last_guest_tsc = tsc;
2635 
2636 	kvm_vcpu_write_tsc_offset(vcpu, offset);
2637 
2638 	if (!matched) {
2639 		/*
2640 		 * We split periods of matched TSC writes into generations.
2641 		 * For each generation, we track the original measured
2642 		 * nanosecond time, offset, and write, so if TSCs are in
2643 		 * sync, we can match exact offset, and if not, we can match
2644 		 * exact software computation in compute_guest_tsc()
2645 		 *
2646 		 * These values are tracked in kvm->arch.cur_xxx variables.
2647 		 */
2648 		kvm->arch.cur_tsc_generation++;
2649 		kvm->arch.cur_tsc_nsec = ns;
2650 		kvm->arch.cur_tsc_write = tsc;
2651 		kvm->arch.cur_tsc_offset = offset;
2652 		kvm->arch.nr_vcpus_matched_tsc = 0;
2653 	} else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {
2654 		kvm->arch.nr_vcpus_matched_tsc++;
2655 	}
2656 
2657 	/* Keep track of which generation this VCPU has synchronized to */
2658 	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
2659 	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
2660 	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
2661 
2662 	kvm_track_tsc_matching(vcpu);
2663 }
2664 
kvm_synchronize_tsc(struct kvm_vcpu * vcpu,u64 data)2665 static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data)
2666 {
2667 	struct kvm *kvm = vcpu->kvm;
2668 	u64 offset, ns, elapsed;
2669 	unsigned long flags;
2670 	bool matched = false;
2671 	bool synchronizing = false;
2672 
2673 	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
2674 	offset = kvm_compute_l1_tsc_offset(vcpu, data);
2675 	ns = get_kvmclock_base_ns();
2676 	elapsed = ns - kvm->arch.last_tsc_nsec;
2677 
2678 	if (vcpu->arch.virtual_tsc_khz) {
2679 		if (data == 0) {
2680 			/*
2681 			 * detection of vcpu initialization -- need to sync
2682 			 * with other vCPUs. This particularly helps to keep
2683 			 * kvm_clock stable after CPU hotplug
2684 			 */
2685 			synchronizing = true;
2686 		} else {
2687 			u64 tsc_exp = kvm->arch.last_tsc_write +
2688 						nsec_to_cycles(vcpu, elapsed);
2689 			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
2690 			/*
2691 			 * Special case: TSC write with a small delta (1 second)
2692 			 * of virtual cycle time against real time is
2693 			 * interpreted as an attempt to synchronize the CPU.
2694 			 */
2695 			synchronizing = data < tsc_exp + tsc_hz &&
2696 					data + tsc_hz > tsc_exp;
2697 		}
2698 	}
2699 
2700 	/*
2701 	 * For a reliable TSC, we can match TSC offsets, and for an unstable
2702 	 * TSC, we add elapsed time in this computation.  We could let the
2703 	 * compensation code attempt to catch up if we fall behind, but
2704 	 * it's better to try to match offsets from the beginning.
2705          */
2706 	if (synchronizing &&
2707 	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
2708 		if (!kvm_check_tsc_unstable()) {
2709 			offset = kvm->arch.cur_tsc_offset;
2710 		} else {
2711 			u64 delta = nsec_to_cycles(vcpu, elapsed);
2712 			data += delta;
2713 			offset = kvm_compute_l1_tsc_offset(vcpu, data);
2714 		}
2715 		matched = true;
2716 	}
2717 
2718 	__kvm_synchronize_tsc(vcpu, offset, data, ns, matched);
2719 	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
2720 }
2721 
adjust_tsc_offset_guest(struct kvm_vcpu * vcpu,s64 adjustment)2722 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
2723 					   s64 adjustment)
2724 {
2725 	u64 tsc_offset = vcpu->arch.l1_tsc_offset;
2726 	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
2727 }
2728 
adjust_tsc_offset_host(struct kvm_vcpu * vcpu,s64 adjustment)2729 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
2730 {
2731 	if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio)
2732 		WARN_ON(adjustment < 0);
2733 	adjustment = kvm_scale_tsc((u64) adjustment,
2734 				   vcpu->arch.l1_tsc_scaling_ratio);
2735 	adjust_tsc_offset_guest(vcpu, adjustment);
2736 }
2737 
2738 #ifdef CONFIG_X86_64
2739 
read_tsc(void)2740 static u64 read_tsc(void)
2741 {
2742 	u64 ret = (u64)rdtsc_ordered();
2743 	u64 last = pvclock_gtod_data.clock.cycle_last;
2744 
2745 	if (likely(ret >= last))
2746 		return ret;
2747 
2748 	/*
2749 	 * GCC likes to generate cmov here, but this branch is extremely
2750 	 * predictable (it's just a function of time and the likely is
2751 	 * very likely) and there's a data dependence, so force GCC
2752 	 * to generate a branch instead.  I don't barrier() because
2753 	 * we don't actually need a barrier, and if this function
2754 	 * ever gets inlined it will generate worse code.
2755 	 */
2756 	asm volatile ("");
2757 	return last;
2758 }
2759 
vgettsc(struct pvclock_clock * clock,u64 * tsc_timestamp,int * mode)2760 static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
2761 			  int *mode)
2762 {
2763 	long v;
2764 	u64 tsc_pg_val;
2765 
2766 	switch (clock->vclock_mode) {
2767 	case VDSO_CLOCKMODE_HVCLOCK:
2768 		tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(),
2769 						  tsc_timestamp);
2770 		if (tsc_pg_val != U64_MAX) {
2771 			/* TSC page valid */
2772 			*mode = VDSO_CLOCKMODE_HVCLOCK;
2773 			v = (tsc_pg_val - clock->cycle_last) &
2774 				clock->mask;
2775 		} else {
2776 			/* TSC page invalid */
2777 			*mode = VDSO_CLOCKMODE_NONE;
2778 		}
2779 		break;
2780 	case VDSO_CLOCKMODE_TSC:
2781 		*mode = VDSO_CLOCKMODE_TSC;
2782 		*tsc_timestamp = read_tsc();
2783 		v = (*tsc_timestamp - clock->cycle_last) &
2784 			clock->mask;
2785 		break;
2786 	default:
2787 		*mode = VDSO_CLOCKMODE_NONE;
2788 	}
2789 
2790 	if (*mode == VDSO_CLOCKMODE_NONE)
2791 		*tsc_timestamp = v = 0;
2792 
2793 	return v * clock->mult;
2794 }
2795 
do_monotonic_raw(s64 * t,u64 * tsc_timestamp)2796 static int do_monotonic_raw(s64 *t, u64 *tsc_timestamp)
2797 {
2798 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2799 	unsigned long seq;
2800 	int mode;
2801 	u64 ns;
2802 
2803 	do {
2804 		seq = read_seqcount_begin(&gtod->seq);
2805 		ns = gtod->raw_clock.base_cycles;
2806 		ns += vgettsc(&gtod->raw_clock, tsc_timestamp, &mode);
2807 		ns >>= gtod->raw_clock.shift;
2808 		ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot));
2809 	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
2810 	*t = ns;
2811 
2812 	return mode;
2813 }
2814 
do_realtime(struct timespec64 * ts,u64 * tsc_timestamp)2815 static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
2816 {
2817 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2818 	unsigned long seq;
2819 	int mode;
2820 	u64 ns;
2821 
2822 	do {
2823 		seq = read_seqcount_begin(&gtod->seq);
2824 		ts->tv_sec = gtod->wall_time_sec;
2825 		ns = gtod->clock.base_cycles;
2826 		ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
2827 		ns >>= gtod->clock.shift;
2828 	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
2829 
2830 	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
2831 	ts->tv_nsec = ns;
2832 
2833 	return mode;
2834 }
2835 
2836 /* returns true if host is using TSC based clocksource */
kvm_get_time_and_clockread(s64 * kernel_ns,u64 * tsc_timestamp)2837 static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
2838 {
2839 	/* checked again under seqlock below */
2840 	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2841 		return false;
2842 
2843 	return gtod_is_based_on_tsc(do_monotonic_raw(kernel_ns,
2844 						      tsc_timestamp));
2845 }
2846 
2847 /* returns true if host is using TSC based clocksource */
kvm_get_walltime_and_clockread(struct timespec64 * ts,u64 * tsc_timestamp)2848 static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
2849 					   u64 *tsc_timestamp)
2850 {
2851 	/* checked again under seqlock below */
2852 	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2853 		return false;
2854 
2855 	return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
2856 }
2857 #endif
2858 
2859 /*
2860  *
2861  * Assuming a stable TSC across physical CPUS, and a stable TSC
2862  * across virtual CPUs, the following condition is possible.
2863  * Each numbered line represents an event visible to both
2864  * CPUs at the next numbered event.
2865  *
2866  * "timespecX" represents host monotonic time. "tscX" represents
2867  * RDTSC value.
2868  *
2869  * 		VCPU0 on CPU0		|	VCPU1 on CPU1
2870  *
2871  * 1.  read timespec0,tsc0
2872  * 2.					| timespec1 = timespec0 + N
2873  * 					| tsc1 = tsc0 + M
2874  * 3. transition to guest		| transition to guest
2875  * 4. ret0 = timespec0 + (rdtsc - tsc0) |
2876  * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
2877  * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
2878  *
2879  * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
2880  *
2881  * 	- ret0 < ret1
2882  *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
2883  *		...
2884  *	- 0 < N - M => M < N
2885  *
2886  * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
2887  * always the case (the difference between two distinct xtime instances
2888  * might be smaller then the difference between corresponding TSC reads,
2889  * when updating guest vcpus pvclock areas).
2890  *
2891  * To avoid that problem, do not allow visibility of distinct
2892  * system_timestamp/tsc_timestamp values simultaneously: use a master
2893  * copy of host monotonic time values. Update that master copy
2894  * in lockstep.
2895  *
2896  * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
2897  *
2898  */
2899 
pvclock_update_vm_gtod_copy(struct kvm * kvm)2900 static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
2901 {
2902 #ifdef CONFIG_X86_64
2903 	struct kvm_arch *ka = &kvm->arch;
2904 	int vclock_mode;
2905 	bool host_tsc_clocksource, vcpus_matched;
2906 
2907 	lockdep_assert_held(&kvm->arch.tsc_write_lock);
2908 	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
2909 			atomic_read(&kvm->online_vcpus));
2910 
2911 	/*
2912 	 * If the host uses TSC clock, then passthrough TSC as stable
2913 	 * to the guest.
2914 	 */
2915 	host_tsc_clocksource = kvm_get_time_and_clockread(
2916 					&ka->master_kernel_ns,
2917 					&ka->master_cycle_now);
2918 
2919 	ka->use_master_clock = host_tsc_clocksource && vcpus_matched
2920 				&& !ka->backwards_tsc_observed
2921 				&& !ka->boot_vcpu_runs_old_kvmclock;
2922 
2923 	if (ka->use_master_clock)
2924 		atomic_set(&kvm_guest_has_master_clock, 1);
2925 
2926 	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
2927 	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
2928 					vcpus_matched);
2929 #endif
2930 }
2931 
kvm_make_mclock_inprogress_request(struct kvm * kvm)2932 static void kvm_make_mclock_inprogress_request(struct kvm *kvm)
2933 {
2934 	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
2935 }
2936 
__kvm_start_pvclock_update(struct kvm * kvm)2937 static void __kvm_start_pvclock_update(struct kvm *kvm)
2938 {
2939 	raw_spin_lock_irq(&kvm->arch.tsc_write_lock);
2940 	write_seqcount_begin(&kvm->arch.pvclock_sc);
2941 }
2942 
kvm_start_pvclock_update(struct kvm * kvm)2943 static void kvm_start_pvclock_update(struct kvm *kvm)
2944 {
2945 	kvm_make_mclock_inprogress_request(kvm);
2946 
2947 	/* no guest entries from this point */
2948 	__kvm_start_pvclock_update(kvm);
2949 }
2950 
kvm_end_pvclock_update(struct kvm * kvm)2951 static void kvm_end_pvclock_update(struct kvm *kvm)
2952 {
2953 	struct kvm_arch *ka = &kvm->arch;
2954 	struct kvm_vcpu *vcpu;
2955 	unsigned long i;
2956 
2957 	write_seqcount_end(&ka->pvclock_sc);
2958 	raw_spin_unlock_irq(&ka->tsc_write_lock);
2959 	kvm_for_each_vcpu(i, vcpu, kvm)
2960 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2961 
2962 	/* guest entries allowed */
2963 	kvm_for_each_vcpu(i, vcpu, kvm)
2964 		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
2965 }
2966 
kvm_update_masterclock(struct kvm * kvm)2967 static void kvm_update_masterclock(struct kvm *kvm)
2968 {
2969 	kvm_hv_request_tsc_page_update(kvm);
2970 	kvm_start_pvclock_update(kvm);
2971 	pvclock_update_vm_gtod_copy(kvm);
2972 	kvm_end_pvclock_update(kvm);
2973 }
2974 
2975 /* Called within read_seqcount_begin/retry for kvm->pvclock_sc.  */
__get_kvmclock(struct kvm * kvm,struct kvm_clock_data * data)2976 static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
2977 {
2978 	struct kvm_arch *ka = &kvm->arch;
2979 	struct pvclock_vcpu_time_info hv_clock;
2980 
2981 	/* both __this_cpu_read() and rdtsc() should be on the same cpu */
2982 	get_cpu();
2983 
2984 	data->flags = 0;
2985 	if (ka->use_master_clock && __this_cpu_read(cpu_tsc_khz)) {
2986 #ifdef CONFIG_X86_64
2987 		struct timespec64 ts;
2988 
2989 		if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) {
2990 			data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec;
2991 			data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC;
2992 		} else
2993 #endif
2994 		data->host_tsc = rdtsc();
2995 
2996 		data->flags |= KVM_CLOCK_TSC_STABLE;
2997 		hv_clock.tsc_timestamp = ka->master_cycle_now;
2998 		hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
2999 		kvm_get_time_scale(NSEC_PER_SEC, __this_cpu_read(cpu_tsc_khz) * 1000LL,
3000 				   &hv_clock.tsc_shift,
3001 				   &hv_clock.tsc_to_system_mul);
3002 		data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc);
3003 	} else {
3004 		data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset;
3005 	}
3006 
3007 	put_cpu();
3008 }
3009 
get_kvmclock(struct kvm * kvm,struct kvm_clock_data * data)3010 static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
3011 {
3012 	struct kvm_arch *ka = &kvm->arch;
3013 	unsigned seq;
3014 
3015 	do {
3016 		seq = read_seqcount_begin(&ka->pvclock_sc);
3017 		__get_kvmclock(kvm, data);
3018 	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
3019 }
3020 
get_kvmclock_ns(struct kvm * kvm)3021 u64 get_kvmclock_ns(struct kvm *kvm)
3022 {
3023 	struct kvm_clock_data data;
3024 
3025 	get_kvmclock(kvm, &data);
3026 	return data.clock;
3027 }
3028 
kvm_setup_guest_pvclock(struct kvm_vcpu * v,struct gfn_to_pfn_cache * gpc,unsigned int offset)3029 static void kvm_setup_guest_pvclock(struct kvm_vcpu *v,
3030 				    struct gfn_to_pfn_cache *gpc,
3031 				    unsigned int offset)
3032 {
3033 	struct kvm_vcpu_arch *vcpu = &v->arch;
3034 	struct pvclock_vcpu_time_info *guest_hv_clock;
3035 	unsigned long flags;
3036 
3037 	read_lock_irqsave(&gpc->lock, flags);
3038 	while (!kvm_gfn_to_pfn_cache_check(v->kvm, gpc, gpc->gpa,
3039 					   offset + sizeof(*guest_hv_clock))) {
3040 		read_unlock_irqrestore(&gpc->lock, flags);
3041 
3042 		if (kvm_gfn_to_pfn_cache_refresh(v->kvm, gpc, gpc->gpa,
3043 						 offset + sizeof(*guest_hv_clock)))
3044 			return;
3045 
3046 		read_lock_irqsave(&gpc->lock, flags);
3047 	}
3048 
3049 	guest_hv_clock = (void *)(gpc->khva + offset);
3050 
3051 	/*
3052 	 * This VCPU is paused, but it's legal for a guest to read another
3053 	 * VCPU's kvmclock, so we really have to follow the specification where
3054 	 * it says that version is odd if data is being modified, and even after
3055 	 * it is consistent.
3056 	 */
3057 
3058 	guest_hv_clock->version = vcpu->hv_clock.version = (guest_hv_clock->version + 1) | 1;
3059 	smp_wmb();
3060 
3061 	/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
3062 	vcpu->hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED);
3063 
3064 	if (vcpu->pvclock_set_guest_stopped_request) {
3065 		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
3066 		vcpu->pvclock_set_guest_stopped_request = false;
3067 	}
3068 
3069 	memcpy(guest_hv_clock, &vcpu->hv_clock, sizeof(*guest_hv_clock));
3070 	smp_wmb();
3071 
3072 	guest_hv_clock->version = ++vcpu->hv_clock.version;
3073 
3074 	mark_page_dirty_in_slot(v->kvm, gpc->memslot, gpc->gpa >> PAGE_SHIFT);
3075 	read_unlock_irqrestore(&gpc->lock, flags);
3076 
3077 	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);
3078 }
3079 
kvm_guest_time_update(struct kvm_vcpu * v)3080 static int kvm_guest_time_update(struct kvm_vcpu *v)
3081 {
3082 	unsigned long flags, tgt_tsc_khz;
3083 	unsigned seq;
3084 	struct kvm_vcpu_arch *vcpu = &v->arch;
3085 	struct kvm_arch *ka = &v->kvm->arch;
3086 	s64 kernel_ns;
3087 	u64 tsc_timestamp, host_tsc;
3088 	u8 pvclock_flags;
3089 	bool use_master_clock;
3090 
3091 	kernel_ns = 0;
3092 	host_tsc = 0;
3093 
3094 	/*
3095 	 * If the host uses TSC clock, then passthrough TSC as stable
3096 	 * to the guest.
3097 	 */
3098 	do {
3099 		seq = read_seqcount_begin(&ka->pvclock_sc);
3100 		use_master_clock = ka->use_master_clock;
3101 		if (use_master_clock) {
3102 			host_tsc = ka->master_cycle_now;
3103 			kernel_ns = ka->master_kernel_ns;
3104 		}
3105 	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
3106 
3107 	/* Keep irq disabled to prevent changes to the clock */
3108 	local_irq_save(flags);
3109 	tgt_tsc_khz = __this_cpu_read(cpu_tsc_khz);
3110 	if (unlikely(tgt_tsc_khz == 0)) {
3111 		local_irq_restore(flags);
3112 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
3113 		return 1;
3114 	}
3115 	if (!use_master_clock) {
3116 		host_tsc = rdtsc();
3117 		kernel_ns = get_kvmclock_base_ns();
3118 	}
3119 
3120 	tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
3121 
3122 	/*
3123 	 * We may have to catch up the TSC to match elapsed wall clock
3124 	 * time for two reasons, even if kvmclock is used.
3125 	 *   1) CPU could have been running below the maximum TSC rate
3126 	 *   2) Broken TSC compensation resets the base at each VCPU
3127 	 *      entry to avoid unknown leaps of TSC even when running
3128 	 *      again on the same CPU.  This may cause apparent elapsed
3129 	 *      time to disappear, and the guest to stand still or run
3130 	 *	very slowly.
3131 	 */
3132 	if (vcpu->tsc_catchup) {
3133 		u64 tsc = compute_guest_tsc(v, kernel_ns);
3134 		if (tsc > tsc_timestamp) {
3135 			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
3136 			tsc_timestamp = tsc;
3137 		}
3138 	}
3139 
3140 	local_irq_restore(flags);
3141 
3142 	/* With all the info we got, fill in the values */
3143 
3144 	if (kvm_caps.has_tsc_control)
3145 		tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz,
3146 					    v->arch.l1_tsc_scaling_ratio);
3147 
3148 	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
3149 		kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
3150 				   &vcpu->hv_clock.tsc_shift,
3151 				   &vcpu->hv_clock.tsc_to_system_mul);
3152 		vcpu->hw_tsc_khz = tgt_tsc_khz;
3153 	}
3154 
3155 	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
3156 	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
3157 	vcpu->last_guest_tsc = tsc_timestamp;
3158 
3159 	/* If the host uses TSC clocksource, then it is stable */
3160 	pvclock_flags = 0;
3161 	if (use_master_clock)
3162 		pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;
3163 
3164 	vcpu->hv_clock.flags = pvclock_flags;
3165 
3166 	if (vcpu->pv_time.active)
3167 		kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0);
3168 	if (vcpu->xen.vcpu_info_cache.active)
3169 		kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache,
3170 					offsetof(struct compat_vcpu_info, time));
3171 	if (vcpu->xen.vcpu_time_info_cache.active)
3172 		kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0);
3173 	kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
3174 	return 0;
3175 }
3176 
3177 /*
3178  * kvmclock updates which are isolated to a given vcpu, such as
3179  * vcpu->cpu migration, should not allow system_timestamp from
3180  * the rest of the vcpus to remain static. Otherwise ntp frequency
3181  * correction applies to one vcpu's system_timestamp but not
3182  * the others.
3183  *
3184  * So in those cases, request a kvmclock update for all vcpus.
3185  * We need to rate-limit these requests though, as they can
3186  * considerably slow guests that have a large number of vcpus.
3187  * The time for a remote vcpu to update its kvmclock is bound
3188  * by the delay we use to rate-limit the updates.
3189  */
3190 
3191 #define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)
3192 
kvmclock_update_fn(struct work_struct * work)3193 static void kvmclock_update_fn(struct work_struct *work)
3194 {
3195 	unsigned long i;
3196 	struct delayed_work *dwork = to_delayed_work(work);
3197 	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
3198 					   kvmclock_update_work);
3199 	struct kvm *kvm = container_of(ka, struct kvm, arch);
3200 	struct kvm_vcpu *vcpu;
3201 
3202 	kvm_for_each_vcpu(i, vcpu, kvm) {
3203 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3204 		kvm_vcpu_kick(vcpu);
3205 	}
3206 }
3207 
kvm_gen_kvmclock_update(struct kvm_vcpu * v)3208 static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
3209 {
3210 	struct kvm *kvm = v->kvm;
3211 
3212 	kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
3213 	schedule_delayed_work(&kvm->arch.kvmclock_update_work,
3214 					KVMCLOCK_UPDATE_DELAY);
3215 }
3216 
3217 #define KVMCLOCK_SYNC_PERIOD (300 * HZ)
3218 
kvmclock_sync_fn(struct work_struct * work)3219 static void kvmclock_sync_fn(struct work_struct *work)
3220 {
3221 	struct delayed_work *dwork = to_delayed_work(work);
3222 	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
3223 					   kvmclock_sync_work);
3224 	struct kvm *kvm = container_of(ka, struct kvm, arch);
3225 
3226 	if (!kvmclock_periodic_sync)
3227 		return;
3228 
3229 	schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
3230 	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
3231 					KVMCLOCK_SYNC_PERIOD);
3232 }
3233 
3234 /* These helpers are safe iff @msr is known to be an MCx bank MSR. */
is_mci_control_msr(u32 msr)3235 static bool is_mci_control_msr(u32 msr)
3236 {
3237 	return (msr & 3) == 0;
3238 }
is_mci_status_msr(u32 msr)3239 static bool is_mci_status_msr(u32 msr)
3240 {
3241 	return (msr & 3) == 1;
3242 }
3243 
3244 /*
3245  * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP.
3246  */
can_set_mci_status(struct kvm_vcpu * vcpu)3247 static bool can_set_mci_status(struct kvm_vcpu *vcpu)
3248 {
3249 	/* McStatusWrEn enabled? */
3250 	if (guest_cpuid_is_amd_or_hygon(vcpu))
3251 		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));
3252 
3253 	return false;
3254 }
3255 
set_msr_mce(struct kvm_vcpu * vcpu,struct msr_data * msr_info)3256 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3257 {
3258 	u64 mcg_cap = vcpu->arch.mcg_cap;
3259 	unsigned bank_num = mcg_cap & 0xff;
3260 	u32 msr = msr_info->index;
3261 	u64 data = msr_info->data;
3262 	u32 offset, last_msr;
3263 
3264 	switch (msr) {
3265 	case MSR_IA32_MCG_STATUS:
3266 		vcpu->arch.mcg_status = data;
3267 		break;
3268 	case MSR_IA32_MCG_CTL:
3269 		if (!(mcg_cap & MCG_CTL_P) &&
3270 		    (data || !msr_info->host_initiated))
3271 			return 1;
3272 		if (data != 0 && data != ~(u64)0)
3273 			return 1;
3274 		vcpu->arch.mcg_ctl = data;
3275 		break;
3276 	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
3277 		last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1;
3278 		if (msr > last_msr)
3279 			return 1;
3280 
3281 		if (!(mcg_cap & MCG_CMCI_P) && (data || !msr_info->host_initiated))
3282 			return 1;
3283 		/* An attempt to write a 1 to a reserved bit raises #GP */
3284 		if (data & ~(MCI_CTL2_CMCI_EN | MCI_CTL2_CMCI_THRESHOLD_MASK))
3285 			return 1;
3286 		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2,
3287 					    last_msr + 1 - MSR_IA32_MC0_CTL2);
3288 		vcpu->arch.mci_ctl2_banks[offset] = data;
3289 		break;
3290 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3291 		last_msr = MSR_IA32_MCx_CTL(bank_num) - 1;
3292 		if (msr > last_msr)
3293 			return 1;
3294 
3295 		/*
3296 		 * Only 0 or all 1s can be written to IA32_MCi_CTL, all other
3297 		 * values are architecturally undefined.  But, some Linux
3298 		 * kernels clear bit 10 in bank 4 to workaround a BIOS/GART TLB
3299 		 * issue on AMD K8s, allow bit 10 to be clear when setting all
3300 		 * other bits in order to avoid an uncaught #GP in the guest.
3301 		 *
3302 		 * UNIXWARE clears bit 0 of MC1_CTL to ignore correctable,
3303 		 * single-bit ECC data errors.
3304 		 */
3305 		if (is_mci_control_msr(msr) &&
3306 		    data != 0 && (data | (1 << 10) | 1) != ~(u64)0)
3307 			return 1;
3308 
3309 		/*
3310 		 * All CPUs allow writing 0 to MCi_STATUS MSRs to clear the MSR.
3311 		 * AMD-based CPUs allow non-zero values, but if and only if
3312 		 * HWCR[McStatusWrEn] is set.
3313 		 */
3314 		if (!msr_info->host_initiated && is_mci_status_msr(msr) &&
3315 		    data != 0 && !can_set_mci_status(vcpu))
3316 			return 1;
3317 
3318 		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL,
3319 					    last_msr + 1 - MSR_IA32_MC0_CTL);
3320 		vcpu->arch.mce_banks[offset] = data;
3321 		break;
3322 	default:
3323 		return 1;
3324 	}
3325 	return 0;
3326 }
3327 
kvm_pv_async_pf_enabled(struct kvm_vcpu * vcpu)3328 static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
3329 {
3330 	u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT;
3331 
3332 	return (vcpu->arch.apf.msr_en_val & mask) == mask;
3333 }
3334 
kvm_pv_enable_async_pf(struct kvm_vcpu * vcpu,u64 data)3335 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
3336 {
3337 	gpa_t gpa = data & ~0x3f;
3338 
3339 	/* Bits 4:5 are reserved, Should be zero */
3340 	if (data & 0x30)
3341 		return 1;
3342 
3343 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) &&
3344 	    (data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT))
3345 		return 1;
3346 
3347 	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) &&
3348 	    (data & KVM_ASYNC_PF_DELIVERY_AS_INT))
3349 		return 1;
3350 
3351 	if (!lapic_in_kernel(vcpu))
3352 		return data ? 1 : 0;
3353 
3354 	vcpu->arch.apf.msr_en_val = data;
3355 
3356 	if (!kvm_pv_async_pf_enabled(vcpu)) {
3357 		kvm_clear_async_pf_completion_queue(vcpu);
3358 		kvm_async_pf_hash_reset(vcpu);
3359 		return 0;
3360 	}
3361 
3362 	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
3363 					sizeof(u64)))
3364 		return 1;
3365 
3366 	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
3367 	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
3368 
3369 	kvm_async_pf_wakeup_all(vcpu);
3370 
3371 	return 0;
3372 }
3373 
kvm_pv_enable_async_pf_int(struct kvm_vcpu * vcpu,u64 data)3374 static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)
3375 {
3376 	/* Bits 8-63 are reserved */
3377 	if (data >> 8)
3378 		return 1;
3379 
3380 	if (!lapic_in_kernel(vcpu))
3381 		return 1;
3382 
3383 	vcpu->arch.apf.msr_int_val = data;
3384 
3385 	vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;
3386 
3387 	return 0;
3388 }
3389 
kvmclock_reset(struct kvm_vcpu * vcpu)3390 static void kvmclock_reset(struct kvm_vcpu *vcpu)
3391 {
3392 	kvm_gpc_deactivate(vcpu->kvm, &vcpu->arch.pv_time);
3393 	vcpu->arch.time = 0;
3394 }
3395 
kvm_vcpu_flush_tlb_all(struct kvm_vcpu * vcpu)3396 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)
3397 {
3398 	++vcpu->stat.tlb_flush;
3399 	static_call(kvm_x86_flush_tlb_all)(vcpu);
3400 }
3401 
kvm_vcpu_flush_tlb_guest(struct kvm_vcpu * vcpu)3402 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
3403 {
3404 	++vcpu->stat.tlb_flush;
3405 
3406 	if (!tdp_enabled) {
3407 		/*
3408 		 * A TLB flush on behalf of the guest is equivalent to
3409 		 * INVPCID(all), toggling CR4.PGE, etc., which requires
3410 		 * a forced sync of the shadow page tables.  Ensure all the
3411 		 * roots are synced and the guest TLB in hardware is clean.
3412 		 */
3413 		kvm_mmu_sync_roots(vcpu);
3414 		kvm_mmu_sync_prev_roots(vcpu);
3415 	}
3416 
3417 	static_call(kvm_x86_flush_tlb_guest)(vcpu);
3418 }
3419 
3420 
kvm_vcpu_flush_tlb_current(struct kvm_vcpu * vcpu)3421 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)
3422 {
3423 	++vcpu->stat.tlb_flush;
3424 	static_call(kvm_x86_flush_tlb_current)(vcpu);
3425 }
3426 
3427 /*
3428  * Service "local" TLB flush requests, which are specific to the current MMU
3429  * context.  In addition to the generic event handling in vcpu_enter_guest(),
3430  * TLB flushes that are targeted at an MMU context also need to be serviced
3431  * prior before nested VM-Enter/VM-Exit.
3432  */
kvm_service_local_tlb_flush_requests(struct kvm_vcpu * vcpu)3433 void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu)
3434 {
3435 	if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
3436 		kvm_vcpu_flush_tlb_current(vcpu);
3437 
3438 	if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))
3439 		kvm_vcpu_flush_tlb_guest(vcpu);
3440 }
3441 EXPORT_SYMBOL_GPL(kvm_service_local_tlb_flush_requests);
3442 
record_steal_time(struct kvm_vcpu * vcpu)3443 static void record_steal_time(struct kvm_vcpu *vcpu)
3444 {
3445 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
3446 	struct kvm_steal_time __user *st;
3447 	struct kvm_memslots *slots;
3448 	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
3449 	u64 steal;
3450 	u32 version;
3451 
3452 	if (kvm_xen_msr_enabled(vcpu->kvm)) {
3453 		kvm_xen_runstate_set_running(vcpu);
3454 		return;
3455 	}
3456 
3457 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
3458 		return;
3459 
3460 	if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))
3461 		return;
3462 
3463 	slots = kvm_memslots(vcpu->kvm);
3464 
3465 	if (unlikely(slots->generation != ghc->generation ||
3466 		     gpa != ghc->gpa ||
3467 		     kvm_is_error_hva(ghc->hva) || !ghc->memslot)) {
3468 		/* We rely on the fact that it fits in a single page. */
3469 		BUILD_BUG_ON((sizeof(*st) - 1) & KVM_STEAL_VALID_BITS);
3470 
3471 		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) ||
3472 		    kvm_is_error_hva(ghc->hva) || !ghc->memslot)
3473 			return;
3474 	}
3475 
3476 	st = (struct kvm_steal_time __user *)ghc->hva;
3477 	/*
3478 	 * Doing a TLB flush here, on the guest's behalf, can avoid
3479 	 * expensive IPIs.
3480 	 */
3481 	if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {
3482 		u8 st_preempted = 0;
3483 		int err = -EFAULT;
3484 
3485 		if (!user_access_begin(st, sizeof(*st)))
3486 			return;
3487 
3488 		asm volatile("1: xchgb %0, %2\n"
3489 			     "xor %1, %1\n"
3490 			     "2:\n"
3491 			     _ASM_EXTABLE_UA(1b, 2b)
3492 			     : "+q" (st_preempted),
3493 			       "+&r" (err),
3494 			       "+m" (st->preempted));
3495 		if (err)
3496 			goto out;
3497 
3498 		user_access_end();
3499 
3500 		vcpu->arch.st.preempted = 0;
3501 
3502 		trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
3503 				       st_preempted & KVM_VCPU_FLUSH_TLB);
3504 		if (st_preempted & KVM_VCPU_FLUSH_TLB)
3505 			kvm_vcpu_flush_tlb_guest(vcpu);
3506 
3507 		if (!user_access_begin(st, sizeof(*st)))
3508 			goto dirty;
3509 	} else {
3510 		if (!user_access_begin(st, sizeof(*st)))
3511 			return;
3512 
3513 		unsafe_put_user(0, &st->preempted, out);
3514 		vcpu->arch.st.preempted = 0;
3515 	}
3516 
3517 	unsafe_get_user(version, &st->version, out);
3518 	if (version & 1)
3519 		version += 1;  /* first time write, random junk */
3520 
3521 	version += 1;
3522 	unsafe_put_user(version, &st->version, out);
3523 
3524 	smp_wmb();
3525 
3526 	unsafe_get_user(steal, &st->steal, out);
3527 	steal += current->sched_info.run_delay -
3528 		vcpu->arch.st.last_steal;
3529 	vcpu->arch.st.last_steal = current->sched_info.run_delay;
3530 	unsafe_put_user(steal, &st->steal, out);
3531 
3532 	version += 1;
3533 	unsafe_put_user(version, &st->version, out);
3534 
3535  out:
3536 	user_access_end();
3537  dirty:
3538 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
3539 }
3540 
kvm_set_msr_common(struct kvm_vcpu * vcpu,struct msr_data * msr_info)3541 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3542 {
3543 	bool pr = false;
3544 	u32 msr = msr_info->index;
3545 	u64 data = msr_info->data;
3546 
3547 	if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr)
3548 		return kvm_xen_write_hypercall_page(vcpu, data);
3549 
3550 	switch (msr) {
3551 	case MSR_AMD64_NB_CFG:
3552 	case MSR_IA32_UCODE_WRITE:
3553 	case MSR_VM_HSAVE_PA:
3554 	case MSR_AMD64_PATCH_LOADER:
3555 	case MSR_AMD64_BU_CFG2:
3556 	case MSR_AMD64_DC_CFG:
3557 	case MSR_F15H_EX_CFG:
3558 		break;
3559 
3560 	case MSR_IA32_UCODE_REV:
3561 		if (msr_info->host_initiated)
3562 			vcpu->arch.microcode_version = data;
3563 		break;
3564 	case MSR_IA32_ARCH_CAPABILITIES:
3565 		if (!msr_info->host_initiated)
3566 			return 1;
3567 		vcpu->arch.arch_capabilities = data;
3568 		break;
3569 	case MSR_IA32_PERF_CAPABILITIES: {
3570 		struct kvm_msr_entry msr_ent = {.index = msr, .data = 0};
3571 
3572 		if (!msr_info->host_initiated)
3573 			return 1;
3574 		if (kvm_get_msr_feature(&msr_ent))
3575 			return 1;
3576 		if (data & ~msr_ent.data)
3577 			return 1;
3578 
3579 		vcpu->arch.perf_capabilities = data;
3580 		kvm_pmu_refresh(vcpu);
3581 		return 0;
3582 	}
3583 	case MSR_EFER:
3584 		return set_efer(vcpu, msr_info);
3585 	case MSR_K7_HWCR:
3586 		data &= ~(u64)0x40;	/* ignore flush filter disable */
3587 		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
3588 		data &= ~(u64)0x8;	/* ignore TLB cache disable */
3589 
3590 		/* Handle McStatusWrEn */
3591 		if (data == BIT_ULL(18)) {
3592 			vcpu->arch.msr_hwcr = data;
3593 		} else if (data != 0) {
3594 			vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
3595 				    data);
3596 			return 1;
3597 		}
3598 		break;
3599 	case MSR_FAM10H_MMIO_CONF_BASE:
3600 		if (data != 0) {
3601 			vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
3602 				    "0x%llx\n", data);
3603 			return 1;
3604 		}
3605 		break;
3606 	case 0x200 ... MSR_IA32_MC0_CTL2 - 1:
3607 	case MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) ... 0x2ff:
3608 		return kvm_mtrr_set_msr(vcpu, msr, data);
3609 	case MSR_IA32_APICBASE:
3610 		return kvm_set_apic_base(vcpu, msr_info);
3611 	case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
3612 		return kvm_x2apic_msr_write(vcpu, msr, data);
3613 	case MSR_IA32_TSC_DEADLINE:
3614 		kvm_set_lapic_tscdeadline_msr(vcpu, data);
3615 		break;
3616 	case MSR_IA32_TSC_ADJUST:
3617 		if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
3618 			if (!msr_info->host_initiated) {
3619 				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
3620 				adjust_tsc_offset_guest(vcpu, adj);
3621 				/* Before back to guest, tsc_timestamp must be adjusted
3622 				 * as well, otherwise guest's percpu pvclock time could jump.
3623 				 */
3624 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3625 			}
3626 			vcpu->arch.ia32_tsc_adjust_msr = data;
3627 		}
3628 		break;
3629 	case MSR_IA32_MISC_ENABLE: {
3630 		u64 old_val = vcpu->arch.ia32_misc_enable_msr;
3631 
3632 		if (!msr_info->host_initiated) {
3633 			/* RO bits */
3634 			if ((old_val ^ data) & MSR_IA32_MISC_ENABLE_PMU_RO_MASK)
3635 				return 1;
3636 
3637 			/* R bits, i.e. writes are ignored, but don't fault. */
3638 			data = data & ~MSR_IA32_MISC_ENABLE_EMON;
3639 			data |= old_val & MSR_IA32_MISC_ENABLE_EMON;
3640 		}
3641 
3642 		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
3643 		    ((old_val ^ data)  & MSR_IA32_MISC_ENABLE_MWAIT)) {
3644 			if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
3645 				return 1;
3646 			vcpu->arch.ia32_misc_enable_msr = data;
3647 			kvm_update_cpuid_runtime(vcpu);
3648 		} else {
3649 			vcpu->arch.ia32_misc_enable_msr = data;
3650 		}
3651 		break;
3652 	}
3653 	case MSR_IA32_SMBASE:
3654 		if (!msr_info->host_initiated)
3655 			return 1;
3656 		vcpu->arch.smbase = data;
3657 		break;
3658 	case MSR_IA32_POWER_CTL:
3659 		vcpu->arch.msr_ia32_power_ctl = data;
3660 		break;
3661 	case MSR_IA32_TSC:
3662 		if (msr_info->host_initiated) {
3663 			kvm_synchronize_tsc(vcpu, data);
3664 		} else {
3665 			u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;
3666 			adjust_tsc_offset_guest(vcpu, adj);
3667 			vcpu->arch.ia32_tsc_adjust_msr += adj;
3668 		}
3669 		break;
3670 	case MSR_IA32_XSS:
3671 		if (!msr_info->host_initiated &&
3672 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
3673 			return 1;
3674 		/*
3675 		 * KVM supports exposing PT to the guest, but does not support
3676 		 * IA32_XSS[bit 8]. Guests have to use RDMSR/WRMSR rather than
3677 		 * XSAVES/XRSTORS to save/restore PT MSRs.
3678 		 */
3679 		if (data & ~kvm_caps.supported_xss)
3680 			return 1;
3681 		vcpu->arch.ia32_xss = data;
3682 		kvm_update_cpuid_runtime(vcpu);
3683 		break;
3684 	case MSR_SMI_COUNT:
3685 		if (!msr_info->host_initiated)
3686 			return 1;
3687 		vcpu->arch.smi_count = data;
3688 		break;
3689 	case MSR_KVM_WALL_CLOCK_NEW:
3690 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3691 			return 1;
3692 
3693 		vcpu->kvm->arch.wall_clock = data;
3694 		kvm_write_wall_clock(vcpu->kvm, data, 0);
3695 		break;
3696 	case MSR_KVM_WALL_CLOCK:
3697 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3698 			return 1;
3699 
3700 		vcpu->kvm->arch.wall_clock = data;
3701 		kvm_write_wall_clock(vcpu->kvm, data, 0);
3702 		break;
3703 	case MSR_KVM_SYSTEM_TIME_NEW:
3704 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3705 			return 1;
3706 
3707 		kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);
3708 		break;
3709 	case MSR_KVM_SYSTEM_TIME:
3710 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3711 			return 1;
3712 
3713 		kvm_write_system_time(vcpu, data, true,  msr_info->host_initiated);
3714 		break;
3715 	case MSR_KVM_ASYNC_PF_EN:
3716 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
3717 			return 1;
3718 
3719 		if (kvm_pv_enable_async_pf(vcpu, data))
3720 			return 1;
3721 		break;
3722 	case MSR_KVM_ASYNC_PF_INT:
3723 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
3724 			return 1;
3725 
3726 		if (kvm_pv_enable_async_pf_int(vcpu, data))
3727 			return 1;
3728 		break;
3729 	case MSR_KVM_ASYNC_PF_ACK:
3730 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
3731 			return 1;
3732 		if (data & 0x1) {
3733 			vcpu->arch.apf.pageready_pending = false;
3734 			kvm_check_async_pf_completion(vcpu);
3735 		}
3736 		break;
3737 	case MSR_KVM_STEAL_TIME:
3738 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
3739 			return 1;
3740 
3741 		if (unlikely(!sched_info_on()))
3742 			return 1;
3743 
3744 		if (data & KVM_STEAL_RESERVED_MASK)
3745 			return 1;
3746 
3747 		vcpu->arch.st.msr_val = data;
3748 
3749 		if (!(data & KVM_MSR_ENABLED))
3750 			break;
3751 
3752 		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
3753 
3754 		break;
3755 	case MSR_KVM_PV_EOI_EN:
3756 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
3757 			return 1;
3758 
3759 		if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8)))
3760 			return 1;
3761 		break;
3762 
3763 	case MSR_KVM_POLL_CONTROL:
3764 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
3765 			return 1;
3766 
3767 		/* only enable bit supported */
3768 		if (data & (-1ULL << 1))
3769 			return 1;
3770 
3771 		vcpu->arch.msr_kvm_poll_control = data;
3772 		break;
3773 
3774 	case MSR_IA32_MCG_CTL:
3775 	case MSR_IA32_MCG_STATUS:
3776 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3777 	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
3778 		return set_msr_mce(vcpu, msr_info);
3779 
3780 	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
3781 	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
3782 		pr = true;
3783 		fallthrough;
3784 	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
3785 	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
3786 		if (kvm_pmu_is_valid_msr(vcpu, msr))
3787 			return kvm_pmu_set_msr(vcpu, msr_info);
3788 
3789 		if (pr || data != 0)
3790 			vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
3791 				    "0x%x data 0x%llx\n", msr, data);
3792 		break;
3793 	case MSR_K7_CLK_CTL:
3794 		/*
3795 		 * Ignore all writes to this no longer documented MSR.
3796 		 * Writes are only relevant for old K7 processors,
3797 		 * all pre-dating SVM, but a recommended workaround from
3798 		 * AMD for these chips. It is possible to specify the
3799 		 * affected processor models on the command line, hence
3800 		 * the need to ignore the workaround.
3801 		 */
3802 		break;
3803 	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
3804 	case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
3805 	case HV_X64_MSR_SYNDBG_OPTIONS:
3806 	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
3807 	case HV_X64_MSR_CRASH_CTL:
3808 	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
3809 	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
3810 	case HV_X64_MSR_TSC_EMULATION_CONTROL:
3811 	case HV_X64_MSR_TSC_EMULATION_STATUS:
3812 		return kvm_hv_set_msr_common(vcpu, msr, data,
3813 					     msr_info->host_initiated);
3814 	case MSR_IA32_BBL_CR_CTL3:
3815 		/* Drop writes to this legacy MSR -- see rdmsr
3816 		 * counterpart for further detail.
3817 		 */
3818 		if (report_ignored_msrs)
3819 			vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n",
3820 				msr, data);
3821 		break;
3822 	case MSR_AMD64_OSVW_ID_LENGTH:
3823 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3824 			return 1;
3825 		vcpu->arch.osvw.length = data;
3826 		break;
3827 	case MSR_AMD64_OSVW_STATUS:
3828 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3829 			return 1;
3830 		vcpu->arch.osvw.status = data;
3831 		break;
3832 	case MSR_PLATFORM_INFO:
3833 		if (!msr_info->host_initiated ||
3834 		    (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) &&
3835 		     cpuid_fault_enabled(vcpu)))
3836 			return 1;
3837 		vcpu->arch.msr_platform_info = data;
3838 		break;
3839 	case MSR_MISC_FEATURES_ENABLES:
3840 		if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
3841 		    (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3842 		     !supports_cpuid_fault(vcpu)))
3843 			return 1;
3844 		vcpu->arch.msr_misc_features_enables = data;
3845 		break;
3846 #ifdef CONFIG_X86_64
3847 	case MSR_IA32_XFD:
3848 		if (!msr_info->host_initiated &&
3849 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
3850 			return 1;
3851 
3852 		if (data & ~kvm_guest_supported_xfd(vcpu))
3853 			return 1;
3854 
3855 		fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data);
3856 		break;
3857 	case MSR_IA32_XFD_ERR:
3858 		if (!msr_info->host_initiated &&
3859 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
3860 			return 1;
3861 
3862 		if (data & ~kvm_guest_supported_xfd(vcpu))
3863 			return 1;
3864 
3865 		vcpu->arch.guest_fpu.xfd_err = data;
3866 		break;
3867 #endif
3868 	case MSR_IA32_PEBS_ENABLE:
3869 	case MSR_IA32_DS_AREA:
3870 	case MSR_PEBS_DATA_CFG:
3871 	case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
3872 		if (kvm_pmu_is_valid_msr(vcpu, msr))
3873 			return kvm_pmu_set_msr(vcpu, msr_info);
3874 		/*
3875 		 * Userspace is allowed to write '0' to MSRs that KVM reports
3876 		 * as to-be-saved, even if an MSRs isn't fully supported.
3877 		 */
3878 		return !msr_info->host_initiated || data;
3879 	default:
3880 		if (kvm_pmu_is_valid_msr(vcpu, msr))
3881 			return kvm_pmu_set_msr(vcpu, msr_info);
3882 		return KVM_MSR_RET_INVALID;
3883 	}
3884 	return 0;
3885 }
3886 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
3887 
get_msr_mce(struct kvm_vcpu * vcpu,u32 msr,u64 * pdata,bool host)3888 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
3889 {
3890 	u64 data;
3891 	u64 mcg_cap = vcpu->arch.mcg_cap;
3892 	unsigned bank_num = mcg_cap & 0xff;
3893 	u32 offset, last_msr;
3894 
3895 	switch (msr) {
3896 	case MSR_IA32_P5_MC_ADDR:
3897 	case MSR_IA32_P5_MC_TYPE:
3898 		data = 0;
3899 		break;
3900 	case MSR_IA32_MCG_CAP:
3901 		data = vcpu->arch.mcg_cap;
3902 		break;
3903 	case MSR_IA32_MCG_CTL:
3904 		if (!(mcg_cap & MCG_CTL_P) && !host)
3905 			return 1;
3906 		data = vcpu->arch.mcg_ctl;
3907 		break;
3908 	case MSR_IA32_MCG_STATUS:
3909 		data = vcpu->arch.mcg_status;
3910 		break;
3911 	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
3912 		last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1;
3913 		if (msr > last_msr)
3914 			return 1;
3915 
3916 		if (!(mcg_cap & MCG_CMCI_P) && !host)
3917 			return 1;
3918 		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2,
3919 					    last_msr + 1 - MSR_IA32_MC0_CTL2);
3920 		data = vcpu->arch.mci_ctl2_banks[offset];
3921 		break;
3922 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3923 		last_msr = MSR_IA32_MCx_CTL(bank_num) - 1;
3924 		if (msr > last_msr)
3925 			return 1;
3926 
3927 		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL,
3928 					    last_msr + 1 - MSR_IA32_MC0_CTL);
3929 		data = vcpu->arch.mce_banks[offset];
3930 		break;
3931 	default:
3932 		return 1;
3933 	}
3934 	*pdata = data;
3935 	return 0;
3936 }
3937 
kvm_get_msr_common(struct kvm_vcpu * vcpu,struct msr_data * msr_info)3938 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3939 {
3940 	switch (msr_info->index) {
3941 	case MSR_IA32_PLATFORM_ID:
3942 	case MSR_IA32_EBL_CR_POWERON:
3943 	case MSR_IA32_LASTBRANCHFROMIP:
3944 	case MSR_IA32_LASTBRANCHTOIP:
3945 	case MSR_IA32_LASTINTFROMIP:
3946 	case MSR_IA32_LASTINTTOIP:
3947 	case MSR_AMD64_SYSCFG:
3948 	case MSR_K8_TSEG_ADDR:
3949 	case MSR_K8_TSEG_MASK:
3950 	case MSR_VM_HSAVE_PA:
3951 	case MSR_K8_INT_PENDING_MSG:
3952 	case MSR_AMD64_NB_CFG:
3953 	case MSR_FAM10H_MMIO_CONF_BASE:
3954 	case MSR_AMD64_BU_CFG2:
3955 	case MSR_IA32_PERF_CTL:
3956 	case MSR_AMD64_DC_CFG:
3957 	case MSR_F15H_EX_CFG:
3958 	/*
3959 	 * Intel Sandy Bridge CPUs must support the RAPL (running average power
3960 	 * limit) MSRs. Just return 0, as we do not want to expose the host
3961 	 * data here. Do not conditionalize this on CPUID, as KVM does not do
3962 	 * so for existing CPU-specific MSRs.
3963 	 */
3964 	case MSR_RAPL_POWER_UNIT:
3965 	case MSR_PP0_ENERGY_STATUS:	/* Power plane 0 (core) */
3966 	case MSR_PP1_ENERGY_STATUS:	/* Power plane 1 (graphics uncore) */
3967 	case MSR_PKG_ENERGY_STATUS:	/* Total package */
3968 	case MSR_DRAM_ENERGY_STATUS:	/* DRAM controller */
3969 		msr_info->data = 0;
3970 		break;
3971 	case MSR_IA32_PEBS_ENABLE:
3972 	case MSR_IA32_DS_AREA:
3973 	case MSR_PEBS_DATA_CFG:
3974 	case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
3975 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3976 			return kvm_pmu_get_msr(vcpu, msr_info);
3977 		/*
3978 		 * Userspace is allowed to read MSRs that KVM reports as
3979 		 * to-be-saved, even if an MSR isn't fully supported.
3980 		 */
3981 		if (!msr_info->host_initiated)
3982 			return 1;
3983 		msr_info->data = 0;
3984 		break;
3985 	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
3986 	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
3987 	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
3988 	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
3989 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3990 			return kvm_pmu_get_msr(vcpu, msr_info);
3991 		msr_info->data = 0;
3992 		break;
3993 	case MSR_IA32_UCODE_REV:
3994 		msr_info->data = vcpu->arch.microcode_version;
3995 		break;
3996 	case MSR_IA32_ARCH_CAPABILITIES:
3997 		if (!msr_info->host_initiated &&
3998 		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
3999 			return 1;
4000 		msr_info->data = vcpu->arch.arch_capabilities;
4001 		break;
4002 	case MSR_IA32_PERF_CAPABILITIES:
4003 		if (!msr_info->host_initiated &&
4004 		    !guest_cpuid_has(vcpu, X86_FEATURE_PDCM))
4005 			return 1;
4006 		msr_info->data = vcpu->arch.perf_capabilities;
4007 		break;
4008 	case MSR_IA32_POWER_CTL:
4009 		msr_info->data = vcpu->arch.msr_ia32_power_ctl;
4010 		break;
4011 	case MSR_IA32_TSC: {
4012 		/*
4013 		 * Intel SDM states that MSR_IA32_TSC read adds the TSC offset
4014 		 * even when not intercepted. AMD manual doesn't explicitly
4015 		 * state this but appears to behave the same.
4016 		 *
4017 		 * On userspace reads and writes, however, we unconditionally
4018 		 * return L1's TSC value to ensure backwards-compatible
4019 		 * behavior for migration.
4020 		 */
4021 		u64 offset, ratio;
4022 
4023 		if (msr_info->host_initiated) {
4024 			offset = vcpu->arch.l1_tsc_offset;
4025 			ratio = vcpu->arch.l1_tsc_scaling_ratio;
4026 		} else {
4027 			offset = vcpu->arch.tsc_offset;
4028 			ratio = vcpu->arch.tsc_scaling_ratio;
4029 		}
4030 
4031 		msr_info->data = kvm_scale_tsc(rdtsc(), ratio) + offset;
4032 		break;
4033 	}
4034 	case MSR_MTRRcap:
4035 	case 0x200 ... MSR_IA32_MC0_CTL2 - 1:
4036 	case MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) ... 0x2ff:
4037 		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
4038 	case 0xcd: /* fsb frequency */
4039 		msr_info->data = 3;
4040 		break;
4041 		/*
4042 		 * MSR_EBC_FREQUENCY_ID
4043 		 * Conservative value valid for even the basic CPU models.
4044 		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
4045 		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
4046 		 * and 266MHz for model 3, or 4. Set Core Clock
4047 		 * Frequency to System Bus Frequency Ratio to 1 (bits
4048 		 * 31:24) even though these are only valid for CPU
4049 		 * models > 2, however guests may end up dividing or
4050 		 * multiplying by zero otherwise.
4051 		 */
4052 	case MSR_EBC_FREQUENCY_ID:
4053 		msr_info->data = 1 << 24;
4054 		break;
4055 	case MSR_IA32_APICBASE:
4056 		msr_info->data = kvm_get_apic_base(vcpu);
4057 		break;
4058 	case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
4059 		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
4060 	case MSR_IA32_TSC_DEADLINE:
4061 		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
4062 		break;
4063 	case MSR_IA32_TSC_ADJUST:
4064 		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
4065 		break;
4066 	case MSR_IA32_MISC_ENABLE:
4067 		msr_info->data = vcpu->arch.ia32_misc_enable_msr;
4068 		break;
4069 	case MSR_IA32_SMBASE:
4070 		if (!msr_info->host_initiated)
4071 			return 1;
4072 		msr_info->data = vcpu->arch.smbase;
4073 		break;
4074 	case MSR_SMI_COUNT:
4075 		msr_info->data = vcpu->arch.smi_count;
4076 		break;
4077 	case MSR_IA32_PERF_STATUS:
4078 		/* TSC increment by tick */
4079 		msr_info->data = 1000ULL;
4080 		/* CPU multiplier */
4081 		msr_info->data |= (((uint64_t)4ULL) << 40);
4082 		break;
4083 	case MSR_EFER:
4084 		msr_info->data = vcpu->arch.efer;
4085 		break;
4086 	case MSR_KVM_WALL_CLOCK:
4087 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
4088 			return 1;
4089 
4090 		msr_info->data = vcpu->kvm->arch.wall_clock;
4091 		break;
4092 	case MSR_KVM_WALL_CLOCK_NEW:
4093 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
4094 			return 1;
4095 
4096 		msr_info->data = vcpu->kvm->arch.wall_clock;
4097 		break;
4098 	case MSR_KVM_SYSTEM_TIME:
4099 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
4100 			return 1;
4101 
4102 		msr_info->data = vcpu->arch.time;
4103 		break;
4104 	case MSR_KVM_SYSTEM_TIME_NEW:
4105 		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
4106 			return 1;
4107 
4108 		msr_info->data = vcpu->arch.time;
4109 		break;
4110 	case MSR_KVM_ASYNC_PF_EN:
4111 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
4112 			return 1;
4113 
4114 		msr_info->data = vcpu->arch.apf.msr_en_val;
4115 		break;
4116 	case MSR_KVM_ASYNC_PF_INT:
4117 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
4118 			return 1;
4119 
4120 		msr_info->data = vcpu->arch.apf.msr_int_val;
4121 		break;
4122 	case MSR_KVM_ASYNC_PF_ACK:
4123 		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
4124 			return 1;
4125 
4126 		msr_info->data = 0;
4127 		break;
4128 	case MSR_KVM_STEAL_TIME:
4129 		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
4130 			return 1;
4131 
4132 		msr_info->data = vcpu->arch.st.msr_val;
4133 		break;
4134 	case MSR_KVM_PV_EOI_EN:
4135 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
4136 			return 1;
4137 
4138 		msr_info->data = vcpu->arch.pv_eoi.msr_val;
4139 		break;
4140 	case MSR_KVM_POLL_CONTROL:
4141 		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
4142 			return 1;
4143 
4144 		msr_info->data = vcpu->arch.msr_kvm_poll_control;
4145 		break;
4146 	case MSR_IA32_P5_MC_ADDR:
4147 	case MSR_IA32_P5_MC_TYPE:
4148 	case MSR_IA32_MCG_CAP:
4149 	case MSR_IA32_MCG_CTL:
4150 	case MSR_IA32_MCG_STATUS:
4151 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
4152 	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
4153 		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
4154 				   msr_info->host_initiated);
4155 	case MSR_IA32_XSS:
4156 		if (!msr_info->host_initiated &&
4157 		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
4158 			return 1;
4159 		msr_info->data = vcpu->arch.ia32_xss;
4160 		break;
4161 	case MSR_K7_CLK_CTL:
4162 		/*
4163 		 * Provide expected ramp-up count for K7. All other
4164 		 * are set to zero, indicating minimum divisors for
4165 		 * every field.
4166 		 *
4167 		 * This prevents guest kernels on AMD host with CPU
4168 		 * type 6, model 8 and higher from exploding due to
4169 		 * the rdmsr failing.
4170 		 */
4171 		msr_info->data = 0x20000000;
4172 		break;
4173 	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
4174 	case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
4175 	case HV_X64_MSR_SYNDBG_OPTIONS:
4176 	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
4177 	case HV_X64_MSR_CRASH_CTL:
4178 	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
4179 	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
4180 	case HV_X64_MSR_TSC_EMULATION_CONTROL:
4181 	case HV_X64_MSR_TSC_EMULATION_STATUS:
4182 		return kvm_hv_get_msr_common(vcpu,
4183 					     msr_info->index, &msr_info->data,
4184 					     msr_info->host_initiated);
4185 	case MSR_IA32_BBL_CR_CTL3:
4186 		/* This legacy MSR exists but isn't fully documented in current
4187 		 * silicon.  It is however accessed by winxp in very narrow
4188 		 * scenarios where it sets bit #19, itself documented as
4189 		 * a "reserved" bit.  Best effort attempt to source coherent
4190 		 * read data here should the balance of the register be
4191 		 * interpreted by the guest:
4192 		 *
4193 		 * L2 cache control register 3: 64GB range, 256KB size,
4194 		 * enabled, latency 0x1, configured
4195 		 */
4196 		msr_info->data = 0xbe702111;
4197 		break;
4198 	case MSR_AMD64_OSVW_ID_LENGTH:
4199 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
4200 			return 1;
4201 		msr_info->data = vcpu->arch.osvw.length;
4202 		break;
4203 	case MSR_AMD64_OSVW_STATUS:
4204 		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
4205 			return 1;
4206 		msr_info->data = vcpu->arch.osvw.status;
4207 		break;
4208 	case MSR_PLATFORM_INFO:
4209 		if (!msr_info->host_initiated &&
4210 		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)
4211 			return 1;
4212 		msr_info->data = vcpu->arch.msr_platform_info;
4213 		break;
4214 	case MSR_MISC_FEATURES_ENABLES:
4215 		msr_info->data = vcpu->arch.msr_misc_features_enables;
4216 		break;
4217 	case MSR_K7_HWCR:
4218 		msr_info->data = vcpu->arch.msr_hwcr;
4219 		break;
4220 #ifdef CONFIG_X86_64
4221 	case MSR_IA32_XFD:
4222 		if (!msr_info->host_initiated &&
4223 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4224 			return 1;
4225 
4226 		msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd;
4227 		break;
4228 	case MSR_IA32_XFD_ERR:
4229 		if (!msr_info->host_initiated &&
4230 		    !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4231 			return 1;
4232 
4233 		msr_info->data = vcpu->arch.guest_fpu.xfd_err;
4234 		break;
4235 #endif
4236 	default:
4237 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
4238 			return kvm_pmu_get_msr(vcpu, msr_info);
4239 		return KVM_MSR_RET_INVALID;
4240 	}
4241 	return 0;
4242 }
4243 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
4244 
4245 /*
4246  * Read or write a bunch of msrs. All parameters are kernel addresses.
4247  *
4248  * @return number of msrs set successfully.
4249  */
__msr_io(struct kvm_vcpu * vcpu,struct kvm_msrs * msrs,struct kvm_msr_entry * entries,int (* do_msr)(struct kvm_vcpu * vcpu,unsigned index,u64 * data))4250 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
4251 		    struct kvm_msr_entry *entries,
4252 		    int (*do_msr)(struct kvm_vcpu *vcpu,
4253 				  unsigned index, u64 *data))
4254 {
4255 	int i;
4256 
4257 	for (i = 0; i < msrs->nmsrs; ++i)
4258 		if (do_msr(vcpu, entries[i].index, &entries[i].data))
4259 			break;
4260 
4261 	return i;
4262 }
4263 
4264 /*
4265  * Read or write a bunch of msrs. Parameters are user addresses.
4266  *
4267  * @return number of msrs set successfully.
4268  */
msr_io(struct kvm_vcpu * vcpu,struct kvm_msrs __user * user_msrs,int (* do_msr)(struct kvm_vcpu * vcpu,unsigned index,u64 * data),int writeback)4269 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
4270 		  int (*do_msr)(struct kvm_vcpu *vcpu,
4271 				unsigned index, u64 *data),
4272 		  int writeback)
4273 {
4274 	struct kvm_msrs msrs;
4275 	struct kvm_msr_entry *entries;
4276 	int r, n;
4277 	unsigned size;
4278 
4279 	r = -EFAULT;
4280 	if (copy_from_user(&msrs, user_msrs, sizeof(msrs)))
4281 		goto out;
4282 
4283 	r = -E2BIG;
4284 	if (msrs.nmsrs >= MAX_IO_MSRS)
4285 		goto out;
4286 
4287 	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
4288 	entries = memdup_user(user_msrs->entries, size);
4289 	if (IS_ERR(entries)) {
4290 		r = PTR_ERR(entries);
4291 		goto out;
4292 	}
4293 
4294 	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
4295 	if (r < 0)
4296 		goto out_free;
4297 
4298 	r = -EFAULT;
4299 	if (writeback && copy_to_user(user_msrs->entries, entries, size))
4300 		goto out_free;
4301 
4302 	r = n;
4303 
4304 out_free:
4305 	kfree(entries);
4306 out:
4307 	return r;
4308 }
4309 
kvm_can_mwait_in_guest(void)4310 static inline bool kvm_can_mwait_in_guest(void)
4311 {
4312 	return boot_cpu_has(X86_FEATURE_MWAIT) &&
4313 		!boot_cpu_has_bug(X86_BUG_MONITOR) &&
4314 		boot_cpu_has(X86_FEATURE_ARAT);
4315 }
4316 
kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu * vcpu,struct kvm_cpuid2 __user * cpuid_arg)4317 static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu,
4318 					    struct kvm_cpuid2 __user *cpuid_arg)
4319 {
4320 	struct kvm_cpuid2 cpuid;
4321 	int r;
4322 
4323 	r = -EFAULT;
4324 	if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4325 		return r;
4326 
4327 	r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries);
4328 	if (r)
4329 		return r;
4330 
4331 	r = -EFAULT;
4332 	if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4333 		return r;
4334 
4335 	return 0;
4336 }
4337 
kvm_vm_ioctl_check_extension(struct kvm * kvm,long ext)4338 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
4339 {
4340 	int r = 0;
4341 
4342 	switch (ext) {
4343 	case KVM_CAP_IRQCHIP:
4344 	case KVM_CAP_HLT:
4345 	case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
4346 	case KVM_CAP_SET_TSS_ADDR:
4347 	case KVM_CAP_EXT_CPUID:
4348 	case KVM_CAP_EXT_EMUL_CPUID:
4349 	case KVM_CAP_CLOCKSOURCE:
4350 	case KVM_CAP_PIT:
4351 	case KVM_CAP_NOP_IO_DELAY:
4352 	case KVM_CAP_MP_STATE:
4353 	case KVM_CAP_SYNC_MMU:
4354 	case KVM_CAP_USER_NMI:
4355 	case KVM_CAP_REINJECT_CONTROL:
4356 	case KVM_CAP_IRQ_INJECT_STATUS:
4357 	case KVM_CAP_IOEVENTFD:
4358 	case KVM_CAP_IOEVENTFD_NO_LENGTH:
4359 	case KVM_CAP_PIT2:
4360 	case KVM_CAP_PIT_STATE2:
4361 	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
4362 	case KVM_CAP_VCPU_EVENTS:
4363 	case KVM_CAP_HYPERV:
4364 	case KVM_CAP_HYPERV_VAPIC:
4365 	case KVM_CAP_HYPERV_SPIN:
4366 	case KVM_CAP_HYPERV_SYNIC:
4367 	case KVM_CAP_HYPERV_SYNIC2:
4368 	case KVM_CAP_HYPERV_VP_INDEX:
4369 	case KVM_CAP_HYPERV_EVENTFD:
4370 	case KVM_CAP_HYPERV_TLBFLUSH:
4371 	case KVM_CAP_HYPERV_SEND_IPI:
4372 	case KVM_CAP_HYPERV_CPUID:
4373 	case KVM_CAP_HYPERV_ENFORCE_CPUID:
4374 	case KVM_CAP_SYS_HYPERV_CPUID:
4375 	case KVM_CAP_PCI_SEGMENT:
4376 	case KVM_CAP_DEBUGREGS:
4377 	case KVM_CAP_X86_ROBUST_SINGLESTEP:
4378 	case KVM_CAP_XSAVE:
4379 	case KVM_CAP_ASYNC_PF:
4380 	case KVM_CAP_ASYNC_PF_INT:
4381 	case KVM_CAP_GET_TSC_KHZ:
4382 	case KVM_CAP_KVMCLOCK_CTRL:
4383 	case KVM_CAP_READONLY_MEM:
4384 	case KVM_CAP_HYPERV_TIME:
4385 	case KVM_CAP_IOAPIC_POLARITY_IGNORED:
4386 	case KVM_CAP_TSC_DEADLINE_TIMER:
4387 	case KVM_CAP_DISABLE_QUIRKS:
4388 	case KVM_CAP_SET_BOOT_CPU_ID:
4389  	case KVM_CAP_SPLIT_IRQCHIP:
4390 	case KVM_CAP_IMMEDIATE_EXIT:
4391 	case KVM_CAP_PMU_EVENT_FILTER:
4392 	case KVM_CAP_GET_MSR_FEATURES:
4393 	case KVM_CAP_MSR_PLATFORM_INFO:
4394 	case KVM_CAP_EXCEPTION_PAYLOAD:
4395 	case KVM_CAP_X86_TRIPLE_FAULT_EVENT:
4396 	case KVM_CAP_SET_GUEST_DEBUG:
4397 	case KVM_CAP_LAST_CPU:
4398 	case KVM_CAP_X86_USER_SPACE_MSR:
4399 	case KVM_CAP_X86_MSR_FILTER:
4400 	case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
4401 #ifdef CONFIG_X86_SGX_KVM
4402 	case KVM_CAP_SGX_ATTRIBUTE:
4403 #endif
4404 	case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM:
4405 	case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM:
4406 	case KVM_CAP_SREGS2:
4407 	case KVM_CAP_EXIT_ON_EMULATION_FAILURE:
4408 	case KVM_CAP_VCPU_ATTRIBUTES:
4409 	case KVM_CAP_SYS_ATTRIBUTES:
4410 	case KVM_CAP_VAPIC:
4411 	case KVM_CAP_ENABLE_CAP:
4412 	case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES:
4413 		r = 1;
4414 		break;
4415 	case KVM_CAP_EXIT_HYPERCALL:
4416 		r = KVM_EXIT_HYPERCALL_VALID_MASK;
4417 		break;
4418 	case KVM_CAP_SET_GUEST_DEBUG2:
4419 		return KVM_GUESTDBG_VALID_MASK;
4420 #ifdef CONFIG_KVM_XEN
4421 	case KVM_CAP_XEN_HVM:
4422 		r = KVM_XEN_HVM_CONFIG_HYPERCALL_MSR |
4423 		    KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL |
4424 		    KVM_XEN_HVM_CONFIG_SHARED_INFO |
4425 		    KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL |
4426 		    KVM_XEN_HVM_CONFIG_EVTCHN_SEND;
4427 		if (sched_info_on())
4428 			r |= KVM_XEN_HVM_CONFIG_RUNSTATE;
4429 		break;
4430 #endif
4431 	case KVM_CAP_SYNC_REGS:
4432 		r = KVM_SYNC_X86_VALID_FIELDS;
4433 		break;
4434 	case KVM_CAP_ADJUST_CLOCK:
4435 		r = KVM_CLOCK_VALID_FLAGS;
4436 		break;
4437 	case KVM_CAP_X86_DISABLE_EXITS:
4438 		r |=  KVM_X86_DISABLE_EXITS_HLT | KVM_X86_DISABLE_EXITS_PAUSE |
4439 		      KVM_X86_DISABLE_EXITS_CSTATE;
4440 		if(kvm_can_mwait_in_guest())
4441 			r |= KVM_X86_DISABLE_EXITS_MWAIT;
4442 		break;
4443 	case KVM_CAP_X86_SMM:
4444 		/* SMBASE is usually relocated above 1M on modern chipsets,
4445 		 * and SMM handlers might indeed rely on 4G segment limits,
4446 		 * so do not report SMM to be available if real mode is
4447 		 * emulated via vm86 mode.  Still, do not go to great lengths
4448 		 * to avoid userspace's usage of the feature, because it is a
4449 		 * fringe case that is not enabled except via specific settings
4450 		 * of the module parameters.
4451 		 */
4452 		r = static_call(kvm_x86_has_emulated_msr)(kvm, MSR_IA32_SMBASE);
4453 		break;
4454 	case KVM_CAP_NR_VCPUS:
4455 		r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS);
4456 		break;
4457 	case KVM_CAP_MAX_VCPUS:
4458 		r = KVM_MAX_VCPUS;
4459 		break;
4460 	case KVM_CAP_MAX_VCPU_ID:
4461 		r = KVM_MAX_VCPU_IDS;
4462 		break;
4463 	case KVM_CAP_PV_MMU:	/* obsolete */
4464 		r = 0;
4465 		break;
4466 	case KVM_CAP_MCE:
4467 		r = KVM_MAX_MCE_BANKS;
4468 		break;
4469 	case KVM_CAP_XCRS:
4470 		r = boot_cpu_has(X86_FEATURE_XSAVE);
4471 		break;
4472 	case KVM_CAP_TSC_CONTROL:
4473 	case KVM_CAP_VM_TSC_CONTROL:
4474 		r = kvm_caps.has_tsc_control;
4475 		break;
4476 	case KVM_CAP_X2APIC_API:
4477 		r = KVM_X2APIC_API_VALID_FLAGS;
4478 		break;
4479 	case KVM_CAP_NESTED_STATE:
4480 		r = kvm_x86_ops.nested_ops->get_state ?
4481 			kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0;
4482 		break;
4483 	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
4484 		r = kvm_x86_ops.enable_direct_tlbflush != NULL;
4485 		break;
4486 	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
4487 		r = kvm_x86_ops.nested_ops->enable_evmcs != NULL;
4488 		break;
4489 	case KVM_CAP_SMALLER_MAXPHYADDR:
4490 		r = (int) allow_smaller_maxphyaddr;
4491 		break;
4492 	case KVM_CAP_STEAL_TIME:
4493 		r = sched_info_on();
4494 		break;
4495 	case KVM_CAP_X86_BUS_LOCK_EXIT:
4496 		if (kvm_caps.has_bus_lock_exit)
4497 			r = KVM_BUS_LOCK_DETECTION_OFF |
4498 			    KVM_BUS_LOCK_DETECTION_EXIT;
4499 		else
4500 			r = 0;
4501 		break;
4502 	case KVM_CAP_XSAVE2: {
4503 		u64 guest_perm = xstate_get_guest_group_perm();
4504 
4505 		r = xstate_required_size(kvm_caps.supported_xcr0 & guest_perm, false);
4506 		if (r < sizeof(struct kvm_xsave))
4507 			r = sizeof(struct kvm_xsave);
4508 		break;
4509 	}
4510 	case KVM_CAP_PMU_CAPABILITY:
4511 		r = enable_pmu ? KVM_CAP_PMU_VALID_MASK : 0;
4512 		break;
4513 	case KVM_CAP_DISABLE_QUIRKS2:
4514 		r = KVM_X86_VALID_QUIRKS;
4515 		break;
4516 	case KVM_CAP_X86_NOTIFY_VMEXIT:
4517 		r = kvm_caps.has_notify_vmexit;
4518 		break;
4519 	default:
4520 		break;
4521 	}
4522 	return r;
4523 }
4524 
kvm_get_attr_addr(struct kvm_device_attr * attr)4525 static inline void __user *kvm_get_attr_addr(struct kvm_device_attr *attr)
4526 {
4527 	void __user *uaddr = (void __user*)(unsigned long)attr->addr;
4528 
4529 	if ((u64)(unsigned long)uaddr != attr->addr)
4530 		return ERR_PTR_USR(-EFAULT);
4531 	return uaddr;
4532 }
4533 
kvm_x86_dev_get_attr(struct kvm_device_attr * attr)4534 static int kvm_x86_dev_get_attr(struct kvm_device_attr *attr)
4535 {
4536 	u64 __user *uaddr = kvm_get_attr_addr(attr);
4537 
4538 	if (attr->group)
4539 		return -ENXIO;
4540 
4541 	if (IS_ERR(uaddr))
4542 		return PTR_ERR(uaddr);
4543 
4544 	switch (attr->attr) {
4545 	case KVM_X86_XCOMP_GUEST_SUPP:
4546 		if (put_user(kvm_caps.supported_xcr0, uaddr))
4547 			return -EFAULT;
4548 		return 0;
4549 	default:
4550 		return -ENXIO;
4551 		break;
4552 	}
4553 }
4554 
kvm_x86_dev_has_attr(struct kvm_device_attr * attr)4555 static int kvm_x86_dev_has_attr(struct kvm_device_attr *attr)
4556 {
4557 	if (attr->group)
4558 		return -ENXIO;
4559 
4560 	switch (attr->attr) {
4561 	case KVM_X86_XCOMP_GUEST_SUPP:
4562 		return 0;
4563 	default:
4564 		return -ENXIO;
4565 	}
4566 }
4567 
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)4568 long kvm_arch_dev_ioctl(struct file *filp,
4569 			unsigned int ioctl, unsigned long arg)
4570 {
4571 	void __user *argp = (void __user *)arg;
4572 	long r;
4573 
4574 	switch (ioctl) {
4575 	case KVM_GET_MSR_INDEX_LIST: {
4576 		struct kvm_msr_list __user *user_msr_list = argp;
4577 		struct kvm_msr_list msr_list;
4578 		unsigned n;
4579 
4580 		r = -EFAULT;
4581 		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
4582 			goto out;
4583 		n = msr_list.nmsrs;
4584 		msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs;
4585 		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
4586 			goto out;
4587 		r = -E2BIG;
4588 		if (n < msr_list.nmsrs)
4589 			goto out;
4590 		r = -EFAULT;
4591 		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
4592 				 num_msrs_to_save * sizeof(u32)))
4593 			goto out;
4594 		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
4595 				 &emulated_msrs,
4596 				 num_emulated_msrs * sizeof(u32)))
4597 			goto out;
4598 		r = 0;
4599 		break;
4600 	}
4601 	case KVM_GET_SUPPORTED_CPUID:
4602 	case KVM_GET_EMULATED_CPUID: {
4603 		struct kvm_cpuid2 __user *cpuid_arg = argp;
4604 		struct kvm_cpuid2 cpuid;
4605 
4606 		r = -EFAULT;
4607 		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4608 			goto out;
4609 
4610 		r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
4611 					    ioctl);
4612 		if (r)
4613 			goto out;
4614 
4615 		r = -EFAULT;
4616 		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4617 			goto out;
4618 		r = 0;
4619 		break;
4620 	}
4621 	case KVM_X86_GET_MCE_CAP_SUPPORTED:
4622 		r = -EFAULT;
4623 		if (copy_to_user(argp, &kvm_caps.supported_mce_cap,
4624 				 sizeof(kvm_caps.supported_mce_cap)))
4625 			goto out;
4626 		r = 0;
4627 		break;
4628 	case KVM_GET_MSR_FEATURE_INDEX_LIST: {
4629 		struct kvm_msr_list __user *user_msr_list = argp;
4630 		struct kvm_msr_list msr_list;
4631 		unsigned int n;
4632 
4633 		r = -EFAULT;
4634 		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
4635 			goto out;
4636 		n = msr_list.nmsrs;
4637 		msr_list.nmsrs = num_msr_based_features;
4638 		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
4639 			goto out;
4640 		r = -E2BIG;
4641 		if (n < msr_list.nmsrs)
4642 			goto out;
4643 		r = -EFAULT;
4644 		if (copy_to_user(user_msr_list->indices, &msr_based_features,
4645 				 num_msr_based_features * sizeof(u32)))
4646 			goto out;
4647 		r = 0;
4648 		break;
4649 	}
4650 	case KVM_GET_MSRS:
4651 		r = msr_io(NULL, argp, do_get_msr_feature, 1);
4652 		break;
4653 	case KVM_GET_SUPPORTED_HV_CPUID:
4654 		r = kvm_ioctl_get_supported_hv_cpuid(NULL, argp);
4655 		break;
4656 	case KVM_GET_DEVICE_ATTR: {
4657 		struct kvm_device_attr attr;
4658 		r = -EFAULT;
4659 		if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
4660 			break;
4661 		r = kvm_x86_dev_get_attr(&attr);
4662 		break;
4663 	}
4664 	case KVM_HAS_DEVICE_ATTR: {
4665 		struct kvm_device_attr attr;
4666 		r = -EFAULT;
4667 		if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
4668 			break;
4669 		r = kvm_x86_dev_has_attr(&attr);
4670 		break;
4671 	}
4672 	default:
4673 		r = -EINVAL;
4674 		break;
4675 	}
4676 out:
4677 	return r;
4678 }
4679 
wbinvd_ipi(void * garbage)4680 static void wbinvd_ipi(void *garbage)
4681 {
4682 	wbinvd();
4683 }
4684 
need_emulate_wbinvd(struct kvm_vcpu * vcpu)4685 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
4686 {
4687 	return kvm_arch_has_noncoherent_dma(vcpu->kvm);
4688 }
4689 
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)4690 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
4691 {
4692 	/* Address WBINVD may be executed by guest */
4693 	if (need_emulate_wbinvd(vcpu)) {
4694 		if (static_call(kvm_x86_has_wbinvd_exit)())
4695 			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
4696 		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
4697 			smp_call_function_single(vcpu->cpu,
4698 					wbinvd_ipi, NULL, 1);
4699 	}
4700 
4701 	static_call(kvm_x86_vcpu_load)(vcpu, cpu);
4702 
4703 	/* Save host pkru register if supported */
4704 	vcpu->arch.host_pkru = read_pkru();
4705 
4706 	/* Apply any externally detected TSC adjustments (due to suspend) */
4707 	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
4708 		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
4709 		vcpu->arch.tsc_offset_adjustment = 0;
4710 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
4711 	}
4712 
4713 	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
4714 		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
4715 				rdtsc() - vcpu->arch.last_host_tsc;
4716 		if (tsc_delta < 0)
4717 			mark_tsc_unstable("KVM discovered backwards TSC");
4718 
4719 		if (kvm_check_tsc_unstable()) {
4720 			u64 offset = kvm_compute_l1_tsc_offset(vcpu,
4721 						vcpu->arch.last_guest_tsc);
4722 			kvm_vcpu_write_tsc_offset(vcpu, offset);
4723 			vcpu->arch.tsc_catchup = 1;
4724 		}
4725 
4726 		if (kvm_lapic_hv_timer_in_use(vcpu))
4727 			kvm_lapic_restart_hv_timer(vcpu);
4728 
4729 		/*
4730 		 * On a host with synchronized TSC, there is no need to update
4731 		 * kvmclock on vcpu->cpu migration
4732 		 */
4733 		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
4734 			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
4735 		if (vcpu->cpu != cpu)
4736 			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
4737 		vcpu->cpu = cpu;
4738 	}
4739 
4740 	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
4741 }
4742 
kvm_steal_time_set_preempted(struct kvm_vcpu * vcpu)4743 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
4744 {
4745 	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
4746 	struct kvm_steal_time __user *st;
4747 	struct kvm_memslots *slots;
4748 	static const u8 preempted = KVM_VCPU_PREEMPTED;
4749 	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
4750 
4751 	/*
4752 	 * The vCPU can be marked preempted if and only if the VM-Exit was on
4753 	 * an instruction boundary and will not trigger guest emulation of any
4754 	 * kind (see vcpu_run).  Vendor specific code controls (conservatively)
4755 	 * when this is true, for example allowing the vCPU to be marked
4756 	 * preempted if and only if the VM-Exit was due to a host interrupt.
4757 	 */
4758 	if (!vcpu->arch.at_instruction_boundary) {
4759 		vcpu->stat.preemption_other++;
4760 		return;
4761 	}
4762 
4763 	vcpu->stat.preemption_reported++;
4764 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
4765 		return;
4766 
4767 	if (vcpu->arch.st.preempted)
4768 		return;
4769 
4770 	/* This happens on process exit */
4771 	if (unlikely(current->mm != vcpu->kvm->mm))
4772 		return;
4773 
4774 	slots = kvm_memslots(vcpu->kvm);
4775 
4776 	if (unlikely(slots->generation != ghc->generation ||
4777 		     gpa != ghc->gpa ||
4778 		     kvm_is_error_hva(ghc->hva) || !ghc->memslot))
4779 		return;
4780 
4781 	st = (struct kvm_steal_time __user *)ghc->hva;
4782 	BUILD_BUG_ON(sizeof(st->preempted) != sizeof(preempted));
4783 
4784 	if (!copy_to_user_nofault(&st->preempted, &preempted, sizeof(preempted)))
4785 		vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
4786 
4787 	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
4788 }
4789 
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)4790 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
4791 {
4792 	int idx;
4793 
4794 	if (vcpu->preempted) {
4795 		if (!vcpu->arch.guest_state_protected)
4796 			vcpu->arch.preempted_in_kernel = !static_call(kvm_x86_get_cpl)(vcpu);
4797 
4798 		/*
4799 		 * Take the srcu lock as memslots will be accessed to check the gfn
4800 		 * cache generation against the memslots generation.
4801 		 */
4802 		idx = srcu_read_lock(&vcpu->kvm->srcu);
4803 		if (kvm_xen_msr_enabled(vcpu->kvm))
4804 			kvm_xen_runstate_set_preempted(vcpu);
4805 		else
4806 			kvm_steal_time_set_preempted(vcpu);
4807 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4808 	}
4809 
4810 	static_call(kvm_x86_vcpu_put)(vcpu);
4811 	vcpu->arch.last_host_tsc = rdtsc();
4812 }
4813 
kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu * vcpu,struct kvm_lapic_state * s)4814 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
4815 				    struct kvm_lapic_state *s)
4816 {
4817 	static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
4818 
4819 	return kvm_apic_get_state(vcpu, s);
4820 }
4821 
kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu * vcpu,struct kvm_lapic_state * s)4822 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
4823 				    struct kvm_lapic_state *s)
4824 {
4825 	int r;
4826 
4827 	r = kvm_apic_set_state(vcpu, s);
4828 	if (r)
4829 		return r;
4830 	update_cr8_intercept(vcpu);
4831 
4832 	return 0;
4833 }
4834 
kvm_cpu_accept_dm_intr(struct kvm_vcpu * vcpu)4835 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
4836 {
4837 	/*
4838 	 * We can accept userspace's request for interrupt injection
4839 	 * as long as we have a place to store the interrupt number.
4840 	 * The actual injection will happen when the CPU is able to
4841 	 * deliver the interrupt.
4842 	 */
4843 	if (kvm_cpu_has_extint(vcpu))
4844 		return false;
4845 
4846 	/* Acknowledging ExtINT does not happen if LINT0 is masked.  */
4847 	return (!lapic_in_kernel(vcpu) ||
4848 		kvm_apic_accept_pic_intr(vcpu));
4849 }
4850 
kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu * vcpu)4851 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
4852 {
4853 	/*
4854 	 * Do not cause an interrupt window exit if an exception
4855 	 * is pending or an event needs reinjection; userspace
4856 	 * might want to inject the interrupt manually using KVM_SET_REGS
4857 	 * or KVM_SET_SREGS.  For that to work, we must be at an
4858 	 * instruction boundary and with no events half-injected.
4859 	 */
4860 	return (kvm_arch_interrupt_allowed(vcpu) &&
4861 		kvm_cpu_accept_dm_intr(vcpu) &&
4862 		!kvm_event_needs_reinjection(vcpu) &&
4863 		!kvm_is_exception_pending(vcpu));
4864 }
4865 
kvm_vcpu_ioctl_interrupt(struct kvm_vcpu * vcpu,struct kvm_interrupt * irq)4866 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
4867 				    struct kvm_interrupt *irq)
4868 {
4869 	if (irq->irq >= KVM_NR_INTERRUPTS)
4870 		return -EINVAL;
4871 
4872 	if (!irqchip_in_kernel(vcpu->kvm)) {
4873 		kvm_queue_interrupt(vcpu, irq->irq, false);
4874 		kvm_make_request(KVM_REQ_EVENT, vcpu);
4875 		return 0;
4876 	}
4877 
4878 	/*
4879 	 * With in-kernel LAPIC, we only use this to inject EXTINT, so
4880 	 * fail for in-kernel 8259.
4881 	 */
4882 	if (pic_in_kernel(vcpu->kvm))
4883 		return -ENXIO;
4884 
4885 	if (vcpu->arch.pending_external_vector != -1)
4886 		return -EEXIST;
4887 
4888 	vcpu->arch.pending_external_vector = irq->irq;
4889 	kvm_make_request(KVM_REQ_EVENT, vcpu);
4890 	return 0;
4891 }
4892 
kvm_vcpu_ioctl_nmi(struct kvm_vcpu * vcpu)4893 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
4894 {
4895 	kvm_inject_nmi(vcpu);
4896 
4897 	return 0;
4898 }
4899 
kvm_vcpu_ioctl_smi(struct kvm_vcpu * vcpu)4900 static int kvm_vcpu_ioctl_smi(struct kvm_vcpu *vcpu)
4901 {
4902 	kvm_make_request(KVM_REQ_SMI, vcpu);
4903 
4904 	return 0;
4905 }
4906 
vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu * vcpu,struct kvm_tpr_access_ctl * tac)4907 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
4908 					   struct kvm_tpr_access_ctl *tac)
4909 {
4910 	if (tac->flags)
4911 		return -EINVAL;
4912 	vcpu->arch.tpr_access_reporting = !!tac->enabled;
4913 	return 0;
4914 }
4915 
kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu * vcpu,u64 mcg_cap)4916 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
4917 					u64 mcg_cap)
4918 {
4919 	int r;
4920 	unsigned bank_num = mcg_cap & 0xff, bank;
4921 
4922 	r = -EINVAL;
4923 	if (!bank_num || bank_num > KVM_MAX_MCE_BANKS)
4924 		goto out;
4925 	if (mcg_cap & ~(kvm_caps.supported_mce_cap | 0xff | 0xff0000))
4926 		goto out;
4927 	r = 0;
4928 	vcpu->arch.mcg_cap = mcg_cap;
4929 	/* Init IA32_MCG_CTL to all 1s */
4930 	if (mcg_cap & MCG_CTL_P)
4931 		vcpu->arch.mcg_ctl = ~(u64)0;
4932 	/* Init IA32_MCi_CTL to all 1s, IA32_MCi_CTL2 to all 0s */
4933 	for (bank = 0; bank < bank_num; bank++) {
4934 		vcpu->arch.mce_banks[bank*4] = ~(u64)0;
4935 		if (mcg_cap & MCG_CMCI_P)
4936 			vcpu->arch.mci_ctl2_banks[bank] = 0;
4937 	}
4938 
4939 	kvm_apic_after_set_mcg_cap(vcpu);
4940 
4941 	static_call(kvm_x86_setup_mce)(vcpu);
4942 out:
4943 	return r;
4944 }
4945 
4946 /*
4947  * Validate this is an UCNA (uncorrectable no action) error by checking the
4948  * MCG_STATUS and MCi_STATUS registers:
4949  * - none of the bits for Machine Check Exceptions are set
4950  * - both the VAL (valid) and UC (uncorrectable) bits are set
4951  * MCI_STATUS_PCC - Processor Context Corrupted
4952  * MCI_STATUS_S - Signaled as a Machine Check Exception
4953  * MCI_STATUS_AR - Software recoverable Action Required
4954  */
is_ucna(struct kvm_x86_mce * mce)4955 static bool is_ucna(struct kvm_x86_mce *mce)
4956 {
4957 	return	!mce->mcg_status &&
4958 		!(mce->status & (MCI_STATUS_PCC | MCI_STATUS_S | MCI_STATUS_AR)) &&
4959 		(mce->status & MCI_STATUS_VAL) &&
4960 		(mce->status & MCI_STATUS_UC);
4961 }
4962 
kvm_vcpu_x86_set_ucna(struct kvm_vcpu * vcpu,struct kvm_x86_mce * mce,u64 * banks)4963 static int kvm_vcpu_x86_set_ucna(struct kvm_vcpu *vcpu, struct kvm_x86_mce *mce, u64* banks)
4964 {
4965 	u64 mcg_cap = vcpu->arch.mcg_cap;
4966 
4967 	banks[1] = mce->status;
4968 	banks[2] = mce->addr;
4969 	banks[3] = mce->misc;
4970 	vcpu->arch.mcg_status = mce->mcg_status;
4971 
4972 	if (!(mcg_cap & MCG_CMCI_P) ||
4973 	    !(vcpu->arch.mci_ctl2_banks[mce->bank] & MCI_CTL2_CMCI_EN))
4974 		return 0;
4975 
4976 	if (lapic_in_kernel(vcpu))
4977 		kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTCMCI);
4978 
4979 	return 0;
4980 }
4981 
kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu * vcpu,struct kvm_x86_mce * mce)4982 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
4983 				      struct kvm_x86_mce *mce)
4984 {
4985 	u64 mcg_cap = vcpu->arch.mcg_cap;
4986 	unsigned bank_num = mcg_cap & 0xff;
4987 	u64 *banks = vcpu->arch.mce_banks;
4988 
4989 	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
4990 		return -EINVAL;
4991 
4992 	banks += array_index_nospec(4 * mce->bank, 4 * bank_num);
4993 
4994 	if (is_ucna(mce))
4995 		return kvm_vcpu_x86_set_ucna(vcpu, mce, banks);
4996 
4997 	/*
4998 	 * if IA32_MCG_CTL is not all 1s, the uncorrected error
4999 	 * reporting is disabled
5000 	 */
5001 	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
5002 	    vcpu->arch.mcg_ctl != ~(u64)0)
5003 		return 0;
5004 	/*
5005 	 * if IA32_MCi_CTL is not all 1s, the uncorrected error
5006 	 * reporting is disabled for the bank
5007 	 */
5008 	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
5009 		return 0;
5010 	if (mce->status & MCI_STATUS_UC) {
5011 		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
5012 		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
5013 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5014 			return 0;
5015 		}
5016 		if (banks[1] & MCI_STATUS_VAL)
5017 			mce->status |= MCI_STATUS_OVER;
5018 		banks[2] = mce->addr;
5019 		banks[3] = mce->misc;
5020 		vcpu->arch.mcg_status = mce->mcg_status;
5021 		banks[1] = mce->status;
5022 		kvm_queue_exception(vcpu, MC_VECTOR);
5023 	} else if (!(banks[1] & MCI_STATUS_VAL)
5024 		   || !(banks[1] & MCI_STATUS_UC)) {
5025 		if (banks[1] & MCI_STATUS_VAL)
5026 			mce->status |= MCI_STATUS_OVER;
5027 		banks[2] = mce->addr;
5028 		banks[3] = mce->misc;
5029 		banks[1] = mce->status;
5030 	} else
5031 		banks[1] |= MCI_STATUS_OVER;
5032 	return 0;
5033 }
5034 
kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)5035 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
5036 					       struct kvm_vcpu_events *events)
5037 {
5038 	struct kvm_queued_exception *ex;
5039 
5040 	process_nmi(vcpu);
5041 
5042 	if (kvm_check_request(KVM_REQ_SMI, vcpu))
5043 		process_smi(vcpu);
5044 
5045 	/*
5046 	 * KVM's ABI only allows for one exception to be migrated.  Luckily,
5047 	 * the only time there can be two queued exceptions is if there's a
5048 	 * non-exiting _injected_ exception, and a pending exiting exception.
5049 	 * In that case, ignore the VM-Exiting exception as it's an extension
5050 	 * of the injected exception.
5051 	 */
5052 	if (vcpu->arch.exception_vmexit.pending &&
5053 	    !vcpu->arch.exception.pending &&
5054 	    !vcpu->arch.exception.injected)
5055 		ex = &vcpu->arch.exception_vmexit;
5056 	else
5057 		ex = &vcpu->arch.exception;
5058 
5059 	/*
5060 	 * In guest mode, payload delivery should be deferred if the exception
5061 	 * will be intercepted by L1, e.g. KVM should not modifying CR2 if L1
5062 	 * intercepts #PF, ditto for DR6 and #DBs.  If the per-VM capability,
5063 	 * KVM_CAP_EXCEPTION_PAYLOAD, is not set, userspace may or may not
5064 	 * propagate the payload and so it cannot be safely deferred.  Deliver
5065 	 * the payload if the capability hasn't been requested.
5066 	 */
5067 	if (!vcpu->kvm->arch.exception_payload_enabled &&
5068 	    ex->pending && ex->has_payload)
5069 		kvm_deliver_exception_payload(vcpu, ex);
5070 
5071 	/*
5072 	 * The API doesn't provide the instruction length for software
5073 	 * exceptions, so don't report them. As long as the guest RIP
5074 	 * isn't advanced, we should expect to encounter the exception
5075 	 * again.
5076 	 */
5077 	if (kvm_exception_is_soft(ex->vector)) {
5078 		events->exception.injected = 0;
5079 		events->exception.pending = 0;
5080 	} else {
5081 		events->exception.injected = ex->injected;
5082 		events->exception.pending = ex->pending;
5083 		/*
5084 		 * For ABI compatibility, deliberately conflate
5085 		 * pending and injected exceptions when
5086 		 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled.
5087 		 */
5088 		if (!vcpu->kvm->arch.exception_payload_enabled)
5089 			events->exception.injected |= ex->pending;
5090 	}
5091 	events->exception.nr = ex->vector;
5092 	events->exception.has_error_code = ex->has_error_code;
5093 	events->exception.error_code = ex->error_code;
5094 	events->exception_has_payload = ex->has_payload;
5095 	events->exception_payload = ex->payload;
5096 
5097 	events->interrupt.injected =
5098 		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
5099 	events->interrupt.nr = vcpu->arch.interrupt.nr;
5100 	events->interrupt.soft = 0;
5101 	events->interrupt.shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
5102 
5103 	events->nmi.injected = vcpu->arch.nmi_injected;
5104 	events->nmi.pending = vcpu->arch.nmi_pending != 0;
5105 	events->nmi.masked = static_call(kvm_x86_get_nmi_mask)(vcpu);
5106 	events->nmi.pad = 0;
5107 
5108 	events->sipi_vector = 0; /* never valid when reporting to user space */
5109 
5110 	events->smi.smm = is_smm(vcpu);
5111 	events->smi.pending = vcpu->arch.smi_pending;
5112 	events->smi.smm_inside_nmi =
5113 		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
5114 	events->smi.latched_init = kvm_lapic_latched_init(vcpu);
5115 
5116 	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
5117 			 | KVM_VCPUEVENT_VALID_SHADOW
5118 			 | KVM_VCPUEVENT_VALID_SMM);
5119 	if (vcpu->kvm->arch.exception_payload_enabled)
5120 		events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;
5121 	if (vcpu->kvm->arch.triple_fault_event) {
5122 		events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5123 		events->flags |= KVM_VCPUEVENT_VALID_TRIPLE_FAULT;
5124 	}
5125 
5126 	memset(&events->reserved, 0, sizeof(events->reserved));
5127 }
5128 
5129 static void kvm_smm_changed(struct kvm_vcpu *vcpu, bool entering_smm);
5130 
kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)5131 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
5132 					      struct kvm_vcpu_events *events)
5133 {
5134 	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
5135 			      | KVM_VCPUEVENT_VALID_SIPI_VECTOR
5136 			      | KVM_VCPUEVENT_VALID_SHADOW
5137 			      | KVM_VCPUEVENT_VALID_SMM
5138 			      | KVM_VCPUEVENT_VALID_PAYLOAD
5139 			      | KVM_VCPUEVENT_VALID_TRIPLE_FAULT))
5140 		return -EINVAL;
5141 
5142 	if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
5143 		if (!vcpu->kvm->arch.exception_payload_enabled)
5144 			return -EINVAL;
5145 		if (events->exception.pending)
5146 			events->exception.injected = 0;
5147 		else
5148 			events->exception_has_payload = 0;
5149 	} else {
5150 		events->exception.pending = 0;
5151 		events->exception_has_payload = 0;
5152 	}
5153 
5154 	if ((events->exception.injected || events->exception.pending) &&
5155 	    (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR))
5156 		return -EINVAL;
5157 
5158 	/* INITs are latched while in SMM */
5159 	if (events->flags & KVM_VCPUEVENT_VALID_SMM &&
5160 	    (events->smi.smm || events->smi.pending) &&
5161 	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
5162 		return -EINVAL;
5163 
5164 	process_nmi(vcpu);
5165 
5166 	/*
5167 	 * Flag that userspace is stuffing an exception, the next KVM_RUN will
5168 	 * morph the exception to a VM-Exit if appropriate.  Do this only for
5169 	 * pending exceptions, already-injected exceptions are not subject to
5170 	 * intercpetion.  Note, userspace that conflates pending and injected
5171 	 * is hosed, and will incorrectly convert an injected exception into a
5172 	 * pending exception, which in turn may cause a spurious VM-Exit.
5173 	 */
5174 	vcpu->arch.exception_from_userspace = events->exception.pending;
5175 
5176 	vcpu->arch.exception_vmexit.pending = false;
5177 
5178 	vcpu->arch.exception.injected = events->exception.injected;
5179 	vcpu->arch.exception.pending = events->exception.pending;
5180 	vcpu->arch.exception.vector = events->exception.nr;
5181 	vcpu->arch.exception.has_error_code = events->exception.has_error_code;
5182 	vcpu->arch.exception.error_code = events->exception.error_code;
5183 	vcpu->arch.exception.has_payload = events->exception_has_payload;
5184 	vcpu->arch.exception.payload = events->exception_payload;
5185 
5186 	vcpu->arch.interrupt.injected = events->interrupt.injected;
5187 	vcpu->arch.interrupt.nr = events->interrupt.nr;
5188 	vcpu->arch.interrupt.soft = events->interrupt.soft;
5189 	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
5190 		static_call(kvm_x86_set_interrupt_shadow)(vcpu,
5191 						events->interrupt.shadow);
5192 
5193 	vcpu->arch.nmi_injected = events->nmi.injected;
5194 	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
5195 		vcpu->arch.nmi_pending = events->nmi.pending;
5196 	static_call(kvm_x86_set_nmi_mask)(vcpu, events->nmi.masked);
5197 
5198 	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
5199 	    lapic_in_kernel(vcpu))
5200 		vcpu->arch.apic->sipi_vector = events->sipi_vector;
5201 
5202 	if (events->flags & KVM_VCPUEVENT_VALID_SMM) {
5203 		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
5204 			kvm_leave_nested(vcpu);
5205 			kvm_smm_changed(vcpu, events->smi.smm);
5206 		}
5207 
5208 		vcpu->arch.smi_pending = events->smi.pending;
5209 
5210 		if (events->smi.smm) {
5211 			if (events->smi.smm_inside_nmi)
5212 				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
5213 			else
5214 				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
5215 		}
5216 
5217 		if (lapic_in_kernel(vcpu)) {
5218 			if (events->smi.latched_init)
5219 				set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
5220 			else
5221 				clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
5222 		}
5223 	}
5224 
5225 	if (events->flags & KVM_VCPUEVENT_VALID_TRIPLE_FAULT) {
5226 		if (!vcpu->kvm->arch.triple_fault_event)
5227 			return -EINVAL;
5228 		if (events->triple_fault.pending)
5229 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5230 		else
5231 			kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5232 	}
5233 
5234 	kvm_make_request(KVM_REQ_EVENT, vcpu);
5235 
5236 	return 0;
5237 }
5238 
kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu * vcpu,struct kvm_debugregs * dbgregs)5239 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
5240 					     struct kvm_debugregs *dbgregs)
5241 {
5242 	unsigned long val;
5243 
5244 	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
5245 	kvm_get_dr(vcpu, 6, &val);
5246 	dbgregs->dr6 = val;
5247 	dbgregs->dr7 = vcpu->arch.dr7;
5248 	dbgregs->flags = 0;
5249 	memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
5250 }
5251 
kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu * vcpu,struct kvm_debugregs * dbgregs)5252 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
5253 					    struct kvm_debugregs *dbgregs)
5254 {
5255 	if (dbgregs->flags)
5256 		return -EINVAL;
5257 
5258 	if (!kvm_dr6_valid(dbgregs->dr6))
5259 		return -EINVAL;
5260 	if (!kvm_dr7_valid(dbgregs->dr7))
5261 		return -EINVAL;
5262 
5263 	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
5264 	kvm_update_dr0123(vcpu);
5265 	vcpu->arch.dr6 = dbgregs->dr6;
5266 	vcpu->arch.dr7 = dbgregs->dr7;
5267 	kvm_update_dr7(vcpu);
5268 
5269 	return 0;
5270 }
5271 
kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu * vcpu,struct kvm_xsave * guest_xsave)5272 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
5273 					 struct kvm_xsave *guest_xsave)
5274 {
5275 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
5276 		return;
5277 
5278 	fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu,
5279 				       guest_xsave->region,
5280 				       sizeof(guest_xsave->region),
5281 				       vcpu->arch.pkru);
5282 }
5283 
kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu * vcpu,u8 * state,unsigned int size)5284 static void kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu,
5285 					  u8 *state, unsigned int size)
5286 {
5287 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
5288 		return;
5289 
5290 	fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu,
5291 				       state, size, vcpu->arch.pkru);
5292 }
5293 
kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu * vcpu,struct kvm_xsave * guest_xsave)5294 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
5295 					struct kvm_xsave *guest_xsave)
5296 {
5297 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
5298 		return 0;
5299 
5300 	return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,
5301 					      guest_xsave->region,
5302 					      kvm_caps.supported_xcr0,
5303 					      &vcpu->arch.pkru);
5304 }
5305 
kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu * vcpu,struct kvm_xcrs * guest_xcrs)5306 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
5307 					struct kvm_xcrs *guest_xcrs)
5308 {
5309 	if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
5310 		guest_xcrs->nr_xcrs = 0;
5311 		return;
5312 	}
5313 
5314 	guest_xcrs->nr_xcrs = 1;
5315 	guest_xcrs->flags = 0;
5316 	guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
5317 	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
5318 }
5319 
kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu * vcpu,struct kvm_xcrs * guest_xcrs)5320 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
5321 				       struct kvm_xcrs *guest_xcrs)
5322 {
5323 	int i, r = 0;
5324 
5325 	if (!boot_cpu_has(X86_FEATURE_XSAVE))
5326 		return -EINVAL;
5327 
5328 	if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
5329 		return -EINVAL;
5330 
5331 	for (i = 0; i < guest_xcrs->nr_xcrs; i++)
5332 		/* Only support XCR0 currently */
5333 		if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {
5334 			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
5335 				guest_xcrs->xcrs[i].value);
5336 			break;
5337 		}
5338 	if (r)
5339 		r = -EINVAL;
5340 	return r;
5341 }
5342 
5343 /*
5344  * kvm_set_guest_paused() indicates to the guest kernel that it has been
5345  * stopped by the hypervisor.  This function will be called from the host only.
5346  * EINVAL is returned when the host attempts to set the flag for a guest that
5347  * does not support pv clocks.
5348  */
kvm_set_guest_paused(struct kvm_vcpu * vcpu)5349 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
5350 {
5351 	if (!vcpu->arch.pv_time.active)
5352 		return -EINVAL;
5353 	vcpu->arch.pvclock_set_guest_stopped_request = true;
5354 	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
5355 	return 0;
5356 }
5357 
kvm_arch_tsc_has_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)5358 static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu,
5359 				 struct kvm_device_attr *attr)
5360 {
5361 	int r;
5362 
5363 	switch (attr->attr) {
5364 	case KVM_VCPU_TSC_OFFSET:
5365 		r = 0;
5366 		break;
5367 	default:
5368 		r = -ENXIO;
5369 	}
5370 
5371 	return r;
5372 }
5373 
kvm_arch_tsc_get_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)5374 static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu,
5375 				 struct kvm_device_attr *attr)
5376 {
5377 	u64 __user *uaddr = kvm_get_attr_addr(attr);
5378 	int r;
5379 
5380 	if (IS_ERR(uaddr))
5381 		return PTR_ERR(uaddr);
5382 
5383 	switch (attr->attr) {
5384 	case KVM_VCPU_TSC_OFFSET:
5385 		r = -EFAULT;
5386 		if (put_user(vcpu->arch.l1_tsc_offset, uaddr))
5387 			break;
5388 		r = 0;
5389 		break;
5390 	default:
5391 		r = -ENXIO;
5392 	}
5393 
5394 	return r;
5395 }
5396 
kvm_arch_tsc_set_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)5397 static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu,
5398 				 struct kvm_device_attr *attr)
5399 {
5400 	u64 __user *uaddr = kvm_get_attr_addr(attr);
5401 	struct kvm *kvm = vcpu->kvm;
5402 	int r;
5403 
5404 	if (IS_ERR(uaddr))
5405 		return PTR_ERR(uaddr);
5406 
5407 	switch (attr->attr) {
5408 	case KVM_VCPU_TSC_OFFSET: {
5409 		u64 offset, tsc, ns;
5410 		unsigned long flags;
5411 		bool matched;
5412 
5413 		r = -EFAULT;
5414 		if (get_user(offset, uaddr))
5415 			break;
5416 
5417 		raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
5418 
5419 		matched = (vcpu->arch.virtual_tsc_khz &&
5420 			   kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz &&
5421 			   kvm->arch.last_tsc_offset == offset);
5422 
5423 		tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset;
5424 		ns = get_kvmclock_base_ns();
5425 
5426 		__kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched);
5427 		raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
5428 
5429 		r = 0;
5430 		break;
5431 	}
5432 	default:
5433 		r = -ENXIO;
5434 	}
5435 
5436 	return r;
5437 }
5438 
kvm_vcpu_ioctl_device_attr(struct kvm_vcpu * vcpu,unsigned int ioctl,void __user * argp)5439 static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu,
5440 				      unsigned int ioctl,
5441 				      void __user *argp)
5442 {
5443 	struct kvm_device_attr attr;
5444 	int r;
5445 
5446 	if (copy_from_user(&attr, argp, sizeof(attr)))
5447 		return -EFAULT;
5448 
5449 	if (attr.group != KVM_VCPU_TSC_CTRL)
5450 		return -ENXIO;
5451 
5452 	switch (ioctl) {
5453 	case KVM_HAS_DEVICE_ATTR:
5454 		r = kvm_arch_tsc_has_attr(vcpu, &attr);
5455 		break;
5456 	case KVM_GET_DEVICE_ATTR:
5457 		r = kvm_arch_tsc_get_attr(vcpu, &attr);
5458 		break;
5459 	case KVM_SET_DEVICE_ATTR:
5460 		r = kvm_arch_tsc_set_attr(vcpu, &attr);
5461 		break;
5462 	}
5463 
5464 	return r;
5465 }
5466 
kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu * vcpu,struct kvm_enable_cap * cap)5467 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
5468 				     struct kvm_enable_cap *cap)
5469 {
5470 	int r;
5471 	uint16_t vmcs_version;
5472 	void __user *user_ptr;
5473 
5474 	if (cap->flags)
5475 		return -EINVAL;
5476 
5477 	switch (cap->cap) {
5478 	case KVM_CAP_HYPERV_SYNIC2:
5479 		if (cap->args[0])
5480 			return -EINVAL;
5481 		fallthrough;
5482 
5483 	case KVM_CAP_HYPERV_SYNIC:
5484 		if (!irqchip_in_kernel(vcpu->kvm))
5485 			return -EINVAL;
5486 		return kvm_hv_activate_synic(vcpu, cap->cap ==
5487 					     KVM_CAP_HYPERV_SYNIC2);
5488 	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
5489 		if (!kvm_x86_ops.nested_ops->enable_evmcs)
5490 			return -ENOTTY;
5491 		r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);
5492 		if (!r) {
5493 			user_ptr = (void __user *)(uintptr_t)cap->args[0];
5494 			if (copy_to_user(user_ptr, &vmcs_version,
5495 					 sizeof(vmcs_version)))
5496 				r = -EFAULT;
5497 		}
5498 		return r;
5499 	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
5500 		if (!kvm_x86_ops.enable_direct_tlbflush)
5501 			return -ENOTTY;
5502 
5503 		return static_call(kvm_x86_enable_direct_tlbflush)(vcpu);
5504 
5505 	case KVM_CAP_HYPERV_ENFORCE_CPUID:
5506 		return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]);
5507 
5508 	case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
5509 		vcpu->arch.pv_cpuid.enforce = cap->args[0];
5510 		if (vcpu->arch.pv_cpuid.enforce)
5511 			kvm_update_pv_runtime(vcpu);
5512 
5513 		return 0;
5514 	default:
5515 		return -EINVAL;
5516 	}
5517 }
5518 
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5519 long kvm_arch_vcpu_ioctl(struct file *filp,
5520 			 unsigned int ioctl, unsigned long arg)
5521 {
5522 	struct kvm_vcpu *vcpu = filp->private_data;
5523 	void __user *argp = (void __user *)arg;
5524 	int r;
5525 	union {
5526 		struct kvm_sregs2 *sregs2;
5527 		struct kvm_lapic_state *lapic;
5528 		struct kvm_xsave *xsave;
5529 		struct kvm_xcrs *xcrs;
5530 		void *buffer;
5531 	} u;
5532 
5533 	vcpu_load(vcpu);
5534 
5535 	u.buffer = NULL;
5536 	switch (ioctl) {
5537 	case KVM_GET_LAPIC: {
5538 		r = -EINVAL;
5539 		if (!lapic_in_kernel(vcpu))
5540 			goto out;
5541 		u.lapic = kzalloc(sizeof(struct kvm_lapic_state),
5542 				GFP_KERNEL_ACCOUNT);
5543 
5544 		r = -ENOMEM;
5545 		if (!u.lapic)
5546 			goto out;
5547 		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
5548 		if (r)
5549 			goto out;
5550 		r = -EFAULT;
5551 		if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
5552 			goto out;
5553 		r = 0;
5554 		break;
5555 	}
5556 	case KVM_SET_LAPIC: {
5557 		r = -EINVAL;
5558 		if (!lapic_in_kernel(vcpu))
5559 			goto out;
5560 		u.lapic = memdup_user(argp, sizeof(*u.lapic));
5561 		if (IS_ERR(u.lapic)) {
5562 			r = PTR_ERR(u.lapic);
5563 			goto out_nofree;
5564 		}
5565 
5566 		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
5567 		break;
5568 	}
5569 	case KVM_INTERRUPT: {
5570 		struct kvm_interrupt irq;
5571 
5572 		r = -EFAULT;
5573 		if (copy_from_user(&irq, argp, sizeof(irq)))
5574 			goto out;
5575 		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
5576 		break;
5577 	}
5578 	case KVM_NMI: {
5579 		r = kvm_vcpu_ioctl_nmi(vcpu);
5580 		break;
5581 	}
5582 	case KVM_SMI: {
5583 		r = kvm_vcpu_ioctl_smi(vcpu);
5584 		break;
5585 	}
5586 	case KVM_SET_CPUID: {
5587 		struct kvm_cpuid __user *cpuid_arg = argp;
5588 		struct kvm_cpuid cpuid;
5589 
5590 		r = -EFAULT;
5591 		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
5592 			goto out;
5593 		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
5594 		break;
5595 	}
5596 	case KVM_SET_CPUID2: {
5597 		struct kvm_cpuid2 __user *cpuid_arg = argp;
5598 		struct kvm_cpuid2 cpuid;
5599 
5600 		r = -EFAULT;
5601 		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
5602 			goto out;
5603 		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
5604 					      cpuid_arg->entries);
5605 		break;
5606 	}
5607 	case KVM_GET_CPUID2: {
5608 		struct kvm_cpuid2 __user *cpuid_arg = argp;
5609 		struct kvm_cpuid2 cpuid;
5610 
5611 		r = -EFAULT;
5612 		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
5613 			goto out;
5614 		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
5615 					      cpuid_arg->entries);
5616 		if (r)
5617 			goto out;
5618 		r = -EFAULT;
5619 		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
5620 			goto out;
5621 		r = 0;
5622 		break;
5623 	}
5624 	case KVM_GET_MSRS: {
5625 		int idx = srcu_read_lock(&vcpu->kvm->srcu);
5626 		r = msr_io(vcpu, argp, do_get_msr, 1);
5627 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
5628 		break;
5629 	}
5630 	case KVM_SET_MSRS: {
5631 		int idx = srcu_read_lock(&vcpu->kvm->srcu);
5632 		r = msr_io(vcpu, argp, do_set_msr, 0);
5633 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
5634 		break;
5635 	}
5636 	case KVM_TPR_ACCESS_REPORTING: {
5637 		struct kvm_tpr_access_ctl tac;
5638 
5639 		r = -EFAULT;
5640 		if (copy_from_user(&tac, argp, sizeof(tac)))
5641 			goto out;
5642 		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
5643 		if (r)
5644 			goto out;
5645 		r = -EFAULT;
5646 		if (copy_to_user(argp, &tac, sizeof(tac)))
5647 			goto out;
5648 		r = 0;
5649 		break;
5650 	};
5651 	case KVM_SET_VAPIC_ADDR: {
5652 		struct kvm_vapic_addr va;
5653 		int idx;
5654 
5655 		r = -EINVAL;
5656 		if (!lapic_in_kernel(vcpu))
5657 			goto out;
5658 		r = -EFAULT;
5659 		if (copy_from_user(&va, argp, sizeof(va)))
5660 			goto out;
5661 		idx = srcu_read_lock(&vcpu->kvm->srcu);
5662 		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
5663 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
5664 		break;
5665 	}
5666 	case KVM_X86_SETUP_MCE: {
5667 		u64 mcg_cap;
5668 
5669 		r = -EFAULT;
5670 		if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap)))
5671 			goto out;
5672 		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
5673 		break;
5674 	}
5675 	case KVM_X86_SET_MCE: {
5676 		struct kvm_x86_mce mce;
5677 
5678 		r = -EFAULT;
5679 		if (copy_from_user(&mce, argp, sizeof(mce)))
5680 			goto out;
5681 		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
5682 		break;
5683 	}
5684 	case KVM_GET_VCPU_EVENTS: {
5685 		struct kvm_vcpu_events events;
5686 
5687 		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
5688 
5689 		r = -EFAULT;
5690 		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
5691 			break;
5692 		r = 0;
5693 		break;
5694 	}
5695 	case KVM_SET_VCPU_EVENTS: {
5696 		struct kvm_vcpu_events events;
5697 
5698 		r = -EFAULT;
5699 		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
5700 			break;
5701 
5702 		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
5703 		break;
5704 	}
5705 	case KVM_GET_DEBUGREGS: {
5706 		struct kvm_debugregs dbgregs;
5707 
5708 		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);
5709 
5710 		r = -EFAULT;
5711 		if (copy_to_user(argp, &dbgregs,
5712 				 sizeof(struct kvm_debugregs)))
5713 			break;
5714 		r = 0;
5715 		break;
5716 	}
5717 	case KVM_SET_DEBUGREGS: {
5718 		struct kvm_debugregs dbgregs;
5719 
5720 		r = -EFAULT;
5721 		if (copy_from_user(&dbgregs, argp,
5722 				   sizeof(struct kvm_debugregs)))
5723 			break;
5724 
5725 		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
5726 		break;
5727 	}
5728 	case KVM_GET_XSAVE: {
5729 		r = -EINVAL;
5730 		if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave))
5731 			break;
5732 
5733 		u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT);
5734 		r = -ENOMEM;
5735 		if (!u.xsave)
5736 			break;
5737 
5738 		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
5739 
5740 		r = -EFAULT;
5741 		if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
5742 			break;
5743 		r = 0;
5744 		break;
5745 	}
5746 	case KVM_SET_XSAVE: {
5747 		int size = vcpu->arch.guest_fpu.uabi_size;
5748 
5749 		u.xsave = memdup_user(argp, size);
5750 		if (IS_ERR(u.xsave)) {
5751 			r = PTR_ERR(u.xsave);
5752 			goto out_nofree;
5753 		}
5754 
5755 		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
5756 		break;
5757 	}
5758 
5759 	case KVM_GET_XSAVE2: {
5760 		int size = vcpu->arch.guest_fpu.uabi_size;
5761 
5762 		u.xsave = kzalloc(size, GFP_KERNEL_ACCOUNT);
5763 		r = -ENOMEM;
5764 		if (!u.xsave)
5765 			break;
5766 
5767 		kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size);
5768 
5769 		r = -EFAULT;
5770 		if (copy_to_user(argp, u.xsave, size))
5771 			break;
5772 
5773 		r = 0;
5774 		break;
5775 	}
5776 
5777 	case KVM_GET_XCRS: {
5778 		u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT);
5779 		r = -ENOMEM;
5780 		if (!u.xcrs)
5781 			break;
5782 
5783 		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
5784 
5785 		r = -EFAULT;
5786 		if (copy_to_user(argp, u.xcrs,
5787 				 sizeof(struct kvm_xcrs)))
5788 			break;
5789 		r = 0;
5790 		break;
5791 	}
5792 	case KVM_SET_XCRS: {
5793 		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
5794 		if (IS_ERR(u.xcrs)) {
5795 			r = PTR_ERR(u.xcrs);
5796 			goto out_nofree;
5797 		}
5798 
5799 		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
5800 		break;
5801 	}
5802 	case KVM_SET_TSC_KHZ: {
5803 		u32 user_tsc_khz;
5804 
5805 		r = -EINVAL;
5806 		user_tsc_khz = (u32)arg;
5807 
5808 		if (kvm_caps.has_tsc_control &&
5809 		    user_tsc_khz >= kvm_caps.max_guest_tsc_khz)
5810 			goto out;
5811 
5812 		if (user_tsc_khz == 0)
5813 			user_tsc_khz = tsc_khz;
5814 
5815 		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
5816 			r = 0;
5817 
5818 		goto out;
5819 	}
5820 	case KVM_GET_TSC_KHZ: {
5821 		r = vcpu->arch.virtual_tsc_khz;
5822 		goto out;
5823 	}
5824 	case KVM_KVMCLOCK_CTRL: {
5825 		r = kvm_set_guest_paused(vcpu);
5826 		goto out;
5827 	}
5828 	case KVM_ENABLE_CAP: {
5829 		struct kvm_enable_cap cap;
5830 
5831 		r = -EFAULT;
5832 		if (copy_from_user(&cap, argp, sizeof(cap)))
5833 			goto out;
5834 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
5835 		break;
5836 	}
5837 	case KVM_GET_NESTED_STATE: {
5838 		struct kvm_nested_state __user *user_kvm_nested_state = argp;
5839 		u32 user_data_size;
5840 
5841 		r = -EINVAL;
5842 		if (!kvm_x86_ops.nested_ops->get_state)
5843 			break;
5844 
5845 		BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
5846 		r = -EFAULT;
5847 		if (get_user(user_data_size, &user_kvm_nested_state->size))
5848 			break;
5849 
5850 		r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,
5851 						     user_data_size);
5852 		if (r < 0)
5853 			break;
5854 
5855 		if (r > user_data_size) {
5856 			if (put_user(r, &user_kvm_nested_state->size))
5857 				r = -EFAULT;
5858 			else
5859 				r = -E2BIG;
5860 			break;
5861 		}
5862 
5863 		r = 0;
5864 		break;
5865 	}
5866 	case KVM_SET_NESTED_STATE: {
5867 		struct kvm_nested_state __user *user_kvm_nested_state = argp;
5868 		struct kvm_nested_state kvm_state;
5869 		int idx;
5870 
5871 		r = -EINVAL;
5872 		if (!kvm_x86_ops.nested_ops->set_state)
5873 			break;
5874 
5875 		r = -EFAULT;
5876 		if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state)))
5877 			break;
5878 
5879 		r = -EINVAL;
5880 		if (kvm_state.size < sizeof(kvm_state))
5881 			break;
5882 
5883 		if (kvm_state.flags &
5884 		    ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE
5885 		      | KVM_STATE_NESTED_EVMCS | KVM_STATE_NESTED_MTF_PENDING
5886 		      | KVM_STATE_NESTED_GIF_SET))
5887 			break;
5888 
5889 		/* nested_run_pending implies guest_mode.  */
5890 		if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING)
5891 		    && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE))
5892 			break;
5893 
5894 		idx = srcu_read_lock(&vcpu->kvm->srcu);
5895 		r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);
5896 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
5897 		break;
5898 	}
5899 	case KVM_GET_SUPPORTED_HV_CPUID:
5900 		r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp);
5901 		break;
5902 #ifdef CONFIG_KVM_XEN
5903 	case KVM_XEN_VCPU_GET_ATTR: {
5904 		struct kvm_xen_vcpu_attr xva;
5905 
5906 		r = -EFAULT;
5907 		if (copy_from_user(&xva, argp, sizeof(xva)))
5908 			goto out;
5909 		r = kvm_xen_vcpu_get_attr(vcpu, &xva);
5910 		if (!r && copy_to_user(argp, &xva, sizeof(xva)))
5911 			r = -EFAULT;
5912 		break;
5913 	}
5914 	case KVM_XEN_VCPU_SET_ATTR: {
5915 		struct kvm_xen_vcpu_attr xva;
5916 
5917 		r = -EFAULT;
5918 		if (copy_from_user(&xva, argp, sizeof(xva)))
5919 			goto out;
5920 		r = kvm_xen_vcpu_set_attr(vcpu, &xva);
5921 		break;
5922 	}
5923 #endif
5924 	case KVM_GET_SREGS2: {
5925 		u.sregs2 = kzalloc(sizeof(struct kvm_sregs2), GFP_KERNEL);
5926 		r = -ENOMEM;
5927 		if (!u.sregs2)
5928 			goto out;
5929 		__get_sregs2(vcpu, u.sregs2);
5930 		r = -EFAULT;
5931 		if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2)))
5932 			goto out;
5933 		r = 0;
5934 		break;
5935 	}
5936 	case KVM_SET_SREGS2: {
5937 		u.sregs2 = memdup_user(argp, sizeof(struct kvm_sregs2));
5938 		if (IS_ERR(u.sregs2)) {
5939 			r = PTR_ERR(u.sregs2);
5940 			u.sregs2 = NULL;
5941 			goto out;
5942 		}
5943 		r = __set_sregs2(vcpu, u.sregs2);
5944 		break;
5945 	}
5946 	case KVM_HAS_DEVICE_ATTR:
5947 	case KVM_GET_DEVICE_ATTR:
5948 	case KVM_SET_DEVICE_ATTR:
5949 		r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp);
5950 		break;
5951 	default:
5952 		r = -EINVAL;
5953 	}
5954 out:
5955 	kfree(u.buffer);
5956 out_nofree:
5957 	vcpu_put(vcpu);
5958 	return r;
5959 }
5960 
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)5961 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
5962 {
5963 	return VM_FAULT_SIGBUS;
5964 }
5965 
kvm_vm_ioctl_set_tss_addr(struct kvm * kvm,unsigned long addr)5966 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
5967 {
5968 	int ret;
5969 
5970 	if (addr > (unsigned int)(-3 * PAGE_SIZE))
5971 		return -EINVAL;
5972 	ret = static_call(kvm_x86_set_tss_addr)(kvm, addr);
5973 	return ret;
5974 }
5975 
kvm_vm_ioctl_set_identity_map_addr(struct kvm * kvm,u64 ident_addr)5976 static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
5977 					      u64 ident_addr)
5978 {
5979 	return static_call(kvm_x86_set_identity_map_addr)(kvm, ident_addr);
5980 }
5981 
kvm_vm_ioctl_set_nr_mmu_pages(struct kvm * kvm,unsigned long kvm_nr_mmu_pages)5982 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
5983 					 unsigned long kvm_nr_mmu_pages)
5984 {
5985 	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
5986 		return -EINVAL;
5987 
5988 	mutex_lock(&kvm->slots_lock);
5989 
5990 	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
5991 	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
5992 
5993 	mutex_unlock(&kvm->slots_lock);
5994 	return 0;
5995 }
5996 
kvm_vm_ioctl_get_nr_mmu_pages(struct kvm * kvm)5997 static unsigned long kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
5998 {
5999 	return kvm->arch.n_max_mmu_pages;
6000 }
6001 
kvm_vm_ioctl_get_irqchip(struct kvm * kvm,struct kvm_irqchip * chip)6002 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
6003 {
6004 	struct kvm_pic *pic = kvm->arch.vpic;
6005 	int r;
6006 
6007 	r = 0;
6008 	switch (chip->chip_id) {
6009 	case KVM_IRQCHIP_PIC_MASTER:
6010 		memcpy(&chip->chip.pic, &pic->pics[0],
6011 			sizeof(struct kvm_pic_state));
6012 		break;
6013 	case KVM_IRQCHIP_PIC_SLAVE:
6014 		memcpy(&chip->chip.pic, &pic->pics[1],
6015 			sizeof(struct kvm_pic_state));
6016 		break;
6017 	case KVM_IRQCHIP_IOAPIC:
6018 		kvm_get_ioapic(kvm, &chip->chip.ioapic);
6019 		break;
6020 	default:
6021 		r = -EINVAL;
6022 		break;
6023 	}
6024 	return r;
6025 }
6026 
kvm_vm_ioctl_set_irqchip(struct kvm * kvm,struct kvm_irqchip * chip)6027 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
6028 {
6029 	struct kvm_pic *pic = kvm->arch.vpic;
6030 	int r;
6031 
6032 	r = 0;
6033 	switch (chip->chip_id) {
6034 	case KVM_IRQCHIP_PIC_MASTER:
6035 		spin_lock(&pic->lock);
6036 		memcpy(&pic->pics[0], &chip->chip.pic,
6037 			sizeof(struct kvm_pic_state));
6038 		spin_unlock(&pic->lock);
6039 		break;
6040 	case KVM_IRQCHIP_PIC_SLAVE:
6041 		spin_lock(&pic->lock);
6042 		memcpy(&pic->pics[1], &chip->chip.pic,
6043 			sizeof(struct kvm_pic_state));
6044 		spin_unlock(&pic->lock);
6045 		break;
6046 	case KVM_IRQCHIP_IOAPIC:
6047 		kvm_set_ioapic(kvm, &chip->chip.ioapic);
6048 		break;
6049 	default:
6050 		r = -EINVAL;
6051 		break;
6052 	}
6053 	kvm_pic_update_irq(pic);
6054 	return r;
6055 }
6056 
kvm_vm_ioctl_get_pit(struct kvm * kvm,struct kvm_pit_state * ps)6057 static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
6058 {
6059 	struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;
6060 
6061 	BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels));
6062 
6063 	mutex_lock(&kps->lock);
6064 	memcpy(ps, &kps->channels, sizeof(*ps));
6065 	mutex_unlock(&kps->lock);
6066 	return 0;
6067 }
6068 
kvm_vm_ioctl_set_pit(struct kvm * kvm,struct kvm_pit_state * ps)6069 static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
6070 {
6071 	int i;
6072 	struct kvm_pit *pit = kvm->arch.vpit;
6073 
6074 	mutex_lock(&pit->pit_state.lock);
6075 	memcpy(&pit->pit_state.channels, ps, sizeof(*ps));
6076 	for (i = 0; i < 3; i++)
6077 		kvm_pit_load_count(pit, i, ps->channels[i].count, 0);
6078 	mutex_unlock(&pit->pit_state.lock);
6079 	return 0;
6080 }
6081 
kvm_vm_ioctl_get_pit2(struct kvm * kvm,struct kvm_pit_state2 * ps)6082 static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
6083 {
6084 	mutex_lock(&kvm->arch.vpit->pit_state.lock);
6085 	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
6086 		sizeof(ps->channels));
6087 	ps->flags = kvm->arch.vpit->pit_state.flags;
6088 	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
6089 	memset(&ps->reserved, 0, sizeof(ps->reserved));
6090 	return 0;
6091 }
6092 
kvm_vm_ioctl_set_pit2(struct kvm * kvm,struct kvm_pit_state2 * ps)6093 static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
6094 {
6095 	int start = 0;
6096 	int i;
6097 	u32 prev_legacy, cur_legacy;
6098 	struct kvm_pit *pit = kvm->arch.vpit;
6099 
6100 	mutex_lock(&pit->pit_state.lock);
6101 	prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
6102 	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
6103 	if (!prev_legacy && cur_legacy)
6104 		start = 1;
6105 	memcpy(&pit->pit_state.channels, &ps->channels,
6106 	       sizeof(pit->pit_state.channels));
6107 	pit->pit_state.flags = ps->flags;
6108 	for (i = 0; i < 3; i++)
6109 		kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count,
6110 				   start && i == 0);
6111 	mutex_unlock(&pit->pit_state.lock);
6112 	return 0;
6113 }
6114 
kvm_vm_ioctl_reinject(struct kvm * kvm,struct kvm_reinject_control * control)6115 static int kvm_vm_ioctl_reinject(struct kvm *kvm,
6116 				 struct kvm_reinject_control *control)
6117 {
6118 	struct kvm_pit *pit = kvm->arch.vpit;
6119 
6120 	/* pit->pit_state.lock was overloaded to prevent userspace from getting
6121 	 * an inconsistent state after running multiple KVM_REINJECT_CONTROL
6122 	 * ioctls in parallel.  Use a separate lock if that ioctl isn't rare.
6123 	 */
6124 	mutex_lock(&pit->pit_state.lock);
6125 	kvm_pit_set_reinject(pit, control->pit_reinject);
6126 	mutex_unlock(&pit->pit_state.lock);
6127 
6128 	return 0;
6129 }
6130 
kvm_arch_sync_dirty_log(struct kvm * kvm,struct kvm_memory_slot * memslot)6131 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
6132 {
6133 
6134 	/*
6135 	 * Flush all CPUs' dirty log buffers to the  dirty_bitmap.  Called
6136 	 * before reporting dirty_bitmap to userspace.  KVM flushes the buffers
6137 	 * on all VM-Exits, thus we only need to kick running vCPUs to force a
6138 	 * VM-Exit.
6139 	 */
6140 	struct kvm_vcpu *vcpu;
6141 	unsigned long i;
6142 
6143 	kvm_for_each_vcpu(i, vcpu, kvm)
6144 		kvm_vcpu_kick(vcpu);
6145 }
6146 
kvm_vm_ioctl_irq_line(struct kvm * kvm,struct kvm_irq_level * irq_event,bool line_status)6147 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
6148 			bool line_status)
6149 {
6150 	if (!irqchip_in_kernel(kvm))
6151 		return -ENXIO;
6152 
6153 	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
6154 					irq_event->irq, irq_event->level,
6155 					line_status);
6156 	return 0;
6157 }
6158 
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)6159 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
6160 			    struct kvm_enable_cap *cap)
6161 {
6162 	int r;
6163 
6164 	if (cap->flags)
6165 		return -EINVAL;
6166 
6167 	switch (cap->cap) {
6168 	case KVM_CAP_DISABLE_QUIRKS2:
6169 		r = -EINVAL;
6170 		if (cap->args[0] & ~KVM_X86_VALID_QUIRKS)
6171 			break;
6172 		fallthrough;
6173 	case KVM_CAP_DISABLE_QUIRKS:
6174 		kvm->arch.disabled_quirks = cap->args[0];
6175 		r = 0;
6176 		break;
6177 	case KVM_CAP_SPLIT_IRQCHIP: {
6178 		mutex_lock(&kvm->lock);
6179 		r = -EINVAL;
6180 		if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
6181 			goto split_irqchip_unlock;
6182 		r = -EEXIST;
6183 		if (irqchip_in_kernel(kvm))
6184 			goto split_irqchip_unlock;
6185 		if (kvm->created_vcpus)
6186 			goto split_irqchip_unlock;
6187 		r = kvm_setup_empty_irq_routing(kvm);
6188 		if (r)
6189 			goto split_irqchip_unlock;
6190 		/* Pairs with irqchip_in_kernel. */
6191 		smp_wmb();
6192 		kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;
6193 		kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
6194 		kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT);
6195 		r = 0;
6196 split_irqchip_unlock:
6197 		mutex_unlock(&kvm->lock);
6198 		break;
6199 	}
6200 	case KVM_CAP_X2APIC_API:
6201 		r = -EINVAL;
6202 		if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)
6203 			break;
6204 
6205 		if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)
6206 			kvm->arch.x2apic_format = true;
6207 		if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
6208 			kvm->arch.x2apic_broadcast_quirk_disabled = true;
6209 
6210 		r = 0;
6211 		break;
6212 	case KVM_CAP_X86_DISABLE_EXITS:
6213 		r = -EINVAL;
6214 		if (cap->args[0] & ~KVM_X86_DISABLE_VALID_EXITS)
6215 			break;
6216 
6217 		if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) &&
6218 			kvm_can_mwait_in_guest())
6219 			kvm->arch.mwait_in_guest = true;
6220 		if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT)
6221 			kvm->arch.hlt_in_guest = true;
6222 		if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE)
6223 			kvm->arch.pause_in_guest = true;
6224 		if (cap->args[0] & KVM_X86_DISABLE_EXITS_CSTATE)
6225 			kvm->arch.cstate_in_guest = true;
6226 		r = 0;
6227 		break;
6228 	case KVM_CAP_MSR_PLATFORM_INFO:
6229 		kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
6230 		r = 0;
6231 		break;
6232 	case KVM_CAP_EXCEPTION_PAYLOAD:
6233 		kvm->arch.exception_payload_enabled = cap->args[0];
6234 		r = 0;
6235 		break;
6236 	case KVM_CAP_X86_TRIPLE_FAULT_EVENT:
6237 		kvm->arch.triple_fault_event = cap->args[0];
6238 		r = 0;
6239 		break;
6240 	case KVM_CAP_X86_USER_SPACE_MSR:
6241 		r = -EINVAL;
6242 		if (cap->args[0] & ~(KVM_MSR_EXIT_REASON_INVAL |
6243 				     KVM_MSR_EXIT_REASON_UNKNOWN |
6244 				     KVM_MSR_EXIT_REASON_FILTER))
6245 			break;
6246 		kvm->arch.user_space_msr_mask = cap->args[0];
6247 		r = 0;
6248 		break;
6249 	case KVM_CAP_X86_BUS_LOCK_EXIT:
6250 		r = -EINVAL;
6251 		if (cap->args[0] & ~KVM_BUS_LOCK_DETECTION_VALID_MODE)
6252 			break;
6253 
6254 		if ((cap->args[0] & KVM_BUS_LOCK_DETECTION_OFF) &&
6255 		    (cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT))
6256 			break;
6257 
6258 		if (kvm_caps.has_bus_lock_exit &&
6259 		    cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT)
6260 			kvm->arch.bus_lock_detection_enabled = true;
6261 		r = 0;
6262 		break;
6263 #ifdef CONFIG_X86_SGX_KVM
6264 	case KVM_CAP_SGX_ATTRIBUTE: {
6265 		unsigned long allowed_attributes = 0;
6266 
6267 		r = sgx_set_attribute(&allowed_attributes, cap->args[0]);
6268 		if (r)
6269 			break;
6270 
6271 		/* KVM only supports the PROVISIONKEY privileged attribute. */
6272 		if ((allowed_attributes & SGX_ATTR_PROVISIONKEY) &&
6273 		    !(allowed_attributes & ~SGX_ATTR_PROVISIONKEY))
6274 			kvm->arch.sgx_provisioning_allowed = true;
6275 		else
6276 			r = -EINVAL;
6277 		break;
6278 	}
6279 #endif
6280 	case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM:
6281 		r = -EINVAL;
6282 		if (!kvm_x86_ops.vm_copy_enc_context_from)
6283 			break;
6284 
6285 		r = static_call(kvm_x86_vm_copy_enc_context_from)(kvm, cap->args[0]);
6286 		break;
6287 	case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM:
6288 		r = -EINVAL;
6289 		if (!kvm_x86_ops.vm_move_enc_context_from)
6290 			break;
6291 
6292 		r = static_call(kvm_x86_vm_move_enc_context_from)(kvm, cap->args[0]);
6293 		break;
6294 	case KVM_CAP_EXIT_HYPERCALL:
6295 		if (cap->args[0] & ~KVM_EXIT_HYPERCALL_VALID_MASK) {
6296 			r = -EINVAL;
6297 			break;
6298 		}
6299 		kvm->arch.hypercall_exit_enabled = cap->args[0];
6300 		r = 0;
6301 		break;
6302 	case KVM_CAP_EXIT_ON_EMULATION_FAILURE:
6303 		r = -EINVAL;
6304 		if (cap->args[0] & ~1)
6305 			break;
6306 		kvm->arch.exit_on_emulation_error = cap->args[0];
6307 		r = 0;
6308 		break;
6309 	case KVM_CAP_PMU_CAPABILITY:
6310 		r = -EINVAL;
6311 		if (!enable_pmu || (cap->args[0] & ~KVM_CAP_PMU_VALID_MASK))
6312 			break;
6313 
6314 		mutex_lock(&kvm->lock);
6315 		if (!kvm->created_vcpus) {
6316 			kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE);
6317 			r = 0;
6318 		}
6319 		mutex_unlock(&kvm->lock);
6320 		break;
6321 	case KVM_CAP_MAX_VCPU_ID:
6322 		r = -EINVAL;
6323 		if (cap->args[0] > KVM_MAX_VCPU_IDS)
6324 			break;
6325 
6326 		mutex_lock(&kvm->lock);
6327 		if (kvm->arch.max_vcpu_ids == cap->args[0]) {
6328 			r = 0;
6329 		} else if (!kvm->arch.max_vcpu_ids) {
6330 			kvm->arch.max_vcpu_ids = cap->args[0];
6331 			r = 0;
6332 		}
6333 		mutex_unlock(&kvm->lock);
6334 		break;
6335 	case KVM_CAP_X86_NOTIFY_VMEXIT:
6336 		r = -EINVAL;
6337 		if ((u32)cap->args[0] & ~KVM_X86_NOTIFY_VMEXIT_VALID_BITS)
6338 			break;
6339 		if (!kvm_caps.has_notify_vmexit)
6340 			break;
6341 		if (!((u32)cap->args[0] & KVM_X86_NOTIFY_VMEXIT_ENABLED))
6342 			break;
6343 		mutex_lock(&kvm->lock);
6344 		if (!kvm->created_vcpus) {
6345 			kvm->arch.notify_window = cap->args[0] >> 32;
6346 			kvm->arch.notify_vmexit_flags = (u32)cap->args[0];
6347 			r = 0;
6348 		}
6349 		mutex_unlock(&kvm->lock);
6350 		break;
6351 	case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES:
6352 		r = -EINVAL;
6353 
6354 		/*
6355 		 * Since the risk of disabling NX hugepages is a guest crashing
6356 		 * the system, ensure the userspace process has permission to
6357 		 * reboot the system.
6358 		 *
6359 		 * Note that unlike the reboot() syscall, the process must have
6360 		 * this capability in the root namespace because exposing
6361 		 * /dev/kvm into a container does not limit the scope of the
6362 		 * iTLB multihit bug to that container. In other words,
6363 		 * this must use capable(), not ns_capable().
6364 		 */
6365 		if (!capable(CAP_SYS_BOOT)) {
6366 			r = -EPERM;
6367 			break;
6368 		}
6369 
6370 		if (cap->args[0])
6371 			break;
6372 
6373 		mutex_lock(&kvm->lock);
6374 		if (!kvm->created_vcpus) {
6375 			kvm->arch.disable_nx_huge_pages = true;
6376 			r = 0;
6377 		}
6378 		mutex_unlock(&kvm->lock);
6379 		break;
6380 	default:
6381 		r = -EINVAL;
6382 		break;
6383 	}
6384 	return r;
6385 }
6386 
kvm_alloc_msr_filter(bool default_allow)6387 static struct kvm_x86_msr_filter *kvm_alloc_msr_filter(bool default_allow)
6388 {
6389 	struct kvm_x86_msr_filter *msr_filter;
6390 
6391 	msr_filter = kzalloc(sizeof(*msr_filter), GFP_KERNEL_ACCOUNT);
6392 	if (!msr_filter)
6393 		return NULL;
6394 
6395 	msr_filter->default_allow = default_allow;
6396 	return msr_filter;
6397 }
6398 
kvm_free_msr_filter(struct kvm_x86_msr_filter * msr_filter)6399 static void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter)
6400 {
6401 	u32 i;
6402 
6403 	if (!msr_filter)
6404 		return;
6405 
6406 	for (i = 0; i < msr_filter->count; i++)
6407 		kfree(msr_filter->ranges[i].bitmap);
6408 
6409 	kfree(msr_filter);
6410 }
6411 
kvm_add_msr_filter(struct kvm_x86_msr_filter * msr_filter,struct kvm_msr_filter_range * user_range)6412 static int kvm_add_msr_filter(struct kvm_x86_msr_filter *msr_filter,
6413 			      struct kvm_msr_filter_range *user_range)
6414 {
6415 	unsigned long *bitmap = NULL;
6416 	size_t bitmap_size;
6417 
6418 	if (!user_range->nmsrs)
6419 		return 0;
6420 
6421 	if (user_range->flags & ~(KVM_MSR_FILTER_READ | KVM_MSR_FILTER_WRITE))
6422 		return -EINVAL;
6423 
6424 	if (!user_range->flags)
6425 		return -EINVAL;
6426 
6427 	bitmap_size = BITS_TO_LONGS(user_range->nmsrs) * sizeof(long);
6428 	if (!bitmap_size || bitmap_size > KVM_MSR_FILTER_MAX_BITMAP_SIZE)
6429 		return -EINVAL;
6430 
6431 	bitmap = memdup_user((__user u8*)user_range->bitmap, bitmap_size);
6432 	if (IS_ERR(bitmap))
6433 		return PTR_ERR(bitmap);
6434 
6435 	msr_filter->ranges[msr_filter->count] = (struct msr_bitmap_range) {
6436 		.flags = user_range->flags,
6437 		.base = user_range->base,
6438 		.nmsrs = user_range->nmsrs,
6439 		.bitmap = bitmap,
6440 	};
6441 
6442 	msr_filter->count++;
6443 	return 0;
6444 }
6445 
kvm_vm_ioctl_set_msr_filter(struct kvm * kvm,struct kvm_msr_filter * filter)6446 static int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm,
6447 				       struct kvm_msr_filter *filter)
6448 {
6449 	struct kvm_x86_msr_filter *new_filter, *old_filter;
6450 	bool default_allow;
6451 	bool empty = true;
6452 	int r = 0;
6453 	u32 i;
6454 
6455 	if (filter->flags & ~KVM_MSR_FILTER_DEFAULT_DENY)
6456 		return -EINVAL;
6457 
6458 	for (i = 0; i < ARRAY_SIZE(filter->ranges); i++)
6459 		empty &= !filter->ranges[i].nmsrs;
6460 
6461 	default_allow = !(filter->flags & KVM_MSR_FILTER_DEFAULT_DENY);
6462 	if (empty && !default_allow)
6463 		return -EINVAL;
6464 
6465 	new_filter = kvm_alloc_msr_filter(default_allow);
6466 	if (!new_filter)
6467 		return -ENOMEM;
6468 
6469 	for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) {
6470 		r = kvm_add_msr_filter(new_filter, &filter->ranges[i]);
6471 		if (r) {
6472 			kvm_free_msr_filter(new_filter);
6473 			return r;
6474 		}
6475 	}
6476 
6477 	mutex_lock(&kvm->lock);
6478 
6479 	/* The per-VM filter is protected by kvm->lock... */
6480 	old_filter = srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1);
6481 
6482 	rcu_assign_pointer(kvm->arch.msr_filter, new_filter);
6483 	synchronize_srcu(&kvm->srcu);
6484 
6485 	kvm_free_msr_filter(old_filter);
6486 
6487 	kvm_make_all_cpus_request(kvm, KVM_REQ_MSR_FILTER_CHANGED);
6488 	mutex_unlock(&kvm->lock);
6489 
6490 	return 0;
6491 }
6492 
6493 #ifdef CONFIG_KVM_COMPAT
6494 /* for KVM_X86_SET_MSR_FILTER */
6495 struct kvm_msr_filter_range_compat {
6496 	__u32 flags;
6497 	__u32 nmsrs;
6498 	__u32 base;
6499 	__u32 bitmap;
6500 };
6501 
6502 struct kvm_msr_filter_compat {
6503 	__u32 flags;
6504 	struct kvm_msr_filter_range_compat ranges[KVM_MSR_FILTER_MAX_RANGES];
6505 };
6506 
6507 #define KVM_X86_SET_MSR_FILTER_COMPAT _IOW(KVMIO, 0xc6, struct kvm_msr_filter_compat)
6508 
kvm_arch_vm_compat_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)6509 long kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl,
6510 			      unsigned long arg)
6511 {
6512 	void __user *argp = (void __user *)arg;
6513 	struct kvm *kvm = filp->private_data;
6514 	long r = -ENOTTY;
6515 
6516 	switch (ioctl) {
6517 	case KVM_X86_SET_MSR_FILTER_COMPAT: {
6518 		struct kvm_msr_filter __user *user_msr_filter = argp;
6519 		struct kvm_msr_filter_compat filter_compat;
6520 		struct kvm_msr_filter filter;
6521 		int i;
6522 
6523 		if (copy_from_user(&filter_compat, user_msr_filter,
6524 				   sizeof(filter_compat)))
6525 			return -EFAULT;
6526 
6527 		filter.flags = filter_compat.flags;
6528 		for (i = 0; i < ARRAY_SIZE(filter.ranges); i++) {
6529 			struct kvm_msr_filter_range_compat *cr;
6530 
6531 			cr = &filter_compat.ranges[i];
6532 			filter.ranges[i] = (struct kvm_msr_filter_range) {
6533 				.flags = cr->flags,
6534 				.nmsrs = cr->nmsrs,
6535 				.base = cr->base,
6536 				.bitmap = (__u8 *)(ulong)cr->bitmap,
6537 			};
6538 		}
6539 
6540 		r = kvm_vm_ioctl_set_msr_filter(kvm, &filter);
6541 		break;
6542 	}
6543 	}
6544 
6545 	return r;
6546 }
6547 #endif
6548 
6549 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
kvm_arch_suspend_notifier(struct kvm * kvm)6550 static int kvm_arch_suspend_notifier(struct kvm *kvm)
6551 {
6552 	struct kvm_vcpu *vcpu;
6553 	unsigned long i;
6554 	int ret = 0;
6555 
6556 	mutex_lock(&kvm->lock);
6557 	kvm_for_each_vcpu(i, vcpu, kvm) {
6558 		if (!vcpu->arch.pv_time.active)
6559 			continue;
6560 
6561 		ret = kvm_set_guest_paused(vcpu);
6562 		if (ret) {
6563 			kvm_err("Failed to pause guest VCPU%d: %d\n",
6564 				vcpu->vcpu_id, ret);
6565 			break;
6566 		}
6567 	}
6568 	mutex_unlock(&kvm->lock);
6569 
6570 	return ret ? NOTIFY_BAD : NOTIFY_DONE;
6571 }
6572 
kvm_arch_pm_notifier(struct kvm * kvm,unsigned long state)6573 int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state)
6574 {
6575 	switch (state) {
6576 	case PM_HIBERNATION_PREPARE:
6577 	case PM_SUSPEND_PREPARE:
6578 		return kvm_arch_suspend_notifier(kvm);
6579 	}
6580 
6581 	return NOTIFY_DONE;
6582 }
6583 #endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */
6584 
kvm_vm_ioctl_get_clock(struct kvm * kvm,void __user * argp)6585 static int kvm_vm_ioctl_get_clock(struct kvm *kvm, void __user *argp)
6586 {
6587 	struct kvm_clock_data data = { 0 };
6588 
6589 	get_kvmclock(kvm, &data);
6590 	if (copy_to_user(argp, &data, sizeof(data)))
6591 		return -EFAULT;
6592 
6593 	return 0;
6594 }
6595 
kvm_vm_ioctl_set_clock(struct kvm * kvm,void __user * argp)6596 static int kvm_vm_ioctl_set_clock(struct kvm *kvm, void __user *argp)
6597 {
6598 	struct kvm_arch *ka = &kvm->arch;
6599 	struct kvm_clock_data data;
6600 	u64 now_raw_ns;
6601 
6602 	if (copy_from_user(&data, argp, sizeof(data)))
6603 		return -EFAULT;
6604 
6605 	/*
6606 	 * Only KVM_CLOCK_REALTIME is used, but allow passing the
6607 	 * result of KVM_GET_CLOCK back to KVM_SET_CLOCK.
6608 	 */
6609 	if (data.flags & ~KVM_CLOCK_VALID_FLAGS)
6610 		return -EINVAL;
6611 
6612 	kvm_hv_request_tsc_page_update(kvm);
6613 	kvm_start_pvclock_update(kvm);
6614 	pvclock_update_vm_gtod_copy(kvm);
6615 
6616 	/*
6617 	 * This pairs with kvm_guest_time_update(): when masterclock is
6618 	 * in use, we use master_kernel_ns + kvmclock_offset to set
6619 	 * unsigned 'system_time' so if we use get_kvmclock_ns() (which
6620 	 * is slightly ahead) here we risk going negative on unsigned
6621 	 * 'system_time' when 'data.clock' is very small.
6622 	 */
6623 	if (data.flags & KVM_CLOCK_REALTIME) {
6624 		u64 now_real_ns = ktime_get_real_ns();
6625 
6626 		/*
6627 		 * Avoid stepping the kvmclock backwards.
6628 		 */
6629 		if (now_real_ns > data.realtime)
6630 			data.clock += now_real_ns - data.realtime;
6631 	}
6632 
6633 	if (ka->use_master_clock)
6634 		now_raw_ns = ka->master_kernel_ns;
6635 	else
6636 		now_raw_ns = get_kvmclock_base_ns();
6637 	ka->kvmclock_offset = data.clock - now_raw_ns;
6638 	kvm_end_pvclock_update(kvm);
6639 	return 0;
6640 }
6641 
kvm_arch_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)6642 long kvm_arch_vm_ioctl(struct file *filp,
6643 		       unsigned int ioctl, unsigned long arg)
6644 {
6645 	struct kvm *kvm = filp->private_data;
6646 	void __user *argp = (void __user *)arg;
6647 	int r = -ENOTTY;
6648 	/*
6649 	 * This union makes it completely explicit to gcc-3.x
6650 	 * that these two variables' stack usage should be
6651 	 * combined, not added together.
6652 	 */
6653 	union {
6654 		struct kvm_pit_state ps;
6655 		struct kvm_pit_state2 ps2;
6656 		struct kvm_pit_config pit_config;
6657 	} u;
6658 
6659 	switch (ioctl) {
6660 	case KVM_SET_TSS_ADDR:
6661 		r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
6662 		break;
6663 	case KVM_SET_IDENTITY_MAP_ADDR: {
6664 		u64 ident_addr;
6665 
6666 		mutex_lock(&kvm->lock);
6667 		r = -EINVAL;
6668 		if (kvm->created_vcpus)
6669 			goto set_identity_unlock;
6670 		r = -EFAULT;
6671 		if (copy_from_user(&ident_addr, argp, sizeof(ident_addr)))
6672 			goto set_identity_unlock;
6673 		r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
6674 set_identity_unlock:
6675 		mutex_unlock(&kvm->lock);
6676 		break;
6677 	}
6678 	case KVM_SET_NR_MMU_PAGES:
6679 		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
6680 		break;
6681 	case KVM_GET_NR_MMU_PAGES:
6682 		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
6683 		break;
6684 	case KVM_CREATE_IRQCHIP: {
6685 		mutex_lock(&kvm->lock);
6686 
6687 		r = -EEXIST;
6688 		if (irqchip_in_kernel(kvm))
6689 			goto create_irqchip_unlock;
6690 
6691 		r = -EINVAL;
6692 		if (kvm->created_vcpus)
6693 			goto create_irqchip_unlock;
6694 
6695 		r = kvm_pic_init(kvm);
6696 		if (r)
6697 			goto create_irqchip_unlock;
6698 
6699 		r = kvm_ioapic_init(kvm);
6700 		if (r) {
6701 			kvm_pic_destroy(kvm);
6702 			goto create_irqchip_unlock;
6703 		}
6704 
6705 		r = kvm_setup_default_irq_routing(kvm);
6706 		if (r) {
6707 			kvm_ioapic_destroy(kvm);
6708 			kvm_pic_destroy(kvm);
6709 			goto create_irqchip_unlock;
6710 		}
6711 		/* Write kvm->irq_routing before enabling irqchip_in_kernel. */
6712 		smp_wmb();
6713 		kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;
6714 		kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT);
6715 	create_irqchip_unlock:
6716 		mutex_unlock(&kvm->lock);
6717 		break;
6718 	}
6719 	case KVM_CREATE_PIT:
6720 		u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
6721 		goto create_pit;
6722 	case KVM_CREATE_PIT2:
6723 		r = -EFAULT;
6724 		if (copy_from_user(&u.pit_config, argp,
6725 				   sizeof(struct kvm_pit_config)))
6726 			goto out;
6727 	create_pit:
6728 		mutex_lock(&kvm->lock);
6729 		r = -EEXIST;
6730 		if (kvm->arch.vpit)
6731 			goto create_pit_unlock;
6732 		r = -ENOMEM;
6733 		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
6734 		if (kvm->arch.vpit)
6735 			r = 0;
6736 	create_pit_unlock:
6737 		mutex_unlock(&kvm->lock);
6738 		break;
6739 	case KVM_GET_IRQCHIP: {
6740 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
6741 		struct kvm_irqchip *chip;
6742 
6743 		chip = memdup_user(argp, sizeof(*chip));
6744 		if (IS_ERR(chip)) {
6745 			r = PTR_ERR(chip);
6746 			goto out;
6747 		}
6748 
6749 		r = -ENXIO;
6750 		if (!irqchip_kernel(kvm))
6751 			goto get_irqchip_out;
6752 		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
6753 		if (r)
6754 			goto get_irqchip_out;
6755 		r = -EFAULT;
6756 		if (copy_to_user(argp, chip, sizeof(*chip)))
6757 			goto get_irqchip_out;
6758 		r = 0;
6759 	get_irqchip_out:
6760 		kfree(chip);
6761 		break;
6762 	}
6763 	case KVM_SET_IRQCHIP: {
6764 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
6765 		struct kvm_irqchip *chip;
6766 
6767 		chip = memdup_user(argp, sizeof(*chip));
6768 		if (IS_ERR(chip)) {
6769 			r = PTR_ERR(chip);
6770 			goto out;
6771 		}
6772 
6773 		r = -ENXIO;
6774 		if (!irqchip_kernel(kvm))
6775 			goto set_irqchip_out;
6776 		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
6777 	set_irqchip_out:
6778 		kfree(chip);
6779 		break;
6780 	}
6781 	case KVM_GET_PIT: {
6782 		r = -EFAULT;
6783 		if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
6784 			goto out;
6785 		r = -ENXIO;
6786 		if (!kvm->arch.vpit)
6787 			goto out;
6788 		r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
6789 		if (r)
6790 			goto out;
6791 		r = -EFAULT;
6792 		if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
6793 			goto out;
6794 		r = 0;
6795 		break;
6796 	}
6797 	case KVM_SET_PIT: {
6798 		r = -EFAULT;
6799 		if (copy_from_user(&u.ps, argp, sizeof(u.ps)))
6800 			goto out;
6801 		mutex_lock(&kvm->lock);
6802 		r = -ENXIO;
6803 		if (!kvm->arch.vpit)
6804 			goto set_pit_out;
6805 		r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
6806 set_pit_out:
6807 		mutex_unlock(&kvm->lock);
6808 		break;
6809 	}
6810 	case KVM_GET_PIT2: {
6811 		r = -ENXIO;
6812 		if (!kvm->arch.vpit)
6813 			goto out;
6814 		r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
6815 		if (r)
6816 			goto out;
6817 		r = -EFAULT;
6818 		if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
6819 			goto out;
6820 		r = 0;
6821 		break;
6822 	}
6823 	case KVM_SET_PIT2: {
6824 		r = -EFAULT;
6825 		if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
6826 			goto out;
6827 		mutex_lock(&kvm->lock);
6828 		r = -ENXIO;
6829 		if (!kvm->arch.vpit)
6830 			goto set_pit2_out;
6831 		r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
6832 set_pit2_out:
6833 		mutex_unlock(&kvm->lock);
6834 		break;
6835 	}
6836 	case KVM_REINJECT_CONTROL: {
6837 		struct kvm_reinject_control control;
6838 		r =  -EFAULT;
6839 		if (copy_from_user(&control, argp, sizeof(control)))
6840 			goto out;
6841 		r = -ENXIO;
6842 		if (!kvm->arch.vpit)
6843 			goto out;
6844 		r = kvm_vm_ioctl_reinject(kvm, &control);
6845 		break;
6846 	}
6847 	case KVM_SET_BOOT_CPU_ID:
6848 		r = 0;
6849 		mutex_lock(&kvm->lock);
6850 		if (kvm->created_vcpus)
6851 			r = -EBUSY;
6852 		else
6853 			kvm->arch.bsp_vcpu_id = arg;
6854 		mutex_unlock(&kvm->lock);
6855 		break;
6856 #ifdef CONFIG_KVM_XEN
6857 	case KVM_XEN_HVM_CONFIG: {
6858 		struct kvm_xen_hvm_config xhc;
6859 		r = -EFAULT;
6860 		if (copy_from_user(&xhc, argp, sizeof(xhc)))
6861 			goto out;
6862 		r = kvm_xen_hvm_config(kvm, &xhc);
6863 		break;
6864 	}
6865 	case KVM_XEN_HVM_GET_ATTR: {
6866 		struct kvm_xen_hvm_attr xha;
6867 
6868 		r = -EFAULT;
6869 		if (copy_from_user(&xha, argp, sizeof(xha)))
6870 			goto out;
6871 		r = kvm_xen_hvm_get_attr(kvm, &xha);
6872 		if (!r && copy_to_user(argp, &xha, sizeof(xha)))
6873 			r = -EFAULT;
6874 		break;
6875 	}
6876 	case KVM_XEN_HVM_SET_ATTR: {
6877 		struct kvm_xen_hvm_attr xha;
6878 
6879 		r = -EFAULT;
6880 		if (copy_from_user(&xha, argp, sizeof(xha)))
6881 			goto out;
6882 		r = kvm_xen_hvm_set_attr(kvm, &xha);
6883 		break;
6884 	}
6885 	case KVM_XEN_HVM_EVTCHN_SEND: {
6886 		struct kvm_irq_routing_xen_evtchn uxe;
6887 
6888 		r = -EFAULT;
6889 		if (copy_from_user(&uxe, argp, sizeof(uxe)))
6890 			goto out;
6891 		r = kvm_xen_hvm_evtchn_send(kvm, &uxe);
6892 		break;
6893 	}
6894 #endif
6895 	case KVM_SET_CLOCK:
6896 		r = kvm_vm_ioctl_set_clock(kvm, argp);
6897 		break;
6898 	case KVM_GET_CLOCK:
6899 		r = kvm_vm_ioctl_get_clock(kvm, argp);
6900 		break;
6901 	case KVM_SET_TSC_KHZ: {
6902 		u32 user_tsc_khz;
6903 
6904 		r = -EINVAL;
6905 		user_tsc_khz = (u32)arg;
6906 
6907 		if (kvm_caps.has_tsc_control &&
6908 		    user_tsc_khz >= kvm_caps.max_guest_tsc_khz)
6909 			goto out;
6910 
6911 		if (user_tsc_khz == 0)
6912 			user_tsc_khz = tsc_khz;
6913 
6914 		WRITE_ONCE(kvm->arch.default_tsc_khz, user_tsc_khz);
6915 		r = 0;
6916 
6917 		goto out;
6918 	}
6919 	case KVM_GET_TSC_KHZ: {
6920 		r = READ_ONCE(kvm->arch.default_tsc_khz);
6921 		goto out;
6922 	}
6923 	case KVM_MEMORY_ENCRYPT_OP: {
6924 		r = -ENOTTY;
6925 		if (!kvm_x86_ops.mem_enc_ioctl)
6926 			goto out;
6927 
6928 		r = static_call(kvm_x86_mem_enc_ioctl)(kvm, argp);
6929 		break;
6930 	}
6931 	case KVM_MEMORY_ENCRYPT_REG_REGION: {
6932 		struct kvm_enc_region region;
6933 
6934 		r = -EFAULT;
6935 		if (copy_from_user(&region, argp, sizeof(region)))
6936 			goto out;
6937 
6938 		r = -ENOTTY;
6939 		if (!kvm_x86_ops.mem_enc_register_region)
6940 			goto out;
6941 
6942 		r = static_call(kvm_x86_mem_enc_register_region)(kvm, &region);
6943 		break;
6944 	}
6945 	case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
6946 		struct kvm_enc_region region;
6947 
6948 		r = -EFAULT;
6949 		if (copy_from_user(&region, argp, sizeof(region)))
6950 			goto out;
6951 
6952 		r = -ENOTTY;
6953 		if (!kvm_x86_ops.mem_enc_unregister_region)
6954 			goto out;
6955 
6956 		r = static_call(kvm_x86_mem_enc_unregister_region)(kvm, &region);
6957 		break;
6958 	}
6959 	case KVM_HYPERV_EVENTFD: {
6960 		struct kvm_hyperv_eventfd hvevfd;
6961 
6962 		r = -EFAULT;
6963 		if (copy_from_user(&hvevfd, argp, sizeof(hvevfd)))
6964 			goto out;
6965 		r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd);
6966 		break;
6967 	}
6968 	case KVM_SET_PMU_EVENT_FILTER:
6969 		r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp);
6970 		break;
6971 	case KVM_X86_SET_MSR_FILTER: {
6972 		struct kvm_msr_filter __user *user_msr_filter = argp;
6973 		struct kvm_msr_filter filter;
6974 
6975 		if (copy_from_user(&filter, user_msr_filter, sizeof(filter)))
6976 			return -EFAULT;
6977 
6978 		r = kvm_vm_ioctl_set_msr_filter(kvm, &filter);
6979 		break;
6980 	}
6981 	default:
6982 		r = -ENOTTY;
6983 	}
6984 out:
6985 	return r;
6986 }
6987 
kvm_init_msr_list(void)6988 static void kvm_init_msr_list(void)
6989 {
6990 	u32 dummy[2];
6991 	unsigned i;
6992 
6993 	BUILD_BUG_ON_MSG(KVM_PMC_MAX_FIXED != 3,
6994 			 "Please update the fixed PMCs in msrs_to_saved_all[]");
6995 
6996 	num_msrs_to_save = 0;
6997 	num_emulated_msrs = 0;
6998 	num_msr_based_features = 0;
6999 
7000 	for (i = 0; i < ARRAY_SIZE(msrs_to_save_all); i++) {
7001 		if (rdmsr_safe(msrs_to_save_all[i], &dummy[0], &dummy[1]) < 0)
7002 			continue;
7003 
7004 		/*
7005 		 * Even MSRs that are valid in the host may not be exposed
7006 		 * to the guests in some cases.
7007 		 */
7008 		switch (msrs_to_save_all[i]) {
7009 		case MSR_IA32_BNDCFGS:
7010 			if (!kvm_mpx_supported())
7011 				continue;
7012 			break;
7013 		case MSR_TSC_AUX:
7014 			if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP) &&
7015 			    !kvm_cpu_cap_has(X86_FEATURE_RDPID))
7016 				continue;
7017 			break;
7018 		case MSR_IA32_UMWAIT_CONTROL:
7019 			if (!kvm_cpu_cap_has(X86_FEATURE_WAITPKG))
7020 				continue;
7021 			break;
7022 		case MSR_IA32_RTIT_CTL:
7023 		case MSR_IA32_RTIT_STATUS:
7024 			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT))
7025 				continue;
7026 			break;
7027 		case MSR_IA32_RTIT_CR3_MATCH:
7028 			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
7029 			    !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering))
7030 				continue;
7031 			break;
7032 		case MSR_IA32_RTIT_OUTPUT_BASE:
7033 		case MSR_IA32_RTIT_OUTPUT_MASK:
7034 			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
7035 				(!intel_pt_validate_hw_cap(PT_CAP_topa_output) &&
7036 				 !intel_pt_validate_hw_cap(PT_CAP_single_range_output)))
7037 				continue;
7038 			break;
7039 		case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
7040 			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
7041 				msrs_to_save_all[i] - MSR_IA32_RTIT_ADDR0_A >=
7042 				intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)
7043 				continue;
7044 			break;
7045 		case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR_MAX:
7046 			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_PERFCTR0 >=
7047 			    min(KVM_INTEL_PMC_MAX_GENERIC, kvm_pmu_cap.num_counters_gp))
7048 				continue;
7049 			break;
7050 		case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL_MAX:
7051 			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_EVENTSEL0 >=
7052 			    min(KVM_INTEL_PMC_MAX_GENERIC, kvm_pmu_cap.num_counters_gp))
7053 				continue;
7054 			break;
7055 		case MSR_IA32_XFD:
7056 		case MSR_IA32_XFD_ERR:
7057 			if (!kvm_cpu_cap_has(X86_FEATURE_XFD))
7058 				continue;
7059 			break;
7060 		default:
7061 			break;
7062 		}
7063 
7064 		msrs_to_save[num_msrs_to_save++] = msrs_to_save_all[i];
7065 	}
7066 
7067 	for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {
7068 		if (!static_call(kvm_x86_has_emulated_msr)(NULL, emulated_msrs_all[i]))
7069 			continue;
7070 
7071 		emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];
7072 	}
7073 
7074 	for (i = 0; i < ARRAY_SIZE(msr_based_features_all); i++) {
7075 		struct kvm_msr_entry msr;
7076 
7077 		msr.index = msr_based_features_all[i];
7078 		if (kvm_get_msr_feature(&msr))
7079 			continue;
7080 
7081 		msr_based_features[num_msr_based_features++] = msr_based_features_all[i];
7082 	}
7083 }
7084 
vcpu_mmio_write(struct kvm_vcpu * vcpu,gpa_t addr,int len,const void * v)7085 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
7086 			   const void *v)
7087 {
7088 	int handled = 0;
7089 	int n;
7090 
7091 	do {
7092 		n = min(len, 8);
7093 		if (!(lapic_in_kernel(vcpu) &&
7094 		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
7095 		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
7096 			break;
7097 		handled += n;
7098 		addr += n;
7099 		len -= n;
7100 		v += n;
7101 	} while (len);
7102 
7103 	return handled;
7104 }
7105 
vcpu_mmio_read(struct kvm_vcpu * vcpu,gpa_t addr,int len,void * v)7106 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
7107 {
7108 	int handled = 0;
7109 	int n;
7110 
7111 	do {
7112 		n = min(len, 8);
7113 		if (!(lapic_in_kernel(vcpu) &&
7114 		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
7115 					 addr, n, v))
7116 		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
7117 			break;
7118 		trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v);
7119 		handled += n;
7120 		addr += n;
7121 		len -= n;
7122 		v += n;
7123 	} while (len);
7124 
7125 	return handled;
7126 }
7127 
kvm_set_segment(struct kvm_vcpu * vcpu,struct kvm_segment * var,int seg)7128 static void kvm_set_segment(struct kvm_vcpu *vcpu,
7129 			struct kvm_segment *var, int seg)
7130 {
7131 	static_call(kvm_x86_set_segment)(vcpu, var, seg);
7132 }
7133 
kvm_get_segment(struct kvm_vcpu * vcpu,struct kvm_segment * var,int seg)7134 void kvm_get_segment(struct kvm_vcpu *vcpu,
7135 		     struct kvm_segment *var, int seg)
7136 {
7137 	static_call(kvm_x86_get_segment)(vcpu, var, seg);
7138 }
7139 
translate_nested_gpa(struct kvm_vcpu * vcpu,gpa_t gpa,u64 access,struct x86_exception * exception)7140 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access,
7141 			   struct x86_exception *exception)
7142 {
7143 	struct kvm_mmu *mmu = vcpu->arch.mmu;
7144 	gpa_t t_gpa;
7145 
7146 	BUG_ON(!mmu_is_nested(vcpu));
7147 
7148 	/* NPT walks are always user-walks */
7149 	access |= PFERR_USER_MASK;
7150 	t_gpa  = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);
7151 
7152 	return t_gpa;
7153 }
7154 
kvm_mmu_gva_to_gpa_read(struct kvm_vcpu * vcpu,gva_t gva,struct x86_exception * exception)7155 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
7156 			      struct x86_exception *exception)
7157 {
7158 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7159 
7160 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
7161 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
7162 }
7163 EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_read);
7164 
kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu * vcpu,gva_t gva,struct x86_exception * exception)7165  gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
7166 				struct x86_exception *exception)
7167 {
7168 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7169 
7170 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
7171 	access |= PFERR_FETCH_MASK;
7172 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
7173 }
7174 
kvm_mmu_gva_to_gpa_write(struct kvm_vcpu * vcpu,gva_t gva,struct x86_exception * exception)7175 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
7176 			       struct x86_exception *exception)
7177 {
7178 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7179 
7180 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
7181 	access |= PFERR_WRITE_MASK;
7182 	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
7183 }
7184 EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_write);
7185 
7186 /* uses this to access any guest's mapped memory without checking CPL */
kvm_mmu_gva_to_gpa_system(struct kvm_vcpu * vcpu,gva_t gva,struct x86_exception * exception)7187 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
7188 				struct x86_exception *exception)
7189 {
7190 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7191 
7192 	return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception);
7193 }
7194 
kvm_read_guest_virt_helper(gva_t addr,void * val,unsigned int bytes,struct kvm_vcpu * vcpu,u64 access,struct x86_exception * exception)7195 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
7196 				      struct kvm_vcpu *vcpu, u64 access,
7197 				      struct x86_exception *exception)
7198 {
7199 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7200 	void *data = val;
7201 	int r = X86EMUL_CONTINUE;
7202 
7203 	while (bytes) {
7204 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
7205 		unsigned offset = addr & (PAGE_SIZE-1);
7206 		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
7207 		int ret;
7208 
7209 		if (gpa == INVALID_GPA)
7210 			return X86EMUL_PROPAGATE_FAULT;
7211 		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,
7212 					       offset, toread);
7213 		if (ret < 0) {
7214 			r = X86EMUL_IO_NEEDED;
7215 			goto out;
7216 		}
7217 
7218 		bytes -= toread;
7219 		data += toread;
7220 		addr += toread;
7221 	}
7222 out:
7223 	return r;
7224 }
7225 
7226 /* used for instruction fetching */
kvm_fetch_guest_virt(struct x86_emulate_ctxt * ctxt,gva_t addr,void * val,unsigned int bytes,struct x86_exception * exception)7227 static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
7228 				gva_t addr, void *val, unsigned int bytes,
7229 				struct x86_exception *exception)
7230 {
7231 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7232 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7233 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
7234 	unsigned offset;
7235 	int ret;
7236 
7237 	/* Inline kvm_read_guest_virt_helper for speed.  */
7238 	gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK,
7239 				    exception);
7240 	if (unlikely(gpa == INVALID_GPA))
7241 		return X86EMUL_PROPAGATE_FAULT;
7242 
7243 	offset = addr & (PAGE_SIZE-1);
7244 	if (WARN_ON(offset + bytes > PAGE_SIZE))
7245 		bytes = (unsigned)PAGE_SIZE - offset;
7246 	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,
7247 				       offset, bytes);
7248 	if (unlikely(ret < 0))
7249 		return X86EMUL_IO_NEEDED;
7250 
7251 	return X86EMUL_CONTINUE;
7252 }
7253 
kvm_read_guest_virt(struct kvm_vcpu * vcpu,gva_t addr,void * val,unsigned int bytes,struct x86_exception * exception)7254 int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
7255 			       gva_t addr, void *val, unsigned int bytes,
7256 			       struct x86_exception *exception)
7257 {
7258 	u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
7259 
7260 	/*
7261 	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
7262 	 * is returned, but our callers are not ready for that and they blindly
7263 	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
7264 	 * uninitialized kernel stack memory into cr2 and error code.
7265 	 */
7266 	memset(exception, 0, sizeof(*exception));
7267 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
7268 					  exception);
7269 }
7270 EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
7271 
emulator_read_std(struct x86_emulate_ctxt * ctxt,gva_t addr,void * val,unsigned int bytes,struct x86_exception * exception,bool system)7272 static int emulator_read_std(struct x86_emulate_ctxt *ctxt,
7273 			     gva_t addr, void *val, unsigned int bytes,
7274 			     struct x86_exception *exception, bool system)
7275 {
7276 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7277 	u64 access = 0;
7278 
7279 	if (system)
7280 		access |= PFERR_IMPLICIT_ACCESS;
7281 	else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)
7282 		access |= PFERR_USER_MASK;
7283 
7284 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
7285 }
7286 
kvm_read_guest_phys_system(struct x86_emulate_ctxt * ctxt,unsigned long addr,void * val,unsigned int bytes)7287 static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt,
7288 		unsigned long addr, void *val, unsigned int bytes)
7289 {
7290 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7291 	int r = kvm_vcpu_read_guest(vcpu, addr, val, bytes);
7292 
7293 	return r < 0 ? X86EMUL_IO_NEEDED : X86EMUL_CONTINUE;
7294 }
7295 
kvm_write_guest_virt_helper(gva_t addr,void * val,unsigned int bytes,struct kvm_vcpu * vcpu,u64 access,struct x86_exception * exception)7296 static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
7297 				      struct kvm_vcpu *vcpu, u64 access,
7298 				      struct x86_exception *exception)
7299 {
7300 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7301 	void *data = val;
7302 	int r = X86EMUL_CONTINUE;
7303 
7304 	while (bytes) {
7305 		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
7306 		unsigned offset = addr & (PAGE_SIZE-1);
7307 		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
7308 		int ret;
7309 
7310 		if (gpa == INVALID_GPA)
7311 			return X86EMUL_PROPAGATE_FAULT;
7312 		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
7313 		if (ret < 0) {
7314 			r = X86EMUL_IO_NEEDED;
7315 			goto out;
7316 		}
7317 
7318 		bytes -= towrite;
7319 		data += towrite;
7320 		addr += towrite;
7321 	}
7322 out:
7323 	return r;
7324 }
7325 
emulator_write_std(struct x86_emulate_ctxt * ctxt,gva_t addr,void * val,unsigned int bytes,struct x86_exception * exception,bool system)7326 static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val,
7327 			      unsigned int bytes, struct x86_exception *exception,
7328 			      bool system)
7329 {
7330 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7331 	u64 access = PFERR_WRITE_MASK;
7332 
7333 	if (system)
7334 		access |= PFERR_IMPLICIT_ACCESS;
7335 	else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)
7336 		access |= PFERR_USER_MASK;
7337 
7338 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
7339 					   access, exception);
7340 }
7341 
kvm_write_guest_virt_system(struct kvm_vcpu * vcpu,gva_t addr,void * val,unsigned int bytes,struct x86_exception * exception)7342 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
7343 				unsigned int bytes, struct x86_exception *exception)
7344 {
7345 	/* kvm_write_guest_virt_system can pull in tons of pages. */
7346 	vcpu->arch.l1tf_flush_l1d = true;
7347 
7348 	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
7349 					   PFERR_WRITE_MASK, exception);
7350 }
7351 EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
7352 
kvm_can_emulate_insn(struct kvm_vcpu * vcpu,int emul_type,void * insn,int insn_len)7353 static int kvm_can_emulate_insn(struct kvm_vcpu *vcpu, int emul_type,
7354 				void *insn, int insn_len)
7355 {
7356 	return static_call(kvm_x86_can_emulate_instruction)(vcpu, emul_type,
7357 							    insn, insn_len);
7358 }
7359 
handle_ud(struct kvm_vcpu * vcpu)7360 int handle_ud(struct kvm_vcpu *vcpu)
7361 {
7362 	static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX };
7363 	int fep_flags = READ_ONCE(force_emulation_prefix);
7364 	int emul_type = EMULTYPE_TRAP_UD;
7365 	char sig[5]; /* ud2; .ascii "kvm" */
7366 	struct x86_exception e;
7367 
7368 	if (unlikely(!kvm_can_emulate_insn(vcpu, emul_type, NULL, 0)))
7369 		return 1;
7370 
7371 	if (fep_flags &&
7372 	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
7373 				sig, sizeof(sig), &e) == 0 &&
7374 	    memcmp(sig, kvm_emulate_prefix, sizeof(sig)) == 0) {
7375 		if (fep_flags & KVM_FEP_CLEAR_RFLAGS_RF)
7376 			kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF);
7377 		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
7378 		emul_type = EMULTYPE_TRAP_UD_FORCED;
7379 	}
7380 
7381 	return kvm_emulate_instruction(vcpu, emul_type);
7382 }
7383 EXPORT_SYMBOL_GPL(handle_ud);
7384 
vcpu_is_mmio_gpa(struct kvm_vcpu * vcpu,unsigned long gva,gpa_t gpa,bool write)7385 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
7386 			    gpa_t gpa, bool write)
7387 {
7388 	/* For APIC access vmexit */
7389 	if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
7390 		return 1;
7391 
7392 	if (vcpu_match_mmio_gpa(vcpu, gpa)) {
7393 		trace_vcpu_match_mmio(gva, gpa, write, true);
7394 		return 1;
7395 	}
7396 
7397 	return 0;
7398 }
7399 
vcpu_mmio_gva_to_gpa(struct kvm_vcpu * vcpu,unsigned long gva,gpa_t * gpa,struct x86_exception * exception,bool write)7400 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
7401 				gpa_t *gpa, struct x86_exception *exception,
7402 				bool write)
7403 {
7404 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7405 	u64 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)
7406 		| (write ? PFERR_WRITE_MASK : 0);
7407 
7408 	/*
7409 	 * currently PKRU is only applied to ept enabled guest so
7410 	 * there is no pkey in EPT page table for L1 guest or EPT
7411 	 * shadow page table for L2 guest.
7412 	 */
7413 	if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) ||
7414 	    !permission_fault(vcpu, vcpu->arch.walk_mmu,
7415 			      vcpu->arch.mmio_access, 0, access))) {
7416 		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
7417 					(gva & (PAGE_SIZE - 1));
7418 		trace_vcpu_match_mmio(gva, *gpa, write, false);
7419 		return 1;
7420 	}
7421 
7422 	*gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
7423 
7424 	if (*gpa == INVALID_GPA)
7425 		return -1;
7426 
7427 	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
7428 }
7429 
emulator_write_phys(struct kvm_vcpu * vcpu,gpa_t gpa,const void * val,int bytes)7430 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
7431 			const void *val, int bytes)
7432 {
7433 	int ret;
7434 
7435 	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
7436 	if (ret < 0)
7437 		return 0;
7438 	kvm_page_track_write(vcpu, gpa, val, bytes);
7439 	return 1;
7440 }
7441 
7442 struct read_write_emulator_ops {
7443 	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
7444 				  int bytes);
7445 	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
7446 				  void *val, int bytes);
7447 	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7448 			       int bytes, void *val);
7449 	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7450 				    void *val, int bytes);
7451 	bool write;
7452 };
7453 
read_prepare(struct kvm_vcpu * vcpu,void * val,int bytes)7454 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
7455 {
7456 	if (vcpu->mmio_read_completed) {
7457 		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
7458 			       vcpu->mmio_fragments[0].gpa, val);
7459 		vcpu->mmio_read_completed = 0;
7460 		return 1;
7461 	}
7462 
7463 	return 0;
7464 }
7465 
read_emulate(struct kvm_vcpu * vcpu,gpa_t gpa,void * val,int bytes)7466 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
7467 			void *val, int bytes)
7468 {
7469 	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
7470 }
7471 
write_emulate(struct kvm_vcpu * vcpu,gpa_t gpa,void * val,int bytes)7472 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
7473 			 void *val, int bytes)
7474 {
7475 	return emulator_write_phys(vcpu, gpa, val, bytes);
7476 }
7477 
write_mmio(struct kvm_vcpu * vcpu,gpa_t gpa,int bytes,void * val)7478 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
7479 {
7480 	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);
7481 	return vcpu_mmio_write(vcpu, gpa, bytes, val);
7482 }
7483 
read_exit_mmio(struct kvm_vcpu * vcpu,gpa_t gpa,void * val,int bytes)7484 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
7485 			  void *val, int bytes)
7486 {
7487 	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);
7488 	return X86EMUL_IO_NEEDED;
7489 }
7490 
write_exit_mmio(struct kvm_vcpu * vcpu,gpa_t gpa,void * val,int bytes)7491 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
7492 			   void *val, int bytes)
7493 {
7494 	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];
7495 
7496 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
7497 	return X86EMUL_CONTINUE;
7498 }
7499 
7500 static const struct read_write_emulator_ops read_emultor = {
7501 	.read_write_prepare = read_prepare,
7502 	.read_write_emulate = read_emulate,
7503 	.read_write_mmio = vcpu_mmio_read,
7504 	.read_write_exit_mmio = read_exit_mmio,
7505 };
7506 
7507 static const struct read_write_emulator_ops write_emultor = {
7508 	.read_write_emulate = write_emulate,
7509 	.read_write_mmio = write_mmio,
7510 	.read_write_exit_mmio = write_exit_mmio,
7511 	.write = true,
7512 };
7513 
emulator_read_write_onepage(unsigned long addr,void * val,unsigned int bytes,struct x86_exception * exception,struct kvm_vcpu * vcpu,const struct read_write_emulator_ops * ops)7514 static int emulator_read_write_onepage(unsigned long addr, void *val,
7515 				       unsigned int bytes,
7516 				       struct x86_exception *exception,
7517 				       struct kvm_vcpu *vcpu,
7518 				       const struct read_write_emulator_ops *ops)
7519 {
7520 	gpa_t gpa;
7521 	int handled, ret;
7522 	bool write = ops->write;
7523 	struct kvm_mmio_fragment *frag;
7524 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7525 
7526 	/*
7527 	 * If the exit was due to a NPF we may already have a GPA.
7528 	 * If the GPA is present, use it to avoid the GVA to GPA table walk.
7529 	 * Note, this cannot be used on string operations since string
7530 	 * operation using rep will only have the initial GPA from the NPF
7531 	 * occurred.
7532 	 */
7533 	if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) &&
7534 	    (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) {
7535 		gpa = ctxt->gpa_val;
7536 		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
7537 	} else {
7538 		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
7539 		if (ret < 0)
7540 			return X86EMUL_PROPAGATE_FAULT;
7541 	}
7542 
7543 	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
7544 		return X86EMUL_CONTINUE;
7545 
7546 	/*
7547 	 * Is this MMIO handled locally?
7548 	 */
7549 	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
7550 	if (handled == bytes)
7551 		return X86EMUL_CONTINUE;
7552 
7553 	gpa += handled;
7554 	bytes -= handled;
7555 	val += handled;
7556 
7557 	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
7558 	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
7559 	frag->gpa = gpa;
7560 	frag->data = val;
7561 	frag->len = bytes;
7562 	return X86EMUL_CONTINUE;
7563 }
7564 
emulator_read_write(struct x86_emulate_ctxt * ctxt,unsigned long addr,void * val,unsigned int bytes,struct x86_exception * exception,const struct read_write_emulator_ops * ops)7565 static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
7566 			unsigned long addr,
7567 			void *val, unsigned int bytes,
7568 			struct x86_exception *exception,
7569 			const struct read_write_emulator_ops *ops)
7570 {
7571 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7572 	gpa_t gpa;
7573 	int rc;
7574 
7575 	if (ops->read_write_prepare &&
7576 		  ops->read_write_prepare(vcpu, val, bytes))
7577 		return X86EMUL_CONTINUE;
7578 
7579 	vcpu->mmio_nr_fragments = 0;
7580 
7581 	/* Crossing a page boundary? */
7582 	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
7583 		int now;
7584 
7585 		now = -addr & ~PAGE_MASK;
7586 		rc = emulator_read_write_onepage(addr, val, now, exception,
7587 						 vcpu, ops);
7588 
7589 		if (rc != X86EMUL_CONTINUE)
7590 			return rc;
7591 		addr += now;
7592 		if (ctxt->mode != X86EMUL_MODE_PROT64)
7593 			addr = (u32)addr;
7594 		val += now;
7595 		bytes -= now;
7596 	}
7597 
7598 	rc = emulator_read_write_onepage(addr, val, bytes, exception,
7599 					 vcpu, ops);
7600 	if (rc != X86EMUL_CONTINUE)
7601 		return rc;
7602 
7603 	if (!vcpu->mmio_nr_fragments)
7604 		return rc;
7605 
7606 	gpa = vcpu->mmio_fragments[0].gpa;
7607 
7608 	vcpu->mmio_needed = 1;
7609 	vcpu->mmio_cur_fragment = 0;
7610 
7611 	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
7612 	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
7613 	vcpu->run->exit_reason = KVM_EXIT_MMIO;
7614 	vcpu->run->mmio.phys_addr = gpa;
7615 
7616 	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
7617 }
7618 
emulator_read_emulated(struct x86_emulate_ctxt * ctxt,unsigned long addr,void * val,unsigned int bytes,struct x86_exception * exception)7619 static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
7620 				  unsigned long addr,
7621 				  void *val,
7622 				  unsigned int bytes,
7623 				  struct x86_exception *exception)
7624 {
7625 	return emulator_read_write(ctxt, addr, val, bytes,
7626 				   exception, &read_emultor);
7627 }
7628 
emulator_write_emulated(struct x86_emulate_ctxt * ctxt,unsigned long addr,const void * val,unsigned int bytes,struct x86_exception * exception)7629 static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
7630 			    unsigned long addr,
7631 			    const void *val,
7632 			    unsigned int bytes,
7633 			    struct x86_exception *exception)
7634 {
7635 	return emulator_read_write(ctxt, addr, (void *)val, bytes,
7636 				   exception, &write_emultor);
7637 }
7638 
7639 #define emulator_try_cmpxchg_user(t, ptr, old, new) \
7640 	(__try_cmpxchg_user((t __user *)(ptr), (t *)(old), *(t *)(new), efault ## t))
7641 
emulator_cmpxchg_emulated(struct x86_emulate_ctxt * ctxt,unsigned long addr,const void * old,const void * new,unsigned int bytes,struct x86_exception * exception)7642 static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
7643 				     unsigned long addr,
7644 				     const void *old,
7645 				     const void *new,
7646 				     unsigned int bytes,
7647 				     struct x86_exception *exception)
7648 {
7649 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7650 	u64 page_line_mask;
7651 	unsigned long hva;
7652 	gpa_t gpa;
7653 	int r;
7654 
7655 	/* guests cmpxchg8b have to be emulated atomically */
7656 	if (bytes > 8 || (bytes & (bytes - 1)))
7657 		goto emul_write;
7658 
7659 	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
7660 
7661 	if (gpa == INVALID_GPA ||
7662 	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
7663 		goto emul_write;
7664 
7665 	/*
7666 	 * Emulate the atomic as a straight write to avoid #AC if SLD is
7667 	 * enabled in the host and the access splits a cache line.
7668 	 */
7669 	if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT))
7670 		page_line_mask = ~(cache_line_size() - 1);
7671 	else
7672 		page_line_mask = PAGE_MASK;
7673 
7674 	if (((gpa + bytes - 1) & page_line_mask) != (gpa & page_line_mask))
7675 		goto emul_write;
7676 
7677 	hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa));
7678 	if (kvm_is_error_hva(hva))
7679 		goto emul_write;
7680 
7681 	hva += offset_in_page(gpa);
7682 
7683 	switch (bytes) {
7684 	case 1:
7685 		r = emulator_try_cmpxchg_user(u8, hva, old, new);
7686 		break;
7687 	case 2:
7688 		r = emulator_try_cmpxchg_user(u16, hva, old, new);
7689 		break;
7690 	case 4:
7691 		r = emulator_try_cmpxchg_user(u32, hva, old, new);
7692 		break;
7693 	case 8:
7694 		r = emulator_try_cmpxchg_user(u64, hva, old, new);
7695 		break;
7696 	default:
7697 		BUG();
7698 	}
7699 
7700 	if (r < 0)
7701 		return X86EMUL_UNHANDLEABLE;
7702 	if (r)
7703 		return X86EMUL_CMPXCHG_FAILED;
7704 
7705 	kvm_page_track_write(vcpu, gpa, new, bytes);
7706 
7707 	return X86EMUL_CONTINUE;
7708 
7709 emul_write:
7710 	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
7711 
7712 	return emulator_write_emulated(ctxt, addr, new, bytes, exception);
7713 }
7714 
emulator_pio_in_out(struct kvm_vcpu * vcpu,int size,unsigned short port,void * data,unsigned int count,bool in)7715 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
7716 			       unsigned short port, void *data,
7717 			       unsigned int count, bool in)
7718 {
7719 	unsigned i;
7720 	int r;
7721 
7722 	WARN_ON_ONCE(vcpu->arch.pio.count);
7723 	for (i = 0; i < count; i++) {
7724 		if (in)
7725 			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, port, size, data);
7726 		else
7727 			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS, port, size, data);
7728 
7729 		if (r) {
7730 			if (i == 0)
7731 				goto userspace_io;
7732 
7733 			/*
7734 			 * Userspace must have unregistered the device while PIO
7735 			 * was running.  Drop writes / read as 0.
7736 			 */
7737 			if (in)
7738 				memset(data, 0, size * (count - i));
7739 			break;
7740 		}
7741 
7742 		data += size;
7743 	}
7744 	return 1;
7745 
7746 userspace_io:
7747 	vcpu->arch.pio.port = port;
7748 	vcpu->arch.pio.in = in;
7749 	vcpu->arch.pio.count = count;
7750 	vcpu->arch.pio.size = size;
7751 
7752 	if (in)
7753 		memset(vcpu->arch.pio_data, 0, size * count);
7754 	else
7755 		memcpy(vcpu->arch.pio_data, data, size * count);
7756 
7757 	vcpu->run->exit_reason = KVM_EXIT_IO;
7758 	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
7759 	vcpu->run->io.size = size;
7760 	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
7761 	vcpu->run->io.count = count;
7762 	vcpu->run->io.port = port;
7763 	return 0;
7764 }
7765 
emulator_pio_in(struct kvm_vcpu * vcpu,int size,unsigned short port,void * val,unsigned int count)7766 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,
7767       			   unsigned short port, void *val, unsigned int count)
7768 {
7769 	int r = emulator_pio_in_out(vcpu, size, port, val, count, true);
7770 	if (r)
7771 		trace_kvm_pio(KVM_PIO_IN, port, size, count, val);
7772 
7773 	return r;
7774 }
7775 
complete_emulator_pio_in(struct kvm_vcpu * vcpu,void * val)7776 static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val)
7777 {
7778 	int size = vcpu->arch.pio.size;
7779 	unsigned int count = vcpu->arch.pio.count;
7780 	memcpy(val, vcpu->arch.pio_data, size * count);
7781 	trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data);
7782 	vcpu->arch.pio.count = 0;
7783 }
7784 
emulator_pio_in_emulated(struct x86_emulate_ctxt * ctxt,int size,unsigned short port,void * val,unsigned int count)7785 static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
7786 				    int size, unsigned short port, void *val,
7787 				    unsigned int count)
7788 {
7789 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7790 	if (vcpu->arch.pio.count) {
7791 		/*
7792 		 * Complete a previous iteration that required userspace I/O.
7793 		 * Note, @count isn't guaranteed to match pio.count as userspace
7794 		 * can modify ECX before rerunning the vCPU.  Ignore any such
7795 		 * shenanigans as KVM doesn't support modifying the rep count,
7796 		 * and the emulator ensures @count doesn't overflow the buffer.
7797 		 */
7798 		complete_emulator_pio_in(vcpu, val);
7799 		return 1;
7800 	}
7801 
7802 	return emulator_pio_in(vcpu, size, port, val, count);
7803 }
7804 
emulator_pio_out(struct kvm_vcpu * vcpu,int size,unsigned short port,const void * val,unsigned int count)7805 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,
7806 			    unsigned short port, const void *val,
7807 			    unsigned int count)
7808 {
7809 	trace_kvm_pio(KVM_PIO_OUT, port, size, count, val);
7810 	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
7811 }
7812 
emulator_pio_out_emulated(struct x86_emulate_ctxt * ctxt,int size,unsigned short port,const void * val,unsigned int count)7813 static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
7814 				     int size, unsigned short port,
7815 				     const void *val, unsigned int count)
7816 {
7817 	return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count);
7818 }
7819 
get_segment_base(struct kvm_vcpu * vcpu,int seg)7820 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
7821 {
7822 	return static_call(kvm_x86_get_segment_base)(vcpu, seg);
7823 }
7824 
emulator_invlpg(struct x86_emulate_ctxt * ctxt,ulong address)7825 static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
7826 {
7827 	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
7828 }
7829 
kvm_emulate_wbinvd_noskip(struct kvm_vcpu * vcpu)7830 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
7831 {
7832 	if (!need_emulate_wbinvd(vcpu))
7833 		return X86EMUL_CONTINUE;
7834 
7835 	if (static_call(kvm_x86_has_wbinvd_exit)()) {
7836 		int cpu = get_cpu();
7837 
7838 		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
7839 		on_each_cpu_mask(vcpu->arch.wbinvd_dirty_mask,
7840 				wbinvd_ipi, NULL, 1);
7841 		put_cpu();
7842 		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
7843 	} else
7844 		wbinvd();
7845 	return X86EMUL_CONTINUE;
7846 }
7847 
kvm_emulate_wbinvd(struct kvm_vcpu * vcpu)7848 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
7849 {
7850 	kvm_emulate_wbinvd_noskip(vcpu);
7851 	return kvm_skip_emulated_instruction(vcpu);
7852 }
7853 EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
7854 
7855 
7856 
emulator_wbinvd(struct x86_emulate_ctxt * ctxt)7857 static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
7858 {
7859 	kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
7860 }
7861 
emulator_get_dr(struct x86_emulate_ctxt * ctxt,int dr,unsigned long * dest)7862 static void emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
7863 			    unsigned long *dest)
7864 {
7865 	kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
7866 }
7867 
emulator_set_dr(struct x86_emulate_ctxt * ctxt,int dr,unsigned long value)7868 static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
7869 			   unsigned long value)
7870 {
7871 
7872 	return kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
7873 }
7874 
mk_cr_64(u64 curr_cr,u32 new_val)7875 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
7876 {
7877 	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
7878 }
7879 
emulator_get_cr(struct x86_emulate_ctxt * ctxt,int cr)7880 static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
7881 {
7882 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7883 	unsigned long value;
7884 
7885 	switch (cr) {
7886 	case 0:
7887 		value = kvm_read_cr0(vcpu);
7888 		break;
7889 	case 2:
7890 		value = vcpu->arch.cr2;
7891 		break;
7892 	case 3:
7893 		value = kvm_read_cr3(vcpu);
7894 		break;
7895 	case 4:
7896 		value = kvm_read_cr4(vcpu);
7897 		break;
7898 	case 8:
7899 		value = kvm_get_cr8(vcpu);
7900 		break;
7901 	default:
7902 		kvm_err("%s: unexpected cr %u\n", __func__, cr);
7903 		return 0;
7904 	}
7905 
7906 	return value;
7907 }
7908 
emulator_set_cr(struct x86_emulate_ctxt * ctxt,int cr,ulong val)7909 static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
7910 {
7911 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7912 	int res = 0;
7913 
7914 	switch (cr) {
7915 	case 0:
7916 		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
7917 		break;
7918 	case 2:
7919 		vcpu->arch.cr2 = val;
7920 		break;
7921 	case 3:
7922 		res = kvm_set_cr3(vcpu, val);
7923 		break;
7924 	case 4:
7925 		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
7926 		break;
7927 	case 8:
7928 		res = kvm_set_cr8(vcpu, val);
7929 		break;
7930 	default:
7931 		kvm_err("%s: unexpected cr %u\n", __func__, cr);
7932 		res = -1;
7933 	}
7934 
7935 	return res;
7936 }
7937 
emulator_get_cpl(struct x86_emulate_ctxt * ctxt)7938 static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
7939 {
7940 	return static_call(kvm_x86_get_cpl)(emul_to_vcpu(ctxt));
7941 }
7942 
emulator_get_gdt(struct x86_emulate_ctxt * ctxt,struct desc_ptr * dt)7943 static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
7944 {
7945 	static_call(kvm_x86_get_gdt)(emul_to_vcpu(ctxt), dt);
7946 }
7947 
emulator_get_idt(struct x86_emulate_ctxt * ctxt,struct desc_ptr * dt)7948 static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
7949 {
7950 	static_call(kvm_x86_get_idt)(emul_to_vcpu(ctxt), dt);
7951 }
7952 
emulator_set_gdt(struct x86_emulate_ctxt * ctxt,struct desc_ptr * dt)7953 static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
7954 {
7955 	static_call(kvm_x86_set_gdt)(emul_to_vcpu(ctxt), dt);
7956 }
7957 
emulator_set_idt(struct x86_emulate_ctxt * ctxt,struct desc_ptr * dt)7958 static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
7959 {
7960 	static_call(kvm_x86_set_idt)(emul_to_vcpu(ctxt), dt);
7961 }
7962 
emulator_get_cached_segment_base(struct x86_emulate_ctxt * ctxt,int seg)7963 static unsigned long emulator_get_cached_segment_base(
7964 	struct x86_emulate_ctxt *ctxt, int seg)
7965 {
7966 	return get_segment_base(emul_to_vcpu(ctxt), seg);
7967 }
7968 
emulator_get_segment(struct x86_emulate_ctxt * ctxt,u16 * selector,struct desc_struct * desc,u32 * base3,int seg)7969 static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
7970 				 struct desc_struct *desc, u32 *base3,
7971 				 int seg)
7972 {
7973 	struct kvm_segment var;
7974 
7975 	kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
7976 	*selector = var.selector;
7977 
7978 	if (var.unusable) {
7979 		memset(desc, 0, sizeof(*desc));
7980 		if (base3)
7981 			*base3 = 0;
7982 		return false;
7983 	}
7984 
7985 	if (var.g)
7986 		var.limit >>= 12;
7987 	set_desc_limit(desc, var.limit);
7988 	set_desc_base(desc, (unsigned long)var.base);
7989 #ifdef CONFIG_X86_64
7990 	if (base3)
7991 		*base3 = var.base >> 32;
7992 #endif
7993 	desc->type = var.type;
7994 	desc->s = var.s;
7995 	desc->dpl = var.dpl;
7996 	desc->p = var.present;
7997 	desc->avl = var.avl;
7998 	desc->l = var.l;
7999 	desc->d = var.db;
8000 	desc->g = var.g;
8001 
8002 	return true;
8003 }
8004 
emulator_set_segment(struct x86_emulate_ctxt * ctxt,u16 selector,struct desc_struct * desc,u32 base3,int seg)8005 static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
8006 				 struct desc_struct *desc, u32 base3,
8007 				 int seg)
8008 {
8009 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8010 	struct kvm_segment var;
8011 
8012 	var.selector = selector;
8013 	var.base = get_desc_base(desc);
8014 #ifdef CONFIG_X86_64
8015 	var.base |= ((u64)base3) << 32;
8016 #endif
8017 	var.limit = get_desc_limit(desc);
8018 	if (desc->g)
8019 		var.limit = (var.limit << 12) | 0xfff;
8020 	var.type = desc->type;
8021 	var.dpl = desc->dpl;
8022 	var.db = desc->d;
8023 	var.s = desc->s;
8024 	var.l = desc->l;
8025 	var.g = desc->g;
8026 	var.avl = desc->avl;
8027 	var.present = desc->p;
8028 	var.unusable = !var.present;
8029 	var.padding = 0;
8030 
8031 	kvm_set_segment(vcpu, &var, seg);
8032 	return;
8033 }
8034 
emulator_get_msr_with_filter(struct x86_emulate_ctxt * ctxt,u32 msr_index,u64 * pdata)8035 static int emulator_get_msr_with_filter(struct x86_emulate_ctxt *ctxt,
8036 					u32 msr_index, u64 *pdata)
8037 {
8038 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8039 	int r;
8040 
8041 	r = kvm_get_msr_with_filter(vcpu, msr_index, pdata);
8042 	if (r < 0)
8043 		return X86EMUL_UNHANDLEABLE;
8044 
8045 	if (r) {
8046 		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0,
8047 				       complete_emulated_rdmsr, r))
8048 			return X86EMUL_IO_NEEDED;
8049 
8050 		trace_kvm_msr_read_ex(msr_index);
8051 		return X86EMUL_PROPAGATE_FAULT;
8052 	}
8053 
8054 	trace_kvm_msr_read(msr_index, *pdata);
8055 	return X86EMUL_CONTINUE;
8056 }
8057 
emulator_set_msr_with_filter(struct x86_emulate_ctxt * ctxt,u32 msr_index,u64 data)8058 static int emulator_set_msr_with_filter(struct x86_emulate_ctxt *ctxt,
8059 					u32 msr_index, u64 data)
8060 {
8061 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8062 	int r;
8063 
8064 	r = kvm_set_msr_with_filter(vcpu, msr_index, data);
8065 	if (r < 0)
8066 		return X86EMUL_UNHANDLEABLE;
8067 
8068 	if (r) {
8069 		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data,
8070 				       complete_emulated_msr_access, r))
8071 			return X86EMUL_IO_NEEDED;
8072 
8073 		trace_kvm_msr_write_ex(msr_index, data);
8074 		return X86EMUL_PROPAGATE_FAULT;
8075 	}
8076 
8077 	trace_kvm_msr_write(msr_index, data);
8078 	return X86EMUL_CONTINUE;
8079 }
8080 
emulator_get_msr(struct x86_emulate_ctxt * ctxt,u32 msr_index,u64 * pdata)8081 static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
8082 			    u32 msr_index, u64 *pdata)
8083 {
8084 	return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
8085 }
8086 
emulator_set_msr(struct x86_emulate_ctxt * ctxt,u32 msr_index,u64 data)8087 static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
8088 			    u32 msr_index, u64 data)
8089 {
8090 	return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);
8091 }
8092 
emulator_get_smbase(struct x86_emulate_ctxt * ctxt)8093 static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)
8094 {
8095 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8096 
8097 	return vcpu->arch.smbase;
8098 }
8099 
emulator_set_smbase(struct x86_emulate_ctxt * ctxt,u64 smbase)8100 static void emulator_set_smbase(struct x86_emulate_ctxt *ctxt, u64 smbase)
8101 {
8102 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8103 
8104 	vcpu->arch.smbase = smbase;
8105 }
8106 
emulator_check_pmc(struct x86_emulate_ctxt * ctxt,u32 pmc)8107 static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt,
8108 			      u32 pmc)
8109 {
8110 	if (kvm_pmu_is_valid_rdpmc_ecx(emul_to_vcpu(ctxt), pmc))
8111 		return 0;
8112 	return -EINVAL;
8113 }
8114 
emulator_read_pmc(struct x86_emulate_ctxt * ctxt,u32 pmc,u64 * pdata)8115 static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
8116 			     u32 pmc, u64 *pdata)
8117 {
8118 	return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata);
8119 }
8120 
emulator_halt(struct x86_emulate_ctxt * ctxt)8121 static void emulator_halt(struct x86_emulate_ctxt *ctxt)
8122 {
8123 	emul_to_vcpu(ctxt)->arch.halt_request = 1;
8124 }
8125 
emulator_intercept(struct x86_emulate_ctxt * ctxt,struct x86_instruction_info * info,enum x86_intercept_stage stage)8126 static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
8127 			      struct x86_instruction_info *info,
8128 			      enum x86_intercept_stage stage)
8129 {
8130 	return static_call(kvm_x86_check_intercept)(emul_to_vcpu(ctxt), info, stage,
8131 					    &ctxt->exception);
8132 }
8133 
emulator_get_cpuid(struct x86_emulate_ctxt * ctxt,u32 * eax,u32 * ebx,u32 * ecx,u32 * edx,bool exact_only)8134 static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
8135 			      u32 *eax, u32 *ebx, u32 *ecx, u32 *edx,
8136 			      bool exact_only)
8137 {
8138 	return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, exact_only);
8139 }
8140 
emulator_guest_has_long_mode(struct x86_emulate_ctxt * ctxt)8141 static bool emulator_guest_has_long_mode(struct x86_emulate_ctxt *ctxt)
8142 {
8143 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_LM);
8144 }
8145 
emulator_guest_has_movbe(struct x86_emulate_ctxt * ctxt)8146 static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt)
8147 {
8148 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_MOVBE);
8149 }
8150 
emulator_guest_has_fxsr(struct x86_emulate_ctxt * ctxt)8151 static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt)
8152 {
8153 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR);
8154 }
8155 
emulator_guest_has_rdpid(struct x86_emulate_ctxt * ctxt)8156 static bool emulator_guest_has_rdpid(struct x86_emulate_ctxt *ctxt)
8157 {
8158 	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_RDPID);
8159 }
8160 
emulator_read_gpr(struct x86_emulate_ctxt * ctxt,unsigned reg)8161 static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
8162 {
8163 	return kvm_register_read_raw(emul_to_vcpu(ctxt), reg);
8164 }
8165 
emulator_write_gpr(struct x86_emulate_ctxt * ctxt,unsigned reg,ulong val)8166 static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
8167 {
8168 	kvm_register_write_raw(emul_to_vcpu(ctxt), reg, val);
8169 }
8170 
emulator_set_nmi_mask(struct x86_emulate_ctxt * ctxt,bool masked)8171 static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)
8172 {
8173 	static_call(kvm_x86_set_nmi_mask)(emul_to_vcpu(ctxt), masked);
8174 }
8175 
emulator_get_hflags(struct x86_emulate_ctxt * ctxt)8176 static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)
8177 {
8178 	return emul_to_vcpu(ctxt)->arch.hflags;
8179 }
8180 
emulator_exiting_smm(struct x86_emulate_ctxt * ctxt)8181 static void emulator_exiting_smm(struct x86_emulate_ctxt *ctxt)
8182 {
8183 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8184 
8185 	kvm_smm_changed(vcpu, false);
8186 }
8187 
emulator_leave_smm(struct x86_emulate_ctxt * ctxt,const char * smstate)8188 static int emulator_leave_smm(struct x86_emulate_ctxt *ctxt,
8189 				  const char *smstate)
8190 {
8191 	return static_call(kvm_x86_leave_smm)(emul_to_vcpu(ctxt), smstate);
8192 }
8193 
emulator_triple_fault(struct x86_emulate_ctxt * ctxt)8194 static void emulator_triple_fault(struct x86_emulate_ctxt *ctxt)
8195 {
8196 	kvm_make_request(KVM_REQ_TRIPLE_FAULT, emul_to_vcpu(ctxt));
8197 }
8198 
emulator_set_xcr(struct x86_emulate_ctxt * ctxt,u32 index,u64 xcr)8199 static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)
8200 {
8201 	return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);
8202 }
8203 
emulator_vm_bugged(struct x86_emulate_ctxt * ctxt)8204 static void emulator_vm_bugged(struct x86_emulate_ctxt *ctxt)
8205 {
8206 	struct kvm *kvm = emul_to_vcpu(ctxt)->kvm;
8207 
8208 	if (!kvm->vm_bugged)
8209 		kvm_vm_bugged(kvm);
8210 }
8211 
8212 static const struct x86_emulate_ops emulate_ops = {
8213 	.vm_bugged           = emulator_vm_bugged,
8214 	.read_gpr            = emulator_read_gpr,
8215 	.write_gpr           = emulator_write_gpr,
8216 	.read_std            = emulator_read_std,
8217 	.write_std           = emulator_write_std,
8218 	.read_phys           = kvm_read_guest_phys_system,
8219 	.fetch               = kvm_fetch_guest_virt,
8220 	.read_emulated       = emulator_read_emulated,
8221 	.write_emulated      = emulator_write_emulated,
8222 	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
8223 	.invlpg              = emulator_invlpg,
8224 	.pio_in_emulated     = emulator_pio_in_emulated,
8225 	.pio_out_emulated    = emulator_pio_out_emulated,
8226 	.get_segment         = emulator_get_segment,
8227 	.set_segment         = emulator_set_segment,
8228 	.get_cached_segment_base = emulator_get_cached_segment_base,
8229 	.get_gdt             = emulator_get_gdt,
8230 	.get_idt	     = emulator_get_idt,
8231 	.set_gdt             = emulator_set_gdt,
8232 	.set_idt	     = emulator_set_idt,
8233 	.get_cr              = emulator_get_cr,
8234 	.set_cr              = emulator_set_cr,
8235 	.cpl                 = emulator_get_cpl,
8236 	.get_dr              = emulator_get_dr,
8237 	.set_dr              = emulator_set_dr,
8238 	.get_smbase          = emulator_get_smbase,
8239 	.set_smbase          = emulator_set_smbase,
8240 	.set_msr_with_filter = emulator_set_msr_with_filter,
8241 	.get_msr_with_filter = emulator_get_msr_with_filter,
8242 	.set_msr             = emulator_set_msr,
8243 	.get_msr             = emulator_get_msr,
8244 	.check_pmc	     = emulator_check_pmc,
8245 	.read_pmc            = emulator_read_pmc,
8246 	.halt                = emulator_halt,
8247 	.wbinvd              = emulator_wbinvd,
8248 	.fix_hypercall       = emulator_fix_hypercall,
8249 	.intercept           = emulator_intercept,
8250 	.get_cpuid           = emulator_get_cpuid,
8251 	.guest_has_long_mode = emulator_guest_has_long_mode,
8252 	.guest_has_movbe     = emulator_guest_has_movbe,
8253 	.guest_has_fxsr      = emulator_guest_has_fxsr,
8254 	.guest_has_rdpid     = emulator_guest_has_rdpid,
8255 	.set_nmi_mask        = emulator_set_nmi_mask,
8256 	.get_hflags          = emulator_get_hflags,
8257 	.exiting_smm         = emulator_exiting_smm,
8258 	.leave_smm           = emulator_leave_smm,
8259 	.triple_fault        = emulator_triple_fault,
8260 	.set_xcr             = emulator_set_xcr,
8261 };
8262 
toggle_interruptibility(struct kvm_vcpu * vcpu,u32 mask)8263 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
8264 {
8265 	u32 int_shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
8266 	/*
8267 	 * an sti; sti; sequence only disable interrupts for the first
8268 	 * instruction. So, if the last instruction, be it emulated or
8269 	 * not, left the system with the INT_STI flag enabled, it
8270 	 * means that the last instruction is an sti. We should not
8271 	 * leave the flag on in this case. The same goes for mov ss
8272 	 */
8273 	if (int_shadow & mask)
8274 		mask = 0;
8275 	if (unlikely(int_shadow || mask)) {
8276 		static_call(kvm_x86_set_interrupt_shadow)(vcpu, mask);
8277 		if (!mask)
8278 			kvm_make_request(KVM_REQ_EVENT, vcpu);
8279 	}
8280 }
8281 
inject_emulated_exception(struct kvm_vcpu * vcpu)8282 static void inject_emulated_exception(struct kvm_vcpu *vcpu)
8283 {
8284 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8285 
8286 	if (ctxt->exception.vector == PF_VECTOR)
8287 		kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);
8288 	else if (ctxt->exception.error_code_valid)
8289 		kvm_queue_exception_e(vcpu, ctxt->exception.vector,
8290 				      ctxt->exception.error_code);
8291 	else
8292 		kvm_queue_exception(vcpu, ctxt->exception.vector);
8293 }
8294 
alloc_emulate_ctxt(struct kvm_vcpu * vcpu)8295 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)
8296 {
8297 	struct x86_emulate_ctxt *ctxt;
8298 
8299 	ctxt = kmem_cache_zalloc(x86_emulator_cache, GFP_KERNEL_ACCOUNT);
8300 	if (!ctxt) {
8301 		pr_err("kvm: failed to allocate vcpu's emulator\n");
8302 		return NULL;
8303 	}
8304 
8305 	ctxt->vcpu = vcpu;
8306 	ctxt->ops = &emulate_ops;
8307 	vcpu->arch.emulate_ctxt = ctxt;
8308 
8309 	return ctxt;
8310 }
8311 
init_emulate_ctxt(struct kvm_vcpu * vcpu)8312 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
8313 {
8314 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8315 	int cs_db, cs_l;
8316 
8317 	static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
8318 
8319 	ctxt->gpa_available = false;
8320 	ctxt->eflags = kvm_get_rflags(vcpu);
8321 	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
8322 
8323 	ctxt->eip = kvm_rip_read(vcpu);
8324 	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :
8325 		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :
8326 		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :
8327 		     cs_db				? X86EMUL_MODE_PROT32 :
8328 							  X86EMUL_MODE_PROT16;
8329 	BUILD_BUG_ON(HF_GUEST_MASK != X86EMUL_GUEST_MASK);
8330 	BUILD_BUG_ON(HF_SMM_MASK != X86EMUL_SMM_MASK);
8331 	BUILD_BUG_ON(HF_SMM_INSIDE_NMI_MASK != X86EMUL_SMM_INSIDE_NMI_MASK);
8332 
8333 	ctxt->interruptibility = 0;
8334 	ctxt->have_exception = false;
8335 	ctxt->exception.vector = -1;
8336 	ctxt->perm_ok = false;
8337 
8338 	init_decode_cache(ctxt);
8339 	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
8340 }
8341 
kvm_inject_realmode_interrupt(struct kvm_vcpu * vcpu,int irq,int inc_eip)8342 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
8343 {
8344 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8345 	int ret;
8346 
8347 	init_emulate_ctxt(vcpu);
8348 
8349 	ctxt->op_bytes = 2;
8350 	ctxt->ad_bytes = 2;
8351 	ctxt->_eip = ctxt->eip + inc_eip;
8352 	ret = emulate_int_real(ctxt, irq);
8353 
8354 	if (ret != X86EMUL_CONTINUE) {
8355 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
8356 	} else {
8357 		ctxt->eip = ctxt->_eip;
8358 		kvm_rip_write(vcpu, ctxt->eip);
8359 		kvm_set_rflags(vcpu, ctxt->eflags);
8360 	}
8361 }
8362 EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);
8363 
prepare_emulation_failure_exit(struct kvm_vcpu * vcpu,u64 * data,u8 ndata,u8 * insn_bytes,u8 insn_size)8364 static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,
8365 					   u8 ndata, u8 *insn_bytes, u8 insn_size)
8366 {
8367 	struct kvm_run *run = vcpu->run;
8368 	u64 info[5];
8369 	u8 info_start;
8370 
8371 	/*
8372 	 * Zero the whole array used to retrieve the exit info, as casting to
8373 	 * u32 for select entries will leave some chunks uninitialized.
8374 	 */
8375 	memset(&info, 0, sizeof(info));
8376 
8377 	static_call(kvm_x86_get_exit_info)(vcpu, (u32 *)&info[0], &info[1],
8378 					   &info[2], (u32 *)&info[3],
8379 					   (u32 *)&info[4]);
8380 
8381 	run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
8382 	run->emulation_failure.suberror = KVM_INTERNAL_ERROR_EMULATION;
8383 
8384 	/*
8385 	 * There's currently space for 13 entries, but 5 are used for the exit
8386 	 * reason and info.  Restrict to 4 to reduce the maintenance burden
8387 	 * when expanding kvm_run.emulation_failure in the future.
8388 	 */
8389 	if (WARN_ON_ONCE(ndata > 4))
8390 		ndata = 4;
8391 
8392 	/* Always include the flags as a 'data' entry. */
8393 	info_start = 1;
8394 	run->emulation_failure.flags = 0;
8395 
8396 	if (insn_size) {
8397 		BUILD_BUG_ON((sizeof(run->emulation_failure.insn_size) +
8398 			      sizeof(run->emulation_failure.insn_bytes) != 16));
8399 		info_start += 2;
8400 		run->emulation_failure.flags |=
8401 			KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES;
8402 		run->emulation_failure.insn_size = insn_size;
8403 		memset(run->emulation_failure.insn_bytes, 0x90,
8404 		       sizeof(run->emulation_failure.insn_bytes));
8405 		memcpy(run->emulation_failure.insn_bytes, insn_bytes, insn_size);
8406 	}
8407 
8408 	memcpy(&run->internal.data[info_start], info, sizeof(info));
8409 	memcpy(&run->internal.data[info_start + ARRAY_SIZE(info)], data,
8410 	       ndata * sizeof(data[0]));
8411 
8412 	run->emulation_failure.ndata = info_start + ARRAY_SIZE(info) + ndata;
8413 }
8414 
prepare_emulation_ctxt_failure_exit(struct kvm_vcpu * vcpu)8415 static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu)
8416 {
8417 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8418 
8419 	prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data,
8420 				       ctxt->fetch.end - ctxt->fetch.data);
8421 }
8422 
__kvm_prepare_emulation_failure_exit(struct kvm_vcpu * vcpu,u64 * data,u8 ndata)8423 void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,
8424 					  u8 ndata)
8425 {
8426 	prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0);
8427 }
8428 EXPORT_SYMBOL_GPL(__kvm_prepare_emulation_failure_exit);
8429 
kvm_prepare_emulation_failure_exit(struct kvm_vcpu * vcpu)8430 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu)
8431 {
8432 	__kvm_prepare_emulation_failure_exit(vcpu, NULL, 0);
8433 }
8434 EXPORT_SYMBOL_GPL(kvm_prepare_emulation_failure_exit);
8435 
handle_emulation_failure(struct kvm_vcpu * vcpu,int emulation_type)8436 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
8437 {
8438 	struct kvm *kvm = vcpu->kvm;
8439 
8440 	++vcpu->stat.insn_emulation_fail;
8441 	trace_kvm_emulate_insn_failed(vcpu);
8442 
8443 	if (emulation_type & EMULTYPE_VMWARE_GP) {
8444 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
8445 		return 1;
8446 	}
8447 
8448 	if (kvm->arch.exit_on_emulation_error ||
8449 	    (emulation_type & EMULTYPE_SKIP)) {
8450 		prepare_emulation_ctxt_failure_exit(vcpu);
8451 		return 0;
8452 	}
8453 
8454 	kvm_queue_exception(vcpu, UD_VECTOR);
8455 
8456 	if (!is_guest_mode(vcpu) && static_call(kvm_x86_get_cpl)(vcpu) == 0) {
8457 		prepare_emulation_ctxt_failure_exit(vcpu);
8458 		return 0;
8459 	}
8460 
8461 	return 1;
8462 }
8463 
reexecute_instruction(struct kvm_vcpu * vcpu,gpa_t cr2_or_gpa,bool write_fault_to_shadow_pgtable,int emulation_type)8464 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
8465 				  bool write_fault_to_shadow_pgtable,
8466 				  int emulation_type)
8467 {
8468 	gpa_t gpa = cr2_or_gpa;
8469 	kvm_pfn_t pfn;
8470 
8471 	if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF))
8472 		return false;
8473 
8474 	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
8475 	    WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))
8476 		return false;
8477 
8478 	if (!vcpu->arch.mmu->root_role.direct) {
8479 		/*
8480 		 * Write permission should be allowed since only
8481 		 * write access need to be emulated.
8482 		 */
8483 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
8484 
8485 		/*
8486 		 * If the mapping is invalid in guest, let cpu retry
8487 		 * it to generate fault.
8488 		 */
8489 		if (gpa == INVALID_GPA)
8490 			return true;
8491 	}
8492 
8493 	/*
8494 	 * Do not retry the unhandleable instruction if it faults on the
8495 	 * readonly host memory, otherwise it will goto a infinite loop:
8496 	 * retry instruction -> write #PF -> emulation fail -> retry
8497 	 * instruction -> ...
8498 	 */
8499 	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
8500 
8501 	/*
8502 	 * If the instruction failed on the error pfn, it can not be fixed,
8503 	 * report the error to userspace.
8504 	 */
8505 	if (is_error_noslot_pfn(pfn))
8506 		return false;
8507 
8508 	kvm_release_pfn_clean(pfn);
8509 
8510 	/* The instructions are well-emulated on direct mmu. */
8511 	if (vcpu->arch.mmu->root_role.direct) {
8512 		unsigned int indirect_shadow_pages;
8513 
8514 		write_lock(&vcpu->kvm->mmu_lock);
8515 		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
8516 		write_unlock(&vcpu->kvm->mmu_lock);
8517 
8518 		if (indirect_shadow_pages)
8519 			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
8520 
8521 		return true;
8522 	}
8523 
8524 	/*
8525 	 * if emulation was due to access to shadowed page table
8526 	 * and it failed try to unshadow page and re-enter the
8527 	 * guest to let CPU execute the instruction.
8528 	 */
8529 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
8530 
8531 	/*
8532 	 * If the access faults on its page table, it can not
8533 	 * be fixed by unprotecting shadow page and it should
8534 	 * be reported to userspace.
8535 	 */
8536 	return !write_fault_to_shadow_pgtable;
8537 }
8538 
retry_instruction(struct x86_emulate_ctxt * ctxt,gpa_t cr2_or_gpa,int emulation_type)8539 static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
8540 			      gpa_t cr2_or_gpa,  int emulation_type)
8541 {
8542 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8543 	unsigned long last_retry_eip, last_retry_addr, gpa = cr2_or_gpa;
8544 
8545 	last_retry_eip = vcpu->arch.last_retry_eip;
8546 	last_retry_addr = vcpu->arch.last_retry_addr;
8547 
8548 	/*
8549 	 * If the emulation is caused by #PF and it is non-page_table
8550 	 * writing instruction, it means the VM-EXIT is caused by shadow
8551 	 * page protected, we can zap the shadow page and retry this
8552 	 * instruction directly.
8553 	 *
8554 	 * Note: if the guest uses a non-page-table modifying instruction
8555 	 * on the PDE that points to the instruction, then we will unmap
8556 	 * the instruction and go to an infinite loop. So, we cache the
8557 	 * last retried eip and the last fault address, if we meet the eip
8558 	 * and the address again, we can break out of the potential infinite
8559 	 * loop.
8560 	 */
8561 	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;
8562 
8563 	if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF))
8564 		return false;
8565 
8566 	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
8567 	    WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))
8568 		return false;
8569 
8570 	if (x86_page_table_writing_insn(ctxt))
8571 		return false;
8572 
8573 	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2_or_gpa)
8574 		return false;
8575 
8576 	vcpu->arch.last_retry_eip = ctxt->eip;
8577 	vcpu->arch.last_retry_addr = cr2_or_gpa;
8578 
8579 	if (!vcpu->arch.mmu->root_role.direct)
8580 		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
8581 
8582 	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
8583 
8584 	return true;
8585 }
8586 
8587 static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
8588 static int complete_emulated_pio(struct kvm_vcpu *vcpu);
8589 
kvm_smm_changed(struct kvm_vcpu * vcpu,bool entering_smm)8590 static void kvm_smm_changed(struct kvm_vcpu *vcpu, bool entering_smm)
8591 {
8592 	trace_kvm_smm_transition(vcpu->vcpu_id, vcpu->arch.smbase, entering_smm);
8593 
8594 	if (entering_smm) {
8595 		vcpu->arch.hflags |= HF_SMM_MASK;
8596 	} else {
8597 		vcpu->arch.hflags &= ~(HF_SMM_MASK | HF_SMM_INSIDE_NMI_MASK);
8598 
8599 		/* Process a latched INIT or SMI, if any.  */
8600 		kvm_make_request(KVM_REQ_EVENT, vcpu);
8601 
8602 		/*
8603 		 * Even if KVM_SET_SREGS2 loaded PDPTRs out of band,
8604 		 * on SMM exit we still need to reload them from
8605 		 * guest memory
8606 		 */
8607 		vcpu->arch.pdptrs_from_userspace = false;
8608 	}
8609 
8610 	kvm_mmu_reset_context(vcpu);
8611 }
8612 
kvm_vcpu_check_hw_bp(unsigned long addr,u32 type,u32 dr7,unsigned long * db)8613 static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
8614 				unsigned long *db)
8615 {
8616 	u32 dr6 = 0;
8617 	int i;
8618 	u32 enable, rwlen;
8619 
8620 	enable = dr7;
8621 	rwlen = dr7 >> 16;
8622 	for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4)
8623 		if ((enable & 3) && (rwlen & 15) == type && db[i] == addr)
8624 			dr6 |= (1 << i);
8625 	return dr6;
8626 }
8627 
kvm_vcpu_do_singlestep(struct kvm_vcpu * vcpu)8628 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)
8629 {
8630 	struct kvm_run *kvm_run = vcpu->run;
8631 
8632 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
8633 		kvm_run->debug.arch.dr6 = DR6_BS | DR6_ACTIVE_LOW;
8634 		kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
8635 		kvm_run->debug.arch.exception = DB_VECTOR;
8636 		kvm_run->exit_reason = KVM_EXIT_DEBUG;
8637 		return 0;
8638 	}
8639 	kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
8640 	return 1;
8641 }
8642 
kvm_skip_emulated_instruction(struct kvm_vcpu * vcpu)8643 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
8644 {
8645 	unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
8646 	int r;
8647 
8648 	r = static_call(kvm_x86_skip_emulated_instruction)(vcpu);
8649 	if (unlikely(!r))
8650 		return 0;
8651 
8652 	kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);
8653 
8654 	/*
8655 	 * rflags is the old, "raw" value of the flags.  The new value has
8656 	 * not been saved yet.
8657 	 *
8658 	 * This is correct even for TF set by the guest, because "the
8659 	 * processor will not generate this exception after the instruction
8660 	 * that sets the TF flag".
8661 	 */
8662 	if (unlikely(rflags & X86_EFLAGS_TF))
8663 		r = kvm_vcpu_do_singlestep(vcpu);
8664 	return r;
8665 }
8666 EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction);
8667 
kvm_is_code_breakpoint_inhibited(struct kvm_vcpu * vcpu)8668 static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu)
8669 {
8670 	u32 shadow;
8671 
8672 	if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF)
8673 		return true;
8674 
8675 	/*
8676 	 * Intel CPUs inhibit code #DBs when MOV/POP SS blocking is active,
8677 	 * but AMD CPUs do not.  MOV/POP SS blocking is rare, check that first
8678 	 * to avoid the relatively expensive CPUID lookup.
8679 	 */
8680 	shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
8681 	return (shadow & KVM_X86_SHADOW_INT_MOV_SS) &&
8682 	       guest_cpuid_is_intel(vcpu);
8683 }
8684 
kvm_vcpu_check_code_breakpoint(struct kvm_vcpu * vcpu,int emulation_type,int * r)8685 static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu,
8686 					   int emulation_type, int *r)
8687 {
8688 	WARN_ON_ONCE(emulation_type & EMULTYPE_NO_DECODE);
8689 
8690 	/*
8691 	 * Do not check for code breakpoints if hardware has already done the
8692 	 * checks, as inferred from the emulation type.  On NO_DECODE and SKIP,
8693 	 * the instruction has passed all exception checks, and all intercepted
8694 	 * exceptions that trigger emulation have lower priority than code
8695 	 * breakpoints, i.e. the fact that the intercepted exception occurred
8696 	 * means any code breakpoints have already been serviced.
8697 	 *
8698 	 * Note, KVM needs to check for code #DBs on EMULTYPE_TRAP_UD_FORCED as
8699 	 * hardware has checked the RIP of the magic prefix, but not the RIP of
8700 	 * the instruction being emulated.  The intent of forced emulation is
8701 	 * to behave as if KVM intercepted the instruction without an exception
8702 	 * and without a prefix.
8703 	 */
8704 	if (emulation_type & (EMULTYPE_NO_DECODE | EMULTYPE_SKIP |
8705 			      EMULTYPE_TRAP_UD | EMULTYPE_VMWARE_GP | EMULTYPE_PF))
8706 		return false;
8707 
8708 	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
8709 	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
8710 		struct kvm_run *kvm_run = vcpu->run;
8711 		unsigned long eip = kvm_get_linear_rip(vcpu);
8712 		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
8713 					   vcpu->arch.guest_debug_dr7,
8714 					   vcpu->arch.eff_db);
8715 
8716 		if (dr6 != 0) {
8717 			kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW;
8718 			kvm_run->debug.arch.pc = eip;
8719 			kvm_run->debug.arch.exception = DB_VECTOR;
8720 			kvm_run->exit_reason = KVM_EXIT_DEBUG;
8721 			*r = 0;
8722 			return true;
8723 		}
8724 	}
8725 
8726 	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
8727 	    !kvm_is_code_breakpoint_inhibited(vcpu)) {
8728 		unsigned long eip = kvm_get_linear_rip(vcpu);
8729 		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
8730 					   vcpu->arch.dr7,
8731 					   vcpu->arch.db);
8732 
8733 		if (dr6 != 0) {
8734 			kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
8735 			*r = 1;
8736 			return true;
8737 		}
8738 	}
8739 
8740 	return false;
8741 }
8742 
is_vmware_backdoor_opcode(struct x86_emulate_ctxt * ctxt)8743 static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt)
8744 {
8745 	switch (ctxt->opcode_len) {
8746 	case 1:
8747 		switch (ctxt->b) {
8748 		case 0xe4:	/* IN */
8749 		case 0xe5:
8750 		case 0xec:
8751 		case 0xed:
8752 		case 0xe6:	/* OUT */
8753 		case 0xe7:
8754 		case 0xee:
8755 		case 0xef:
8756 		case 0x6c:	/* INS */
8757 		case 0x6d:
8758 		case 0x6e:	/* OUTS */
8759 		case 0x6f:
8760 			return true;
8761 		}
8762 		break;
8763 	case 2:
8764 		switch (ctxt->b) {
8765 		case 0x33:	/* RDPMC */
8766 			return true;
8767 		}
8768 		break;
8769 	}
8770 
8771 	return false;
8772 }
8773 
8774 /*
8775  * Decode an instruction for emulation.  The caller is responsible for handling
8776  * code breakpoints.  Note, manually detecting code breakpoints is unnecessary
8777  * (and wrong) when emulating on an intercepted fault-like exception[*], as
8778  * code breakpoints have higher priority and thus have already been done by
8779  * hardware.
8780  *
8781  * [*] Except #MC, which is higher priority, but KVM should never emulate in
8782  *     response to a machine check.
8783  */
x86_decode_emulated_instruction(struct kvm_vcpu * vcpu,int emulation_type,void * insn,int insn_len)8784 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,
8785 				    void *insn, int insn_len)
8786 {
8787 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8788 	int r;
8789 
8790 	init_emulate_ctxt(vcpu);
8791 
8792 	r = x86_decode_insn(ctxt, insn, insn_len, emulation_type);
8793 
8794 	trace_kvm_emulate_insn_start(vcpu);
8795 	++vcpu->stat.insn_emulation;
8796 
8797 	return r;
8798 }
8799 EXPORT_SYMBOL_GPL(x86_decode_emulated_instruction);
8800 
x86_emulate_instruction(struct kvm_vcpu * vcpu,gpa_t cr2_or_gpa,int emulation_type,void * insn,int insn_len)8801 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
8802 			    int emulation_type, void *insn, int insn_len)
8803 {
8804 	int r;
8805 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8806 	bool writeback = true;
8807 	bool write_fault_to_spt;
8808 
8809 	if (unlikely(!kvm_can_emulate_insn(vcpu, emulation_type, insn, insn_len)))
8810 		return 1;
8811 
8812 	vcpu->arch.l1tf_flush_l1d = true;
8813 
8814 	/*
8815 	 * Clear write_fault_to_shadow_pgtable here to ensure it is
8816 	 * never reused.
8817 	 */
8818 	write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
8819 	vcpu->arch.write_fault_to_shadow_pgtable = false;
8820 
8821 	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
8822 		kvm_clear_exception_queue(vcpu);
8823 
8824 		/*
8825 		 * Return immediately if RIP hits a code breakpoint, such #DBs
8826 		 * are fault-like and are higher priority than any faults on
8827 		 * the code fetch itself.
8828 		 */
8829 		if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r))
8830 			return r;
8831 
8832 		r = x86_decode_emulated_instruction(vcpu, emulation_type,
8833 						    insn, insn_len);
8834 		if (r != EMULATION_OK)  {
8835 			if ((emulation_type & EMULTYPE_TRAP_UD) ||
8836 			    (emulation_type & EMULTYPE_TRAP_UD_FORCED)) {
8837 				kvm_queue_exception(vcpu, UD_VECTOR);
8838 				return 1;
8839 			}
8840 			if (reexecute_instruction(vcpu, cr2_or_gpa,
8841 						  write_fault_to_spt,
8842 						  emulation_type))
8843 				return 1;
8844 			if (ctxt->have_exception) {
8845 				/*
8846 				 * #UD should result in just EMULATION_FAILED, and trap-like
8847 				 * exception should not be encountered during decode.
8848 				 */
8849 				WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||
8850 					     exception_type(ctxt->exception.vector) == EXCPT_TRAP);
8851 				inject_emulated_exception(vcpu);
8852 				return 1;
8853 			}
8854 			return handle_emulation_failure(vcpu, emulation_type);
8855 		}
8856 	}
8857 
8858 	if ((emulation_type & EMULTYPE_VMWARE_GP) &&
8859 	    !is_vmware_backdoor_opcode(ctxt)) {
8860 		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
8861 		return 1;
8862 	}
8863 
8864 	/*
8865 	 * EMULTYPE_SKIP without EMULTYPE_COMPLETE_USER_EXIT is intended for
8866 	 * use *only* by vendor callbacks for kvm_skip_emulated_instruction().
8867 	 * The caller is responsible for updating interruptibility state and
8868 	 * injecting single-step #DBs.
8869 	 */
8870 	if (emulation_type & EMULTYPE_SKIP) {
8871 		if (ctxt->mode != X86EMUL_MODE_PROT64)
8872 			ctxt->eip = (u32)ctxt->_eip;
8873 		else
8874 			ctxt->eip = ctxt->_eip;
8875 
8876 		if (emulation_type & EMULTYPE_COMPLETE_USER_EXIT) {
8877 			r = 1;
8878 			goto writeback;
8879 		}
8880 
8881 		kvm_rip_write(vcpu, ctxt->eip);
8882 		if (ctxt->eflags & X86_EFLAGS_RF)
8883 			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
8884 		return 1;
8885 	}
8886 
8887 	if (retry_instruction(ctxt, cr2_or_gpa, emulation_type))
8888 		return 1;
8889 
8890 	/* this is needed for vmware backdoor interface to work since it
8891 	   changes registers values  during IO operation */
8892 	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
8893 		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
8894 		emulator_invalidate_register_cache(ctxt);
8895 	}
8896 
8897 restart:
8898 	if (emulation_type & EMULTYPE_PF) {
8899 		/* Save the faulting GPA (cr2) in the address field */
8900 		ctxt->exception.address = cr2_or_gpa;
8901 
8902 		/* With shadow page tables, cr2 contains a GVA or nGPA. */
8903 		if (vcpu->arch.mmu->root_role.direct) {
8904 			ctxt->gpa_available = true;
8905 			ctxt->gpa_val = cr2_or_gpa;
8906 		}
8907 	} else {
8908 		/* Sanitize the address out of an abundance of paranoia. */
8909 		ctxt->exception.address = 0;
8910 	}
8911 
8912 	r = x86_emulate_insn(ctxt);
8913 
8914 	if (r == EMULATION_INTERCEPTED)
8915 		return 1;
8916 
8917 	if (r == EMULATION_FAILED) {
8918 		if (reexecute_instruction(vcpu, cr2_or_gpa, write_fault_to_spt,
8919 					emulation_type))
8920 			return 1;
8921 
8922 		return handle_emulation_failure(vcpu, emulation_type);
8923 	}
8924 
8925 	if (ctxt->have_exception) {
8926 		r = 1;
8927 		inject_emulated_exception(vcpu);
8928 	} else if (vcpu->arch.pio.count) {
8929 		if (!vcpu->arch.pio.in) {
8930 			/* FIXME: return into emulator if single-stepping.  */
8931 			vcpu->arch.pio.count = 0;
8932 		} else {
8933 			writeback = false;
8934 			vcpu->arch.complete_userspace_io = complete_emulated_pio;
8935 		}
8936 		r = 0;
8937 	} else if (vcpu->mmio_needed) {
8938 		++vcpu->stat.mmio_exits;
8939 
8940 		if (!vcpu->mmio_is_write)
8941 			writeback = false;
8942 		r = 0;
8943 		vcpu->arch.complete_userspace_io = complete_emulated_mmio;
8944 	} else if (vcpu->arch.complete_userspace_io) {
8945 		writeback = false;
8946 		r = 0;
8947 	} else if (r == EMULATION_RESTART)
8948 		goto restart;
8949 	else
8950 		r = 1;
8951 
8952 writeback:
8953 	if (writeback) {
8954 		unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
8955 		toggle_interruptibility(vcpu, ctxt->interruptibility);
8956 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
8957 
8958 		/*
8959 		 * Note, EXCPT_DB is assumed to be fault-like as the emulator
8960 		 * only supports code breakpoints and general detect #DB, both
8961 		 * of which are fault-like.
8962 		 */
8963 		if (!ctxt->have_exception ||
8964 		    exception_type(ctxt->exception.vector) == EXCPT_TRAP) {
8965 			kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);
8966 			if (ctxt->is_branch)
8967 				kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
8968 			kvm_rip_write(vcpu, ctxt->eip);
8969 			if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))
8970 				r = kvm_vcpu_do_singlestep(vcpu);
8971 			static_call_cond(kvm_x86_update_emulated_instruction)(vcpu);
8972 			__kvm_set_rflags(vcpu, ctxt->eflags);
8973 		}
8974 
8975 		/*
8976 		 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will
8977 		 * do nothing, and it will be requested again as soon as
8978 		 * the shadow expires.  But we still need to check here,
8979 		 * because POPF has no interrupt shadow.
8980 		 */
8981 		if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))
8982 			kvm_make_request(KVM_REQ_EVENT, vcpu);
8983 	} else
8984 		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
8985 
8986 	return r;
8987 }
8988 
kvm_emulate_instruction(struct kvm_vcpu * vcpu,int emulation_type)8989 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
8990 {
8991 	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
8992 }
8993 EXPORT_SYMBOL_GPL(kvm_emulate_instruction);
8994 
kvm_emulate_instruction_from_buffer(struct kvm_vcpu * vcpu,void * insn,int insn_len)8995 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
8996 					void *insn, int insn_len)
8997 {
8998 	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
8999 }
9000 EXPORT_SYMBOL_GPL(kvm_emulate_instruction_from_buffer);
9001 
complete_fast_pio_out_port_0x7e(struct kvm_vcpu * vcpu)9002 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
9003 {
9004 	vcpu->arch.pio.count = 0;
9005 	return 1;
9006 }
9007 
complete_fast_pio_out(struct kvm_vcpu * vcpu)9008 static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
9009 {
9010 	vcpu->arch.pio.count = 0;
9011 
9012 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
9013 		return 1;
9014 
9015 	return kvm_skip_emulated_instruction(vcpu);
9016 }
9017 
kvm_fast_pio_out(struct kvm_vcpu * vcpu,int size,unsigned short port)9018 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
9019 			    unsigned short port)
9020 {
9021 	unsigned long val = kvm_rax_read(vcpu);
9022 	int ret = emulator_pio_out(vcpu, size, port, &val, 1);
9023 
9024 	if (ret)
9025 		return ret;
9026 
9027 	/*
9028 	 * Workaround userspace that relies on old KVM behavior of %rip being
9029 	 * incremented prior to exiting to userspace to handle "OUT 0x7e".
9030 	 */
9031 	if (port == 0x7e &&
9032 	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
9033 		vcpu->arch.complete_userspace_io =
9034 			complete_fast_pio_out_port_0x7e;
9035 		kvm_skip_emulated_instruction(vcpu);
9036 	} else {
9037 		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
9038 		vcpu->arch.complete_userspace_io = complete_fast_pio_out;
9039 	}
9040 	return 0;
9041 }
9042 
complete_fast_pio_in(struct kvm_vcpu * vcpu)9043 static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
9044 {
9045 	unsigned long val;
9046 
9047 	/* We should only ever be called with arch.pio.count equal to 1 */
9048 	BUG_ON(vcpu->arch.pio.count != 1);
9049 
9050 	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
9051 		vcpu->arch.pio.count = 0;
9052 		return 1;
9053 	}
9054 
9055 	/* For size less than 4 we merge, else we zero extend */
9056 	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
9057 
9058 	complete_emulator_pio_in(vcpu, &val);
9059 	kvm_rax_write(vcpu, val);
9060 
9061 	return kvm_skip_emulated_instruction(vcpu);
9062 }
9063 
kvm_fast_pio_in(struct kvm_vcpu * vcpu,int size,unsigned short port)9064 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
9065 			   unsigned short port)
9066 {
9067 	unsigned long val;
9068 	int ret;
9069 
9070 	/* For size less than 4 we merge, else we zero extend */
9071 	val = (size < 4) ? kvm_rax_read(vcpu) : 0;
9072 
9073 	ret = emulator_pio_in(vcpu, size, port, &val, 1);
9074 	if (ret) {
9075 		kvm_rax_write(vcpu, val);
9076 		return ret;
9077 	}
9078 
9079 	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
9080 	vcpu->arch.complete_userspace_io = complete_fast_pio_in;
9081 
9082 	return 0;
9083 }
9084 
kvm_fast_pio(struct kvm_vcpu * vcpu,int size,unsigned short port,int in)9085 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)
9086 {
9087 	int ret;
9088 
9089 	if (in)
9090 		ret = kvm_fast_pio_in(vcpu, size, port);
9091 	else
9092 		ret = kvm_fast_pio_out(vcpu, size, port);
9093 	return ret && kvm_skip_emulated_instruction(vcpu);
9094 }
9095 EXPORT_SYMBOL_GPL(kvm_fast_pio);
9096 
kvmclock_cpu_down_prep(unsigned int cpu)9097 static int kvmclock_cpu_down_prep(unsigned int cpu)
9098 {
9099 	__this_cpu_write(cpu_tsc_khz, 0);
9100 	return 0;
9101 }
9102 
tsc_khz_changed(void * data)9103 static void tsc_khz_changed(void *data)
9104 {
9105 	struct cpufreq_freqs *freq = data;
9106 	unsigned long khz = 0;
9107 
9108 	if (data)
9109 		khz = freq->new;
9110 	else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
9111 		khz = cpufreq_quick_get(raw_smp_processor_id());
9112 	if (!khz)
9113 		khz = tsc_khz;
9114 	__this_cpu_write(cpu_tsc_khz, khz);
9115 }
9116 
9117 #ifdef CONFIG_X86_64
kvm_hyperv_tsc_notifier(void)9118 static void kvm_hyperv_tsc_notifier(void)
9119 {
9120 	struct kvm *kvm;
9121 	int cpu;
9122 
9123 	mutex_lock(&kvm_lock);
9124 	list_for_each_entry(kvm, &vm_list, vm_list)
9125 		kvm_make_mclock_inprogress_request(kvm);
9126 
9127 	/* no guest entries from this point */
9128 	hyperv_stop_tsc_emulation();
9129 
9130 	/* TSC frequency always matches when on Hyper-V */
9131 	for_each_present_cpu(cpu)
9132 		per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
9133 	kvm_caps.max_guest_tsc_khz = tsc_khz;
9134 
9135 	list_for_each_entry(kvm, &vm_list, vm_list) {
9136 		__kvm_start_pvclock_update(kvm);
9137 		pvclock_update_vm_gtod_copy(kvm);
9138 		kvm_end_pvclock_update(kvm);
9139 	}
9140 
9141 	mutex_unlock(&kvm_lock);
9142 }
9143 #endif
9144 
__kvmclock_cpufreq_notifier(struct cpufreq_freqs * freq,int cpu)9145 static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu)
9146 {
9147 	struct kvm *kvm;
9148 	struct kvm_vcpu *vcpu;
9149 	int send_ipi = 0;
9150 	unsigned long i;
9151 
9152 	/*
9153 	 * We allow guests to temporarily run on slowing clocks,
9154 	 * provided we notify them after, or to run on accelerating
9155 	 * clocks, provided we notify them before.  Thus time never
9156 	 * goes backwards.
9157 	 *
9158 	 * However, we have a problem.  We can't atomically update
9159 	 * the frequency of a given CPU from this function; it is
9160 	 * merely a notifier, which can be called from any CPU.
9161 	 * Changing the TSC frequency at arbitrary points in time
9162 	 * requires a recomputation of local variables related to
9163 	 * the TSC for each VCPU.  We must flag these local variables
9164 	 * to be updated and be sure the update takes place with the
9165 	 * new frequency before any guests proceed.
9166 	 *
9167 	 * Unfortunately, the combination of hotplug CPU and frequency
9168 	 * change creates an intractable locking scenario; the order
9169 	 * of when these callouts happen is undefined with respect to
9170 	 * CPU hotplug, and they can race with each other.  As such,
9171 	 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
9172 	 * undefined; you can actually have a CPU frequency change take
9173 	 * place in between the computation of X and the setting of the
9174 	 * variable.  To protect against this problem, all updates of
9175 	 * the per_cpu tsc_khz variable are done in an interrupt
9176 	 * protected IPI, and all callers wishing to update the value
9177 	 * must wait for a synchronous IPI to complete (which is trivial
9178 	 * if the caller is on the CPU already).  This establishes the
9179 	 * necessary total order on variable updates.
9180 	 *
9181 	 * Note that because a guest time update may take place
9182 	 * anytime after the setting of the VCPU's request bit, the
9183 	 * correct TSC value must be set before the request.  However,
9184 	 * to ensure the update actually makes it to any guest which
9185 	 * starts running in hardware virtualization between the set
9186 	 * and the acquisition of the spinlock, we must also ping the
9187 	 * CPU after setting the request bit.
9188 	 *
9189 	 */
9190 
9191 	smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
9192 
9193 	mutex_lock(&kvm_lock);
9194 	list_for_each_entry(kvm, &vm_list, vm_list) {
9195 		kvm_for_each_vcpu(i, vcpu, kvm) {
9196 			if (vcpu->cpu != cpu)
9197 				continue;
9198 			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
9199 			if (vcpu->cpu != raw_smp_processor_id())
9200 				send_ipi = 1;
9201 		}
9202 	}
9203 	mutex_unlock(&kvm_lock);
9204 
9205 	if (freq->old < freq->new && send_ipi) {
9206 		/*
9207 		 * We upscale the frequency.  Must make the guest
9208 		 * doesn't see old kvmclock values while running with
9209 		 * the new frequency, otherwise we risk the guest sees
9210 		 * time go backwards.
9211 		 *
9212 		 * In case we update the frequency for another cpu
9213 		 * (which might be in guest context) send an interrupt
9214 		 * to kick the cpu out of guest context.  Next time
9215 		 * guest context is entered kvmclock will be updated,
9216 		 * so the guest will not see stale values.
9217 		 */
9218 		smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
9219 	}
9220 }
9221 
kvmclock_cpufreq_notifier(struct notifier_block * nb,unsigned long val,void * data)9222 static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
9223 				     void *data)
9224 {
9225 	struct cpufreq_freqs *freq = data;
9226 	int cpu;
9227 
9228 	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
9229 		return 0;
9230 	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
9231 		return 0;
9232 
9233 	for_each_cpu(cpu, freq->policy->cpus)
9234 		__kvmclock_cpufreq_notifier(freq, cpu);
9235 
9236 	return 0;
9237 }
9238 
9239 static struct notifier_block kvmclock_cpufreq_notifier_block = {
9240 	.notifier_call  = kvmclock_cpufreq_notifier
9241 };
9242 
kvmclock_cpu_online(unsigned int cpu)9243 static int kvmclock_cpu_online(unsigned int cpu)
9244 {
9245 	tsc_khz_changed(NULL);
9246 	return 0;
9247 }
9248 
kvm_timer_init(void)9249 static void kvm_timer_init(void)
9250 {
9251 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
9252 		max_tsc_khz = tsc_khz;
9253 
9254 		if (IS_ENABLED(CONFIG_CPU_FREQ)) {
9255 			struct cpufreq_policy *policy;
9256 			int cpu;
9257 
9258 			cpu = get_cpu();
9259 			policy = cpufreq_cpu_get(cpu);
9260 			if (policy) {
9261 				if (policy->cpuinfo.max_freq)
9262 					max_tsc_khz = policy->cpuinfo.max_freq;
9263 				cpufreq_cpu_put(policy);
9264 			}
9265 			put_cpu();
9266 		}
9267 		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
9268 					  CPUFREQ_TRANSITION_NOTIFIER);
9269 	}
9270 
9271 	cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online",
9272 			  kvmclock_cpu_online, kvmclock_cpu_down_prep);
9273 }
9274 
9275 #ifdef CONFIG_X86_64
pvclock_gtod_update_fn(struct work_struct * work)9276 static void pvclock_gtod_update_fn(struct work_struct *work)
9277 {
9278 	struct kvm *kvm;
9279 	struct kvm_vcpu *vcpu;
9280 	unsigned long i;
9281 
9282 	mutex_lock(&kvm_lock);
9283 	list_for_each_entry(kvm, &vm_list, vm_list)
9284 		kvm_for_each_vcpu(i, vcpu, kvm)
9285 			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
9286 	atomic_set(&kvm_guest_has_master_clock, 0);
9287 	mutex_unlock(&kvm_lock);
9288 }
9289 
9290 static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);
9291 
9292 /*
9293  * Indirection to move queue_work() out of the tk_core.seq write held
9294  * region to prevent possible deadlocks against time accessors which
9295  * are invoked with work related locks held.
9296  */
pvclock_irq_work_fn(struct irq_work * w)9297 static void pvclock_irq_work_fn(struct irq_work *w)
9298 {
9299 	queue_work(system_long_wq, &pvclock_gtod_work);
9300 }
9301 
9302 static DEFINE_IRQ_WORK(pvclock_irq_work, pvclock_irq_work_fn);
9303 
9304 /*
9305  * Notification about pvclock gtod data update.
9306  */
pvclock_gtod_notify(struct notifier_block * nb,unsigned long unused,void * priv)9307 static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
9308 			       void *priv)
9309 {
9310 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
9311 	struct timekeeper *tk = priv;
9312 
9313 	update_pvclock_gtod(tk);
9314 
9315 	/*
9316 	 * Disable master clock if host does not trust, or does not use,
9317 	 * TSC based clocksource. Delegate queue_work() to irq_work as
9318 	 * this is invoked with tk_core.seq write held.
9319 	 */
9320 	if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&
9321 	    atomic_read(&kvm_guest_has_master_clock) != 0)
9322 		irq_work_queue(&pvclock_irq_work);
9323 	return 0;
9324 }
9325 
9326 static struct notifier_block pvclock_gtod_notifier = {
9327 	.notifier_call = pvclock_gtod_notify,
9328 };
9329 #endif
9330 
kvm_arch_init(void * opaque)9331 int kvm_arch_init(void *opaque)
9332 {
9333 	struct kvm_x86_init_ops *ops = opaque;
9334 	u64 host_pat;
9335 	int r;
9336 
9337 	if (kvm_x86_ops.hardware_enable) {
9338 		pr_err("kvm: already loaded vendor module '%s'\n", kvm_x86_ops.name);
9339 		return -EEXIST;
9340 	}
9341 
9342 	if (!ops->cpu_has_kvm_support()) {
9343 		pr_err_ratelimited("kvm: no hardware support for '%s'\n",
9344 				   ops->runtime_ops->name);
9345 		return -EOPNOTSUPP;
9346 	}
9347 	if (ops->disabled_by_bios()) {
9348 		pr_err_ratelimited("kvm: support for '%s' disabled by bios\n",
9349 				   ops->runtime_ops->name);
9350 		return -EOPNOTSUPP;
9351 	}
9352 
9353 	/*
9354 	 * KVM explicitly assumes that the guest has an FPU and
9355 	 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the
9356 	 * vCPU's FPU state as a fxregs_state struct.
9357 	 */
9358 	if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) {
9359 		printk(KERN_ERR "kvm: inadequate fpu\n");
9360 		return -EOPNOTSUPP;
9361 	}
9362 
9363 	if (IS_ENABLED(CONFIG_PREEMPT_RT) && !boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
9364 		pr_err("RT requires X86_FEATURE_CONSTANT_TSC\n");
9365 		return -EOPNOTSUPP;
9366 	}
9367 
9368 	/*
9369 	 * KVM assumes that PAT entry '0' encodes WB memtype and simply zeroes
9370 	 * the PAT bits in SPTEs.  Bail if PAT[0] is programmed to something
9371 	 * other than WB.  Note, EPT doesn't utilize the PAT, but don't bother
9372 	 * with an exception.  PAT[0] is set to WB on RESET and also by the
9373 	 * kernel, i.e. failure indicates a kernel bug or broken firmware.
9374 	 */
9375 	if (rdmsrl_safe(MSR_IA32_CR_PAT, &host_pat) ||
9376 	    (host_pat & GENMASK(2, 0)) != 6) {
9377 		pr_err("kvm: host PAT[0] is not WB\n");
9378 		return -EIO;
9379 	}
9380 
9381 	x86_emulator_cache = kvm_alloc_emulator_cache();
9382 	if (!x86_emulator_cache) {
9383 		pr_err("kvm: failed to allocate cache for x86 emulator\n");
9384 		return -ENOMEM;
9385 	}
9386 
9387 	user_return_msrs = alloc_percpu(struct kvm_user_return_msrs);
9388 	if (!user_return_msrs) {
9389 		printk(KERN_ERR "kvm: failed to allocate percpu kvm_user_return_msrs\n");
9390 		r = -ENOMEM;
9391 		goto out_free_x86_emulator_cache;
9392 	}
9393 	kvm_nr_uret_msrs = 0;
9394 
9395 	r = kvm_mmu_vendor_module_init();
9396 	if (r)
9397 		goto out_free_percpu;
9398 
9399 	kvm_timer_init();
9400 
9401 	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
9402 		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
9403 		kvm_caps.supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0;
9404 	}
9405 
9406 	if (pi_inject_timer == -1)
9407 		pi_inject_timer = housekeeping_enabled(HK_TYPE_TIMER);
9408 #ifdef CONFIG_X86_64
9409 	pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
9410 
9411 	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
9412 		set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
9413 #endif
9414 
9415 	return 0;
9416 
9417 out_free_percpu:
9418 	free_percpu(user_return_msrs);
9419 out_free_x86_emulator_cache:
9420 	kmem_cache_destroy(x86_emulator_cache);
9421 	return r;
9422 }
9423 
kvm_arch_exit(void)9424 void kvm_arch_exit(void)
9425 {
9426 #ifdef CONFIG_X86_64
9427 	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
9428 		clear_hv_tscchange_cb();
9429 #endif
9430 	kvm_lapic_exit();
9431 
9432 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
9433 		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
9434 					    CPUFREQ_TRANSITION_NOTIFIER);
9435 	cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
9436 #ifdef CONFIG_X86_64
9437 	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
9438 	irq_work_sync(&pvclock_irq_work);
9439 	cancel_work_sync(&pvclock_gtod_work);
9440 #endif
9441 	kvm_x86_ops.hardware_enable = NULL;
9442 	kvm_mmu_vendor_module_exit();
9443 	free_percpu(user_return_msrs);
9444 	kmem_cache_destroy(x86_emulator_cache);
9445 #ifdef CONFIG_KVM_XEN
9446 	static_key_deferred_flush(&kvm_xen_enabled);
9447 	WARN_ON(static_branch_unlikely(&kvm_xen_enabled.key));
9448 #endif
9449 }
9450 
__kvm_emulate_halt(struct kvm_vcpu * vcpu,int state,int reason)9451 static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason)
9452 {
9453 	/*
9454 	 * The vCPU has halted, e.g. executed HLT.  Update the run state if the
9455 	 * local APIC is in-kernel, the run loop will detect the non-runnable
9456 	 * state and halt the vCPU.  Exit to userspace if the local APIC is
9457 	 * managed by userspace, in which case userspace is responsible for
9458 	 * handling wake events.
9459 	 */
9460 	++vcpu->stat.halt_exits;
9461 	if (lapic_in_kernel(vcpu)) {
9462 		vcpu->arch.mp_state = state;
9463 		return 1;
9464 	} else {
9465 		vcpu->run->exit_reason = reason;
9466 		return 0;
9467 	}
9468 }
9469 
kvm_emulate_halt_noskip(struct kvm_vcpu * vcpu)9470 int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu)
9471 {
9472 	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT);
9473 }
9474 EXPORT_SYMBOL_GPL(kvm_emulate_halt_noskip);
9475 
kvm_emulate_halt(struct kvm_vcpu * vcpu)9476 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
9477 {
9478 	int ret = kvm_skip_emulated_instruction(vcpu);
9479 	/*
9480 	 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered
9481 	 * KVM_EXIT_DEBUG here.
9482 	 */
9483 	return kvm_emulate_halt_noskip(vcpu) && ret;
9484 }
9485 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
9486 
kvm_emulate_ap_reset_hold(struct kvm_vcpu * vcpu)9487 int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu)
9488 {
9489 	int ret = kvm_skip_emulated_instruction(vcpu);
9490 
9491 	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD,
9492 					KVM_EXIT_AP_RESET_HOLD) && ret;
9493 }
9494 EXPORT_SYMBOL_GPL(kvm_emulate_ap_reset_hold);
9495 
9496 #ifdef CONFIG_X86_64
kvm_pv_clock_pairing(struct kvm_vcpu * vcpu,gpa_t paddr,unsigned long clock_type)9497 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
9498 			        unsigned long clock_type)
9499 {
9500 	struct kvm_clock_pairing clock_pairing;
9501 	struct timespec64 ts;
9502 	u64 cycle;
9503 	int ret;
9504 
9505 	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
9506 		return -KVM_EOPNOTSUPP;
9507 
9508 	/*
9509 	 * When tsc is in permanent catchup mode guests won't be able to use
9510 	 * pvclock_read_retry loop to get consistent view of pvclock
9511 	 */
9512 	if (vcpu->arch.tsc_always_catchup)
9513 		return -KVM_EOPNOTSUPP;
9514 
9515 	if (!kvm_get_walltime_and_clockread(&ts, &cycle))
9516 		return -KVM_EOPNOTSUPP;
9517 
9518 	clock_pairing.sec = ts.tv_sec;
9519 	clock_pairing.nsec = ts.tv_nsec;
9520 	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
9521 	clock_pairing.flags = 0;
9522 	memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad));
9523 
9524 	ret = 0;
9525 	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
9526 			    sizeof(struct kvm_clock_pairing)))
9527 		ret = -KVM_EFAULT;
9528 
9529 	return ret;
9530 }
9531 #endif
9532 
9533 /*
9534  * kvm_pv_kick_cpu_op:  Kick a vcpu.
9535  *
9536  * @apicid - apicid of vcpu to be kicked.
9537  */
kvm_pv_kick_cpu_op(struct kvm * kvm,int apicid)9538 static void kvm_pv_kick_cpu_op(struct kvm *kvm, int apicid)
9539 {
9540 	/*
9541 	 * All other fields are unused for APIC_DM_REMRD, but may be consumed by
9542 	 * common code, e.g. for tracing. Defer initialization to the compiler.
9543 	 */
9544 	struct kvm_lapic_irq lapic_irq = {
9545 		.delivery_mode = APIC_DM_REMRD,
9546 		.dest_mode = APIC_DEST_PHYSICAL,
9547 		.shorthand = APIC_DEST_NOSHORT,
9548 		.dest_id = apicid,
9549 	};
9550 
9551 	kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
9552 }
9553 
kvm_apicv_activated(struct kvm * kvm)9554 bool kvm_apicv_activated(struct kvm *kvm)
9555 {
9556 	return (READ_ONCE(kvm->arch.apicv_inhibit_reasons) == 0);
9557 }
9558 EXPORT_SYMBOL_GPL(kvm_apicv_activated);
9559 
kvm_vcpu_apicv_activated(struct kvm_vcpu * vcpu)9560 bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu)
9561 {
9562 	ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons);
9563 	ulong vcpu_reasons = static_call(kvm_x86_vcpu_get_apicv_inhibit_reasons)(vcpu);
9564 
9565 	return (vm_reasons | vcpu_reasons) == 0;
9566 }
9567 EXPORT_SYMBOL_GPL(kvm_vcpu_apicv_activated);
9568 
set_or_clear_apicv_inhibit(unsigned long * inhibits,enum kvm_apicv_inhibit reason,bool set)9569 static void set_or_clear_apicv_inhibit(unsigned long *inhibits,
9570 				       enum kvm_apicv_inhibit reason, bool set)
9571 {
9572 	if (set)
9573 		__set_bit(reason, inhibits);
9574 	else
9575 		__clear_bit(reason, inhibits);
9576 
9577 	trace_kvm_apicv_inhibit_changed(reason, set, *inhibits);
9578 }
9579 
kvm_apicv_init(struct kvm * kvm)9580 static void kvm_apicv_init(struct kvm *kvm)
9581 {
9582 	unsigned long *inhibits = &kvm->arch.apicv_inhibit_reasons;
9583 
9584 	init_rwsem(&kvm->arch.apicv_update_lock);
9585 
9586 	set_or_clear_apicv_inhibit(inhibits, APICV_INHIBIT_REASON_ABSENT, true);
9587 
9588 	if (!enable_apicv)
9589 		set_or_clear_apicv_inhibit(inhibits,
9590 					   APICV_INHIBIT_REASON_DISABLE, true);
9591 }
9592 
kvm_sched_yield(struct kvm_vcpu * vcpu,unsigned long dest_id)9593 static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id)
9594 {
9595 	struct kvm_vcpu *target = NULL;
9596 	struct kvm_apic_map *map;
9597 
9598 	vcpu->stat.directed_yield_attempted++;
9599 
9600 	if (single_task_running())
9601 		goto no_yield;
9602 
9603 	rcu_read_lock();
9604 	map = rcu_dereference(vcpu->kvm->arch.apic_map);
9605 
9606 	if (likely(map) && dest_id <= map->max_apic_id && map->phys_map[dest_id])
9607 		target = map->phys_map[dest_id]->vcpu;
9608 
9609 	rcu_read_unlock();
9610 
9611 	if (!target || !READ_ONCE(target->ready))
9612 		goto no_yield;
9613 
9614 	/* Ignore requests to yield to self */
9615 	if (vcpu == target)
9616 		goto no_yield;
9617 
9618 	if (kvm_vcpu_yield_to(target) <= 0)
9619 		goto no_yield;
9620 
9621 	vcpu->stat.directed_yield_successful++;
9622 
9623 no_yield:
9624 	return;
9625 }
9626 
complete_hypercall_exit(struct kvm_vcpu * vcpu)9627 static int complete_hypercall_exit(struct kvm_vcpu *vcpu)
9628 {
9629 	u64 ret = vcpu->run->hypercall.ret;
9630 
9631 	if (!is_64_bit_mode(vcpu))
9632 		ret = (u32)ret;
9633 	kvm_rax_write(vcpu, ret);
9634 	++vcpu->stat.hypercalls;
9635 	return kvm_skip_emulated_instruction(vcpu);
9636 }
9637 
kvm_emulate_hypercall(struct kvm_vcpu * vcpu)9638 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
9639 {
9640 	unsigned long nr, a0, a1, a2, a3, ret;
9641 	int op_64_bit;
9642 
9643 	if (kvm_xen_hypercall_enabled(vcpu->kvm))
9644 		return kvm_xen_hypercall(vcpu);
9645 
9646 	if (kvm_hv_hypercall_enabled(vcpu))
9647 		return kvm_hv_hypercall(vcpu);
9648 
9649 	nr = kvm_rax_read(vcpu);
9650 	a0 = kvm_rbx_read(vcpu);
9651 	a1 = kvm_rcx_read(vcpu);
9652 	a2 = kvm_rdx_read(vcpu);
9653 	a3 = kvm_rsi_read(vcpu);
9654 
9655 	trace_kvm_hypercall(nr, a0, a1, a2, a3);
9656 
9657 	op_64_bit = is_64_bit_hypercall(vcpu);
9658 	if (!op_64_bit) {
9659 		nr &= 0xFFFFFFFF;
9660 		a0 &= 0xFFFFFFFF;
9661 		a1 &= 0xFFFFFFFF;
9662 		a2 &= 0xFFFFFFFF;
9663 		a3 &= 0xFFFFFFFF;
9664 	}
9665 
9666 	if (static_call(kvm_x86_get_cpl)(vcpu) != 0) {
9667 		ret = -KVM_EPERM;
9668 		goto out;
9669 	}
9670 
9671 	ret = -KVM_ENOSYS;
9672 
9673 	switch (nr) {
9674 	case KVM_HC_VAPIC_POLL_IRQ:
9675 		ret = 0;
9676 		break;
9677 	case KVM_HC_KICK_CPU:
9678 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT))
9679 			break;
9680 
9681 		kvm_pv_kick_cpu_op(vcpu->kvm, a1);
9682 		kvm_sched_yield(vcpu, a1);
9683 		ret = 0;
9684 		break;
9685 #ifdef CONFIG_X86_64
9686 	case KVM_HC_CLOCK_PAIRING:
9687 		ret = kvm_pv_clock_pairing(vcpu, a0, a1);
9688 		break;
9689 #endif
9690 	case KVM_HC_SEND_IPI:
9691 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI))
9692 			break;
9693 
9694 		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
9695 		break;
9696 	case KVM_HC_SCHED_YIELD:
9697 		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD))
9698 			break;
9699 
9700 		kvm_sched_yield(vcpu, a0);
9701 		ret = 0;
9702 		break;
9703 	case KVM_HC_MAP_GPA_RANGE: {
9704 		u64 gpa = a0, npages = a1, attrs = a2;
9705 
9706 		ret = -KVM_ENOSYS;
9707 		if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE)))
9708 			break;
9709 
9710 		if (!PAGE_ALIGNED(gpa) || !npages ||
9711 		    gpa_to_gfn(gpa) + npages <= gpa_to_gfn(gpa)) {
9712 			ret = -KVM_EINVAL;
9713 			break;
9714 		}
9715 
9716 		vcpu->run->exit_reason        = KVM_EXIT_HYPERCALL;
9717 		vcpu->run->hypercall.nr       = KVM_HC_MAP_GPA_RANGE;
9718 		vcpu->run->hypercall.args[0]  = gpa;
9719 		vcpu->run->hypercall.args[1]  = npages;
9720 		vcpu->run->hypercall.args[2]  = attrs;
9721 		vcpu->run->hypercall.longmode = op_64_bit;
9722 		vcpu->arch.complete_userspace_io = complete_hypercall_exit;
9723 		return 0;
9724 	}
9725 	default:
9726 		ret = -KVM_ENOSYS;
9727 		break;
9728 	}
9729 out:
9730 	if (!op_64_bit)
9731 		ret = (u32)ret;
9732 	kvm_rax_write(vcpu, ret);
9733 
9734 	++vcpu->stat.hypercalls;
9735 	return kvm_skip_emulated_instruction(vcpu);
9736 }
9737 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
9738 
emulator_fix_hypercall(struct x86_emulate_ctxt * ctxt)9739 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
9740 {
9741 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
9742 	char instruction[3];
9743 	unsigned long rip = kvm_rip_read(vcpu);
9744 
9745 	/*
9746 	 * If the quirk is disabled, synthesize a #UD and let the guest pick up
9747 	 * the pieces.
9748 	 */
9749 	if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) {
9750 		ctxt->exception.error_code_valid = false;
9751 		ctxt->exception.vector = UD_VECTOR;
9752 		ctxt->have_exception = true;
9753 		return X86EMUL_PROPAGATE_FAULT;
9754 	}
9755 
9756 	static_call(kvm_x86_patch_hypercall)(vcpu, instruction);
9757 
9758 	return emulator_write_emulated(ctxt, rip, instruction, 3,
9759 		&ctxt->exception);
9760 }
9761 
dm_request_for_irq_injection(struct kvm_vcpu * vcpu)9762 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
9763 {
9764 	return vcpu->run->request_interrupt_window &&
9765 		likely(!pic_in_kernel(vcpu->kvm));
9766 }
9767 
9768 /* Called within kvm->srcu read side.  */
post_kvm_run_save(struct kvm_vcpu * vcpu)9769 static void post_kvm_run_save(struct kvm_vcpu *vcpu)
9770 {
9771 	struct kvm_run *kvm_run = vcpu->run;
9772 
9773 	kvm_run->if_flag = static_call(kvm_x86_get_if_flag)(vcpu);
9774 	kvm_run->cr8 = kvm_get_cr8(vcpu);
9775 	kvm_run->apic_base = kvm_get_apic_base(vcpu);
9776 
9777 	kvm_run->ready_for_interrupt_injection =
9778 		pic_in_kernel(vcpu->kvm) ||
9779 		kvm_vcpu_ready_for_interrupt_injection(vcpu);
9780 
9781 	if (is_smm(vcpu))
9782 		kvm_run->flags |= KVM_RUN_X86_SMM;
9783 }
9784 
update_cr8_intercept(struct kvm_vcpu * vcpu)9785 static void update_cr8_intercept(struct kvm_vcpu *vcpu)
9786 {
9787 	int max_irr, tpr;
9788 
9789 	if (!kvm_x86_ops.update_cr8_intercept)
9790 		return;
9791 
9792 	if (!lapic_in_kernel(vcpu))
9793 		return;
9794 
9795 	if (vcpu->arch.apic->apicv_active)
9796 		return;
9797 
9798 	if (!vcpu->arch.apic->vapic_addr)
9799 		max_irr = kvm_lapic_find_highest_irr(vcpu);
9800 	else
9801 		max_irr = -1;
9802 
9803 	if (max_irr != -1)
9804 		max_irr >>= 4;
9805 
9806 	tpr = kvm_lapic_get_cr8(vcpu);
9807 
9808 	static_call(kvm_x86_update_cr8_intercept)(vcpu, tpr, max_irr);
9809 }
9810 
9811 
kvm_check_nested_events(struct kvm_vcpu * vcpu)9812 int kvm_check_nested_events(struct kvm_vcpu *vcpu)
9813 {
9814 	if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
9815 		kvm_x86_ops.nested_ops->triple_fault(vcpu);
9816 		return 1;
9817 	}
9818 
9819 	return kvm_x86_ops.nested_ops->check_events(vcpu);
9820 }
9821 
kvm_inject_exception(struct kvm_vcpu * vcpu)9822 static void kvm_inject_exception(struct kvm_vcpu *vcpu)
9823 {
9824 	trace_kvm_inj_exception(vcpu->arch.exception.vector,
9825 				vcpu->arch.exception.has_error_code,
9826 				vcpu->arch.exception.error_code,
9827 				vcpu->arch.exception.injected);
9828 
9829 	if (vcpu->arch.exception.error_code && !is_protmode(vcpu))
9830 		vcpu->arch.exception.error_code = false;
9831 	static_call(kvm_x86_inject_exception)(vcpu);
9832 }
9833 
9834 /*
9835  * Check for any event (interrupt or exception) that is ready to be injected,
9836  * and if there is at least one event, inject the event with the highest
9837  * priority.  This handles both "pending" events, i.e. events that have never
9838  * been injected into the guest, and "injected" events, i.e. events that were
9839  * injected as part of a previous VM-Enter, but weren't successfully delivered
9840  * and need to be re-injected.
9841  *
9842  * Note, this is not guaranteed to be invoked on a guest instruction boundary,
9843  * i.e. doesn't guarantee that there's an event window in the guest.  KVM must
9844  * be able to inject exceptions in the "middle" of an instruction, and so must
9845  * also be able to re-inject NMIs and IRQs in the middle of an instruction.
9846  * I.e. for exceptions and re-injected events, NOT invoking this on instruction
9847  * boundaries is necessary and correct.
9848  *
9849  * For simplicity, KVM uses a single path to inject all events (except events
9850  * that are injected directly from L1 to L2) and doesn't explicitly track
9851  * instruction boundaries for asynchronous events.  However, because VM-Exits
9852  * that can occur during instruction execution typically result in KVM skipping
9853  * the instruction or injecting an exception, e.g. instruction and exception
9854  * intercepts, and because pending exceptions have higher priority than pending
9855  * interrupts, KVM still honors instruction boundaries in most scenarios.
9856  *
9857  * But, if a VM-Exit occurs during instruction execution, and KVM does NOT skip
9858  * the instruction or inject an exception, then KVM can incorrecty inject a new
9859  * asynchrounous event if the event became pending after the CPU fetched the
9860  * instruction (in the guest).  E.g. if a page fault (#PF, #NPF, EPT violation)
9861  * occurs and is resolved by KVM, a coincident NMI, SMI, IRQ, etc... can be
9862  * injected on the restarted instruction instead of being deferred until the
9863  * instruction completes.
9864  *
9865  * In practice, this virtualization hole is unlikely to be observed by the
9866  * guest, and even less likely to cause functional problems.  To detect the
9867  * hole, the guest would have to trigger an event on a side effect of an early
9868  * phase of instruction execution, e.g. on the instruction fetch from memory.
9869  * And for it to be a functional problem, the guest would need to depend on the
9870  * ordering between that side effect, the instruction completing, _and_ the
9871  * delivery of the asynchronous event.
9872  */
kvm_check_and_inject_events(struct kvm_vcpu * vcpu,bool * req_immediate_exit)9873 static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu,
9874 				       bool *req_immediate_exit)
9875 {
9876 	bool can_inject;
9877 	int r;
9878 
9879 	/*
9880 	 * Process nested events first, as nested VM-Exit supercedes event
9881 	 * re-injection.  If there's an event queued for re-injection, it will
9882 	 * be saved into the appropriate vmc{b,s}12 fields on nested VM-Exit.
9883 	 */
9884 	if (is_guest_mode(vcpu))
9885 		r = kvm_check_nested_events(vcpu);
9886 	else
9887 		r = 0;
9888 
9889 	/*
9890 	 * Re-inject exceptions and events *especially* if immediate entry+exit
9891 	 * to/from L2 is needed, as any event that has already been injected
9892 	 * into L2 needs to complete its lifecycle before injecting a new event.
9893 	 *
9894 	 * Don't re-inject an NMI or interrupt if there is a pending exception.
9895 	 * This collision arises if an exception occurred while vectoring the
9896 	 * injected event, KVM intercepted said exception, and KVM ultimately
9897 	 * determined the fault belongs to the guest and queues the exception
9898 	 * for injection back into the guest.
9899 	 *
9900 	 * "Injected" interrupts can also collide with pending exceptions if
9901 	 * userspace ignores the "ready for injection" flag and blindly queues
9902 	 * an interrupt.  In that case, prioritizing the exception is correct,
9903 	 * as the exception "occurred" before the exit to userspace.  Trap-like
9904 	 * exceptions, e.g. most #DBs, have higher priority than interrupts.
9905 	 * And while fault-like exceptions, e.g. #GP and #PF, are the lowest
9906 	 * priority, they're only generated (pended) during instruction
9907 	 * execution, and interrupts are recognized at instruction boundaries.
9908 	 * Thus a pending fault-like exception means the fault occurred on the
9909 	 * *previous* instruction and must be serviced prior to recognizing any
9910 	 * new events in order to fully complete the previous instruction.
9911 	 */
9912 	if (vcpu->arch.exception.injected)
9913 		kvm_inject_exception(vcpu);
9914 	else if (kvm_is_exception_pending(vcpu))
9915 		; /* see above */
9916 	else if (vcpu->arch.nmi_injected)
9917 		static_call(kvm_x86_inject_nmi)(vcpu);
9918 	else if (vcpu->arch.interrupt.injected)
9919 		static_call(kvm_x86_inject_irq)(vcpu, true);
9920 
9921 	/*
9922 	 * Exceptions that morph to VM-Exits are handled above, and pending
9923 	 * exceptions on top of injected exceptions that do not VM-Exit should
9924 	 * either morph to #DF or, sadly, override the injected exception.
9925 	 */
9926 	WARN_ON_ONCE(vcpu->arch.exception.injected &&
9927 		     vcpu->arch.exception.pending);
9928 
9929 	/*
9930 	 * Bail if immediate entry+exit to/from the guest is needed to complete
9931 	 * nested VM-Enter or event re-injection so that a different pending
9932 	 * event can be serviced (or if KVM needs to exit to userspace).
9933 	 *
9934 	 * Otherwise, continue processing events even if VM-Exit occurred.  The
9935 	 * VM-Exit will have cleared exceptions that were meant for L2, but
9936 	 * there may now be events that can be injected into L1.
9937 	 */
9938 	if (r < 0)
9939 		goto out;
9940 
9941 	/*
9942 	 * A pending exception VM-Exit should either result in nested VM-Exit
9943 	 * or force an immediate re-entry and exit to/from L2, and exception
9944 	 * VM-Exits cannot be injected (flag should _never_ be set).
9945 	 */
9946 	WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected ||
9947 		     vcpu->arch.exception_vmexit.pending);
9948 
9949 	/*
9950 	 * New events, other than exceptions, cannot be injected if KVM needs
9951 	 * to re-inject a previous event.  See above comments on re-injecting
9952 	 * for why pending exceptions get priority.
9953 	 */
9954 	can_inject = !kvm_event_needs_reinjection(vcpu);
9955 
9956 	if (vcpu->arch.exception.pending) {
9957 		/*
9958 		 * Fault-class exceptions, except #DBs, set RF=1 in the RFLAGS
9959 		 * value pushed on the stack.  Trap-like exception and all #DBs
9960 		 * leave RF as-is (KVM follows Intel's behavior in this regard;
9961 		 * AMD states that code breakpoint #DBs excplitly clear RF=0).
9962 		 *
9963 		 * Note, most versions of Intel's SDM and AMD's APM incorrectly
9964 		 * describe the behavior of General Detect #DBs, which are
9965 		 * fault-like.  They do _not_ set RF, a la code breakpoints.
9966 		 */
9967 		if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT)
9968 			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
9969 					     X86_EFLAGS_RF);
9970 
9971 		if (vcpu->arch.exception.vector == DB_VECTOR) {
9972 			kvm_deliver_exception_payload(vcpu, &vcpu->arch.exception);
9973 			if (vcpu->arch.dr7 & DR7_GD) {
9974 				vcpu->arch.dr7 &= ~DR7_GD;
9975 				kvm_update_dr7(vcpu);
9976 			}
9977 		}
9978 
9979 		kvm_inject_exception(vcpu);
9980 
9981 		vcpu->arch.exception.pending = false;
9982 		vcpu->arch.exception.injected = true;
9983 
9984 		can_inject = false;
9985 	}
9986 
9987 	/* Don't inject interrupts if the user asked to avoid doing so */
9988 	if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ)
9989 		return 0;
9990 
9991 	/*
9992 	 * Finally, inject interrupt events.  If an event cannot be injected
9993 	 * due to architectural conditions (e.g. IF=0) a window-open exit
9994 	 * will re-request KVM_REQ_EVENT.  Sometimes however an event is pending
9995 	 * and can architecturally be injected, but we cannot do it right now:
9996 	 * an interrupt could have arrived just now and we have to inject it
9997 	 * as a vmexit, or there could already an event in the queue, which is
9998 	 * indicated by can_inject.  In that case we request an immediate exit
9999 	 * in order to make progress and get back here for another iteration.
10000 	 * The kvm_x86_ops hooks communicate this by returning -EBUSY.
10001 	 */
10002 	if (vcpu->arch.smi_pending) {
10003 		r = can_inject ? static_call(kvm_x86_smi_allowed)(vcpu, true) : -EBUSY;
10004 		if (r < 0)
10005 			goto out;
10006 		if (r) {
10007 			vcpu->arch.smi_pending = false;
10008 			++vcpu->arch.smi_count;
10009 			enter_smm(vcpu);
10010 			can_inject = false;
10011 		} else
10012 			static_call(kvm_x86_enable_smi_window)(vcpu);
10013 	}
10014 
10015 	if (vcpu->arch.nmi_pending) {
10016 		r = can_inject ? static_call(kvm_x86_nmi_allowed)(vcpu, true) : -EBUSY;
10017 		if (r < 0)
10018 			goto out;
10019 		if (r) {
10020 			--vcpu->arch.nmi_pending;
10021 			vcpu->arch.nmi_injected = true;
10022 			static_call(kvm_x86_inject_nmi)(vcpu);
10023 			can_inject = false;
10024 			WARN_ON(static_call(kvm_x86_nmi_allowed)(vcpu, true) < 0);
10025 		}
10026 		if (vcpu->arch.nmi_pending)
10027 			static_call(kvm_x86_enable_nmi_window)(vcpu);
10028 	}
10029 
10030 	if (kvm_cpu_has_injectable_intr(vcpu)) {
10031 		r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY;
10032 		if (r < 0)
10033 			goto out;
10034 		if (r) {
10035 			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false);
10036 			static_call(kvm_x86_inject_irq)(vcpu, false);
10037 			WARN_ON(static_call(kvm_x86_interrupt_allowed)(vcpu, true) < 0);
10038 		}
10039 		if (kvm_cpu_has_injectable_intr(vcpu))
10040 			static_call(kvm_x86_enable_irq_window)(vcpu);
10041 	}
10042 
10043 	if (is_guest_mode(vcpu) &&
10044 	    kvm_x86_ops.nested_ops->has_events &&
10045 	    kvm_x86_ops.nested_ops->has_events(vcpu))
10046 		*req_immediate_exit = true;
10047 
10048 	/*
10049 	 * KVM must never queue a new exception while injecting an event; KVM
10050 	 * is done emulating and should only propagate the to-be-injected event
10051 	 * to the VMCS/VMCB.  Queueing a new exception can put the vCPU into an
10052 	 * infinite loop as KVM will bail from VM-Enter to inject the pending
10053 	 * exception and start the cycle all over.
10054 	 *
10055 	 * Exempt triple faults as they have special handling and won't put the
10056 	 * vCPU into an infinite loop.  Triple fault can be queued when running
10057 	 * VMX without unrestricted guest, as that requires KVM to emulate Real
10058 	 * Mode events (see kvm_inject_realmode_interrupt()).
10059 	 */
10060 	WARN_ON_ONCE(vcpu->arch.exception.pending ||
10061 		     vcpu->arch.exception_vmexit.pending);
10062 	return 0;
10063 
10064 out:
10065 	if (r == -EBUSY) {
10066 		*req_immediate_exit = true;
10067 		r = 0;
10068 	}
10069 	return r;
10070 }
10071 
process_nmi(struct kvm_vcpu * vcpu)10072 static void process_nmi(struct kvm_vcpu *vcpu)
10073 {
10074 	unsigned limit = 2;
10075 
10076 	/*
10077 	 * x86 is limited to one NMI running, and one NMI pending after it.
10078 	 * If an NMI is already in progress, limit further NMIs to just one.
10079 	 * Otherwise, allow two (and we'll inject the first one immediately).
10080 	 */
10081 	if (static_call(kvm_x86_get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected)
10082 		limit = 1;
10083 
10084 	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
10085 	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
10086 	kvm_make_request(KVM_REQ_EVENT, vcpu);
10087 }
10088 
enter_smm_get_segment_flags(struct kvm_segment * seg)10089 static u32 enter_smm_get_segment_flags(struct kvm_segment *seg)
10090 {
10091 	u32 flags = 0;
10092 	flags |= seg->g       << 23;
10093 	flags |= seg->db      << 22;
10094 	flags |= seg->l       << 21;
10095 	flags |= seg->avl     << 20;
10096 	flags |= seg->present << 15;
10097 	flags |= seg->dpl     << 13;
10098 	flags |= seg->s       << 12;
10099 	flags |= seg->type    << 8;
10100 	return flags;
10101 }
10102 
enter_smm_save_seg_32(struct kvm_vcpu * vcpu,char * buf,int n)10103 static void enter_smm_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n)
10104 {
10105 	struct kvm_segment seg;
10106 	int offset;
10107 
10108 	kvm_get_segment(vcpu, &seg, n);
10109 	put_smstate(u32, buf, 0x7fa8 + n * 4, seg.selector);
10110 
10111 	if (n < 3)
10112 		offset = 0x7f84 + n * 12;
10113 	else
10114 		offset = 0x7f2c + (n - 3) * 12;
10115 
10116 	put_smstate(u32, buf, offset + 8, seg.base);
10117 	put_smstate(u32, buf, offset + 4, seg.limit);
10118 	put_smstate(u32, buf, offset, enter_smm_get_segment_flags(&seg));
10119 }
10120 
10121 #ifdef CONFIG_X86_64
enter_smm_save_seg_64(struct kvm_vcpu * vcpu,char * buf,int n)10122 static void enter_smm_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n)
10123 {
10124 	struct kvm_segment seg;
10125 	int offset;
10126 	u16 flags;
10127 
10128 	kvm_get_segment(vcpu, &seg, n);
10129 	offset = 0x7e00 + n * 16;
10130 
10131 	flags = enter_smm_get_segment_flags(&seg) >> 8;
10132 	put_smstate(u16, buf, offset, seg.selector);
10133 	put_smstate(u16, buf, offset + 2, flags);
10134 	put_smstate(u32, buf, offset + 4, seg.limit);
10135 	put_smstate(u64, buf, offset + 8, seg.base);
10136 }
10137 #endif
10138 
enter_smm_save_state_32(struct kvm_vcpu * vcpu,char * buf)10139 static void enter_smm_save_state_32(struct kvm_vcpu *vcpu, char *buf)
10140 {
10141 	struct desc_ptr dt;
10142 	struct kvm_segment seg;
10143 	unsigned long val;
10144 	int i;
10145 
10146 	put_smstate(u32, buf, 0x7ffc, kvm_read_cr0(vcpu));
10147 	put_smstate(u32, buf, 0x7ff8, kvm_read_cr3(vcpu));
10148 	put_smstate(u32, buf, 0x7ff4, kvm_get_rflags(vcpu));
10149 	put_smstate(u32, buf, 0x7ff0, kvm_rip_read(vcpu));
10150 
10151 	for (i = 0; i < 8; i++)
10152 		put_smstate(u32, buf, 0x7fd0 + i * 4, kvm_register_read_raw(vcpu, i));
10153 
10154 	kvm_get_dr(vcpu, 6, &val);
10155 	put_smstate(u32, buf, 0x7fcc, (u32)val);
10156 	kvm_get_dr(vcpu, 7, &val);
10157 	put_smstate(u32, buf, 0x7fc8, (u32)val);
10158 
10159 	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
10160 	put_smstate(u32, buf, 0x7fc4, seg.selector);
10161 	put_smstate(u32, buf, 0x7f64, seg.base);
10162 	put_smstate(u32, buf, 0x7f60, seg.limit);
10163 	put_smstate(u32, buf, 0x7f5c, enter_smm_get_segment_flags(&seg));
10164 
10165 	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
10166 	put_smstate(u32, buf, 0x7fc0, seg.selector);
10167 	put_smstate(u32, buf, 0x7f80, seg.base);
10168 	put_smstate(u32, buf, 0x7f7c, seg.limit);
10169 	put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg));
10170 
10171 	static_call(kvm_x86_get_gdt)(vcpu, &dt);
10172 	put_smstate(u32, buf, 0x7f74, dt.address);
10173 	put_smstate(u32, buf, 0x7f70, dt.size);
10174 
10175 	static_call(kvm_x86_get_idt)(vcpu, &dt);
10176 	put_smstate(u32, buf, 0x7f58, dt.address);
10177 	put_smstate(u32, buf, 0x7f54, dt.size);
10178 
10179 	for (i = 0; i < 6; i++)
10180 		enter_smm_save_seg_32(vcpu, buf, i);
10181 
10182 	put_smstate(u32, buf, 0x7f14, kvm_read_cr4(vcpu));
10183 
10184 	/* revision id */
10185 	put_smstate(u32, buf, 0x7efc, 0x00020000);
10186 	put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase);
10187 }
10188 
10189 #ifdef CONFIG_X86_64
enter_smm_save_state_64(struct kvm_vcpu * vcpu,char * buf)10190 static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf)
10191 {
10192 	struct desc_ptr dt;
10193 	struct kvm_segment seg;
10194 	unsigned long val;
10195 	int i;
10196 
10197 	for (i = 0; i < 16; i++)
10198 		put_smstate(u64, buf, 0x7ff8 - i * 8, kvm_register_read_raw(vcpu, i));
10199 
10200 	put_smstate(u64, buf, 0x7f78, kvm_rip_read(vcpu));
10201 	put_smstate(u32, buf, 0x7f70, kvm_get_rflags(vcpu));
10202 
10203 	kvm_get_dr(vcpu, 6, &val);
10204 	put_smstate(u64, buf, 0x7f68, val);
10205 	kvm_get_dr(vcpu, 7, &val);
10206 	put_smstate(u64, buf, 0x7f60, val);
10207 
10208 	put_smstate(u64, buf, 0x7f58, kvm_read_cr0(vcpu));
10209 	put_smstate(u64, buf, 0x7f50, kvm_read_cr3(vcpu));
10210 	put_smstate(u64, buf, 0x7f48, kvm_read_cr4(vcpu));
10211 
10212 	put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase);
10213 
10214 	/* revision id */
10215 	put_smstate(u32, buf, 0x7efc, 0x00020064);
10216 
10217 	put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer);
10218 
10219 	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
10220 	put_smstate(u16, buf, 0x7e90, seg.selector);
10221 	put_smstate(u16, buf, 0x7e92, enter_smm_get_segment_flags(&seg) >> 8);
10222 	put_smstate(u32, buf, 0x7e94, seg.limit);
10223 	put_smstate(u64, buf, 0x7e98, seg.base);
10224 
10225 	static_call(kvm_x86_get_idt)(vcpu, &dt);
10226 	put_smstate(u32, buf, 0x7e84, dt.size);
10227 	put_smstate(u64, buf, 0x7e88, dt.address);
10228 
10229 	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
10230 	put_smstate(u16, buf, 0x7e70, seg.selector);
10231 	put_smstate(u16, buf, 0x7e72, enter_smm_get_segment_flags(&seg) >> 8);
10232 	put_smstate(u32, buf, 0x7e74, seg.limit);
10233 	put_smstate(u64, buf, 0x7e78, seg.base);
10234 
10235 	static_call(kvm_x86_get_gdt)(vcpu, &dt);
10236 	put_smstate(u32, buf, 0x7e64, dt.size);
10237 	put_smstate(u64, buf, 0x7e68, dt.address);
10238 
10239 	for (i = 0; i < 6; i++)
10240 		enter_smm_save_seg_64(vcpu, buf, i);
10241 }
10242 #endif
10243 
enter_smm(struct kvm_vcpu * vcpu)10244 static void enter_smm(struct kvm_vcpu *vcpu)
10245 {
10246 	struct kvm_segment cs, ds;
10247 	struct desc_ptr dt;
10248 	unsigned long cr0;
10249 	char buf[512];
10250 
10251 	memset(buf, 0, 512);
10252 #ifdef CONFIG_X86_64
10253 	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
10254 		enter_smm_save_state_64(vcpu, buf);
10255 	else
10256 #endif
10257 		enter_smm_save_state_32(vcpu, buf);
10258 
10259 	/*
10260 	 * Give enter_smm() a chance to make ISA-specific changes to the vCPU
10261 	 * state (e.g. leave guest mode) after we've saved the state into the
10262 	 * SMM state-save area.
10263 	 */
10264 	static_call(kvm_x86_enter_smm)(vcpu, buf);
10265 
10266 	kvm_smm_changed(vcpu, true);
10267 	kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
10268 
10269 	if (static_call(kvm_x86_get_nmi_mask)(vcpu))
10270 		vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
10271 	else
10272 		static_call(kvm_x86_set_nmi_mask)(vcpu, true);
10273 
10274 	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
10275 	kvm_rip_write(vcpu, 0x8000);
10276 
10277 	cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);
10278 	static_call(kvm_x86_set_cr0)(vcpu, cr0);
10279 	vcpu->arch.cr0 = cr0;
10280 
10281 	static_call(kvm_x86_set_cr4)(vcpu, 0);
10282 
10283 	/* Undocumented: IDT limit is set to zero on entry to SMM.  */
10284 	dt.address = dt.size = 0;
10285 	static_call(kvm_x86_set_idt)(vcpu, &dt);
10286 
10287 	kvm_set_dr(vcpu, 7, DR7_FIXED_1);
10288 
10289 	cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
10290 	cs.base = vcpu->arch.smbase;
10291 
10292 	ds.selector = 0;
10293 	ds.base = 0;
10294 
10295 	cs.limit    = ds.limit = 0xffffffff;
10296 	cs.type     = ds.type = 0x3;
10297 	cs.dpl      = ds.dpl = 0;
10298 	cs.db       = ds.db = 0;
10299 	cs.s        = ds.s = 1;
10300 	cs.l        = ds.l = 0;
10301 	cs.g        = ds.g = 1;
10302 	cs.avl      = ds.avl = 0;
10303 	cs.present  = ds.present = 1;
10304 	cs.unusable = ds.unusable = 0;
10305 	cs.padding  = ds.padding = 0;
10306 
10307 	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
10308 	kvm_set_segment(vcpu, &ds, VCPU_SREG_DS);
10309 	kvm_set_segment(vcpu, &ds, VCPU_SREG_ES);
10310 	kvm_set_segment(vcpu, &ds, VCPU_SREG_FS);
10311 	kvm_set_segment(vcpu, &ds, VCPU_SREG_GS);
10312 	kvm_set_segment(vcpu, &ds, VCPU_SREG_SS);
10313 
10314 #ifdef CONFIG_X86_64
10315 	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
10316 		static_call(kvm_x86_set_efer)(vcpu, 0);
10317 #endif
10318 
10319 	kvm_update_cpuid_runtime(vcpu);
10320 	kvm_mmu_reset_context(vcpu);
10321 }
10322 
process_smi(struct kvm_vcpu * vcpu)10323 static void process_smi(struct kvm_vcpu *vcpu)
10324 {
10325 	vcpu->arch.smi_pending = true;
10326 	kvm_make_request(KVM_REQ_EVENT, vcpu);
10327 }
10328 
kvm_make_scan_ioapic_request_mask(struct kvm * kvm,unsigned long * vcpu_bitmap)10329 void kvm_make_scan_ioapic_request_mask(struct kvm *kvm,
10330 				       unsigned long *vcpu_bitmap)
10331 {
10332 	kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC, vcpu_bitmap);
10333 }
10334 
kvm_make_scan_ioapic_request(struct kvm * kvm)10335 void kvm_make_scan_ioapic_request(struct kvm *kvm)
10336 {
10337 	kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
10338 }
10339 
kvm_vcpu_update_apicv(struct kvm_vcpu * vcpu)10340 void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)
10341 {
10342 	struct kvm_lapic *apic = vcpu->arch.apic;
10343 	bool activate;
10344 
10345 	if (!lapic_in_kernel(vcpu))
10346 		return;
10347 
10348 	down_read(&vcpu->kvm->arch.apicv_update_lock);
10349 	preempt_disable();
10350 
10351 	/* Do not activate APICV when APIC is disabled */
10352 	activate = kvm_vcpu_apicv_activated(vcpu) &&
10353 		   (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED);
10354 
10355 	if (apic->apicv_active == activate)
10356 		goto out;
10357 
10358 	apic->apicv_active = activate;
10359 	kvm_apic_update_apicv(vcpu);
10360 	static_call(kvm_x86_refresh_apicv_exec_ctrl)(vcpu);
10361 
10362 	/*
10363 	 * When APICv gets disabled, we may still have injected interrupts
10364 	 * pending. At the same time, KVM_REQ_EVENT may not be set as APICv was
10365 	 * still active when the interrupt got accepted. Make sure
10366 	 * kvm_check_and_inject_events() is called to check for that.
10367 	 */
10368 	if (!apic->apicv_active)
10369 		kvm_make_request(KVM_REQ_EVENT, vcpu);
10370 
10371 out:
10372 	preempt_enable();
10373 	up_read(&vcpu->kvm->arch.apicv_update_lock);
10374 }
10375 EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv);
10376 
__kvm_set_or_clear_apicv_inhibit(struct kvm * kvm,enum kvm_apicv_inhibit reason,bool set)10377 void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
10378 				      enum kvm_apicv_inhibit reason, bool set)
10379 {
10380 	unsigned long old, new;
10381 
10382 	lockdep_assert_held_write(&kvm->arch.apicv_update_lock);
10383 
10384 	if (!static_call(kvm_x86_check_apicv_inhibit_reasons)(reason))
10385 		return;
10386 
10387 	old = new = kvm->arch.apicv_inhibit_reasons;
10388 
10389 	set_or_clear_apicv_inhibit(&new, reason, set);
10390 
10391 	if (!!old != !!new) {
10392 		/*
10393 		 * Kick all vCPUs before setting apicv_inhibit_reasons to avoid
10394 		 * false positives in the sanity check WARN in svm_vcpu_run().
10395 		 * This task will wait for all vCPUs to ack the kick IRQ before
10396 		 * updating apicv_inhibit_reasons, and all other vCPUs will
10397 		 * block on acquiring apicv_update_lock so that vCPUs can't
10398 		 * redo svm_vcpu_run() without seeing the new inhibit state.
10399 		 *
10400 		 * Note, holding apicv_update_lock and taking it in the read
10401 		 * side (handling the request) also prevents other vCPUs from
10402 		 * servicing the request with a stale apicv_inhibit_reasons.
10403 		 */
10404 		kvm_make_all_cpus_request(kvm, KVM_REQ_APICV_UPDATE);
10405 		kvm->arch.apicv_inhibit_reasons = new;
10406 		if (new) {
10407 			unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE);
10408 			int idx = srcu_read_lock(&kvm->srcu);
10409 
10410 			kvm_zap_gfn_range(kvm, gfn, gfn+1);
10411 			srcu_read_unlock(&kvm->srcu, idx);
10412 		}
10413 	} else {
10414 		kvm->arch.apicv_inhibit_reasons = new;
10415 	}
10416 }
10417 
kvm_set_or_clear_apicv_inhibit(struct kvm * kvm,enum kvm_apicv_inhibit reason,bool set)10418 void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
10419 				    enum kvm_apicv_inhibit reason, bool set)
10420 {
10421 	if (!enable_apicv)
10422 		return;
10423 
10424 	down_write(&kvm->arch.apicv_update_lock);
10425 	__kvm_set_or_clear_apicv_inhibit(kvm, reason, set);
10426 	up_write(&kvm->arch.apicv_update_lock);
10427 }
10428 EXPORT_SYMBOL_GPL(kvm_set_or_clear_apicv_inhibit);
10429 
vcpu_scan_ioapic(struct kvm_vcpu * vcpu)10430 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
10431 {
10432 	if (!kvm_apic_present(vcpu))
10433 		return;
10434 
10435 	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
10436 
10437 	if (irqchip_split(vcpu->kvm))
10438 		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
10439 	else {
10440 		static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
10441 		if (ioapic_in_kernel(vcpu->kvm))
10442 			kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
10443 	}
10444 
10445 	if (is_guest_mode(vcpu))
10446 		vcpu->arch.load_eoi_exitmap_pending = true;
10447 	else
10448 		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
10449 }
10450 
vcpu_load_eoi_exitmap(struct kvm_vcpu * vcpu)10451 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
10452 {
10453 	u64 eoi_exit_bitmap[4];
10454 
10455 	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
10456 		return;
10457 
10458 	if (to_hv_vcpu(vcpu)) {
10459 		bitmap_or((ulong *)eoi_exit_bitmap,
10460 			  vcpu->arch.ioapic_handled_vectors,
10461 			  to_hv_synic(vcpu)->vec_bitmap, 256);
10462 		static_call_cond(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap);
10463 		return;
10464 	}
10465 
10466 	static_call_cond(kvm_x86_load_eoi_exitmap)(
10467 		vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors);
10468 }
10469 
kvm_arch_mmu_notifier_invalidate_range(struct kvm * kvm,unsigned long start,unsigned long end)10470 void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
10471 					    unsigned long start, unsigned long end)
10472 {
10473 	unsigned long apic_address;
10474 
10475 	/*
10476 	 * The physical address of apic access page is stored in the VMCS.
10477 	 * Update it when it becomes invalid.
10478 	 */
10479 	apic_address = gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
10480 	if (start <= apic_address && apic_address < end)
10481 		kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
10482 }
10483 
kvm_arch_guest_memory_reclaimed(struct kvm * kvm)10484 void kvm_arch_guest_memory_reclaimed(struct kvm *kvm)
10485 {
10486 	static_call_cond(kvm_x86_guest_memory_reclaimed)(kvm);
10487 }
10488 
kvm_vcpu_reload_apic_access_page(struct kvm_vcpu * vcpu)10489 static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
10490 {
10491 	if (!lapic_in_kernel(vcpu))
10492 		return;
10493 
10494 	static_call_cond(kvm_x86_set_apic_access_page_addr)(vcpu);
10495 }
10496 
__kvm_request_immediate_exit(struct kvm_vcpu * vcpu)10497 void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
10498 {
10499 	smp_send_reschedule(vcpu->cpu);
10500 }
10501 EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit);
10502 
10503 /*
10504  * Called within kvm->srcu read side.
10505  * Returns 1 to let vcpu_run() continue the guest execution loop without
10506  * exiting to the userspace.  Otherwise, the value will be returned to the
10507  * userspace.
10508  */
vcpu_enter_guest(struct kvm_vcpu * vcpu)10509 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
10510 {
10511 	int r;
10512 	bool req_int_win =
10513 		dm_request_for_irq_injection(vcpu) &&
10514 		kvm_cpu_accept_dm_intr(vcpu);
10515 	fastpath_t exit_fastpath;
10516 
10517 	bool req_immediate_exit = false;
10518 
10519 	/* Forbid vmenter if vcpu dirty ring is soft-full */
10520 	if (unlikely(vcpu->kvm->dirty_ring_size &&
10521 		     kvm_dirty_ring_soft_full(&vcpu->dirty_ring))) {
10522 		vcpu->run->exit_reason = KVM_EXIT_DIRTY_RING_FULL;
10523 		trace_kvm_dirty_ring_exit(vcpu);
10524 		r = 0;
10525 		goto out;
10526 	}
10527 
10528 	if (kvm_request_pending(vcpu)) {
10529 		if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {
10530 			r = -EIO;
10531 			goto out;
10532 		}
10533 		if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
10534 			if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {
10535 				r = 0;
10536 				goto out;
10537 			}
10538 		}
10539 		if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))
10540 			kvm_mmu_free_obsolete_roots(vcpu);
10541 		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
10542 			__kvm_migrate_timers(vcpu);
10543 		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
10544 			kvm_update_masterclock(vcpu->kvm);
10545 		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
10546 			kvm_gen_kvmclock_update(vcpu);
10547 		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
10548 			r = kvm_guest_time_update(vcpu);
10549 			if (unlikely(r))
10550 				goto out;
10551 		}
10552 		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
10553 			kvm_mmu_sync_roots(vcpu);
10554 		if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu))
10555 			kvm_mmu_load_pgd(vcpu);
10556 		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
10557 			kvm_vcpu_flush_tlb_all(vcpu);
10558 
10559 			/* Flushing all ASIDs flushes the current ASID... */
10560 			kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
10561 		}
10562 		kvm_service_local_tlb_flush_requests(vcpu);
10563 
10564 		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
10565 			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
10566 			r = 0;
10567 			goto out;
10568 		}
10569 		if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
10570 			if (is_guest_mode(vcpu))
10571 				kvm_x86_ops.nested_ops->triple_fault(vcpu);
10572 
10573 			if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
10574 				vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
10575 				vcpu->mmio_needed = 0;
10576 				r = 0;
10577 				goto out;
10578 			}
10579 		}
10580 		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
10581 			/* Page is swapped out. Do synthetic halt */
10582 			vcpu->arch.apf.halted = true;
10583 			r = 1;
10584 			goto out;
10585 		}
10586 		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
10587 			record_steal_time(vcpu);
10588 		if (kvm_check_request(KVM_REQ_SMI, vcpu))
10589 			process_smi(vcpu);
10590 		if (kvm_check_request(KVM_REQ_NMI, vcpu))
10591 			process_nmi(vcpu);
10592 		if (kvm_check_request(KVM_REQ_PMU, vcpu))
10593 			kvm_pmu_handle_event(vcpu);
10594 		if (kvm_check_request(KVM_REQ_PMI, vcpu))
10595 			kvm_pmu_deliver_pmi(vcpu);
10596 		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
10597 			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
10598 			if (test_bit(vcpu->arch.pending_ioapic_eoi,
10599 				     vcpu->arch.ioapic_handled_vectors)) {
10600 				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
10601 				vcpu->run->eoi.vector =
10602 						vcpu->arch.pending_ioapic_eoi;
10603 				r = 0;
10604 				goto out;
10605 			}
10606 		}
10607 		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
10608 			vcpu_scan_ioapic(vcpu);
10609 		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))
10610 			vcpu_load_eoi_exitmap(vcpu);
10611 		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
10612 			kvm_vcpu_reload_apic_access_page(vcpu);
10613 		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {
10614 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
10615 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;
10616 			vcpu->run->system_event.ndata = 0;
10617 			r = 0;
10618 			goto out;
10619 		}
10620 		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {
10621 			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
10622 			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;
10623 			vcpu->run->system_event.ndata = 0;
10624 			r = 0;
10625 			goto out;
10626 		}
10627 		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
10628 			struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
10629 
10630 			vcpu->run->exit_reason = KVM_EXIT_HYPERV;
10631 			vcpu->run->hyperv = hv_vcpu->exit;
10632 			r = 0;
10633 			goto out;
10634 		}
10635 
10636 		/*
10637 		 * KVM_REQ_HV_STIMER has to be processed after
10638 		 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers
10639 		 * depend on the guest clock being up-to-date
10640 		 */
10641 		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
10642 			kvm_hv_process_stimers(vcpu);
10643 		if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))
10644 			kvm_vcpu_update_apicv(vcpu);
10645 		if (kvm_check_request(KVM_REQ_APF_READY, vcpu))
10646 			kvm_check_async_pf_completion(vcpu);
10647 		if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))
10648 			static_call(kvm_x86_msr_filter_changed)(vcpu);
10649 
10650 		if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu))
10651 			static_call(kvm_x86_update_cpu_dirty_logging)(vcpu);
10652 	}
10653 
10654 	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||
10655 	    kvm_xen_has_interrupt(vcpu)) {
10656 		++vcpu->stat.req_event;
10657 		r = kvm_apic_accept_events(vcpu);
10658 		if (r < 0) {
10659 			r = 0;
10660 			goto out;
10661 		}
10662 		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
10663 			r = 1;
10664 			goto out;
10665 		}
10666 
10667 		r = kvm_check_and_inject_events(vcpu, &req_immediate_exit);
10668 		if (r < 0) {
10669 			r = 0;
10670 			goto out;
10671 		}
10672 		if (req_int_win)
10673 			static_call(kvm_x86_enable_irq_window)(vcpu);
10674 
10675 		if (kvm_lapic_enabled(vcpu)) {
10676 			update_cr8_intercept(vcpu);
10677 			kvm_lapic_sync_to_vapic(vcpu);
10678 		}
10679 	}
10680 
10681 	r = kvm_mmu_reload(vcpu);
10682 	if (unlikely(r)) {
10683 		goto cancel_injection;
10684 	}
10685 
10686 	preempt_disable();
10687 
10688 	static_call(kvm_x86_prepare_switch_to_guest)(vcpu);
10689 
10690 	/*
10691 	 * Disable IRQs before setting IN_GUEST_MODE.  Posted interrupt
10692 	 * IPI are then delayed after guest entry, which ensures that they
10693 	 * result in virtual interrupt delivery.
10694 	 */
10695 	local_irq_disable();
10696 
10697 	/* Store vcpu->apicv_active before vcpu->mode.  */
10698 	smp_store_release(&vcpu->mode, IN_GUEST_MODE);
10699 
10700 	kvm_vcpu_srcu_read_unlock(vcpu);
10701 
10702 	/*
10703 	 * 1) We should set ->mode before checking ->requests.  Please see
10704 	 * the comment in kvm_vcpu_exiting_guest_mode().
10705 	 *
10706 	 * 2) For APICv, we should set ->mode before checking PID.ON. This
10707 	 * pairs with the memory barrier implicit in pi_test_and_set_on
10708 	 * (see vmx_deliver_posted_interrupt).
10709 	 *
10710 	 * 3) This also orders the write to mode from any reads to the page
10711 	 * tables done while the VCPU is running.  Please see the comment
10712 	 * in kvm_flush_remote_tlbs.
10713 	 */
10714 	smp_mb__after_srcu_read_unlock();
10715 
10716 	/*
10717 	 * Process pending posted interrupts to handle the case where the
10718 	 * notification IRQ arrived in the host, or was never sent (because the
10719 	 * target vCPU wasn't running).  Do this regardless of the vCPU's APICv
10720 	 * status, KVM doesn't update assigned devices when APICv is inhibited,
10721 	 * i.e. they can post interrupts even if APICv is temporarily disabled.
10722 	 */
10723 	if (kvm_lapic_enabled(vcpu))
10724 		static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
10725 
10726 	if (kvm_vcpu_exit_request(vcpu)) {
10727 		vcpu->mode = OUTSIDE_GUEST_MODE;
10728 		smp_wmb();
10729 		local_irq_enable();
10730 		preempt_enable();
10731 		kvm_vcpu_srcu_read_lock(vcpu);
10732 		r = 1;
10733 		goto cancel_injection;
10734 	}
10735 
10736 	if (req_immediate_exit) {
10737 		kvm_make_request(KVM_REQ_EVENT, vcpu);
10738 		static_call(kvm_x86_request_immediate_exit)(vcpu);
10739 	}
10740 
10741 	fpregs_assert_state_consistent();
10742 	if (test_thread_flag(TIF_NEED_FPU_LOAD))
10743 		switch_fpu_return();
10744 
10745 	if (vcpu->arch.guest_fpu.xfd_err)
10746 		wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
10747 
10748 	if (unlikely(vcpu->arch.switch_db_regs)) {
10749 		set_debugreg(0, 7);
10750 		set_debugreg(vcpu->arch.eff_db[0], 0);
10751 		set_debugreg(vcpu->arch.eff_db[1], 1);
10752 		set_debugreg(vcpu->arch.eff_db[2], 2);
10753 		set_debugreg(vcpu->arch.eff_db[3], 3);
10754 	} else if (unlikely(hw_breakpoint_active())) {
10755 		set_debugreg(0, 7);
10756 	}
10757 
10758 	guest_timing_enter_irqoff();
10759 
10760 	for (;;) {
10761 		/*
10762 		 * Assert that vCPU vs. VM APICv state is consistent.  An APICv
10763 		 * update must kick and wait for all vCPUs before toggling the
10764 		 * per-VM state, and responsing vCPUs must wait for the update
10765 		 * to complete before servicing KVM_REQ_APICV_UPDATE.
10766 		 */
10767 		WARN_ON_ONCE((kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)) &&
10768 			     (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED));
10769 
10770 		exit_fastpath = static_call(kvm_x86_vcpu_run)(vcpu);
10771 		if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST))
10772 			break;
10773 
10774 		if (kvm_lapic_enabled(vcpu))
10775 			static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
10776 
10777 		if (unlikely(kvm_vcpu_exit_request(vcpu))) {
10778 			exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED;
10779 			break;
10780 		}
10781 	}
10782 
10783 	/*
10784 	 * Do this here before restoring debug registers on the host.  And
10785 	 * since we do this before handling the vmexit, a DR access vmexit
10786 	 * can (a) read the correct value of the debug registers, (b) set
10787 	 * KVM_DEBUGREG_WONT_EXIT again.
10788 	 */
10789 	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
10790 		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
10791 		static_call(kvm_x86_sync_dirty_debug_regs)(vcpu);
10792 		kvm_update_dr0123(vcpu);
10793 		kvm_update_dr7(vcpu);
10794 	}
10795 
10796 	/*
10797 	 * If the guest has used debug registers, at least dr7
10798 	 * will be disabled while returning to the host.
10799 	 * If we don't have active breakpoints in the host, we don't
10800 	 * care about the messed up debug address registers. But if
10801 	 * we have some of them active, restore the old state.
10802 	 */
10803 	if (hw_breakpoint_active())
10804 		hw_breakpoint_restore();
10805 
10806 	vcpu->arch.last_vmentry_cpu = vcpu->cpu;
10807 	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
10808 
10809 	vcpu->mode = OUTSIDE_GUEST_MODE;
10810 	smp_wmb();
10811 
10812 	/*
10813 	 * Sync xfd before calling handle_exit_irqoff() which may
10814 	 * rely on the fact that guest_fpu::xfd is up-to-date (e.g.
10815 	 * in #NM irqoff handler).
10816 	 */
10817 	if (vcpu->arch.xfd_no_write_intercept)
10818 		fpu_sync_guest_vmexit_xfd_state();
10819 
10820 	static_call(kvm_x86_handle_exit_irqoff)(vcpu);
10821 
10822 	if (vcpu->arch.guest_fpu.xfd_err)
10823 		wrmsrl(MSR_IA32_XFD_ERR, 0);
10824 
10825 	/*
10826 	 * Consume any pending interrupts, including the possible source of
10827 	 * VM-Exit on SVM and any ticks that occur between VM-Exit and now.
10828 	 * An instruction is required after local_irq_enable() to fully unblock
10829 	 * interrupts on processors that implement an interrupt shadow, the
10830 	 * stat.exits increment will do nicely.
10831 	 */
10832 	kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
10833 	local_irq_enable();
10834 	++vcpu->stat.exits;
10835 	local_irq_disable();
10836 	kvm_after_interrupt(vcpu);
10837 
10838 	/*
10839 	 * Wait until after servicing IRQs to account guest time so that any
10840 	 * ticks that occurred while running the guest are properly accounted
10841 	 * to the guest.  Waiting until IRQs are enabled degrades the accuracy
10842 	 * of accounting via context tracking, but the loss of accuracy is
10843 	 * acceptable for all known use cases.
10844 	 */
10845 	guest_timing_exit_irqoff();
10846 
10847 	local_irq_enable();
10848 	preempt_enable();
10849 
10850 	kvm_vcpu_srcu_read_lock(vcpu);
10851 
10852 	/*
10853 	 * Profile KVM exit RIPs:
10854 	 */
10855 	if (unlikely(prof_on == KVM_PROFILING)) {
10856 		unsigned long rip = kvm_rip_read(vcpu);
10857 		profile_hit(KVM_PROFILING, (void *)rip);
10858 	}
10859 
10860 	if (unlikely(vcpu->arch.tsc_always_catchup))
10861 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
10862 
10863 	if (vcpu->arch.apic_attention)
10864 		kvm_lapic_sync_from_vapic(vcpu);
10865 
10866 	r = static_call(kvm_x86_handle_exit)(vcpu, exit_fastpath);
10867 	return r;
10868 
10869 cancel_injection:
10870 	if (req_immediate_exit)
10871 		kvm_make_request(KVM_REQ_EVENT, vcpu);
10872 	static_call(kvm_x86_cancel_injection)(vcpu);
10873 	if (unlikely(vcpu->arch.apic_attention))
10874 		kvm_lapic_sync_from_vapic(vcpu);
10875 out:
10876 	return r;
10877 }
10878 
10879 /* Called within kvm->srcu read side.  */
vcpu_block(struct kvm_vcpu * vcpu)10880 static inline int vcpu_block(struct kvm_vcpu *vcpu)
10881 {
10882 	bool hv_timer;
10883 
10884 	if (!kvm_arch_vcpu_runnable(vcpu)) {
10885 		/*
10886 		 * Switch to the software timer before halt-polling/blocking as
10887 		 * the guest's timer may be a break event for the vCPU, and the
10888 		 * hypervisor timer runs only when the CPU is in guest mode.
10889 		 * Switch before halt-polling so that KVM recognizes an expired
10890 		 * timer before blocking.
10891 		 */
10892 		hv_timer = kvm_lapic_hv_timer_in_use(vcpu);
10893 		if (hv_timer)
10894 			kvm_lapic_switch_to_sw_timer(vcpu);
10895 
10896 		kvm_vcpu_srcu_read_unlock(vcpu);
10897 		if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
10898 			kvm_vcpu_halt(vcpu);
10899 		else
10900 			kvm_vcpu_block(vcpu);
10901 		kvm_vcpu_srcu_read_lock(vcpu);
10902 
10903 		if (hv_timer)
10904 			kvm_lapic_switch_to_hv_timer(vcpu);
10905 
10906 		/*
10907 		 * If the vCPU is not runnable, a signal or another host event
10908 		 * of some kind is pending; service it without changing the
10909 		 * vCPU's activity state.
10910 		 */
10911 		if (!kvm_arch_vcpu_runnable(vcpu))
10912 			return 1;
10913 	}
10914 
10915 	/*
10916 	 * Evaluate nested events before exiting the halted state.  This allows
10917 	 * the halt state to be recorded properly in the VMCS12's activity
10918 	 * state field (AMD does not have a similar field and a VM-Exit always
10919 	 * causes a spurious wakeup from HLT).
10920 	 */
10921 	if (is_guest_mode(vcpu)) {
10922 		if (kvm_check_nested_events(vcpu) < 0)
10923 			return 0;
10924 	}
10925 
10926 	if (kvm_apic_accept_events(vcpu) < 0)
10927 		return 0;
10928 	switch(vcpu->arch.mp_state) {
10929 	case KVM_MP_STATE_HALTED:
10930 	case KVM_MP_STATE_AP_RESET_HOLD:
10931 		vcpu->arch.pv.pv_unhalted = false;
10932 		vcpu->arch.mp_state =
10933 			KVM_MP_STATE_RUNNABLE;
10934 		fallthrough;
10935 	case KVM_MP_STATE_RUNNABLE:
10936 		vcpu->arch.apf.halted = false;
10937 		break;
10938 	case KVM_MP_STATE_INIT_RECEIVED:
10939 		break;
10940 	default:
10941 		WARN_ON_ONCE(1);
10942 		break;
10943 	}
10944 	return 1;
10945 }
10946 
kvm_vcpu_running(struct kvm_vcpu * vcpu)10947 static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
10948 {
10949 	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
10950 		!vcpu->arch.apf.halted);
10951 }
10952 
10953 /* Called within kvm->srcu read side.  */
vcpu_run(struct kvm_vcpu * vcpu)10954 static int vcpu_run(struct kvm_vcpu *vcpu)
10955 {
10956 	int r;
10957 
10958 	vcpu->arch.l1tf_flush_l1d = true;
10959 
10960 	for (;;) {
10961 		/*
10962 		 * If another guest vCPU requests a PV TLB flush in the middle
10963 		 * of instruction emulation, the rest of the emulation could
10964 		 * use a stale page translation. Assume that any code after
10965 		 * this point can start executing an instruction.
10966 		 */
10967 		vcpu->arch.at_instruction_boundary = false;
10968 		if (kvm_vcpu_running(vcpu)) {
10969 			r = vcpu_enter_guest(vcpu);
10970 		} else {
10971 			r = vcpu_block(vcpu);
10972 		}
10973 
10974 		if (r <= 0)
10975 			break;
10976 
10977 		kvm_clear_request(KVM_REQ_UNBLOCK, vcpu);
10978 		if (kvm_xen_has_pending_events(vcpu))
10979 			kvm_xen_inject_pending_events(vcpu);
10980 
10981 		if (kvm_cpu_has_pending_timer(vcpu))
10982 			kvm_inject_pending_timer_irqs(vcpu);
10983 
10984 		if (dm_request_for_irq_injection(vcpu) &&
10985 			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {
10986 			r = 0;
10987 			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
10988 			++vcpu->stat.request_irq_exits;
10989 			break;
10990 		}
10991 
10992 		if (__xfer_to_guest_mode_work_pending()) {
10993 			kvm_vcpu_srcu_read_unlock(vcpu);
10994 			r = xfer_to_guest_mode_handle_work(vcpu);
10995 			kvm_vcpu_srcu_read_lock(vcpu);
10996 			if (r)
10997 				return r;
10998 		}
10999 	}
11000 
11001 	return r;
11002 }
11003 
complete_emulated_io(struct kvm_vcpu * vcpu)11004 static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
11005 {
11006 	return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
11007 }
11008 
complete_emulated_pio(struct kvm_vcpu * vcpu)11009 static int complete_emulated_pio(struct kvm_vcpu *vcpu)
11010 {
11011 	BUG_ON(!vcpu->arch.pio.count);
11012 
11013 	return complete_emulated_io(vcpu);
11014 }
11015 
11016 /*
11017  * Implements the following, as a state machine:
11018  *
11019  * read:
11020  *   for each fragment
11021  *     for each mmio piece in the fragment
11022  *       write gpa, len
11023  *       exit
11024  *       copy data
11025  *   execute insn
11026  *
11027  * write:
11028  *   for each fragment
11029  *     for each mmio piece in the fragment
11030  *       write gpa, len
11031  *       copy data
11032  *       exit
11033  */
complete_emulated_mmio(struct kvm_vcpu * vcpu)11034 static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
11035 {
11036 	struct kvm_run *run = vcpu->run;
11037 	struct kvm_mmio_fragment *frag;
11038 	unsigned len;
11039 
11040 	BUG_ON(!vcpu->mmio_needed);
11041 
11042 	/* Complete previous fragment */
11043 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
11044 	len = min(8u, frag->len);
11045 	if (!vcpu->mmio_is_write)
11046 		memcpy(frag->data, run->mmio.data, len);
11047 
11048 	if (frag->len <= 8) {
11049 		/* Switch to the next fragment. */
11050 		frag++;
11051 		vcpu->mmio_cur_fragment++;
11052 	} else {
11053 		/* Go forward to the next mmio piece. */
11054 		frag->data += len;
11055 		frag->gpa += len;
11056 		frag->len -= len;
11057 	}
11058 
11059 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
11060 		vcpu->mmio_needed = 0;
11061 
11062 		/* FIXME: return into emulator if single-stepping.  */
11063 		if (vcpu->mmio_is_write)
11064 			return 1;
11065 		vcpu->mmio_read_completed = 1;
11066 		return complete_emulated_io(vcpu);
11067 	}
11068 
11069 	run->exit_reason = KVM_EXIT_MMIO;
11070 	run->mmio.phys_addr = frag->gpa;
11071 	if (vcpu->mmio_is_write)
11072 		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
11073 	run->mmio.len = min(8u, frag->len);
11074 	run->mmio.is_write = vcpu->mmio_is_write;
11075 	vcpu->arch.complete_userspace_io = complete_emulated_mmio;
11076 	return 0;
11077 }
11078 
11079 /* Swap (qemu) user FPU context for the guest FPU context. */
kvm_load_guest_fpu(struct kvm_vcpu * vcpu)11080 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
11081 {
11082 	/* Exclude PKRU, it's restored separately immediately after VM-Exit. */
11083 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);
11084 	trace_kvm_fpu(1);
11085 }
11086 
11087 /* When vcpu_run ends, restore user space FPU context. */
kvm_put_guest_fpu(struct kvm_vcpu * vcpu)11088 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
11089 {
11090 	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);
11091 	++vcpu->stat.fpu_reload;
11092 	trace_kvm_fpu(0);
11093 }
11094 
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu)11095 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
11096 {
11097 	struct kvm_queued_exception *ex = &vcpu->arch.exception;
11098 	struct kvm_run *kvm_run = vcpu->run;
11099 	int r;
11100 
11101 	vcpu_load(vcpu);
11102 	kvm_sigset_activate(vcpu);
11103 	kvm_run->flags = 0;
11104 	kvm_load_guest_fpu(vcpu);
11105 
11106 	kvm_vcpu_srcu_read_lock(vcpu);
11107 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
11108 		if (kvm_run->immediate_exit) {
11109 			r = -EINTR;
11110 			goto out;
11111 		}
11112 		/*
11113 		 * It should be impossible for the hypervisor timer to be in
11114 		 * use before KVM has ever run the vCPU.
11115 		 */
11116 		WARN_ON_ONCE(kvm_lapic_hv_timer_in_use(vcpu));
11117 
11118 		kvm_vcpu_srcu_read_unlock(vcpu);
11119 		kvm_vcpu_block(vcpu);
11120 		kvm_vcpu_srcu_read_lock(vcpu);
11121 
11122 		if (kvm_apic_accept_events(vcpu) < 0) {
11123 			r = 0;
11124 			goto out;
11125 		}
11126 		r = -EAGAIN;
11127 		if (signal_pending(current)) {
11128 			r = -EINTR;
11129 			kvm_run->exit_reason = KVM_EXIT_INTR;
11130 			++vcpu->stat.signal_exits;
11131 		}
11132 		goto out;
11133 	}
11134 
11135 	if ((kvm_run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) ||
11136 	    (kvm_run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)) {
11137 		r = -EINVAL;
11138 		goto out;
11139 	}
11140 
11141 	if (kvm_run->kvm_dirty_regs) {
11142 		r = sync_regs(vcpu);
11143 		if (r != 0)
11144 			goto out;
11145 	}
11146 
11147 	/* re-sync apic's tpr */
11148 	if (!lapic_in_kernel(vcpu)) {
11149 		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
11150 			r = -EINVAL;
11151 			goto out;
11152 		}
11153 	}
11154 
11155 	/*
11156 	 * If userspace set a pending exception and L2 is active, convert it to
11157 	 * a pending VM-Exit if L1 wants to intercept the exception.
11158 	 */
11159 	if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) &&
11160 	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector,
11161 							ex->error_code)) {
11162 		kvm_queue_exception_vmexit(vcpu, ex->vector,
11163 					   ex->has_error_code, ex->error_code,
11164 					   ex->has_payload, ex->payload);
11165 		ex->injected = false;
11166 		ex->pending = false;
11167 	}
11168 	vcpu->arch.exception_from_userspace = false;
11169 
11170 	if (unlikely(vcpu->arch.complete_userspace_io)) {
11171 		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
11172 		vcpu->arch.complete_userspace_io = NULL;
11173 		r = cui(vcpu);
11174 		if (r <= 0)
11175 			goto out;
11176 	} else {
11177 		WARN_ON_ONCE(vcpu->arch.pio.count);
11178 		WARN_ON_ONCE(vcpu->mmio_needed);
11179 	}
11180 
11181 	if (kvm_run->immediate_exit) {
11182 		r = -EINTR;
11183 		goto out;
11184 	}
11185 
11186 	r = static_call(kvm_x86_vcpu_pre_run)(vcpu);
11187 	if (r <= 0)
11188 		goto out;
11189 
11190 	r = vcpu_run(vcpu);
11191 
11192 out:
11193 	kvm_put_guest_fpu(vcpu);
11194 	if (kvm_run->kvm_valid_regs)
11195 		store_regs(vcpu);
11196 	post_kvm_run_save(vcpu);
11197 	kvm_vcpu_srcu_read_unlock(vcpu);
11198 
11199 	kvm_sigset_deactivate(vcpu);
11200 	vcpu_put(vcpu);
11201 	return r;
11202 }
11203 
__get_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)11204 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11205 {
11206 	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
11207 		/*
11208 		 * We are here if userspace calls get_regs() in the middle of
11209 		 * instruction emulation. Registers state needs to be copied
11210 		 * back from emulation context to vcpu. Userspace shouldn't do
11211 		 * that usually, but some bad designed PV devices (vmware
11212 		 * backdoor interface) need this to work
11213 		 */
11214 		emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);
11215 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
11216 	}
11217 	regs->rax = kvm_rax_read(vcpu);
11218 	regs->rbx = kvm_rbx_read(vcpu);
11219 	regs->rcx = kvm_rcx_read(vcpu);
11220 	regs->rdx = kvm_rdx_read(vcpu);
11221 	regs->rsi = kvm_rsi_read(vcpu);
11222 	regs->rdi = kvm_rdi_read(vcpu);
11223 	regs->rsp = kvm_rsp_read(vcpu);
11224 	regs->rbp = kvm_rbp_read(vcpu);
11225 #ifdef CONFIG_X86_64
11226 	regs->r8 = kvm_r8_read(vcpu);
11227 	regs->r9 = kvm_r9_read(vcpu);
11228 	regs->r10 = kvm_r10_read(vcpu);
11229 	regs->r11 = kvm_r11_read(vcpu);
11230 	regs->r12 = kvm_r12_read(vcpu);
11231 	regs->r13 = kvm_r13_read(vcpu);
11232 	regs->r14 = kvm_r14_read(vcpu);
11233 	regs->r15 = kvm_r15_read(vcpu);
11234 #endif
11235 
11236 	regs->rip = kvm_rip_read(vcpu);
11237 	regs->rflags = kvm_get_rflags(vcpu);
11238 }
11239 
kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)11240 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11241 {
11242 	vcpu_load(vcpu);
11243 	__get_regs(vcpu, regs);
11244 	vcpu_put(vcpu);
11245 	return 0;
11246 }
11247 
__set_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)11248 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11249 {
11250 	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
11251 	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
11252 
11253 	kvm_rax_write(vcpu, regs->rax);
11254 	kvm_rbx_write(vcpu, regs->rbx);
11255 	kvm_rcx_write(vcpu, regs->rcx);
11256 	kvm_rdx_write(vcpu, regs->rdx);
11257 	kvm_rsi_write(vcpu, regs->rsi);
11258 	kvm_rdi_write(vcpu, regs->rdi);
11259 	kvm_rsp_write(vcpu, regs->rsp);
11260 	kvm_rbp_write(vcpu, regs->rbp);
11261 #ifdef CONFIG_X86_64
11262 	kvm_r8_write(vcpu, regs->r8);
11263 	kvm_r9_write(vcpu, regs->r9);
11264 	kvm_r10_write(vcpu, regs->r10);
11265 	kvm_r11_write(vcpu, regs->r11);
11266 	kvm_r12_write(vcpu, regs->r12);
11267 	kvm_r13_write(vcpu, regs->r13);
11268 	kvm_r14_write(vcpu, regs->r14);
11269 	kvm_r15_write(vcpu, regs->r15);
11270 #endif
11271 
11272 	kvm_rip_write(vcpu, regs->rip);
11273 	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
11274 
11275 	vcpu->arch.exception.pending = false;
11276 	vcpu->arch.exception_vmexit.pending = false;
11277 
11278 	kvm_make_request(KVM_REQ_EVENT, vcpu);
11279 }
11280 
kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)11281 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11282 {
11283 	vcpu_load(vcpu);
11284 	__set_regs(vcpu, regs);
11285 	vcpu_put(vcpu);
11286 	return 0;
11287 }
11288 
__get_sregs_common(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)11289 static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11290 {
11291 	struct desc_ptr dt;
11292 
11293 	if (vcpu->arch.guest_state_protected)
11294 		goto skip_protected_regs;
11295 
11296 	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
11297 	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
11298 	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
11299 	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
11300 	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
11301 	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
11302 
11303 	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
11304 	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
11305 
11306 	static_call(kvm_x86_get_idt)(vcpu, &dt);
11307 	sregs->idt.limit = dt.size;
11308 	sregs->idt.base = dt.address;
11309 	static_call(kvm_x86_get_gdt)(vcpu, &dt);
11310 	sregs->gdt.limit = dt.size;
11311 	sregs->gdt.base = dt.address;
11312 
11313 	sregs->cr2 = vcpu->arch.cr2;
11314 	sregs->cr3 = kvm_read_cr3(vcpu);
11315 
11316 skip_protected_regs:
11317 	sregs->cr0 = kvm_read_cr0(vcpu);
11318 	sregs->cr4 = kvm_read_cr4(vcpu);
11319 	sregs->cr8 = kvm_get_cr8(vcpu);
11320 	sregs->efer = vcpu->arch.efer;
11321 	sregs->apic_base = kvm_get_apic_base(vcpu);
11322 }
11323 
__get_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)11324 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11325 {
11326 	__get_sregs_common(vcpu, sregs);
11327 
11328 	if (vcpu->arch.guest_state_protected)
11329 		return;
11330 
11331 	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
11332 		set_bit(vcpu->arch.interrupt.nr,
11333 			(unsigned long *)sregs->interrupt_bitmap);
11334 }
11335 
__get_sregs2(struct kvm_vcpu * vcpu,struct kvm_sregs2 * sregs2)11336 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
11337 {
11338 	int i;
11339 
11340 	__get_sregs_common(vcpu, (struct kvm_sregs *)sregs2);
11341 
11342 	if (vcpu->arch.guest_state_protected)
11343 		return;
11344 
11345 	if (is_pae_paging(vcpu)) {
11346 		for (i = 0 ; i < 4 ; i++)
11347 			sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);
11348 		sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID;
11349 	}
11350 }
11351 
kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)11352 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
11353 				  struct kvm_sregs *sregs)
11354 {
11355 	vcpu_load(vcpu);
11356 	__get_sregs(vcpu, sregs);
11357 	vcpu_put(vcpu);
11358 	return 0;
11359 }
11360 
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)11361 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
11362 				    struct kvm_mp_state *mp_state)
11363 {
11364 	int r;
11365 
11366 	vcpu_load(vcpu);
11367 	if (kvm_mpx_supported())
11368 		kvm_load_guest_fpu(vcpu);
11369 
11370 	r = kvm_apic_accept_events(vcpu);
11371 	if (r < 0)
11372 		goto out;
11373 	r = 0;
11374 
11375 	if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED ||
11376 	     vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) &&
11377 	    vcpu->arch.pv.pv_unhalted)
11378 		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
11379 	else
11380 		mp_state->mp_state = vcpu->arch.mp_state;
11381 
11382 out:
11383 	if (kvm_mpx_supported())
11384 		kvm_put_guest_fpu(vcpu);
11385 	vcpu_put(vcpu);
11386 	return r;
11387 }
11388 
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)11389 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
11390 				    struct kvm_mp_state *mp_state)
11391 {
11392 	int ret = -EINVAL;
11393 
11394 	vcpu_load(vcpu);
11395 
11396 	switch (mp_state->mp_state) {
11397 	case KVM_MP_STATE_UNINITIALIZED:
11398 	case KVM_MP_STATE_HALTED:
11399 	case KVM_MP_STATE_AP_RESET_HOLD:
11400 	case KVM_MP_STATE_INIT_RECEIVED:
11401 	case KVM_MP_STATE_SIPI_RECEIVED:
11402 		if (!lapic_in_kernel(vcpu))
11403 			goto out;
11404 		break;
11405 
11406 	case KVM_MP_STATE_RUNNABLE:
11407 		break;
11408 
11409 	default:
11410 		goto out;
11411 	}
11412 
11413 	/*
11414 	 * Pending INITs are reported using KVM_SET_VCPU_EVENTS, disallow
11415 	 * forcing the guest into INIT/SIPI if those events are supposed to be
11416 	 * blocked.  KVM prioritizes SMI over INIT, so reject INIT/SIPI state
11417 	 * if an SMI is pending as well.
11418 	 */
11419 	if ((!kvm_apic_init_sipi_allowed(vcpu) || vcpu->arch.smi_pending) &&
11420 	    (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED ||
11421 	     mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED))
11422 		goto out;
11423 
11424 	if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
11425 		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
11426 		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
11427 	} else
11428 		vcpu->arch.mp_state = mp_state->mp_state;
11429 	kvm_make_request(KVM_REQ_EVENT, vcpu);
11430 
11431 	ret = 0;
11432 out:
11433 	vcpu_put(vcpu);
11434 	return ret;
11435 }
11436 
kvm_task_switch(struct kvm_vcpu * vcpu,u16 tss_selector,int idt_index,int reason,bool has_error_code,u32 error_code)11437 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
11438 		    int reason, bool has_error_code, u32 error_code)
11439 {
11440 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
11441 	int ret;
11442 
11443 	init_emulate_ctxt(vcpu);
11444 
11445 	ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
11446 				   has_error_code, error_code);
11447 	if (ret) {
11448 		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
11449 		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
11450 		vcpu->run->internal.ndata = 0;
11451 		return 0;
11452 	}
11453 
11454 	kvm_rip_write(vcpu, ctxt->eip);
11455 	kvm_set_rflags(vcpu, ctxt->eflags);
11456 	return 1;
11457 }
11458 EXPORT_SYMBOL_GPL(kvm_task_switch);
11459 
kvm_is_valid_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)11460 static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11461 {
11462 	if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {
11463 		/*
11464 		 * When EFER.LME and CR0.PG are set, the processor is in
11465 		 * 64-bit mode (though maybe in a 32-bit code segment).
11466 		 * CR4.PAE and EFER.LMA must be set.
11467 		 */
11468 		if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA))
11469 			return false;
11470 		if (kvm_vcpu_is_illegal_gpa(vcpu, sregs->cr3))
11471 			return false;
11472 	} else {
11473 		/*
11474 		 * Not in 64-bit mode: EFER.LMA is clear and the code
11475 		 * segment cannot be 64-bit.
11476 		 */
11477 		if (sregs->efer & EFER_LMA || sregs->cs.l)
11478 			return false;
11479 	}
11480 
11481 	return kvm_is_valid_cr4(vcpu, sregs->cr4);
11482 }
11483 
__set_sregs_common(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs,int * mmu_reset_needed,bool update_pdptrs)11484 static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs,
11485 		int *mmu_reset_needed, bool update_pdptrs)
11486 {
11487 	struct msr_data apic_base_msr;
11488 	int idx;
11489 	struct desc_ptr dt;
11490 
11491 	if (!kvm_is_valid_sregs(vcpu, sregs))
11492 		return -EINVAL;
11493 
11494 	apic_base_msr.data = sregs->apic_base;
11495 	apic_base_msr.host_initiated = true;
11496 	if (kvm_set_apic_base(vcpu, &apic_base_msr))
11497 		return -EINVAL;
11498 
11499 	if (vcpu->arch.guest_state_protected)
11500 		return 0;
11501 
11502 	dt.size = sregs->idt.limit;
11503 	dt.address = sregs->idt.base;
11504 	static_call(kvm_x86_set_idt)(vcpu, &dt);
11505 	dt.size = sregs->gdt.limit;
11506 	dt.address = sregs->gdt.base;
11507 	static_call(kvm_x86_set_gdt)(vcpu, &dt);
11508 
11509 	vcpu->arch.cr2 = sregs->cr2;
11510 	*mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
11511 	vcpu->arch.cr3 = sregs->cr3;
11512 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
11513 	static_call_cond(kvm_x86_post_set_cr3)(vcpu, sregs->cr3);
11514 
11515 	kvm_set_cr8(vcpu, sregs->cr8);
11516 
11517 	*mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
11518 	static_call(kvm_x86_set_efer)(vcpu, sregs->efer);
11519 
11520 	*mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
11521 	static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0);
11522 	vcpu->arch.cr0 = sregs->cr0;
11523 
11524 	*mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
11525 	static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4);
11526 
11527 	if (update_pdptrs) {
11528 		idx = srcu_read_lock(&vcpu->kvm->srcu);
11529 		if (is_pae_paging(vcpu)) {
11530 			load_pdptrs(vcpu, kvm_read_cr3(vcpu));
11531 			*mmu_reset_needed = 1;
11532 		}
11533 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
11534 	}
11535 
11536 	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
11537 	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
11538 	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
11539 	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
11540 	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
11541 	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
11542 
11543 	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
11544 	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
11545 
11546 	update_cr8_intercept(vcpu);
11547 
11548 	/* Older userspace won't unhalt the vcpu on reset. */
11549 	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
11550 	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
11551 	    !is_protmode(vcpu))
11552 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
11553 
11554 	return 0;
11555 }
11556 
__set_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)11557 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11558 {
11559 	int pending_vec, max_bits;
11560 	int mmu_reset_needed = 0;
11561 	int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true);
11562 
11563 	if (ret)
11564 		return ret;
11565 
11566 	if (mmu_reset_needed)
11567 		kvm_mmu_reset_context(vcpu);
11568 
11569 	max_bits = KVM_NR_INTERRUPTS;
11570 	pending_vec = find_first_bit(
11571 		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
11572 
11573 	if (pending_vec < max_bits) {
11574 		kvm_queue_interrupt(vcpu, pending_vec, false);
11575 		pr_debug("Set back pending irq %d\n", pending_vec);
11576 		kvm_make_request(KVM_REQ_EVENT, vcpu);
11577 	}
11578 	return 0;
11579 }
11580 
__set_sregs2(struct kvm_vcpu * vcpu,struct kvm_sregs2 * sregs2)11581 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
11582 {
11583 	int mmu_reset_needed = 0;
11584 	bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID;
11585 	bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) &&
11586 		!(sregs2->efer & EFER_LMA);
11587 	int i, ret;
11588 
11589 	if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID)
11590 		return -EINVAL;
11591 
11592 	if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected))
11593 		return -EINVAL;
11594 
11595 	ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2,
11596 				 &mmu_reset_needed, !valid_pdptrs);
11597 	if (ret)
11598 		return ret;
11599 
11600 	if (valid_pdptrs) {
11601 		for (i = 0; i < 4 ; i++)
11602 			kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]);
11603 
11604 		kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
11605 		mmu_reset_needed = 1;
11606 		vcpu->arch.pdptrs_from_userspace = true;
11607 	}
11608 	if (mmu_reset_needed)
11609 		kvm_mmu_reset_context(vcpu);
11610 	return 0;
11611 }
11612 
kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)11613 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
11614 				  struct kvm_sregs *sregs)
11615 {
11616 	int ret;
11617 
11618 	vcpu_load(vcpu);
11619 	ret = __set_sregs(vcpu, sregs);
11620 	vcpu_put(vcpu);
11621 	return ret;
11622 }
11623 
kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm * kvm)11624 static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm)
11625 {
11626 	bool set = false;
11627 	struct kvm_vcpu *vcpu;
11628 	unsigned long i;
11629 
11630 	if (!enable_apicv)
11631 		return;
11632 
11633 	down_write(&kvm->arch.apicv_update_lock);
11634 
11635 	kvm_for_each_vcpu(i, vcpu, kvm) {
11636 		if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) {
11637 			set = true;
11638 			break;
11639 		}
11640 	}
11641 	__kvm_set_or_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_BLOCKIRQ, set);
11642 	up_write(&kvm->arch.apicv_update_lock);
11643 }
11644 
kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu * vcpu,struct kvm_guest_debug * dbg)11645 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
11646 					struct kvm_guest_debug *dbg)
11647 {
11648 	unsigned long rflags;
11649 	int i, r;
11650 
11651 	if (vcpu->arch.guest_state_protected)
11652 		return -EINVAL;
11653 
11654 	vcpu_load(vcpu);
11655 
11656 	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
11657 		r = -EBUSY;
11658 		if (kvm_is_exception_pending(vcpu))
11659 			goto out;
11660 		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
11661 			kvm_queue_exception(vcpu, DB_VECTOR);
11662 		else
11663 			kvm_queue_exception(vcpu, BP_VECTOR);
11664 	}
11665 
11666 	/*
11667 	 * Read rflags as long as potentially injected trace flags are still
11668 	 * filtered out.
11669 	 */
11670 	rflags = kvm_get_rflags(vcpu);
11671 
11672 	vcpu->guest_debug = dbg->control;
11673 	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
11674 		vcpu->guest_debug = 0;
11675 
11676 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
11677 		for (i = 0; i < KVM_NR_DB_REGS; ++i)
11678 			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
11679 		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
11680 	} else {
11681 		for (i = 0; i < KVM_NR_DB_REGS; i++)
11682 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
11683 	}
11684 	kvm_update_dr7(vcpu);
11685 
11686 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
11687 		vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu);
11688 
11689 	/*
11690 	 * Trigger an rflags update that will inject or remove the trace
11691 	 * flags.
11692 	 */
11693 	kvm_set_rflags(vcpu, rflags);
11694 
11695 	static_call(kvm_x86_update_exception_bitmap)(vcpu);
11696 
11697 	kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm);
11698 
11699 	r = 0;
11700 
11701 out:
11702 	vcpu_put(vcpu);
11703 	return r;
11704 }
11705 
11706 /*
11707  * Translate a guest virtual address to a guest physical address.
11708  */
kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu * vcpu,struct kvm_translation * tr)11709 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
11710 				    struct kvm_translation *tr)
11711 {
11712 	unsigned long vaddr = tr->linear_address;
11713 	gpa_t gpa;
11714 	int idx;
11715 
11716 	vcpu_load(vcpu);
11717 
11718 	idx = srcu_read_lock(&vcpu->kvm->srcu);
11719 	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
11720 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
11721 	tr->physical_address = gpa;
11722 	tr->valid = gpa != INVALID_GPA;
11723 	tr->writeable = 1;
11724 	tr->usermode = 0;
11725 
11726 	vcpu_put(vcpu);
11727 	return 0;
11728 }
11729 
kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)11730 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
11731 {
11732 	struct fxregs_state *fxsave;
11733 
11734 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
11735 		return 0;
11736 
11737 	vcpu_load(vcpu);
11738 
11739 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;
11740 	memcpy(fpu->fpr, fxsave->st_space, 128);
11741 	fpu->fcw = fxsave->cwd;
11742 	fpu->fsw = fxsave->swd;
11743 	fpu->ftwx = fxsave->twd;
11744 	fpu->last_opcode = fxsave->fop;
11745 	fpu->last_ip = fxsave->rip;
11746 	fpu->last_dp = fxsave->rdp;
11747 	memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space));
11748 
11749 	vcpu_put(vcpu);
11750 	return 0;
11751 }
11752 
kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)11753 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
11754 {
11755 	struct fxregs_state *fxsave;
11756 
11757 	if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
11758 		return 0;
11759 
11760 	vcpu_load(vcpu);
11761 
11762 	fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;
11763 
11764 	memcpy(fxsave->st_space, fpu->fpr, 128);
11765 	fxsave->cwd = fpu->fcw;
11766 	fxsave->swd = fpu->fsw;
11767 	fxsave->twd = fpu->ftwx;
11768 	fxsave->fop = fpu->last_opcode;
11769 	fxsave->rip = fpu->last_ip;
11770 	fxsave->rdp = fpu->last_dp;
11771 	memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space));
11772 
11773 	vcpu_put(vcpu);
11774 	return 0;
11775 }
11776 
store_regs(struct kvm_vcpu * vcpu)11777 static void store_regs(struct kvm_vcpu *vcpu)
11778 {
11779 	BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES);
11780 
11781 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
11782 		__get_regs(vcpu, &vcpu->run->s.regs.regs);
11783 
11784 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
11785 		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);
11786 
11787 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
11788 		kvm_vcpu_ioctl_x86_get_vcpu_events(
11789 				vcpu, &vcpu->run->s.regs.events);
11790 }
11791 
sync_regs(struct kvm_vcpu * vcpu)11792 static int sync_regs(struct kvm_vcpu *vcpu)
11793 {
11794 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
11795 		__set_regs(vcpu, &vcpu->run->s.regs.regs);
11796 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
11797 	}
11798 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
11799 		if (__set_sregs(vcpu, &vcpu->run->s.regs.sregs))
11800 			return -EINVAL;
11801 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
11802 	}
11803 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
11804 		if (kvm_vcpu_ioctl_x86_set_vcpu_events(
11805 				vcpu, &vcpu->run->s.regs.events))
11806 			return -EINVAL;
11807 		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
11808 	}
11809 
11810 	return 0;
11811 }
11812 
kvm_arch_vcpu_precreate(struct kvm * kvm,unsigned int id)11813 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
11814 {
11815 	if (kvm_check_tsc_unstable() && kvm->created_vcpus)
11816 		pr_warn_once("kvm: SMP vm created on host with unstable TSC; "
11817 			     "guest TSC will not be reliable\n");
11818 
11819 	if (!kvm->arch.max_vcpu_ids)
11820 		kvm->arch.max_vcpu_ids = KVM_MAX_VCPU_IDS;
11821 
11822 	if (id >= kvm->arch.max_vcpu_ids)
11823 		return -EINVAL;
11824 
11825 	return static_call(kvm_x86_vcpu_precreate)(kvm);
11826 }
11827 
kvm_arch_vcpu_create(struct kvm_vcpu * vcpu)11828 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
11829 {
11830 	struct page *page;
11831 	int r;
11832 
11833 	vcpu->arch.last_vmentry_cpu = -1;
11834 	vcpu->arch.regs_avail = ~0;
11835 	vcpu->arch.regs_dirty = ~0;
11836 
11837 	kvm_gpc_init(&vcpu->arch.pv_time);
11838 
11839 	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
11840 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
11841 	else
11842 		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
11843 
11844 	r = kvm_mmu_create(vcpu);
11845 	if (r < 0)
11846 		return r;
11847 
11848 	if (irqchip_in_kernel(vcpu->kvm)) {
11849 		r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
11850 		if (r < 0)
11851 			goto fail_mmu_destroy;
11852 
11853 		/*
11854 		 * Defer evaluating inhibits until the vCPU is first run, as
11855 		 * this vCPU will not get notified of any changes until this
11856 		 * vCPU is visible to other vCPUs (marked online and added to
11857 		 * the set of vCPUs).  Opportunistically mark APICv active as
11858 		 * VMX in particularly is highly unlikely to have inhibits.
11859 		 * Ignore the current per-VM APICv state so that vCPU creation
11860 		 * is guaranteed to run with a deterministic value, the request
11861 		 * will ensure the vCPU gets the correct state before VM-Entry.
11862 		 */
11863 		if (enable_apicv) {
11864 			vcpu->arch.apic->apicv_active = true;
11865 			kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
11866 		}
11867 	} else
11868 		static_branch_inc(&kvm_has_noapic_vcpu);
11869 
11870 	r = -ENOMEM;
11871 
11872 	page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
11873 	if (!page)
11874 		goto fail_free_lapic;
11875 	vcpu->arch.pio_data = page_address(page);
11876 
11877 	vcpu->arch.mce_banks = kcalloc(KVM_MAX_MCE_BANKS * 4, sizeof(u64),
11878 				       GFP_KERNEL_ACCOUNT);
11879 	vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64),
11880 					    GFP_KERNEL_ACCOUNT);
11881 	if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks)
11882 		goto fail_free_mce_banks;
11883 	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
11884 
11885 	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
11886 				GFP_KERNEL_ACCOUNT))
11887 		goto fail_free_mce_banks;
11888 
11889 	if (!alloc_emulate_ctxt(vcpu))
11890 		goto free_wbinvd_dirty_mask;
11891 
11892 	if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) {
11893 		pr_err("kvm: failed to allocate vcpu's fpu\n");
11894 		goto free_emulate_ctxt;
11895 	}
11896 
11897 	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
11898 	vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
11899 
11900 	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;
11901 
11902 	kvm_async_pf_hash_reset(vcpu);
11903 	kvm_pmu_init(vcpu);
11904 
11905 	vcpu->arch.pending_external_vector = -1;
11906 	vcpu->arch.preempted_in_kernel = false;
11907 
11908 #if IS_ENABLED(CONFIG_HYPERV)
11909 	vcpu->arch.hv_root_tdp = INVALID_PAGE;
11910 #endif
11911 
11912 	r = static_call(kvm_x86_vcpu_create)(vcpu);
11913 	if (r)
11914 		goto free_guest_fpu;
11915 
11916 	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
11917 	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
11918 	kvm_xen_init_vcpu(vcpu);
11919 	kvm_vcpu_mtrr_init(vcpu);
11920 	vcpu_load(vcpu);
11921 	kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz);
11922 	kvm_vcpu_reset(vcpu, false);
11923 	kvm_init_mmu(vcpu);
11924 	vcpu_put(vcpu);
11925 	return 0;
11926 
11927 free_guest_fpu:
11928 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);
11929 free_emulate_ctxt:
11930 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
11931 free_wbinvd_dirty_mask:
11932 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
11933 fail_free_mce_banks:
11934 	kfree(vcpu->arch.mce_banks);
11935 	kfree(vcpu->arch.mci_ctl2_banks);
11936 	free_page((unsigned long)vcpu->arch.pio_data);
11937 fail_free_lapic:
11938 	kvm_free_lapic(vcpu);
11939 fail_mmu_destroy:
11940 	kvm_mmu_destroy(vcpu);
11941 	return r;
11942 }
11943 
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)11944 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
11945 {
11946 	struct kvm *kvm = vcpu->kvm;
11947 
11948 	if (mutex_lock_killable(&vcpu->mutex))
11949 		return;
11950 	vcpu_load(vcpu);
11951 	kvm_synchronize_tsc(vcpu, 0);
11952 	vcpu_put(vcpu);
11953 
11954 	/* poll control enabled by default */
11955 	vcpu->arch.msr_kvm_poll_control = 1;
11956 
11957 	mutex_unlock(&vcpu->mutex);
11958 
11959 	if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)
11960 		schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
11961 						KVMCLOCK_SYNC_PERIOD);
11962 }
11963 
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)11964 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
11965 {
11966 	int idx;
11967 
11968 	kvmclock_reset(vcpu);
11969 
11970 	static_call(kvm_x86_vcpu_free)(vcpu);
11971 
11972 	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
11973 	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
11974 	fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);
11975 
11976 	kvm_xen_destroy_vcpu(vcpu);
11977 	kvm_hv_vcpu_uninit(vcpu);
11978 	kvm_pmu_destroy(vcpu);
11979 	kfree(vcpu->arch.mce_banks);
11980 	kfree(vcpu->arch.mci_ctl2_banks);
11981 	kvm_free_lapic(vcpu);
11982 	idx = srcu_read_lock(&vcpu->kvm->srcu);
11983 	kvm_mmu_destroy(vcpu);
11984 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
11985 	free_page((unsigned long)vcpu->arch.pio_data);
11986 	kvfree(vcpu->arch.cpuid_entries);
11987 	if (!lapic_in_kernel(vcpu))
11988 		static_branch_dec(&kvm_has_noapic_vcpu);
11989 }
11990 
kvm_vcpu_reset(struct kvm_vcpu * vcpu,bool init_event)11991 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
11992 {
11993 	struct kvm_cpuid_entry2 *cpuid_0x1;
11994 	unsigned long old_cr0 = kvm_read_cr0(vcpu);
11995 	unsigned long new_cr0;
11996 
11997 	/*
11998 	 * Several of the "set" flows, e.g. ->set_cr0(), read other registers
11999 	 * to handle side effects.  RESET emulation hits those flows and relies
12000 	 * on emulated/virtualized registers, including those that are loaded
12001 	 * into hardware, to be zeroed at vCPU creation.  Use CRs as a sentinel
12002 	 * to detect improper or missing initialization.
12003 	 */
12004 	WARN_ON_ONCE(!init_event &&
12005 		     (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu)));
12006 
12007 	/*
12008 	 * SVM doesn't unconditionally VM-Exit on INIT and SHUTDOWN, thus it's
12009 	 * possible to INIT the vCPU while L2 is active.  Force the vCPU back
12010 	 * into L1 as EFER.SVME is cleared on INIT (along with all other EFER
12011 	 * bits), i.e. virtualization is disabled.
12012 	 */
12013 	if (is_guest_mode(vcpu))
12014 		kvm_leave_nested(vcpu);
12015 
12016 	kvm_lapic_reset(vcpu, init_event);
12017 
12018 	WARN_ON_ONCE(is_guest_mode(vcpu) || is_smm(vcpu));
12019 	vcpu->arch.hflags = 0;
12020 
12021 	vcpu->arch.smi_pending = 0;
12022 	vcpu->arch.smi_count = 0;
12023 	atomic_set(&vcpu->arch.nmi_queued, 0);
12024 	vcpu->arch.nmi_pending = 0;
12025 	vcpu->arch.nmi_injected = false;
12026 	kvm_clear_interrupt_queue(vcpu);
12027 	kvm_clear_exception_queue(vcpu);
12028 
12029 	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
12030 	kvm_update_dr0123(vcpu);
12031 	vcpu->arch.dr6 = DR6_ACTIVE_LOW;
12032 	vcpu->arch.dr7 = DR7_FIXED_1;
12033 	kvm_update_dr7(vcpu);
12034 
12035 	vcpu->arch.cr2 = 0;
12036 
12037 	kvm_make_request(KVM_REQ_EVENT, vcpu);
12038 	vcpu->arch.apf.msr_en_val = 0;
12039 	vcpu->arch.apf.msr_int_val = 0;
12040 	vcpu->arch.st.msr_val = 0;
12041 
12042 	kvmclock_reset(vcpu);
12043 
12044 	kvm_clear_async_pf_completion_queue(vcpu);
12045 	kvm_async_pf_hash_reset(vcpu);
12046 	vcpu->arch.apf.halted = false;
12047 
12048 	if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) {
12049 		struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate;
12050 
12051 		/*
12052 		 * All paths that lead to INIT are required to load the guest's
12053 		 * FPU state (because most paths are buried in KVM_RUN).
12054 		 */
12055 		if (init_event)
12056 			kvm_put_guest_fpu(vcpu);
12057 
12058 		fpstate_clear_xstate_component(fpstate, XFEATURE_BNDREGS);
12059 		fpstate_clear_xstate_component(fpstate, XFEATURE_BNDCSR);
12060 
12061 		if (init_event)
12062 			kvm_load_guest_fpu(vcpu);
12063 	}
12064 
12065 	if (!init_event) {
12066 		kvm_pmu_reset(vcpu);
12067 		vcpu->arch.smbase = 0x30000;
12068 
12069 		vcpu->arch.msr_misc_features_enables = 0;
12070 		vcpu->arch.ia32_misc_enable_msr = MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL |
12071 						  MSR_IA32_MISC_ENABLE_BTS_UNAVAIL;
12072 
12073 		__kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP);
12074 		__kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true);
12075 	}
12076 
12077 	/* All GPRs except RDX (handled below) are zeroed on RESET/INIT. */
12078 	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
12079 	kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP);
12080 
12081 	/*
12082 	 * Fall back to KVM's default Family/Model/Stepping of 0x600 (P6/Athlon)
12083 	 * if no CPUID match is found.  Note, it's impossible to get a match at
12084 	 * RESET since KVM emulates RESET before exposing the vCPU to userspace,
12085 	 * i.e. it's impossible for kvm_find_cpuid_entry() to find a valid entry
12086 	 * on RESET.  But, go through the motions in case that's ever remedied.
12087 	 */
12088 	cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1);
12089 	kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600);
12090 
12091 	static_call(kvm_x86_vcpu_reset)(vcpu, init_event);
12092 
12093 	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
12094 	kvm_rip_write(vcpu, 0xfff0);
12095 
12096 	vcpu->arch.cr3 = 0;
12097 	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
12098 
12099 	/*
12100 	 * CR0.CD/NW are set on RESET, preserved on INIT.  Note, some versions
12101 	 * of Intel's SDM list CD/NW as being set on INIT, but they contradict
12102 	 * (or qualify) that with a footnote stating that CD/NW are preserved.
12103 	 */
12104 	new_cr0 = X86_CR0_ET;
12105 	if (init_event)
12106 		new_cr0 |= (old_cr0 & (X86_CR0_NW | X86_CR0_CD));
12107 	else
12108 		new_cr0 |= X86_CR0_NW | X86_CR0_CD;
12109 
12110 	static_call(kvm_x86_set_cr0)(vcpu, new_cr0);
12111 	static_call(kvm_x86_set_cr4)(vcpu, 0);
12112 	static_call(kvm_x86_set_efer)(vcpu, 0);
12113 	static_call(kvm_x86_update_exception_bitmap)(vcpu);
12114 
12115 	/*
12116 	 * On the standard CR0/CR4/EFER modification paths, there are several
12117 	 * complex conditions determining whether the MMU has to be reset and/or
12118 	 * which PCIDs have to be flushed.  However, CR0.WP and the paging-related
12119 	 * bits in CR4 and EFER are irrelevant if CR0.PG was '0'; and a reset+flush
12120 	 * is needed anyway if CR0.PG was '1' (which can only happen for INIT, as
12121 	 * CR0 will be '0' prior to RESET).  So we only need to check CR0.PG here.
12122 	 */
12123 	if (old_cr0 & X86_CR0_PG) {
12124 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
12125 		kvm_mmu_reset_context(vcpu);
12126 	}
12127 
12128 	/*
12129 	 * Intel's SDM states that all TLB entries are flushed on INIT.  AMD's
12130 	 * APM states the TLBs are untouched by INIT, but it also states that
12131 	 * the TLBs are flushed on "External initialization of the processor."
12132 	 * Flush the guest TLB regardless of vendor, there is no meaningful
12133 	 * benefit in relying on the guest to flush the TLB immediately after
12134 	 * INIT.  A spurious TLB flush is benign and likely negligible from a
12135 	 * performance perspective.
12136 	 */
12137 	if (init_event)
12138 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
12139 }
12140 EXPORT_SYMBOL_GPL(kvm_vcpu_reset);
12141 
kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu * vcpu,u8 vector)12142 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
12143 {
12144 	struct kvm_segment cs;
12145 
12146 	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
12147 	cs.selector = vector << 8;
12148 	cs.base = vector << 12;
12149 	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
12150 	kvm_rip_write(vcpu, 0);
12151 }
12152 EXPORT_SYMBOL_GPL(kvm_vcpu_deliver_sipi_vector);
12153 
kvm_arch_hardware_enable(void)12154 int kvm_arch_hardware_enable(void)
12155 {
12156 	struct kvm *kvm;
12157 	struct kvm_vcpu *vcpu;
12158 	unsigned long i;
12159 	int ret;
12160 	u64 local_tsc;
12161 	u64 max_tsc = 0;
12162 	bool stable, backwards_tsc = false;
12163 
12164 	kvm_user_return_msr_cpu_online();
12165 	ret = static_call(kvm_x86_hardware_enable)();
12166 	if (ret != 0)
12167 		return ret;
12168 
12169 	local_tsc = rdtsc();
12170 	stable = !kvm_check_tsc_unstable();
12171 	list_for_each_entry(kvm, &vm_list, vm_list) {
12172 		kvm_for_each_vcpu(i, vcpu, kvm) {
12173 			if (!stable && vcpu->cpu == smp_processor_id())
12174 				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
12175 			if (stable && vcpu->arch.last_host_tsc > local_tsc) {
12176 				backwards_tsc = true;
12177 				if (vcpu->arch.last_host_tsc > max_tsc)
12178 					max_tsc = vcpu->arch.last_host_tsc;
12179 			}
12180 		}
12181 	}
12182 
12183 	/*
12184 	 * Sometimes, even reliable TSCs go backwards.  This happens on
12185 	 * platforms that reset TSC during suspend or hibernate actions, but
12186 	 * maintain synchronization.  We must compensate.  Fortunately, we can
12187 	 * detect that condition here, which happens early in CPU bringup,
12188 	 * before any KVM threads can be running.  Unfortunately, we can't
12189 	 * bring the TSCs fully up to date with real time, as we aren't yet far
12190 	 * enough into CPU bringup that we know how much real time has actually
12191 	 * elapsed; our helper function, ktime_get_boottime_ns() will be using boot
12192 	 * variables that haven't been updated yet.
12193 	 *
12194 	 * So we simply find the maximum observed TSC above, then record the
12195 	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,
12196 	 * the adjustment will be applied.  Note that we accumulate
12197 	 * adjustments, in case multiple suspend cycles happen before some VCPU
12198 	 * gets a chance to run again.  In the event that no KVM threads get a
12199 	 * chance to run, we will miss the entire elapsed period, as we'll have
12200 	 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
12201 	 * loose cycle time.  This isn't too big a deal, since the loss will be
12202 	 * uniform across all VCPUs (not to mention the scenario is extremely
12203 	 * unlikely). It is possible that a second hibernate recovery happens
12204 	 * much faster than a first, causing the observed TSC here to be
12205 	 * smaller; this would require additional padding adjustment, which is
12206 	 * why we set last_host_tsc to the local tsc observed here.
12207 	 *
12208 	 * N.B. - this code below runs only on platforms with reliable TSC,
12209 	 * as that is the only way backwards_tsc is set above.  Also note
12210 	 * that this runs for ALL vcpus, which is not a bug; all VCPUs should
12211 	 * have the same delta_cyc adjustment applied if backwards_tsc
12212 	 * is detected.  Note further, this adjustment is only done once,
12213 	 * as we reset last_host_tsc on all VCPUs to stop this from being
12214 	 * called multiple times (one for each physical CPU bringup).
12215 	 *
12216 	 * Platforms with unreliable TSCs don't have to deal with this, they
12217 	 * will be compensated by the logic in vcpu_load, which sets the TSC to
12218 	 * catchup mode.  This will catchup all VCPUs to real time, but cannot
12219 	 * guarantee that they stay in perfect synchronization.
12220 	 */
12221 	if (backwards_tsc) {
12222 		u64 delta_cyc = max_tsc - local_tsc;
12223 		list_for_each_entry(kvm, &vm_list, vm_list) {
12224 			kvm->arch.backwards_tsc_observed = true;
12225 			kvm_for_each_vcpu(i, vcpu, kvm) {
12226 				vcpu->arch.tsc_offset_adjustment += delta_cyc;
12227 				vcpu->arch.last_host_tsc = local_tsc;
12228 				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
12229 			}
12230 
12231 			/*
12232 			 * We have to disable TSC offset matching.. if you were
12233 			 * booting a VM while issuing an S4 host suspend....
12234 			 * you may have some problem.  Solving this issue is
12235 			 * left as an exercise to the reader.
12236 			 */
12237 			kvm->arch.last_tsc_nsec = 0;
12238 			kvm->arch.last_tsc_write = 0;
12239 		}
12240 
12241 	}
12242 	return 0;
12243 }
12244 
kvm_arch_hardware_disable(void)12245 void kvm_arch_hardware_disable(void)
12246 {
12247 	static_call(kvm_x86_hardware_disable)();
12248 	drop_user_return_notifiers();
12249 }
12250 
kvm_ops_update(struct kvm_x86_init_ops * ops)12251 static inline void kvm_ops_update(struct kvm_x86_init_ops *ops)
12252 {
12253 	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
12254 
12255 #define __KVM_X86_OP(func) \
12256 	static_call_update(kvm_x86_##func, kvm_x86_ops.func);
12257 #define KVM_X86_OP(func) \
12258 	WARN_ON(!kvm_x86_ops.func); __KVM_X86_OP(func)
12259 #define KVM_X86_OP_OPTIONAL __KVM_X86_OP
12260 #define KVM_X86_OP_OPTIONAL_RET0(func) \
12261 	static_call_update(kvm_x86_##func, (void *)kvm_x86_ops.func ? : \
12262 					   (void *)__static_call_return0);
12263 #include <asm/kvm-x86-ops.h>
12264 #undef __KVM_X86_OP
12265 
12266 	kvm_pmu_ops_update(ops->pmu_ops);
12267 }
12268 
kvm_arch_hardware_setup(void * opaque)12269 int kvm_arch_hardware_setup(void *opaque)
12270 {
12271 	struct kvm_x86_init_ops *ops = opaque;
12272 	int r;
12273 
12274 	rdmsrl_safe(MSR_EFER, &host_efer);
12275 
12276 	if (boot_cpu_has(X86_FEATURE_XSAVES))
12277 		rdmsrl(MSR_IA32_XSS, host_xss);
12278 
12279 	kvm_init_pmu_capability();
12280 
12281 	r = ops->hardware_setup();
12282 	if (r != 0)
12283 		return r;
12284 
12285 	kvm_ops_update(ops);
12286 
12287 	kvm_register_perf_callbacks(ops->handle_intel_pt_intr);
12288 
12289 	if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
12290 		kvm_caps.supported_xss = 0;
12291 
12292 #define __kvm_cpu_cap_has(UNUSED_, f) kvm_cpu_cap_has(f)
12293 	cr4_reserved_bits = __cr4_reserved_bits(__kvm_cpu_cap_has, UNUSED_);
12294 #undef __kvm_cpu_cap_has
12295 
12296 	if (kvm_caps.has_tsc_control) {
12297 		/*
12298 		 * Make sure the user can only configure tsc_khz values that
12299 		 * fit into a signed integer.
12300 		 * A min value is not calculated because it will always
12301 		 * be 1 on all machines.
12302 		 */
12303 		u64 max = min(0x7fffffffULL,
12304 			      __scale_tsc(kvm_caps.max_tsc_scaling_ratio, tsc_khz));
12305 		kvm_caps.max_guest_tsc_khz = max;
12306 	}
12307 	kvm_caps.default_tsc_scaling_ratio = 1ULL << kvm_caps.tsc_scaling_ratio_frac_bits;
12308 	kvm_init_msr_list();
12309 	return 0;
12310 }
12311 
kvm_arch_hardware_unsetup(void)12312 void kvm_arch_hardware_unsetup(void)
12313 {
12314 	kvm_unregister_perf_callbacks();
12315 
12316 	static_call(kvm_x86_hardware_unsetup)();
12317 }
12318 
kvm_arch_check_processor_compat(void * opaque)12319 int kvm_arch_check_processor_compat(void *opaque)
12320 {
12321 	struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());
12322 	struct kvm_x86_init_ops *ops = opaque;
12323 
12324 	WARN_ON(!irqs_disabled());
12325 
12326 	if (__cr4_reserved_bits(cpu_has, c) !=
12327 	    __cr4_reserved_bits(cpu_has, &boot_cpu_data))
12328 		return -EIO;
12329 
12330 	return ops->check_processor_compatibility();
12331 }
12332 
kvm_vcpu_is_reset_bsp(struct kvm_vcpu * vcpu)12333 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
12334 {
12335 	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
12336 }
12337 EXPORT_SYMBOL_GPL(kvm_vcpu_is_reset_bsp);
12338 
kvm_vcpu_is_bsp(struct kvm_vcpu * vcpu)12339 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
12340 {
12341 	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
12342 }
12343 
12344 __read_mostly DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu);
12345 EXPORT_SYMBOL_GPL(kvm_has_noapic_vcpu);
12346 
kvm_arch_sched_in(struct kvm_vcpu * vcpu,int cpu)12347 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
12348 {
12349 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
12350 
12351 	vcpu->arch.l1tf_flush_l1d = true;
12352 	if (pmu->version && unlikely(pmu->event_count)) {
12353 		pmu->need_cleanup = true;
12354 		kvm_make_request(KVM_REQ_PMU, vcpu);
12355 	}
12356 	static_call(kvm_x86_sched_in)(vcpu, cpu);
12357 }
12358 
kvm_arch_free_vm(struct kvm * kvm)12359 void kvm_arch_free_vm(struct kvm *kvm)
12360 {
12361 	kfree(to_kvm_hv(kvm)->hv_pa_pg);
12362 	__kvm_arch_free_vm(kvm);
12363 }
12364 
12365 
kvm_arch_init_vm(struct kvm * kvm,unsigned long type)12366 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
12367 {
12368 	int ret;
12369 	unsigned long flags;
12370 
12371 	if (type)
12372 		return -EINVAL;
12373 
12374 	ret = kvm_page_track_init(kvm);
12375 	if (ret)
12376 		goto out;
12377 
12378 	ret = kvm_mmu_init_vm(kvm);
12379 	if (ret)
12380 		goto out_page_track;
12381 
12382 	ret = static_call(kvm_x86_vm_init)(kvm);
12383 	if (ret)
12384 		goto out_uninit_mmu;
12385 
12386 	INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list);
12387 	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
12388 	atomic_set(&kvm->arch.noncoherent_dma_count, 0);
12389 
12390 	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
12391 	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
12392 	/* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
12393 	set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
12394 		&kvm->arch.irq_sources_bitmap);
12395 
12396 	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
12397 	mutex_init(&kvm->arch.apic_map_lock);
12398 	seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock);
12399 	kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
12400 
12401 	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
12402 	pvclock_update_vm_gtod_copy(kvm);
12403 	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
12404 
12405 	kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz;
12406 	kvm->arch.guest_can_read_msr_platform_info = true;
12407 	kvm->arch.enable_pmu = enable_pmu;
12408 
12409 #if IS_ENABLED(CONFIG_HYPERV)
12410 	spin_lock_init(&kvm->arch.hv_root_tdp_lock);
12411 	kvm->arch.hv_root_tdp = INVALID_PAGE;
12412 #endif
12413 
12414 	INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
12415 	INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
12416 
12417 	kvm_apicv_init(kvm);
12418 	kvm_hv_init_vm(kvm);
12419 	kvm_xen_init_vm(kvm);
12420 
12421 	return 0;
12422 
12423 out_uninit_mmu:
12424 	kvm_mmu_uninit_vm(kvm);
12425 out_page_track:
12426 	kvm_page_track_cleanup(kvm);
12427 out:
12428 	return ret;
12429 }
12430 
kvm_arch_post_init_vm(struct kvm * kvm)12431 int kvm_arch_post_init_vm(struct kvm *kvm)
12432 {
12433 	return kvm_mmu_post_init_vm(kvm);
12434 }
12435 
kvm_unload_vcpu_mmu(struct kvm_vcpu * vcpu)12436 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
12437 {
12438 	vcpu_load(vcpu);
12439 	kvm_mmu_unload(vcpu);
12440 	vcpu_put(vcpu);
12441 }
12442 
kvm_unload_vcpu_mmus(struct kvm * kvm)12443 static void kvm_unload_vcpu_mmus(struct kvm *kvm)
12444 {
12445 	unsigned long i;
12446 	struct kvm_vcpu *vcpu;
12447 
12448 	kvm_for_each_vcpu(i, vcpu, kvm) {
12449 		kvm_clear_async_pf_completion_queue(vcpu);
12450 		kvm_unload_vcpu_mmu(vcpu);
12451 	}
12452 }
12453 
kvm_arch_sync_events(struct kvm * kvm)12454 void kvm_arch_sync_events(struct kvm *kvm)
12455 {
12456 	cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
12457 	cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
12458 	kvm_free_pit(kvm);
12459 }
12460 
12461 /**
12462  * __x86_set_memory_region: Setup KVM internal memory slot
12463  *
12464  * @kvm: the kvm pointer to the VM.
12465  * @id: the slot ID to setup.
12466  * @gpa: the GPA to install the slot (unused when @size == 0).
12467  * @size: the size of the slot. Set to zero to uninstall a slot.
12468  *
12469  * This function helps to setup a KVM internal memory slot.  Specify
12470  * @size > 0 to install a new slot, while @size == 0 to uninstall a
12471  * slot.  The return code can be one of the following:
12472  *
12473  *   HVA:           on success (uninstall will return a bogus HVA)
12474  *   -errno:        on error
12475  *
12476  * The caller should always use IS_ERR() to check the return value
12477  * before use.  Note, the KVM internal memory slots are guaranteed to
12478  * remain valid and unchanged until the VM is destroyed, i.e., the
12479  * GPA->HVA translation will not change.  However, the HVA is a user
12480  * address, i.e. its accessibility is not guaranteed, and must be
12481  * accessed via __copy_{to,from}_user().
12482  */
__x86_set_memory_region(struct kvm * kvm,int id,gpa_t gpa,u32 size)12483 void __user * __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
12484 				      u32 size)
12485 {
12486 	int i, r;
12487 	unsigned long hva, old_npages;
12488 	struct kvm_memslots *slots = kvm_memslots(kvm);
12489 	struct kvm_memory_slot *slot;
12490 
12491 	/* Called with kvm->slots_lock held.  */
12492 	if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))
12493 		return ERR_PTR_USR(-EINVAL);
12494 
12495 	slot = id_to_memslot(slots, id);
12496 	if (size) {
12497 		if (slot && slot->npages)
12498 			return ERR_PTR_USR(-EEXIST);
12499 
12500 		/*
12501 		 * MAP_SHARED to prevent internal slot pages from being moved
12502 		 * by fork()/COW.
12503 		 */
12504 		hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE,
12505 			      MAP_SHARED | MAP_ANONYMOUS, 0);
12506 		if (IS_ERR((void *)hva))
12507 			return (void __user *)hva;
12508 	} else {
12509 		if (!slot || !slot->npages)
12510 			return NULL;
12511 
12512 		old_npages = slot->npages;
12513 		hva = slot->userspace_addr;
12514 	}
12515 
12516 	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
12517 		struct kvm_userspace_memory_region m;
12518 
12519 		m.slot = id | (i << 16);
12520 		m.flags = 0;
12521 		m.guest_phys_addr = gpa;
12522 		m.userspace_addr = hva;
12523 		m.memory_size = size;
12524 		r = __kvm_set_memory_region(kvm, &m);
12525 		if (r < 0)
12526 			return ERR_PTR_USR(r);
12527 	}
12528 
12529 	if (!size)
12530 		vm_munmap(hva, old_npages * PAGE_SIZE);
12531 
12532 	return (void __user *)hva;
12533 }
12534 EXPORT_SYMBOL_GPL(__x86_set_memory_region);
12535 
kvm_arch_pre_destroy_vm(struct kvm * kvm)12536 void kvm_arch_pre_destroy_vm(struct kvm *kvm)
12537 {
12538 	kvm_mmu_pre_destroy_vm(kvm);
12539 }
12540 
kvm_arch_destroy_vm(struct kvm * kvm)12541 void kvm_arch_destroy_vm(struct kvm *kvm)
12542 {
12543 	if (current->mm == kvm->mm) {
12544 		/*
12545 		 * Free memory regions allocated on behalf of userspace,
12546 		 * unless the memory map has changed due to process exit
12547 		 * or fd copying.
12548 		 */
12549 		mutex_lock(&kvm->slots_lock);
12550 		__x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
12551 					0, 0);
12552 		__x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
12553 					0, 0);
12554 		__x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
12555 		mutex_unlock(&kvm->slots_lock);
12556 	}
12557 	kvm_unload_vcpu_mmus(kvm);
12558 	static_call_cond(kvm_x86_vm_destroy)(kvm);
12559 	kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1));
12560 	kvm_pic_destroy(kvm);
12561 	kvm_ioapic_destroy(kvm);
12562 	kvm_destroy_vcpus(kvm);
12563 	kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
12564 	kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
12565 	kvm_mmu_uninit_vm(kvm);
12566 	kvm_page_track_cleanup(kvm);
12567 	kvm_xen_destroy_vm(kvm);
12568 	kvm_hv_destroy_vm(kvm);
12569 }
12570 
memslot_rmap_free(struct kvm_memory_slot * slot)12571 static void memslot_rmap_free(struct kvm_memory_slot *slot)
12572 {
12573 	int i;
12574 
12575 	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
12576 		kvfree(slot->arch.rmap[i]);
12577 		slot->arch.rmap[i] = NULL;
12578 	}
12579 }
12580 
kvm_arch_free_memslot(struct kvm * kvm,struct kvm_memory_slot * slot)12581 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
12582 {
12583 	int i;
12584 
12585 	memslot_rmap_free(slot);
12586 
12587 	for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
12588 		kvfree(slot->arch.lpage_info[i - 1]);
12589 		slot->arch.lpage_info[i - 1] = NULL;
12590 	}
12591 
12592 	kvm_page_track_free_memslot(slot);
12593 }
12594 
memslot_rmap_alloc(struct kvm_memory_slot * slot,unsigned long npages)12595 int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages)
12596 {
12597 	const int sz = sizeof(*slot->arch.rmap[0]);
12598 	int i;
12599 
12600 	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
12601 		int level = i + 1;
12602 		int lpages = __kvm_mmu_slot_lpages(slot, npages, level);
12603 
12604 		if (slot->arch.rmap[i])
12605 			continue;
12606 
12607 		slot->arch.rmap[i] = __vcalloc(lpages, sz, GFP_KERNEL_ACCOUNT);
12608 		if (!slot->arch.rmap[i]) {
12609 			memslot_rmap_free(slot);
12610 			return -ENOMEM;
12611 		}
12612 	}
12613 
12614 	return 0;
12615 }
12616 
kvm_alloc_memslot_metadata(struct kvm * kvm,struct kvm_memory_slot * slot)12617 static int kvm_alloc_memslot_metadata(struct kvm *kvm,
12618 				      struct kvm_memory_slot *slot)
12619 {
12620 	unsigned long npages = slot->npages;
12621 	int i, r;
12622 
12623 	/*
12624 	 * Clear out the previous array pointers for the KVM_MR_MOVE case.  The
12625 	 * old arrays will be freed by __kvm_set_memory_region() if installing
12626 	 * the new memslot is successful.
12627 	 */
12628 	memset(&slot->arch, 0, sizeof(slot->arch));
12629 
12630 	if (kvm_memslots_have_rmaps(kvm)) {
12631 		r = memslot_rmap_alloc(slot, npages);
12632 		if (r)
12633 			return r;
12634 	}
12635 
12636 	for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
12637 		struct kvm_lpage_info *linfo;
12638 		unsigned long ugfn;
12639 		int lpages;
12640 		int level = i + 1;
12641 
12642 		lpages = __kvm_mmu_slot_lpages(slot, npages, level);
12643 
12644 		linfo = __vcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT);
12645 		if (!linfo)
12646 			goto out_free;
12647 
12648 		slot->arch.lpage_info[i - 1] = linfo;
12649 
12650 		if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
12651 			linfo[0].disallow_lpage = 1;
12652 		if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
12653 			linfo[lpages - 1].disallow_lpage = 1;
12654 		ugfn = slot->userspace_addr >> PAGE_SHIFT;
12655 		/*
12656 		 * If the gfn and userspace address are not aligned wrt each
12657 		 * other, disable large page support for this slot.
12658 		 */
12659 		if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1)) {
12660 			unsigned long j;
12661 
12662 			for (j = 0; j < lpages; ++j)
12663 				linfo[j].disallow_lpage = 1;
12664 		}
12665 	}
12666 
12667 	if (kvm_page_track_create_memslot(kvm, slot, npages))
12668 		goto out_free;
12669 
12670 	return 0;
12671 
12672 out_free:
12673 	memslot_rmap_free(slot);
12674 
12675 	for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
12676 		kvfree(slot->arch.lpage_info[i - 1]);
12677 		slot->arch.lpage_info[i - 1] = NULL;
12678 	}
12679 	return -ENOMEM;
12680 }
12681 
kvm_arch_memslots_updated(struct kvm * kvm,u64 gen)12682 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
12683 {
12684 	struct kvm_vcpu *vcpu;
12685 	unsigned long i;
12686 
12687 	/*
12688 	 * memslots->generation has been incremented.
12689 	 * mmio generation may have reached its maximum value.
12690 	 */
12691 	kvm_mmu_invalidate_mmio_sptes(kvm, gen);
12692 
12693 	/* Force re-initialization of steal_time cache */
12694 	kvm_for_each_vcpu(i, vcpu, kvm)
12695 		kvm_vcpu_kick(vcpu);
12696 }
12697 
kvm_arch_prepare_memory_region(struct kvm * kvm,const struct kvm_memory_slot * old,struct kvm_memory_slot * new,enum kvm_mr_change change)12698 int kvm_arch_prepare_memory_region(struct kvm *kvm,
12699 				   const struct kvm_memory_slot *old,
12700 				   struct kvm_memory_slot *new,
12701 				   enum kvm_mr_change change)
12702 {
12703 	if (change == KVM_MR_CREATE || change == KVM_MR_MOVE) {
12704 		if ((new->base_gfn + new->npages - 1) > kvm_mmu_max_gfn())
12705 			return -EINVAL;
12706 
12707 		return kvm_alloc_memslot_metadata(kvm, new);
12708 	}
12709 
12710 	if (change == KVM_MR_FLAGS_ONLY)
12711 		memcpy(&new->arch, &old->arch, sizeof(old->arch));
12712 	else if (WARN_ON_ONCE(change != KVM_MR_DELETE))
12713 		return -EIO;
12714 
12715 	return 0;
12716 }
12717 
12718 
kvm_mmu_update_cpu_dirty_logging(struct kvm * kvm,bool enable)12719 static void kvm_mmu_update_cpu_dirty_logging(struct kvm *kvm, bool enable)
12720 {
12721 	struct kvm_arch *ka = &kvm->arch;
12722 
12723 	if (!kvm_x86_ops.cpu_dirty_log_size)
12724 		return;
12725 
12726 	if ((enable && ++ka->cpu_dirty_logging_count == 1) ||
12727 	    (!enable && --ka->cpu_dirty_logging_count == 0))
12728 		kvm_make_all_cpus_request(kvm, KVM_REQ_UPDATE_CPU_DIRTY_LOGGING);
12729 
12730 	WARN_ON_ONCE(ka->cpu_dirty_logging_count < 0);
12731 }
12732 
kvm_mmu_slot_apply_flags(struct kvm * kvm,struct kvm_memory_slot * old,const struct kvm_memory_slot * new,enum kvm_mr_change change)12733 static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
12734 				     struct kvm_memory_slot *old,
12735 				     const struct kvm_memory_slot *new,
12736 				     enum kvm_mr_change change)
12737 {
12738 	u32 old_flags = old ? old->flags : 0;
12739 	u32 new_flags = new ? new->flags : 0;
12740 	bool log_dirty_pages = new_flags & KVM_MEM_LOG_DIRTY_PAGES;
12741 
12742 	/*
12743 	 * Update CPU dirty logging if dirty logging is being toggled.  This
12744 	 * applies to all operations.
12745 	 */
12746 	if ((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES)
12747 		kvm_mmu_update_cpu_dirty_logging(kvm, log_dirty_pages);
12748 
12749 	/*
12750 	 * Nothing more to do for RO slots (which can't be dirtied and can't be
12751 	 * made writable) or CREATE/MOVE/DELETE of a slot.
12752 	 *
12753 	 * For a memslot with dirty logging disabled:
12754 	 * CREATE:      No dirty mappings will already exist.
12755 	 * MOVE/DELETE: The old mappings will already have been cleaned up by
12756 	 *		kvm_arch_flush_shadow_memslot()
12757 	 *
12758 	 * For a memslot with dirty logging enabled:
12759 	 * CREATE:      No shadow pages exist, thus nothing to write-protect
12760 	 *		and no dirty bits to clear.
12761 	 * MOVE/DELETE: The old mappings will already have been cleaned up by
12762 	 *		kvm_arch_flush_shadow_memslot().
12763 	 */
12764 	if ((change != KVM_MR_FLAGS_ONLY) || (new_flags & KVM_MEM_READONLY))
12765 		return;
12766 
12767 	/*
12768 	 * READONLY and non-flags changes were filtered out above, and the only
12769 	 * other flag is LOG_DIRTY_PAGES, i.e. something is wrong if dirty
12770 	 * logging isn't being toggled on or off.
12771 	 */
12772 	if (WARN_ON_ONCE(!((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES)))
12773 		return;
12774 
12775 	if (!log_dirty_pages) {
12776 		/*
12777 		 * Dirty logging tracks sptes in 4k granularity, meaning that
12778 		 * large sptes have to be split.  If live migration succeeds,
12779 		 * the guest in the source machine will be destroyed and large
12780 		 * sptes will be created in the destination.  However, if the
12781 		 * guest continues to run in the source machine (for example if
12782 		 * live migration fails), small sptes will remain around and
12783 		 * cause bad performance.
12784 		 *
12785 		 * Scan sptes if dirty logging has been stopped, dropping those
12786 		 * which can be collapsed into a single large-page spte.  Later
12787 		 * page faults will create the large-page sptes.
12788 		 */
12789 		kvm_mmu_zap_collapsible_sptes(kvm, new);
12790 	} else {
12791 		/*
12792 		 * Initially-all-set does not require write protecting any page,
12793 		 * because they're all assumed to be dirty.
12794 		 */
12795 		if (kvm_dirty_log_manual_protect_and_init_set(kvm))
12796 			return;
12797 
12798 		if (READ_ONCE(eager_page_split))
12799 			kvm_mmu_slot_try_split_huge_pages(kvm, new, PG_LEVEL_4K);
12800 
12801 		if (kvm_x86_ops.cpu_dirty_log_size) {
12802 			kvm_mmu_slot_leaf_clear_dirty(kvm, new);
12803 			kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_2M);
12804 		} else {
12805 			kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_4K);
12806 		}
12807 
12808 		/*
12809 		 * Unconditionally flush the TLBs after enabling dirty logging.
12810 		 * A flush is almost always going to be necessary (see below),
12811 		 * and unconditionally flushing allows the helpers to omit
12812 		 * the subtly complex checks when removing write access.
12813 		 *
12814 		 * Do the flush outside of mmu_lock to reduce the amount of
12815 		 * time mmu_lock is held.  Flushing after dropping mmu_lock is
12816 		 * safe as KVM only needs to guarantee the slot is fully
12817 		 * write-protected before returning to userspace, i.e. before
12818 		 * userspace can consume the dirty status.
12819 		 *
12820 		 * Flushing outside of mmu_lock requires KVM to be careful when
12821 		 * making decisions based on writable status of an SPTE, e.g. a
12822 		 * !writable SPTE doesn't guarantee a CPU can't perform writes.
12823 		 *
12824 		 * Specifically, KVM also write-protects guest page tables to
12825 		 * monitor changes when using shadow paging, and must guarantee
12826 		 * no CPUs can write to those page before mmu_lock is dropped.
12827 		 * Because CPUs may have stale TLB entries at this point, a
12828 		 * !writable SPTE doesn't guarantee CPUs can't perform writes.
12829 		 *
12830 		 * KVM also allows making SPTES writable outside of mmu_lock,
12831 		 * e.g. to allow dirty logging without taking mmu_lock.
12832 		 *
12833 		 * To handle these scenarios, KVM uses a separate software-only
12834 		 * bit (MMU-writable) to track if a SPTE is !writable due to
12835 		 * a guest page table being write-protected (KVM clears the
12836 		 * MMU-writable flag when write-protecting for shadow paging).
12837 		 *
12838 		 * The use of MMU-writable is also the primary motivation for
12839 		 * the unconditional flush.  Because KVM must guarantee that a
12840 		 * CPU doesn't contain stale, writable TLB entries for a
12841 		 * !MMU-writable SPTE, KVM must flush if it encounters any
12842 		 * MMU-writable SPTE regardless of whether the actual hardware
12843 		 * writable bit was set.  I.e. KVM is almost guaranteed to need
12844 		 * to flush, while unconditionally flushing allows the "remove
12845 		 * write access" helpers to ignore MMU-writable entirely.
12846 		 *
12847 		 * See is_writable_pte() for more details (the case involving
12848 		 * access-tracked SPTEs is particularly relevant).
12849 		 */
12850 		kvm_arch_flush_remote_tlbs_memslot(kvm, new);
12851 	}
12852 }
12853 
kvm_arch_commit_memory_region(struct kvm * kvm,struct kvm_memory_slot * old,const struct kvm_memory_slot * new,enum kvm_mr_change change)12854 void kvm_arch_commit_memory_region(struct kvm *kvm,
12855 				struct kvm_memory_slot *old,
12856 				const struct kvm_memory_slot *new,
12857 				enum kvm_mr_change change)
12858 {
12859 	if (!kvm->arch.n_requested_mmu_pages &&
12860 	    (change == KVM_MR_CREATE || change == KVM_MR_DELETE)) {
12861 		unsigned long nr_mmu_pages;
12862 
12863 		nr_mmu_pages = kvm->nr_memslot_pages / KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO;
12864 		nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES);
12865 		kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
12866 	}
12867 
12868 	kvm_mmu_slot_apply_flags(kvm, old, new, change);
12869 
12870 	/* Free the arrays associated with the old memslot. */
12871 	if (change == KVM_MR_MOVE)
12872 		kvm_arch_free_memslot(kvm, old);
12873 }
12874 
kvm_arch_flush_shadow_all(struct kvm * kvm)12875 void kvm_arch_flush_shadow_all(struct kvm *kvm)
12876 {
12877 	kvm_mmu_zap_all(kvm);
12878 }
12879 
kvm_arch_flush_shadow_memslot(struct kvm * kvm,struct kvm_memory_slot * slot)12880 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
12881 				   struct kvm_memory_slot *slot)
12882 {
12883 	kvm_page_track_flush_slot(kvm, slot);
12884 }
12885 
kvm_guest_apic_has_interrupt(struct kvm_vcpu * vcpu)12886 static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
12887 {
12888 	return (is_guest_mode(vcpu) &&
12889 		static_call(kvm_x86_guest_apic_has_interrupt)(vcpu));
12890 }
12891 
kvm_vcpu_has_events(struct kvm_vcpu * vcpu)12892 static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
12893 {
12894 	if (!list_empty_careful(&vcpu->async_pf.done))
12895 		return true;
12896 
12897 	if (kvm_apic_has_pending_init_or_sipi(vcpu) &&
12898 	    kvm_apic_init_sipi_allowed(vcpu))
12899 		return true;
12900 
12901 	if (vcpu->arch.pv.pv_unhalted)
12902 		return true;
12903 
12904 	if (kvm_is_exception_pending(vcpu))
12905 		return true;
12906 
12907 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
12908 	    (vcpu->arch.nmi_pending &&
12909 	     static_call(kvm_x86_nmi_allowed)(vcpu, false)))
12910 		return true;
12911 
12912 	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||
12913 	    (vcpu->arch.smi_pending &&
12914 	     static_call(kvm_x86_smi_allowed)(vcpu, false)))
12915 		return true;
12916 
12917 	if (kvm_arch_interrupt_allowed(vcpu) &&
12918 	    (kvm_cpu_has_interrupt(vcpu) ||
12919 	    kvm_guest_apic_has_interrupt(vcpu)))
12920 		return true;
12921 
12922 	if (kvm_hv_has_stimer_pending(vcpu))
12923 		return true;
12924 
12925 	if (is_guest_mode(vcpu) &&
12926 	    kvm_x86_ops.nested_ops->has_events &&
12927 	    kvm_x86_ops.nested_ops->has_events(vcpu))
12928 		return true;
12929 
12930 	if (kvm_xen_has_pending_events(vcpu))
12931 		return true;
12932 
12933 	return false;
12934 }
12935 
kvm_arch_vcpu_runnable(struct kvm_vcpu * vcpu)12936 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
12937 {
12938 	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
12939 }
12940 
kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu * vcpu)12941 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu)
12942 {
12943 	if (kvm_vcpu_apicv_active(vcpu) &&
12944 	    static_call(kvm_x86_dy_apicv_has_pending_interrupt)(vcpu))
12945 		return true;
12946 
12947 	return false;
12948 }
12949 
kvm_arch_dy_runnable(struct kvm_vcpu * vcpu)12950 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
12951 {
12952 	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
12953 		return true;
12954 
12955 	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
12956 		kvm_test_request(KVM_REQ_SMI, vcpu) ||
12957 		 kvm_test_request(KVM_REQ_EVENT, vcpu))
12958 		return true;
12959 
12960 	return kvm_arch_dy_has_pending_interrupt(vcpu);
12961 }
12962 
kvm_arch_vcpu_in_kernel(struct kvm_vcpu * vcpu)12963 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
12964 {
12965 	if (vcpu->arch.guest_state_protected)
12966 		return true;
12967 
12968 	return vcpu->arch.preempted_in_kernel;
12969 }
12970 
kvm_arch_vcpu_get_ip(struct kvm_vcpu * vcpu)12971 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
12972 {
12973 	return kvm_rip_read(vcpu);
12974 }
12975 
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)12976 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
12977 {
12978 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
12979 }
12980 
kvm_arch_interrupt_allowed(struct kvm_vcpu * vcpu)12981 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
12982 {
12983 	return static_call(kvm_x86_interrupt_allowed)(vcpu, false);
12984 }
12985 
kvm_get_linear_rip(struct kvm_vcpu * vcpu)12986 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
12987 {
12988 	/* Can't read the RIP when guest state is protected, just return 0 */
12989 	if (vcpu->arch.guest_state_protected)
12990 		return 0;
12991 
12992 	if (is_64_bit_mode(vcpu))
12993 		return kvm_rip_read(vcpu);
12994 	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
12995 		     kvm_rip_read(vcpu));
12996 }
12997 EXPORT_SYMBOL_GPL(kvm_get_linear_rip);
12998 
kvm_is_linear_rip(struct kvm_vcpu * vcpu,unsigned long linear_rip)12999 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
13000 {
13001 	return kvm_get_linear_rip(vcpu) == linear_rip;
13002 }
13003 EXPORT_SYMBOL_GPL(kvm_is_linear_rip);
13004 
kvm_get_rflags(struct kvm_vcpu * vcpu)13005 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
13006 {
13007 	unsigned long rflags;
13008 
13009 	rflags = static_call(kvm_x86_get_rflags)(vcpu);
13010 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
13011 		rflags &= ~X86_EFLAGS_TF;
13012 	return rflags;
13013 }
13014 EXPORT_SYMBOL_GPL(kvm_get_rflags);
13015 
__kvm_set_rflags(struct kvm_vcpu * vcpu,unsigned long rflags)13016 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
13017 {
13018 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
13019 	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
13020 		rflags |= X86_EFLAGS_TF;
13021 	static_call(kvm_x86_set_rflags)(vcpu, rflags);
13022 }
13023 
kvm_set_rflags(struct kvm_vcpu * vcpu,unsigned long rflags)13024 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
13025 {
13026 	__kvm_set_rflags(vcpu, rflags);
13027 	kvm_make_request(KVM_REQ_EVENT, vcpu);
13028 }
13029 EXPORT_SYMBOL_GPL(kvm_set_rflags);
13030 
kvm_async_pf_hash_fn(gfn_t gfn)13031 static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
13032 {
13033 	BUILD_BUG_ON(!is_power_of_2(ASYNC_PF_PER_VCPU));
13034 
13035 	return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
13036 }
13037 
kvm_async_pf_next_probe(u32 key)13038 static inline u32 kvm_async_pf_next_probe(u32 key)
13039 {
13040 	return (key + 1) & (ASYNC_PF_PER_VCPU - 1);
13041 }
13042 
kvm_add_async_pf_gfn(struct kvm_vcpu * vcpu,gfn_t gfn)13043 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
13044 {
13045 	u32 key = kvm_async_pf_hash_fn(gfn);
13046 
13047 	while (vcpu->arch.apf.gfns[key] != ~0)
13048 		key = kvm_async_pf_next_probe(key);
13049 
13050 	vcpu->arch.apf.gfns[key] = gfn;
13051 }
13052 
kvm_async_pf_gfn_slot(struct kvm_vcpu * vcpu,gfn_t gfn)13053 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
13054 {
13055 	int i;
13056 	u32 key = kvm_async_pf_hash_fn(gfn);
13057 
13058 	for (i = 0; i < ASYNC_PF_PER_VCPU &&
13059 		     (vcpu->arch.apf.gfns[key] != gfn &&
13060 		      vcpu->arch.apf.gfns[key] != ~0); i++)
13061 		key = kvm_async_pf_next_probe(key);
13062 
13063 	return key;
13064 }
13065 
kvm_find_async_pf_gfn(struct kvm_vcpu * vcpu,gfn_t gfn)13066 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
13067 {
13068 	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
13069 }
13070 
kvm_del_async_pf_gfn(struct kvm_vcpu * vcpu,gfn_t gfn)13071 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
13072 {
13073 	u32 i, j, k;
13074 
13075 	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
13076 
13077 	if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))
13078 		return;
13079 
13080 	while (true) {
13081 		vcpu->arch.apf.gfns[i] = ~0;
13082 		do {
13083 			j = kvm_async_pf_next_probe(j);
13084 			if (vcpu->arch.apf.gfns[j] == ~0)
13085 				return;
13086 			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
13087 			/*
13088 			 * k lies cyclically in ]i,j]
13089 			 * |    i.k.j |
13090 			 * |....j i.k.| or  |.k..j i...|
13091 			 */
13092 		} while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
13093 		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
13094 		i = j;
13095 	}
13096 }
13097 
apf_put_user_notpresent(struct kvm_vcpu * vcpu)13098 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu)
13099 {
13100 	u32 reason = KVM_PV_REASON_PAGE_NOT_PRESENT;
13101 
13102 	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,
13103 				      sizeof(reason));
13104 }
13105 
apf_put_user_ready(struct kvm_vcpu * vcpu,u32 token)13106 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token)
13107 {
13108 	unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token);
13109 
13110 	return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
13111 					     &token, offset, sizeof(token));
13112 }
13113 
apf_pageready_slot_free(struct kvm_vcpu * vcpu)13114 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)
13115 {
13116 	unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token);
13117 	u32 val;
13118 
13119 	if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
13120 					 &val, offset, sizeof(val)))
13121 		return false;
13122 
13123 	return !val;
13124 }
13125 
kvm_can_deliver_async_pf(struct kvm_vcpu * vcpu)13126 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
13127 {
13128 
13129 	if (!kvm_pv_async_pf_enabled(vcpu))
13130 		return false;
13131 
13132 	if (vcpu->arch.apf.send_user_only &&
13133 	    static_call(kvm_x86_get_cpl)(vcpu) == 0)
13134 		return false;
13135 
13136 	if (is_guest_mode(vcpu)) {
13137 		/*
13138 		 * L1 needs to opt into the special #PF vmexits that are
13139 		 * used to deliver async page faults.
13140 		 */
13141 		return vcpu->arch.apf.delivery_as_pf_vmexit;
13142 	} else {
13143 		/*
13144 		 * Play it safe in case the guest temporarily disables paging.
13145 		 * The real mode IDT in particular is unlikely to have a #PF
13146 		 * exception setup.
13147 		 */
13148 		return is_paging(vcpu);
13149 	}
13150 }
13151 
kvm_can_do_async_pf(struct kvm_vcpu * vcpu)13152 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
13153 {
13154 	if (unlikely(!lapic_in_kernel(vcpu) ||
13155 		     kvm_event_needs_reinjection(vcpu) ||
13156 		     kvm_is_exception_pending(vcpu)))
13157 		return false;
13158 
13159 	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
13160 		return false;
13161 
13162 	/*
13163 	 * If interrupts are off we cannot even use an artificial
13164 	 * halt state.
13165 	 */
13166 	return kvm_arch_interrupt_allowed(vcpu);
13167 }
13168 
kvm_arch_async_page_not_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)13169 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
13170 				     struct kvm_async_pf *work)
13171 {
13172 	struct x86_exception fault;
13173 
13174 	trace_kvm_async_pf_not_present(work->arch.token, work->cr2_or_gpa);
13175 	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
13176 
13177 	if (kvm_can_deliver_async_pf(vcpu) &&
13178 	    !apf_put_user_notpresent(vcpu)) {
13179 		fault.vector = PF_VECTOR;
13180 		fault.error_code_valid = true;
13181 		fault.error_code = 0;
13182 		fault.nested_page_fault = false;
13183 		fault.address = work->arch.token;
13184 		fault.async_page_fault = true;
13185 		kvm_inject_page_fault(vcpu, &fault);
13186 		return true;
13187 	} else {
13188 		/*
13189 		 * It is not possible to deliver a paravirtualized asynchronous
13190 		 * page fault, but putting the guest in an artificial halt state
13191 		 * can be beneficial nevertheless: if an interrupt arrives, we
13192 		 * can deliver it timely and perhaps the guest will schedule
13193 		 * another process.  When the instruction that triggered a page
13194 		 * fault is retried, hopefully the page will be ready in the host.
13195 		 */
13196 		kvm_make_request(KVM_REQ_APF_HALT, vcpu);
13197 		return false;
13198 	}
13199 }
13200 
kvm_arch_async_page_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)13201 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
13202 				 struct kvm_async_pf *work)
13203 {
13204 	struct kvm_lapic_irq irq = {
13205 		.delivery_mode = APIC_DM_FIXED,
13206 		.vector = vcpu->arch.apf.vec
13207 	};
13208 
13209 	if (work->wakeup_all)
13210 		work->arch.token = ~0; /* broadcast wakeup */
13211 	else
13212 		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
13213 	trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa);
13214 
13215 	if ((work->wakeup_all || work->notpresent_injected) &&
13216 	    kvm_pv_async_pf_enabled(vcpu) &&
13217 	    !apf_put_user_ready(vcpu, work->arch.token)) {
13218 		vcpu->arch.apf.pageready_pending = true;
13219 		kvm_apic_set_irq(vcpu, &irq, NULL);
13220 	}
13221 
13222 	vcpu->arch.apf.halted = false;
13223 	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
13224 }
13225 
kvm_arch_async_page_present_queued(struct kvm_vcpu * vcpu)13226 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)
13227 {
13228 	kvm_make_request(KVM_REQ_APF_READY, vcpu);
13229 	if (!vcpu->arch.apf.pageready_pending)
13230 		kvm_vcpu_kick(vcpu);
13231 }
13232 
kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu * vcpu)13233 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
13234 {
13235 	if (!kvm_pv_async_pf_enabled(vcpu))
13236 		return true;
13237 	else
13238 		return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu);
13239 }
13240 
kvm_arch_start_assignment(struct kvm * kvm)13241 void kvm_arch_start_assignment(struct kvm *kvm)
13242 {
13243 	if (atomic_inc_return(&kvm->arch.assigned_device_count) == 1)
13244 		static_call_cond(kvm_x86_pi_start_assignment)(kvm);
13245 }
13246 EXPORT_SYMBOL_GPL(kvm_arch_start_assignment);
13247 
kvm_arch_end_assignment(struct kvm * kvm)13248 void kvm_arch_end_assignment(struct kvm *kvm)
13249 {
13250 	atomic_dec(&kvm->arch.assigned_device_count);
13251 }
13252 EXPORT_SYMBOL_GPL(kvm_arch_end_assignment);
13253 
kvm_arch_has_assigned_device(struct kvm * kvm)13254 bool noinstr kvm_arch_has_assigned_device(struct kvm *kvm)
13255 {
13256 	return arch_atomic_read(&kvm->arch.assigned_device_count);
13257 }
13258 EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);
13259 
kvm_arch_register_noncoherent_dma(struct kvm * kvm)13260 void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
13261 {
13262 	atomic_inc(&kvm->arch.noncoherent_dma_count);
13263 }
13264 EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);
13265 
kvm_arch_unregister_noncoherent_dma(struct kvm * kvm)13266 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
13267 {
13268 	atomic_dec(&kvm->arch.noncoherent_dma_count);
13269 }
13270 EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);
13271 
kvm_arch_has_noncoherent_dma(struct kvm * kvm)13272 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
13273 {
13274 	return atomic_read(&kvm->arch.noncoherent_dma_count);
13275 }
13276 EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);
13277 
kvm_arch_has_irq_bypass(void)13278 bool kvm_arch_has_irq_bypass(void)
13279 {
13280 	return true;
13281 }
13282 
kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)13283 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
13284 				      struct irq_bypass_producer *prod)
13285 {
13286 	struct kvm_kernel_irqfd *irqfd =
13287 		container_of(cons, struct kvm_kernel_irqfd, consumer);
13288 	int ret;
13289 
13290 	irqfd->producer = prod;
13291 	kvm_arch_start_assignment(irqfd->kvm);
13292 	ret = static_call(kvm_x86_pi_update_irte)(irqfd->kvm,
13293 					 prod->irq, irqfd->gsi, 1);
13294 
13295 	if (ret)
13296 		kvm_arch_end_assignment(irqfd->kvm);
13297 
13298 	return ret;
13299 }
13300 
kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)13301 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
13302 				      struct irq_bypass_producer *prod)
13303 {
13304 	int ret;
13305 	struct kvm_kernel_irqfd *irqfd =
13306 		container_of(cons, struct kvm_kernel_irqfd, consumer);
13307 
13308 	WARN_ON(irqfd->producer != prod);
13309 	irqfd->producer = NULL;
13310 
13311 	/*
13312 	 * When producer of consumer is unregistered, we change back to
13313 	 * remapped mode, so we can re-use the current implementation
13314 	 * when the irq is masked/disabled or the consumer side (KVM
13315 	 * int this case doesn't want to receive the interrupts.
13316 	*/
13317 	ret = static_call(kvm_x86_pi_update_irte)(irqfd->kvm, prod->irq, irqfd->gsi, 0);
13318 	if (ret)
13319 		printk(KERN_INFO "irq bypass consumer (token %p) unregistration"
13320 		       " fails: %d\n", irqfd->consumer.token, ret);
13321 
13322 	kvm_arch_end_assignment(irqfd->kvm);
13323 }
13324 
kvm_arch_update_irqfd_routing(struct kvm * kvm,unsigned int host_irq,uint32_t guest_irq,bool set)13325 int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
13326 				   uint32_t guest_irq, bool set)
13327 {
13328 	return static_call(kvm_x86_pi_update_irte)(kvm, host_irq, guest_irq, set);
13329 }
13330 
kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry * old,struct kvm_kernel_irq_routing_entry * new)13331 bool kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry *old,
13332 				  struct kvm_kernel_irq_routing_entry *new)
13333 {
13334 	if (new->type != KVM_IRQ_ROUTING_MSI)
13335 		return true;
13336 
13337 	return !!memcmp(&old->msi, &new->msi, sizeof(new->msi));
13338 }
13339 
kvm_vector_hashing_enabled(void)13340 bool kvm_vector_hashing_enabled(void)
13341 {
13342 	return vector_hashing;
13343 }
13344 
kvm_arch_no_poll(struct kvm_vcpu * vcpu)13345 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
13346 {
13347 	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;
13348 }
13349 EXPORT_SYMBOL_GPL(kvm_arch_no_poll);
13350 
13351 
kvm_spec_ctrl_test_value(u64 value)13352 int kvm_spec_ctrl_test_value(u64 value)
13353 {
13354 	/*
13355 	 * test that setting IA32_SPEC_CTRL to given value
13356 	 * is allowed by the host processor
13357 	 */
13358 
13359 	u64 saved_value;
13360 	unsigned long flags;
13361 	int ret = 0;
13362 
13363 	local_irq_save(flags);
13364 
13365 	if (rdmsrl_safe(MSR_IA32_SPEC_CTRL, &saved_value))
13366 		ret = 1;
13367 	else if (wrmsrl_safe(MSR_IA32_SPEC_CTRL, value))
13368 		ret = 1;
13369 	else
13370 		wrmsrl(MSR_IA32_SPEC_CTRL, saved_value);
13371 
13372 	local_irq_restore(flags);
13373 
13374 	return ret;
13375 }
13376 EXPORT_SYMBOL_GPL(kvm_spec_ctrl_test_value);
13377 
kvm_fixup_and_inject_pf_error(struct kvm_vcpu * vcpu,gva_t gva,u16 error_code)13378 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)
13379 {
13380 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
13381 	struct x86_exception fault;
13382 	u64 access = error_code &
13383 		(PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK);
13384 
13385 	if (!(error_code & PFERR_PRESENT_MASK) ||
13386 	    mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) {
13387 		/*
13388 		 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page
13389 		 * tables probably do not match the TLB.  Just proceed
13390 		 * with the error code that the processor gave.
13391 		 */
13392 		fault.vector = PF_VECTOR;
13393 		fault.error_code_valid = true;
13394 		fault.error_code = error_code;
13395 		fault.nested_page_fault = false;
13396 		fault.address = gva;
13397 		fault.async_page_fault = false;
13398 	}
13399 	vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);
13400 }
13401 EXPORT_SYMBOL_GPL(kvm_fixup_and_inject_pf_error);
13402 
13403 /*
13404  * Handles kvm_read/write_guest_virt*() result and either injects #PF or returns
13405  * KVM_EXIT_INTERNAL_ERROR for cases not currently handled by KVM. Return value
13406  * indicates whether exit to userspace is needed.
13407  */
kvm_handle_memory_failure(struct kvm_vcpu * vcpu,int r,struct x86_exception * e)13408 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,
13409 			      struct x86_exception *e)
13410 {
13411 	if (r == X86EMUL_PROPAGATE_FAULT) {
13412 		kvm_inject_emulated_page_fault(vcpu, e);
13413 		return 1;
13414 	}
13415 
13416 	/*
13417 	 * In case kvm_read/write_guest_virt*() failed with X86EMUL_IO_NEEDED
13418 	 * while handling a VMX instruction KVM could've handled the request
13419 	 * correctly by exiting to userspace and performing I/O but there
13420 	 * doesn't seem to be a real use-case behind such requests, just return
13421 	 * KVM_EXIT_INTERNAL_ERROR for now.
13422 	 */
13423 	kvm_prepare_emulation_failure_exit(vcpu);
13424 
13425 	return 0;
13426 }
13427 EXPORT_SYMBOL_GPL(kvm_handle_memory_failure);
13428 
kvm_handle_invpcid(struct kvm_vcpu * vcpu,unsigned long type,gva_t gva)13429 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
13430 {
13431 	bool pcid_enabled;
13432 	struct x86_exception e;
13433 	struct {
13434 		u64 pcid;
13435 		u64 gla;
13436 	} operand;
13437 	int r;
13438 
13439 	r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
13440 	if (r != X86EMUL_CONTINUE)
13441 		return kvm_handle_memory_failure(vcpu, r, &e);
13442 
13443 	if (operand.pcid >> 12 != 0) {
13444 		kvm_inject_gp(vcpu, 0);
13445 		return 1;
13446 	}
13447 
13448 	pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);
13449 
13450 	switch (type) {
13451 	case INVPCID_TYPE_INDIV_ADDR:
13452 		if ((!pcid_enabled && (operand.pcid != 0)) ||
13453 		    is_noncanonical_address(operand.gla, vcpu)) {
13454 			kvm_inject_gp(vcpu, 0);
13455 			return 1;
13456 		}
13457 		kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);
13458 		return kvm_skip_emulated_instruction(vcpu);
13459 
13460 	case INVPCID_TYPE_SINGLE_CTXT:
13461 		if (!pcid_enabled && (operand.pcid != 0)) {
13462 			kvm_inject_gp(vcpu, 0);
13463 			return 1;
13464 		}
13465 
13466 		kvm_invalidate_pcid(vcpu, operand.pcid);
13467 		return kvm_skip_emulated_instruction(vcpu);
13468 
13469 	case INVPCID_TYPE_ALL_NON_GLOBAL:
13470 		/*
13471 		 * Currently, KVM doesn't mark global entries in the shadow
13472 		 * page tables, so a non-global flush just degenerates to a
13473 		 * global flush. If needed, we could optimize this later by
13474 		 * keeping track of global entries in shadow page tables.
13475 		 */
13476 
13477 		fallthrough;
13478 	case INVPCID_TYPE_ALL_INCL_GLOBAL:
13479 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
13480 		return kvm_skip_emulated_instruction(vcpu);
13481 
13482 	default:
13483 		kvm_inject_gp(vcpu, 0);
13484 		return 1;
13485 	}
13486 }
13487 EXPORT_SYMBOL_GPL(kvm_handle_invpcid);
13488 
complete_sev_es_emulated_mmio(struct kvm_vcpu * vcpu)13489 static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu)
13490 {
13491 	struct kvm_run *run = vcpu->run;
13492 	struct kvm_mmio_fragment *frag;
13493 	unsigned int len;
13494 
13495 	BUG_ON(!vcpu->mmio_needed);
13496 
13497 	/* Complete previous fragment */
13498 	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
13499 	len = min(8u, frag->len);
13500 	if (!vcpu->mmio_is_write)
13501 		memcpy(frag->data, run->mmio.data, len);
13502 
13503 	if (frag->len <= 8) {
13504 		/* Switch to the next fragment. */
13505 		frag++;
13506 		vcpu->mmio_cur_fragment++;
13507 	} else {
13508 		/* Go forward to the next mmio piece. */
13509 		frag->data += len;
13510 		frag->gpa += len;
13511 		frag->len -= len;
13512 	}
13513 
13514 	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
13515 		vcpu->mmio_needed = 0;
13516 
13517 		// VMG change, at this point, we're always done
13518 		// RIP has already been advanced
13519 		return 1;
13520 	}
13521 
13522 	// More MMIO is needed
13523 	run->mmio.phys_addr = frag->gpa;
13524 	run->mmio.len = min(8u, frag->len);
13525 	run->mmio.is_write = vcpu->mmio_is_write;
13526 	if (run->mmio.is_write)
13527 		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
13528 	run->exit_reason = KVM_EXIT_MMIO;
13529 
13530 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
13531 
13532 	return 0;
13533 }
13534 
kvm_sev_es_mmio_write(struct kvm_vcpu * vcpu,gpa_t gpa,unsigned int bytes,void * data)13535 int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
13536 			  void *data)
13537 {
13538 	int handled;
13539 	struct kvm_mmio_fragment *frag;
13540 
13541 	if (!data)
13542 		return -EINVAL;
13543 
13544 	handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data);
13545 	if (handled == bytes)
13546 		return 1;
13547 
13548 	bytes -= handled;
13549 	gpa += handled;
13550 	data += handled;
13551 
13552 	/*TODO: Check if need to increment number of frags */
13553 	frag = vcpu->mmio_fragments;
13554 	vcpu->mmio_nr_fragments = 1;
13555 	frag->len = bytes;
13556 	frag->gpa = gpa;
13557 	frag->data = data;
13558 
13559 	vcpu->mmio_needed = 1;
13560 	vcpu->mmio_cur_fragment = 0;
13561 
13562 	vcpu->run->mmio.phys_addr = gpa;
13563 	vcpu->run->mmio.len = min(8u, frag->len);
13564 	vcpu->run->mmio.is_write = 1;
13565 	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
13566 	vcpu->run->exit_reason = KVM_EXIT_MMIO;
13567 
13568 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
13569 
13570 	return 0;
13571 }
13572 EXPORT_SYMBOL_GPL(kvm_sev_es_mmio_write);
13573 
kvm_sev_es_mmio_read(struct kvm_vcpu * vcpu,gpa_t gpa,unsigned int bytes,void * data)13574 int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
13575 			 void *data)
13576 {
13577 	int handled;
13578 	struct kvm_mmio_fragment *frag;
13579 
13580 	if (!data)
13581 		return -EINVAL;
13582 
13583 	handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data);
13584 	if (handled == bytes)
13585 		return 1;
13586 
13587 	bytes -= handled;
13588 	gpa += handled;
13589 	data += handled;
13590 
13591 	/*TODO: Check if need to increment number of frags */
13592 	frag = vcpu->mmio_fragments;
13593 	vcpu->mmio_nr_fragments = 1;
13594 	frag->len = bytes;
13595 	frag->gpa = gpa;
13596 	frag->data = data;
13597 
13598 	vcpu->mmio_needed = 1;
13599 	vcpu->mmio_cur_fragment = 0;
13600 
13601 	vcpu->run->mmio.phys_addr = gpa;
13602 	vcpu->run->mmio.len = min(8u, frag->len);
13603 	vcpu->run->mmio.is_write = 0;
13604 	vcpu->run->exit_reason = KVM_EXIT_MMIO;
13605 
13606 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
13607 
13608 	return 0;
13609 }
13610 EXPORT_SYMBOL_GPL(kvm_sev_es_mmio_read);
13611 
advance_sev_es_emulated_pio(struct kvm_vcpu * vcpu,unsigned count,int size)13612 static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size)
13613 {
13614 	vcpu->arch.sev_pio_count -= count;
13615 	vcpu->arch.sev_pio_data += count * size;
13616 }
13617 
13618 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
13619 			   unsigned int port);
13620 
complete_sev_es_emulated_outs(struct kvm_vcpu * vcpu)13621 static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu)
13622 {
13623 	int size = vcpu->arch.pio.size;
13624 	int port = vcpu->arch.pio.port;
13625 
13626 	vcpu->arch.pio.count = 0;
13627 	if (vcpu->arch.sev_pio_count)
13628 		return kvm_sev_es_outs(vcpu, size, port);
13629 	return 1;
13630 }
13631 
kvm_sev_es_outs(struct kvm_vcpu * vcpu,unsigned int size,unsigned int port)13632 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
13633 			   unsigned int port)
13634 {
13635 	for (;;) {
13636 		unsigned int count =
13637 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);
13638 		int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count);
13639 
13640 		/* memcpy done already by emulator_pio_out.  */
13641 		advance_sev_es_emulated_pio(vcpu, count, size);
13642 		if (!ret)
13643 			break;
13644 
13645 		/* Emulation done by the kernel.  */
13646 		if (!vcpu->arch.sev_pio_count)
13647 			return 1;
13648 	}
13649 
13650 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs;
13651 	return 0;
13652 }
13653 
13654 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
13655 			  unsigned int port);
13656 
complete_sev_es_emulated_ins(struct kvm_vcpu * vcpu)13657 static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu)
13658 {
13659 	unsigned count = vcpu->arch.pio.count;
13660 	int size = vcpu->arch.pio.size;
13661 	int port = vcpu->arch.pio.port;
13662 
13663 	complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data);
13664 	advance_sev_es_emulated_pio(vcpu, count, size);
13665 	if (vcpu->arch.sev_pio_count)
13666 		return kvm_sev_es_ins(vcpu, size, port);
13667 	return 1;
13668 }
13669 
kvm_sev_es_ins(struct kvm_vcpu * vcpu,unsigned int size,unsigned int port)13670 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
13671 			  unsigned int port)
13672 {
13673 	for (;;) {
13674 		unsigned int count =
13675 			min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);
13676 		if (!emulator_pio_in(vcpu, size, port, vcpu->arch.sev_pio_data, count))
13677 			break;
13678 
13679 		/* Emulation done by the kernel.  */
13680 		advance_sev_es_emulated_pio(vcpu, count, size);
13681 		if (!vcpu->arch.sev_pio_count)
13682 			return 1;
13683 	}
13684 
13685 	vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins;
13686 	return 0;
13687 }
13688 
kvm_sev_es_string_io(struct kvm_vcpu * vcpu,unsigned int size,unsigned int port,void * data,unsigned int count,int in)13689 int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,
13690 			 unsigned int port, void *data,  unsigned int count,
13691 			 int in)
13692 {
13693 	vcpu->arch.sev_pio_data = data;
13694 	vcpu->arch.sev_pio_count = count;
13695 	return in ? kvm_sev_es_ins(vcpu, size, port)
13696 		  : kvm_sev_es_outs(vcpu, size, port);
13697 }
13698 EXPORT_SYMBOL_GPL(kvm_sev_es_string_io);
13699 
13700 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_entry);
13701 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
13702 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio);
13703 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
13704 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
13705 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
13706 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
13707 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter);
13708 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
13709 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
13710 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
13711 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed);
13712 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
13713 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
13714 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
13715 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
13716 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update);
13717 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
13718 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);
13719 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);
13720 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi);
13721 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_ga_log);
13722 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_kick_vcpu_slowpath);
13723 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_doorbell);
13724 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_accept_irq);
13725 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_enter);
13726 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_exit);
13727 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_enter);
13728 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_exit);
13729 
kvm_x86_init(void)13730 static int __init kvm_x86_init(void)
13731 {
13732 	kvm_mmu_x86_module_init();
13733 	return 0;
13734 }
13735 module_init(kvm_x86_init);
13736 
kvm_x86_exit(void)13737 static void __exit kvm_x86_exit(void)
13738 {
13739 	/*
13740 	 * If module_init() is implemented, module_exit() must also be
13741 	 * implemented to allow module unload.
13742 	 */
13743 }
13744 module_exit(kvm_x86_exit);
13745