1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
4  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
5  */
6 
7 #include <linux/bug.h>
8 #include <linux/cpu_pm.h>
9 #include <linux/entry-kvm.h>
10 #include <linux/errno.h>
11 #include <linux/err.h>
12 #include <linux/kvm_host.h>
13 #include <linux/list.h>
14 #include <linux/module.h>
15 #include <linux/vmalloc.h>
16 #include <linux/fs.h>
17 #include <linux/mman.h>
18 #include <linux/sched.h>
19 #include <linux/kmemleak.h>
20 #include <linux/kvm.h>
21 #include <linux/kvm_irqfd.h>
22 #include <linux/irqbypass.h>
23 #include <linux/sched/stat.h>
24 #include <linux/psci.h>
25 #include <trace/events/kvm.h>
26 
27 #define CREATE_TRACE_POINTS
28 #include "trace_arm.h"
29 
30 #include <linux/uaccess.h>
31 #include <asm/ptrace.h>
32 #include <asm/mman.h>
33 #include <asm/tlbflush.h>
34 #include <asm/cacheflush.h>
35 #include <asm/cpufeature.h>
36 #include <asm/virt.h>
37 #include <asm/kvm_arm.h>
38 #include <asm/kvm_asm.h>
39 #include <asm/kvm_mmu.h>
40 #include <asm/kvm_emulate.h>
41 #include <asm/sections.h>
42 
43 #include <kvm/arm_hypercalls.h>
44 #include <kvm/arm_pmu.h>
45 #include <kvm/arm_psci.h>
46 
47 static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
48 DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized);
49 
50 DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
51 
52 DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
53 unsigned long kvm_arm_hyp_percpu_base[NR_CPUS];
54 DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
55 
56 static bool vgic_present;
57 
58 static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled);
59 DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
60 
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)61 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
62 {
63 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
64 }
65 
kvm_arch_hardware_setup(void * opaque)66 int kvm_arch_hardware_setup(void *opaque)
67 {
68 	return 0;
69 }
70 
kvm_arch_check_processor_compat(void * opaque)71 int kvm_arch_check_processor_compat(void *opaque)
72 {
73 	return 0;
74 }
75 
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)76 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
77 			    struct kvm_enable_cap *cap)
78 {
79 	int r;
80 
81 	if (cap->flags)
82 		return -EINVAL;
83 
84 	switch (cap->cap) {
85 	case KVM_CAP_ARM_NISV_TO_USER:
86 		r = 0;
87 		set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER,
88 			&kvm->arch.flags);
89 		break;
90 	case KVM_CAP_ARM_MTE:
91 		mutex_lock(&kvm->lock);
92 		if (!system_supports_mte() || kvm->created_vcpus) {
93 			r = -EINVAL;
94 		} else {
95 			r = 0;
96 			set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags);
97 		}
98 		mutex_unlock(&kvm->lock);
99 		break;
100 	case KVM_CAP_ARM_SYSTEM_SUSPEND:
101 		r = 0;
102 		set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags);
103 		break;
104 	default:
105 		r = -EINVAL;
106 		break;
107 	}
108 
109 	return r;
110 }
111 
kvm_arm_default_max_vcpus(void)112 static int kvm_arm_default_max_vcpus(void)
113 {
114 	return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
115 }
116 
set_default_spectre(struct kvm * kvm)117 static void set_default_spectre(struct kvm *kvm)
118 {
119 	/*
120 	 * The default is to expose CSV2 == 1 if the HW isn't affected.
121 	 * Although this is a per-CPU feature, we make it global because
122 	 * asymmetric systems are just a nuisance.
123 	 *
124 	 * Userspace can override this as long as it doesn't promise
125 	 * the impossible.
126 	 */
127 	if (arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED)
128 		kvm->arch.pfr0_csv2 = 1;
129 	if (arm64_get_meltdown_state() == SPECTRE_UNAFFECTED)
130 		kvm->arch.pfr0_csv3 = 1;
131 }
132 
133 /**
134  * kvm_arch_init_vm - initializes a VM data structure
135  * @kvm:	pointer to the KVM struct
136  */
kvm_arch_init_vm(struct kvm * kvm,unsigned long type)137 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
138 {
139 	int ret;
140 
141 	ret = kvm_arm_setup_stage2(kvm, type);
142 	if (ret)
143 		return ret;
144 
145 	ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu);
146 	if (ret)
147 		return ret;
148 
149 	ret = kvm_share_hyp(kvm, kvm + 1);
150 	if (ret)
151 		goto out_free_stage2_pgd;
152 
153 	if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL)) {
154 		ret = -ENOMEM;
155 		goto out_free_stage2_pgd;
156 	}
157 	cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask);
158 
159 	kvm_vgic_early_init(kvm);
160 
161 	/* The maximum number of VCPUs is limited by the host's GIC model */
162 	kvm->max_vcpus = kvm_arm_default_max_vcpus();
163 
164 	set_default_spectre(kvm);
165 	kvm_arm_init_hypercalls(kvm);
166 
167 	return ret;
168 out_free_stage2_pgd:
169 	kvm_free_stage2_pgd(&kvm->arch.mmu);
170 	return ret;
171 }
172 
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)173 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
174 {
175 	return VM_FAULT_SIGBUS;
176 }
177 
178 
179 /**
180  * kvm_arch_destroy_vm - destroy the VM data structure
181  * @kvm:	pointer to the KVM struct
182  */
kvm_arch_destroy_vm(struct kvm * kvm)183 void kvm_arch_destroy_vm(struct kvm *kvm)
184 {
185 	bitmap_free(kvm->arch.pmu_filter);
186 	free_cpumask_var(kvm->arch.supported_cpus);
187 
188 	kvm_vgic_destroy(kvm);
189 
190 	kvm_destroy_vcpus(kvm);
191 
192 	kvm_unshare_hyp(kvm, kvm + 1);
193 }
194 
kvm_vm_ioctl_check_extension(struct kvm * kvm,long ext)195 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
196 {
197 	int r;
198 	switch (ext) {
199 	case KVM_CAP_IRQCHIP:
200 		r = vgic_present;
201 		break;
202 	case KVM_CAP_IOEVENTFD:
203 	case KVM_CAP_DEVICE_CTRL:
204 	case KVM_CAP_USER_MEMORY:
205 	case KVM_CAP_SYNC_MMU:
206 	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
207 	case KVM_CAP_ONE_REG:
208 	case KVM_CAP_ARM_PSCI:
209 	case KVM_CAP_ARM_PSCI_0_2:
210 	case KVM_CAP_READONLY_MEM:
211 	case KVM_CAP_MP_STATE:
212 	case KVM_CAP_IMMEDIATE_EXIT:
213 	case KVM_CAP_VCPU_EVENTS:
214 	case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
215 	case KVM_CAP_ARM_NISV_TO_USER:
216 	case KVM_CAP_ARM_INJECT_EXT_DABT:
217 	case KVM_CAP_SET_GUEST_DEBUG:
218 	case KVM_CAP_VCPU_ATTRIBUTES:
219 	case KVM_CAP_PTP_KVM:
220 	case KVM_CAP_ARM_SYSTEM_SUSPEND:
221 		r = 1;
222 		break;
223 	case KVM_CAP_SET_GUEST_DEBUG2:
224 		return KVM_GUESTDBG_VALID_MASK;
225 	case KVM_CAP_ARM_SET_DEVICE_ADDR:
226 		r = 1;
227 		break;
228 	case KVM_CAP_NR_VCPUS:
229 		/*
230 		 * ARM64 treats KVM_CAP_NR_CPUS differently from all other
231 		 * architectures, as it does not always bound it to
232 		 * KVM_CAP_MAX_VCPUS. It should not matter much because
233 		 * this is just an advisory value.
234 		 */
235 		r = min_t(unsigned int, num_online_cpus(),
236 			  kvm_arm_default_max_vcpus());
237 		break;
238 	case KVM_CAP_MAX_VCPUS:
239 	case KVM_CAP_MAX_VCPU_ID:
240 		if (kvm)
241 			r = kvm->max_vcpus;
242 		else
243 			r = kvm_arm_default_max_vcpus();
244 		break;
245 	case KVM_CAP_MSI_DEVID:
246 		if (!kvm)
247 			r = -EINVAL;
248 		else
249 			r = kvm->arch.vgic.msis_require_devid;
250 		break;
251 	case KVM_CAP_ARM_USER_IRQ:
252 		/*
253 		 * 1: EL1_VTIMER, EL1_PTIMER, and PMU.
254 		 * (bump this number if adding more devices)
255 		 */
256 		r = 1;
257 		break;
258 	case KVM_CAP_ARM_MTE:
259 		r = system_supports_mte();
260 		break;
261 	case KVM_CAP_STEAL_TIME:
262 		r = kvm_arm_pvtime_supported();
263 		break;
264 	case KVM_CAP_ARM_EL1_32BIT:
265 		r = cpus_have_const_cap(ARM64_HAS_32BIT_EL1);
266 		break;
267 	case KVM_CAP_GUEST_DEBUG_HW_BPS:
268 		r = get_num_brps();
269 		break;
270 	case KVM_CAP_GUEST_DEBUG_HW_WPS:
271 		r = get_num_wrps();
272 		break;
273 	case KVM_CAP_ARM_PMU_V3:
274 		r = kvm_arm_support_pmu_v3();
275 		break;
276 	case KVM_CAP_ARM_INJECT_SERROR_ESR:
277 		r = cpus_have_const_cap(ARM64_HAS_RAS_EXTN);
278 		break;
279 	case KVM_CAP_ARM_VM_IPA_SIZE:
280 		r = get_kvm_ipa_limit();
281 		break;
282 	case KVM_CAP_ARM_SVE:
283 		r = system_supports_sve();
284 		break;
285 	case KVM_CAP_ARM_PTRAUTH_ADDRESS:
286 	case KVM_CAP_ARM_PTRAUTH_GENERIC:
287 		r = system_has_full_ptr_auth();
288 		break;
289 	default:
290 		r = 0;
291 	}
292 
293 	return r;
294 }
295 
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)296 long kvm_arch_dev_ioctl(struct file *filp,
297 			unsigned int ioctl, unsigned long arg)
298 {
299 	return -EINVAL;
300 }
301 
kvm_arch_alloc_vm(void)302 struct kvm *kvm_arch_alloc_vm(void)
303 {
304 	size_t sz = sizeof(struct kvm);
305 
306 	if (!has_vhe())
307 		return kzalloc(sz, GFP_KERNEL_ACCOUNT);
308 
309 	return __vmalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_HIGHMEM | __GFP_ZERO);
310 }
311 
kvm_arch_vcpu_precreate(struct kvm * kvm,unsigned int id)312 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
313 {
314 	if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
315 		return -EBUSY;
316 
317 	if (id >= kvm->max_vcpus)
318 		return -EINVAL;
319 
320 	return 0;
321 }
322 
kvm_arch_vcpu_create(struct kvm_vcpu * vcpu)323 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
324 {
325 	int err;
326 
327 	/* Force users to call KVM_ARM_VCPU_INIT */
328 	vcpu->arch.target = -1;
329 	bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
330 
331 	vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
332 
333 	/*
334 	 * Default value for the FP state, will be overloaded at load
335 	 * time if we support FP (pretty likely)
336 	 */
337 	vcpu->arch.fp_state = FP_STATE_FREE;
338 
339 	/* Set up the timer */
340 	kvm_timer_vcpu_init(vcpu);
341 
342 	kvm_pmu_vcpu_init(vcpu);
343 
344 	kvm_arm_reset_debug_ptr(vcpu);
345 
346 	kvm_arm_pvtime_vcpu_init(&vcpu->arch);
347 
348 	vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
349 
350 	err = kvm_vgic_vcpu_init(vcpu);
351 	if (err)
352 		return err;
353 
354 	return kvm_share_hyp(vcpu, vcpu + 1);
355 }
356 
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)357 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
358 {
359 }
360 
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)361 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
362 {
363 	if (vcpu_has_run_once(vcpu) && unlikely(!irqchip_in_kernel(vcpu->kvm)))
364 		static_branch_dec(&userspace_irqchip_in_use);
365 
366 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
367 	kvm_timer_vcpu_terminate(vcpu);
368 	kvm_pmu_vcpu_destroy(vcpu);
369 
370 	kvm_arm_vcpu_destroy(vcpu);
371 }
372 
kvm_arch_vcpu_blocking(struct kvm_vcpu * vcpu)373 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
374 {
375 
376 }
377 
kvm_arch_vcpu_unblocking(struct kvm_vcpu * vcpu)378 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
379 {
380 
381 }
382 
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)383 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
384 {
385 	struct kvm_s2_mmu *mmu;
386 	int *last_ran;
387 
388 	mmu = vcpu->arch.hw_mmu;
389 	last_ran = this_cpu_ptr(mmu->last_vcpu_ran);
390 
391 	/*
392 	 * We guarantee that both TLBs and I-cache are private to each
393 	 * vcpu. If detecting that a vcpu from the same VM has
394 	 * previously run on the same physical CPU, call into the
395 	 * hypervisor code to nuke the relevant contexts.
396 	 *
397 	 * We might get preempted before the vCPU actually runs, but
398 	 * over-invalidation doesn't affect correctness.
399 	 */
400 	if (*last_ran != vcpu->vcpu_id) {
401 		kvm_call_hyp(__kvm_flush_cpu_context, mmu);
402 		*last_ran = vcpu->vcpu_id;
403 	}
404 
405 	vcpu->cpu = cpu;
406 
407 	kvm_vgic_load(vcpu);
408 	kvm_timer_vcpu_load(vcpu);
409 	if (has_vhe())
410 		kvm_vcpu_load_sysregs_vhe(vcpu);
411 	kvm_arch_vcpu_load_fp(vcpu);
412 	kvm_vcpu_pmu_restore_guest(vcpu);
413 	if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
414 		kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
415 
416 	if (single_task_running())
417 		vcpu_clear_wfx_traps(vcpu);
418 	else
419 		vcpu_set_wfx_traps(vcpu);
420 
421 	if (vcpu_has_ptrauth(vcpu))
422 		vcpu_ptrauth_disable(vcpu);
423 	kvm_arch_vcpu_load_debug_state_flags(vcpu);
424 
425 	if (!cpumask_test_cpu(smp_processor_id(), vcpu->kvm->arch.supported_cpus))
426 		vcpu_set_on_unsupported_cpu(vcpu);
427 }
428 
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)429 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
430 {
431 	kvm_arch_vcpu_put_debug_state_flags(vcpu);
432 	kvm_arch_vcpu_put_fp(vcpu);
433 	if (has_vhe())
434 		kvm_vcpu_put_sysregs_vhe(vcpu);
435 	kvm_timer_vcpu_put(vcpu);
436 	kvm_vgic_put(vcpu);
437 	kvm_vcpu_pmu_restore_host(vcpu);
438 	kvm_arm_vmid_clear_active();
439 
440 	vcpu_clear_on_unsupported_cpu(vcpu);
441 	vcpu->cpu = -1;
442 }
443 
kvm_arm_vcpu_power_off(struct kvm_vcpu * vcpu)444 void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
445 {
446 	vcpu->arch.mp_state.mp_state = KVM_MP_STATE_STOPPED;
447 	kvm_make_request(KVM_REQ_SLEEP, vcpu);
448 	kvm_vcpu_kick(vcpu);
449 }
450 
kvm_arm_vcpu_stopped(struct kvm_vcpu * vcpu)451 bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu)
452 {
453 	return vcpu->arch.mp_state.mp_state == KVM_MP_STATE_STOPPED;
454 }
455 
kvm_arm_vcpu_suspend(struct kvm_vcpu * vcpu)456 static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu)
457 {
458 	vcpu->arch.mp_state.mp_state = KVM_MP_STATE_SUSPENDED;
459 	kvm_make_request(KVM_REQ_SUSPEND, vcpu);
460 	kvm_vcpu_kick(vcpu);
461 }
462 
kvm_arm_vcpu_suspended(struct kvm_vcpu * vcpu)463 static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu)
464 {
465 	return vcpu->arch.mp_state.mp_state == KVM_MP_STATE_SUSPENDED;
466 }
467 
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)468 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
469 				    struct kvm_mp_state *mp_state)
470 {
471 	*mp_state = vcpu->arch.mp_state;
472 
473 	return 0;
474 }
475 
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)476 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
477 				    struct kvm_mp_state *mp_state)
478 {
479 	int ret = 0;
480 
481 	switch (mp_state->mp_state) {
482 	case KVM_MP_STATE_RUNNABLE:
483 		vcpu->arch.mp_state = *mp_state;
484 		break;
485 	case KVM_MP_STATE_STOPPED:
486 		kvm_arm_vcpu_power_off(vcpu);
487 		break;
488 	case KVM_MP_STATE_SUSPENDED:
489 		kvm_arm_vcpu_suspend(vcpu);
490 		break;
491 	default:
492 		ret = -EINVAL;
493 	}
494 
495 	return ret;
496 }
497 
498 /**
499  * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
500  * @v:		The VCPU pointer
501  *
502  * If the guest CPU is not waiting for interrupts or an interrupt line is
503  * asserted, the CPU is by definition runnable.
504  */
kvm_arch_vcpu_runnable(struct kvm_vcpu * v)505 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
506 {
507 	bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF);
508 	return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
509 		&& !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
510 }
511 
kvm_arch_vcpu_in_kernel(struct kvm_vcpu * vcpu)512 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
513 {
514 	return vcpu_mode_priv(vcpu);
515 }
516 
517 #ifdef CONFIG_GUEST_PERF_EVENTS
kvm_arch_vcpu_get_ip(struct kvm_vcpu * vcpu)518 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
519 {
520 	return *vcpu_pc(vcpu);
521 }
522 #endif
523 
kvm_vcpu_initialized(struct kvm_vcpu * vcpu)524 static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
525 {
526 	return vcpu->arch.target >= 0;
527 }
528 
529 /*
530  * Handle both the initialisation that is being done when the vcpu is
531  * run for the first time, as well as the updates that must be
532  * performed each time we get a new thread dealing with this vcpu.
533  */
kvm_arch_vcpu_run_pid_change(struct kvm_vcpu * vcpu)534 int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
535 {
536 	struct kvm *kvm = vcpu->kvm;
537 	int ret;
538 
539 	if (!kvm_vcpu_initialized(vcpu))
540 		return -ENOEXEC;
541 
542 	if (!kvm_arm_vcpu_is_finalized(vcpu))
543 		return -EPERM;
544 
545 	ret = kvm_arch_vcpu_run_map_fp(vcpu);
546 	if (ret)
547 		return ret;
548 
549 	if (likely(vcpu_has_run_once(vcpu)))
550 		return 0;
551 
552 	kvm_arm_vcpu_init_debug(vcpu);
553 
554 	if (likely(irqchip_in_kernel(kvm))) {
555 		/*
556 		 * Map the VGIC hardware resources before running a vcpu the
557 		 * first time on this VM.
558 		 */
559 		ret = kvm_vgic_map_resources(kvm);
560 		if (ret)
561 			return ret;
562 	}
563 
564 	ret = kvm_timer_enable(vcpu);
565 	if (ret)
566 		return ret;
567 
568 	ret = kvm_arm_pmu_v3_enable(vcpu);
569 	if (ret)
570 		return ret;
571 
572 	if (!irqchip_in_kernel(kvm)) {
573 		/*
574 		 * Tell the rest of the code that there are userspace irqchip
575 		 * VMs in the wild.
576 		 */
577 		static_branch_inc(&userspace_irqchip_in_use);
578 	}
579 
580 	/*
581 	 * Initialize traps for protected VMs.
582 	 * NOTE: Move to run in EL2 directly, rather than via a hypercall, once
583 	 * the code is in place for first run initialization at EL2.
584 	 */
585 	if (kvm_vm_is_protected(kvm))
586 		kvm_call_hyp_nvhe(__pkvm_vcpu_init_traps, vcpu);
587 
588 	mutex_lock(&kvm->lock);
589 	set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags);
590 	mutex_unlock(&kvm->lock);
591 
592 	return ret;
593 }
594 
kvm_arch_intc_initialized(struct kvm * kvm)595 bool kvm_arch_intc_initialized(struct kvm *kvm)
596 {
597 	return vgic_initialized(kvm);
598 }
599 
kvm_arm_halt_guest(struct kvm * kvm)600 void kvm_arm_halt_guest(struct kvm *kvm)
601 {
602 	unsigned long i;
603 	struct kvm_vcpu *vcpu;
604 
605 	kvm_for_each_vcpu(i, vcpu, kvm)
606 		vcpu->arch.pause = true;
607 	kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
608 }
609 
kvm_arm_resume_guest(struct kvm * kvm)610 void kvm_arm_resume_guest(struct kvm *kvm)
611 {
612 	unsigned long i;
613 	struct kvm_vcpu *vcpu;
614 
615 	kvm_for_each_vcpu(i, vcpu, kvm) {
616 		vcpu->arch.pause = false;
617 		__kvm_vcpu_wake_up(vcpu);
618 	}
619 }
620 
kvm_vcpu_sleep(struct kvm_vcpu * vcpu)621 static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu)
622 {
623 	struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
624 
625 	rcuwait_wait_event(wait,
626 			   (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause),
627 			   TASK_INTERRUPTIBLE);
628 
629 	if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) {
630 		/* Awaken to handle a signal, request we sleep again later. */
631 		kvm_make_request(KVM_REQ_SLEEP, vcpu);
632 	}
633 
634 	/*
635 	 * Make sure we will observe a potential reset request if we've
636 	 * observed a change to the power state. Pairs with the smp_wmb() in
637 	 * kvm_psci_vcpu_on().
638 	 */
639 	smp_rmb();
640 }
641 
642 /**
643  * kvm_vcpu_wfi - emulate Wait-For-Interrupt behavior
644  * @vcpu:	The VCPU pointer
645  *
646  * Suspend execution of a vCPU until a valid wake event is detected, i.e. until
647  * the vCPU is runnable.  The vCPU may or may not be scheduled out, depending
648  * on when a wake event arrives, e.g. there may already be a pending wake event.
649  */
kvm_vcpu_wfi(struct kvm_vcpu * vcpu)650 void kvm_vcpu_wfi(struct kvm_vcpu *vcpu)
651 {
652 	/*
653 	 * Sync back the state of the GIC CPU interface so that we have
654 	 * the latest PMR and group enables. This ensures that
655 	 * kvm_arch_vcpu_runnable has up-to-date data to decide whether
656 	 * we have pending interrupts, e.g. when determining if the
657 	 * vCPU should block.
658 	 *
659 	 * For the same reason, we want to tell GICv4 that we need
660 	 * doorbells to be signalled, should an interrupt become pending.
661 	 */
662 	preempt_disable();
663 	kvm_vgic_vmcr_sync(vcpu);
664 	vgic_v4_put(vcpu, true);
665 	preempt_enable();
666 
667 	kvm_vcpu_halt(vcpu);
668 	vcpu_clear_flag(vcpu, IN_WFIT);
669 
670 	preempt_disable();
671 	vgic_v4_load(vcpu);
672 	preempt_enable();
673 }
674 
kvm_vcpu_suspend(struct kvm_vcpu * vcpu)675 static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu)
676 {
677 	if (!kvm_arm_vcpu_suspended(vcpu))
678 		return 1;
679 
680 	kvm_vcpu_wfi(vcpu);
681 
682 	/*
683 	 * The suspend state is sticky; we do not leave it until userspace
684 	 * explicitly marks the vCPU as runnable. Request that we suspend again
685 	 * later.
686 	 */
687 	kvm_make_request(KVM_REQ_SUSPEND, vcpu);
688 
689 	/*
690 	 * Check to make sure the vCPU is actually runnable. If so, exit to
691 	 * userspace informing it of the wakeup condition.
692 	 */
693 	if (kvm_arch_vcpu_runnable(vcpu)) {
694 		memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event));
695 		vcpu->run->system_event.type = KVM_SYSTEM_EVENT_WAKEUP;
696 		vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
697 		return 0;
698 	}
699 
700 	/*
701 	 * Otherwise, we were unblocked to process a different event, such as a
702 	 * pending signal. Return 1 and allow kvm_arch_vcpu_ioctl_run() to
703 	 * process the event.
704 	 */
705 	return 1;
706 }
707 
708 /**
709  * check_vcpu_requests - check and handle pending vCPU requests
710  * @vcpu:	the VCPU pointer
711  *
712  * Return: 1 if we should enter the guest
713  *	   0 if we should exit to userspace
714  *	   < 0 if we should exit to userspace, where the return value indicates
715  *	   an error
716  */
check_vcpu_requests(struct kvm_vcpu * vcpu)717 static int check_vcpu_requests(struct kvm_vcpu *vcpu)
718 {
719 	if (kvm_request_pending(vcpu)) {
720 		if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
721 			kvm_vcpu_sleep(vcpu);
722 
723 		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
724 			kvm_reset_vcpu(vcpu);
725 
726 		/*
727 		 * Clear IRQ_PENDING requests that were made to guarantee
728 		 * that a VCPU sees new virtual interrupts.
729 		 */
730 		kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
731 
732 		if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu))
733 			kvm_update_stolen_time(vcpu);
734 
735 		if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) {
736 			/* The distributor enable bits were changed */
737 			preempt_disable();
738 			vgic_v4_put(vcpu, false);
739 			vgic_v4_load(vcpu);
740 			preempt_enable();
741 		}
742 
743 		if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu))
744 			kvm_pmu_handle_pmcr(vcpu,
745 					    __vcpu_sys_reg(vcpu, PMCR_EL0));
746 
747 		if (kvm_check_request(KVM_REQ_SUSPEND, vcpu))
748 			return kvm_vcpu_suspend(vcpu);
749 	}
750 
751 	return 1;
752 }
753 
vcpu_mode_is_bad_32bit(struct kvm_vcpu * vcpu)754 static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu)
755 {
756 	if (likely(!vcpu_mode_is_32bit(vcpu)))
757 		return false;
758 
759 	return !kvm_supports_32bit_el0();
760 }
761 
762 /**
763  * kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest
764  * @vcpu:	The VCPU pointer
765  * @ret:	Pointer to write optional return code
766  *
767  * Returns: true if the VCPU needs to return to a preemptible + interruptible
768  *	    and skip guest entry.
769  *
770  * This function disambiguates between two different types of exits: exits to a
771  * preemptible + interruptible kernel context and exits to userspace. For an
772  * exit to userspace, this function will write the return code to ret and return
773  * true. For an exit to preemptible + interruptible kernel context (i.e. check
774  * for pending work and re-enter), return true without writing to ret.
775  */
kvm_vcpu_exit_request(struct kvm_vcpu * vcpu,int * ret)776 static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret)
777 {
778 	struct kvm_run *run = vcpu->run;
779 
780 	/*
781 	 * If we're using a userspace irqchip, then check if we need
782 	 * to tell a userspace irqchip about timer or PMU level
783 	 * changes and if so, exit to userspace (the actual level
784 	 * state gets updated in kvm_timer_update_run and
785 	 * kvm_pmu_update_run below).
786 	 */
787 	if (static_branch_unlikely(&userspace_irqchip_in_use)) {
788 		if (kvm_timer_should_notify_user(vcpu) ||
789 		    kvm_pmu_should_notify_user(vcpu)) {
790 			*ret = -EINTR;
791 			run->exit_reason = KVM_EXIT_INTR;
792 			return true;
793 		}
794 	}
795 
796 	if (unlikely(vcpu_on_unsupported_cpu(vcpu))) {
797 		run->exit_reason = KVM_EXIT_FAIL_ENTRY;
798 		run->fail_entry.hardware_entry_failure_reason = KVM_EXIT_FAIL_ENTRY_CPU_UNSUPPORTED;
799 		run->fail_entry.cpu = smp_processor_id();
800 		*ret = 0;
801 		return true;
802 	}
803 
804 	return kvm_request_pending(vcpu) ||
805 			xfer_to_guest_mode_work_pending();
806 }
807 
808 /*
809  * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
810  * the vCPU is running.
811  *
812  * This must be noinstr as instrumentation may make use of RCU, and this is not
813  * safe during the EQS.
814  */
kvm_arm_vcpu_enter_exit(struct kvm_vcpu * vcpu)815 static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
816 {
817 	int ret;
818 
819 	guest_state_enter_irqoff();
820 	ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu);
821 	guest_state_exit_irqoff();
822 
823 	return ret;
824 }
825 
826 /**
827  * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
828  * @vcpu:	The VCPU pointer
829  *
830  * This function is called through the VCPU_RUN ioctl called from user space. It
831  * will execute VM code in a loop until the time slice for the process is used
832  * or some emulation is needed from user space in which case the function will
833  * return with return value 0 and with the kvm_run structure filled in with the
834  * required data for the requested emulation.
835  */
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu)836 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
837 {
838 	struct kvm_run *run = vcpu->run;
839 	int ret;
840 
841 	if (run->exit_reason == KVM_EXIT_MMIO) {
842 		ret = kvm_handle_mmio_return(vcpu);
843 		if (ret)
844 			return ret;
845 	}
846 
847 	vcpu_load(vcpu);
848 
849 	if (run->immediate_exit) {
850 		ret = -EINTR;
851 		goto out;
852 	}
853 
854 	kvm_sigset_activate(vcpu);
855 
856 	ret = 1;
857 	run->exit_reason = KVM_EXIT_UNKNOWN;
858 	run->flags = 0;
859 	while (ret > 0) {
860 		/*
861 		 * Check conditions before entering the guest
862 		 */
863 		ret = xfer_to_guest_mode_handle_work(vcpu);
864 		if (!ret)
865 			ret = 1;
866 
867 		if (ret > 0)
868 			ret = check_vcpu_requests(vcpu);
869 
870 		/*
871 		 * Preparing the interrupts to be injected also
872 		 * involves poking the GIC, which must be done in a
873 		 * non-preemptible context.
874 		 */
875 		preempt_disable();
876 
877 		/*
878 		 * The VMID allocator only tracks active VMIDs per
879 		 * physical CPU, and therefore the VMID allocated may not be
880 		 * preserved on VMID roll-over if the task was preempted,
881 		 * making a thread's VMID inactive. So we need to call
882 		 * kvm_arm_vmid_update() in non-premptible context.
883 		 */
884 		kvm_arm_vmid_update(&vcpu->arch.hw_mmu->vmid);
885 
886 		kvm_pmu_flush_hwstate(vcpu);
887 
888 		local_irq_disable();
889 
890 		kvm_vgic_flush_hwstate(vcpu);
891 
892 		kvm_pmu_update_vcpu_events(vcpu);
893 
894 		/*
895 		 * Ensure we set mode to IN_GUEST_MODE after we disable
896 		 * interrupts and before the final VCPU requests check.
897 		 * See the comment in kvm_vcpu_exiting_guest_mode() and
898 		 * Documentation/virt/kvm/vcpu-requests.rst
899 		 */
900 		smp_store_mb(vcpu->mode, IN_GUEST_MODE);
901 
902 		if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) {
903 			vcpu->mode = OUTSIDE_GUEST_MODE;
904 			isb(); /* Ensure work in x_flush_hwstate is committed */
905 			kvm_pmu_sync_hwstate(vcpu);
906 			if (static_branch_unlikely(&userspace_irqchip_in_use))
907 				kvm_timer_sync_user(vcpu);
908 			kvm_vgic_sync_hwstate(vcpu);
909 			local_irq_enable();
910 			preempt_enable();
911 			continue;
912 		}
913 
914 		kvm_arm_setup_debug(vcpu);
915 		kvm_arch_vcpu_ctxflush_fp(vcpu);
916 
917 		/**************************************************************
918 		 * Enter the guest
919 		 */
920 		trace_kvm_entry(*vcpu_pc(vcpu));
921 		guest_timing_enter_irqoff();
922 
923 		ret = kvm_arm_vcpu_enter_exit(vcpu);
924 
925 		vcpu->mode = OUTSIDE_GUEST_MODE;
926 		vcpu->stat.exits++;
927 		/*
928 		 * Back from guest
929 		 *************************************************************/
930 
931 		kvm_arm_clear_debug(vcpu);
932 
933 		/*
934 		 * We must sync the PMU state before the vgic state so
935 		 * that the vgic can properly sample the updated state of the
936 		 * interrupt line.
937 		 */
938 		kvm_pmu_sync_hwstate(vcpu);
939 
940 		/*
941 		 * Sync the vgic state before syncing the timer state because
942 		 * the timer code needs to know if the virtual timer
943 		 * interrupts are active.
944 		 */
945 		kvm_vgic_sync_hwstate(vcpu);
946 
947 		/*
948 		 * Sync the timer hardware state before enabling interrupts as
949 		 * we don't want vtimer interrupts to race with syncing the
950 		 * timer virtual interrupt state.
951 		 */
952 		if (static_branch_unlikely(&userspace_irqchip_in_use))
953 			kvm_timer_sync_user(vcpu);
954 
955 		kvm_arch_vcpu_ctxsync_fp(vcpu);
956 
957 		/*
958 		 * We must ensure that any pending interrupts are taken before
959 		 * we exit guest timing so that timer ticks are accounted as
960 		 * guest time. Transiently unmask interrupts so that any
961 		 * pending interrupts are taken.
962 		 *
963 		 * Per ARM DDI 0487G.b section D1.13.4, an ISB (or other
964 		 * context synchronization event) is necessary to ensure that
965 		 * pending interrupts are taken.
966 		 */
967 		if (ARM_EXCEPTION_CODE(ret) == ARM_EXCEPTION_IRQ) {
968 			local_irq_enable();
969 			isb();
970 			local_irq_disable();
971 		}
972 
973 		guest_timing_exit_irqoff();
974 
975 		local_irq_enable();
976 
977 		trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
978 
979 		/* Exit types that need handling before we can be preempted */
980 		handle_exit_early(vcpu, ret);
981 
982 		preempt_enable();
983 
984 		/*
985 		 * The ARMv8 architecture doesn't give the hypervisor
986 		 * a mechanism to prevent a guest from dropping to AArch32 EL0
987 		 * if implemented by the CPU. If we spot the guest in such
988 		 * state and that we decided it wasn't supposed to do so (like
989 		 * with the asymmetric AArch32 case), return to userspace with
990 		 * a fatal error.
991 		 */
992 		if (vcpu_mode_is_bad_32bit(vcpu)) {
993 			/*
994 			 * As we have caught the guest red-handed, decide that
995 			 * it isn't fit for purpose anymore by making the vcpu
996 			 * invalid. The VMM can try and fix it by issuing  a
997 			 * KVM_ARM_VCPU_INIT if it really wants to.
998 			 */
999 			vcpu->arch.target = -1;
1000 			ret = ARM_EXCEPTION_IL;
1001 		}
1002 
1003 		ret = handle_exit(vcpu, ret);
1004 	}
1005 
1006 	/* Tell userspace about in-kernel device output levels */
1007 	if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
1008 		kvm_timer_update_run(vcpu);
1009 		kvm_pmu_update_run(vcpu);
1010 	}
1011 
1012 	kvm_sigset_deactivate(vcpu);
1013 
1014 out:
1015 	/*
1016 	 * In the unlikely event that we are returning to userspace
1017 	 * with pending exceptions or PC adjustment, commit these
1018 	 * adjustments in order to give userspace a consistent view of
1019 	 * the vcpu state. Note that this relies on __kvm_adjust_pc()
1020 	 * being preempt-safe on VHE.
1021 	 */
1022 	if (unlikely(vcpu_get_flag(vcpu, PENDING_EXCEPTION) ||
1023 		     vcpu_get_flag(vcpu, INCREMENT_PC)))
1024 		kvm_call_hyp(__kvm_adjust_pc, vcpu);
1025 
1026 	vcpu_put(vcpu);
1027 	return ret;
1028 }
1029 
vcpu_interrupt_line(struct kvm_vcpu * vcpu,int number,bool level)1030 static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
1031 {
1032 	int bit_index;
1033 	bool set;
1034 	unsigned long *hcr;
1035 
1036 	if (number == KVM_ARM_IRQ_CPU_IRQ)
1037 		bit_index = __ffs(HCR_VI);
1038 	else /* KVM_ARM_IRQ_CPU_FIQ */
1039 		bit_index = __ffs(HCR_VF);
1040 
1041 	hcr = vcpu_hcr(vcpu);
1042 	if (level)
1043 		set = test_and_set_bit(bit_index, hcr);
1044 	else
1045 		set = test_and_clear_bit(bit_index, hcr);
1046 
1047 	/*
1048 	 * If we didn't change anything, no need to wake up or kick other CPUs
1049 	 */
1050 	if (set == level)
1051 		return 0;
1052 
1053 	/*
1054 	 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
1055 	 * trigger a world-switch round on the running physical CPU to set the
1056 	 * virtual IRQ/FIQ fields in the HCR appropriately.
1057 	 */
1058 	kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
1059 	kvm_vcpu_kick(vcpu);
1060 
1061 	return 0;
1062 }
1063 
kvm_vm_ioctl_irq_line(struct kvm * kvm,struct kvm_irq_level * irq_level,bool line_status)1064 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
1065 			  bool line_status)
1066 {
1067 	u32 irq = irq_level->irq;
1068 	unsigned int irq_type, vcpu_idx, irq_num;
1069 	int nrcpus = atomic_read(&kvm->online_vcpus);
1070 	struct kvm_vcpu *vcpu = NULL;
1071 	bool level = irq_level->level;
1072 
1073 	irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
1074 	vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
1075 	vcpu_idx += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
1076 	irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
1077 
1078 	trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
1079 
1080 	switch (irq_type) {
1081 	case KVM_ARM_IRQ_TYPE_CPU:
1082 		if (irqchip_in_kernel(kvm))
1083 			return -ENXIO;
1084 
1085 		if (vcpu_idx >= nrcpus)
1086 			return -EINVAL;
1087 
1088 		vcpu = kvm_get_vcpu(kvm, vcpu_idx);
1089 		if (!vcpu)
1090 			return -EINVAL;
1091 
1092 		if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
1093 			return -EINVAL;
1094 
1095 		return vcpu_interrupt_line(vcpu, irq_num, level);
1096 	case KVM_ARM_IRQ_TYPE_PPI:
1097 		if (!irqchip_in_kernel(kvm))
1098 			return -ENXIO;
1099 
1100 		if (vcpu_idx >= nrcpus)
1101 			return -EINVAL;
1102 
1103 		vcpu = kvm_get_vcpu(kvm, vcpu_idx);
1104 		if (!vcpu)
1105 			return -EINVAL;
1106 
1107 		if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
1108 			return -EINVAL;
1109 
1110 		return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level, NULL);
1111 	case KVM_ARM_IRQ_TYPE_SPI:
1112 		if (!irqchip_in_kernel(kvm))
1113 			return -ENXIO;
1114 
1115 		if (irq_num < VGIC_NR_PRIVATE_IRQS)
1116 			return -EINVAL;
1117 
1118 		return kvm_vgic_inject_irq(kvm, 0, irq_num, level, NULL);
1119 	}
1120 
1121 	return -EINVAL;
1122 }
1123 
kvm_vcpu_set_target(struct kvm_vcpu * vcpu,const struct kvm_vcpu_init * init)1124 static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1125 			       const struct kvm_vcpu_init *init)
1126 {
1127 	unsigned int i, ret;
1128 	u32 phys_target = kvm_target_cpu();
1129 
1130 	if (init->target != phys_target)
1131 		return -EINVAL;
1132 
1133 	/*
1134 	 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
1135 	 * use the same target.
1136 	 */
1137 	if (vcpu->arch.target != -1 && vcpu->arch.target != init->target)
1138 		return -EINVAL;
1139 
1140 	/* -ENOENT for unknown features, -EINVAL for invalid combinations. */
1141 	for (i = 0; i < sizeof(init->features) * 8; i++) {
1142 		bool set = (init->features[i / 32] & (1 << (i % 32)));
1143 
1144 		if (set && i >= KVM_VCPU_MAX_FEATURES)
1145 			return -ENOENT;
1146 
1147 		/*
1148 		 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
1149 		 * use the same feature set.
1150 		 */
1151 		if (vcpu->arch.target != -1 && i < KVM_VCPU_MAX_FEATURES &&
1152 		    test_bit(i, vcpu->arch.features) != set)
1153 			return -EINVAL;
1154 
1155 		if (set)
1156 			set_bit(i, vcpu->arch.features);
1157 	}
1158 
1159 	vcpu->arch.target = phys_target;
1160 
1161 	/* Now we know what it is, we can reset it. */
1162 	ret = kvm_reset_vcpu(vcpu);
1163 	if (ret) {
1164 		vcpu->arch.target = -1;
1165 		bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
1166 	}
1167 
1168 	return ret;
1169 }
1170 
kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu * vcpu,struct kvm_vcpu_init * init)1171 static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
1172 					 struct kvm_vcpu_init *init)
1173 {
1174 	int ret;
1175 
1176 	ret = kvm_vcpu_set_target(vcpu, init);
1177 	if (ret)
1178 		return ret;
1179 
1180 	/*
1181 	 * Ensure a rebooted VM will fault in RAM pages and detect if the
1182 	 * guest MMU is turned off and flush the caches as needed.
1183 	 *
1184 	 * S2FWB enforces all memory accesses to RAM being cacheable,
1185 	 * ensuring that the data side is always coherent. We still
1186 	 * need to invalidate the I-cache though, as FWB does *not*
1187 	 * imply CTR_EL0.DIC.
1188 	 */
1189 	if (vcpu_has_run_once(vcpu)) {
1190 		if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
1191 			stage2_unmap_vm(vcpu->kvm);
1192 		else
1193 			icache_inval_all_pou();
1194 	}
1195 
1196 	vcpu_reset_hcr(vcpu);
1197 	vcpu->arch.cptr_el2 = CPTR_EL2_DEFAULT;
1198 
1199 	/*
1200 	 * Handle the "start in power-off" case.
1201 	 */
1202 	if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features))
1203 		kvm_arm_vcpu_power_off(vcpu);
1204 	else
1205 		vcpu->arch.mp_state.mp_state = KVM_MP_STATE_RUNNABLE;
1206 
1207 	return 0;
1208 }
1209 
kvm_arm_vcpu_set_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1210 static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
1211 				 struct kvm_device_attr *attr)
1212 {
1213 	int ret = -ENXIO;
1214 
1215 	switch (attr->group) {
1216 	default:
1217 		ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr);
1218 		break;
1219 	}
1220 
1221 	return ret;
1222 }
1223 
kvm_arm_vcpu_get_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1224 static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
1225 				 struct kvm_device_attr *attr)
1226 {
1227 	int ret = -ENXIO;
1228 
1229 	switch (attr->group) {
1230 	default:
1231 		ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr);
1232 		break;
1233 	}
1234 
1235 	return ret;
1236 }
1237 
kvm_arm_vcpu_has_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1238 static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
1239 				 struct kvm_device_attr *attr)
1240 {
1241 	int ret = -ENXIO;
1242 
1243 	switch (attr->group) {
1244 	default:
1245 		ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr);
1246 		break;
1247 	}
1248 
1249 	return ret;
1250 }
1251 
kvm_arm_vcpu_get_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)1252 static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
1253 				   struct kvm_vcpu_events *events)
1254 {
1255 	memset(events, 0, sizeof(*events));
1256 
1257 	return __kvm_arm_vcpu_get_events(vcpu, events);
1258 }
1259 
kvm_arm_vcpu_set_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)1260 static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
1261 				   struct kvm_vcpu_events *events)
1262 {
1263 	int i;
1264 
1265 	/* check whether the reserved field is zero */
1266 	for (i = 0; i < ARRAY_SIZE(events->reserved); i++)
1267 		if (events->reserved[i])
1268 			return -EINVAL;
1269 
1270 	/* check whether the pad field is zero */
1271 	for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++)
1272 		if (events->exception.pad[i])
1273 			return -EINVAL;
1274 
1275 	return __kvm_arm_vcpu_set_events(vcpu, events);
1276 }
1277 
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)1278 long kvm_arch_vcpu_ioctl(struct file *filp,
1279 			 unsigned int ioctl, unsigned long arg)
1280 {
1281 	struct kvm_vcpu *vcpu = filp->private_data;
1282 	void __user *argp = (void __user *)arg;
1283 	struct kvm_device_attr attr;
1284 	long r;
1285 
1286 	switch (ioctl) {
1287 	case KVM_ARM_VCPU_INIT: {
1288 		struct kvm_vcpu_init init;
1289 
1290 		r = -EFAULT;
1291 		if (copy_from_user(&init, argp, sizeof(init)))
1292 			break;
1293 
1294 		r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
1295 		break;
1296 	}
1297 	case KVM_SET_ONE_REG:
1298 	case KVM_GET_ONE_REG: {
1299 		struct kvm_one_reg reg;
1300 
1301 		r = -ENOEXEC;
1302 		if (unlikely(!kvm_vcpu_initialized(vcpu)))
1303 			break;
1304 
1305 		r = -EFAULT;
1306 		if (copy_from_user(&reg, argp, sizeof(reg)))
1307 			break;
1308 
1309 		/*
1310 		 * We could owe a reset due to PSCI. Handle the pending reset
1311 		 * here to ensure userspace register accesses are ordered after
1312 		 * the reset.
1313 		 */
1314 		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1315 			kvm_reset_vcpu(vcpu);
1316 
1317 		if (ioctl == KVM_SET_ONE_REG)
1318 			r = kvm_arm_set_reg(vcpu, &reg);
1319 		else
1320 			r = kvm_arm_get_reg(vcpu, &reg);
1321 		break;
1322 	}
1323 	case KVM_GET_REG_LIST: {
1324 		struct kvm_reg_list __user *user_list = argp;
1325 		struct kvm_reg_list reg_list;
1326 		unsigned n;
1327 
1328 		r = -ENOEXEC;
1329 		if (unlikely(!kvm_vcpu_initialized(vcpu)))
1330 			break;
1331 
1332 		r = -EPERM;
1333 		if (!kvm_arm_vcpu_is_finalized(vcpu))
1334 			break;
1335 
1336 		r = -EFAULT;
1337 		if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
1338 			break;
1339 		n = reg_list.n;
1340 		reg_list.n = kvm_arm_num_regs(vcpu);
1341 		if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
1342 			break;
1343 		r = -E2BIG;
1344 		if (n < reg_list.n)
1345 			break;
1346 		r = kvm_arm_copy_reg_indices(vcpu, user_list->reg);
1347 		break;
1348 	}
1349 	case KVM_SET_DEVICE_ATTR: {
1350 		r = -EFAULT;
1351 		if (copy_from_user(&attr, argp, sizeof(attr)))
1352 			break;
1353 		r = kvm_arm_vcpu_set_attr(vcpu, &attr);
1354 		break;
1355 	}
1356 	case KVM_GET_DEVICE_ATTR: {
1357 		r = -EFAULT;
1358 		if (copy_from_user(&attr, argp, sizeof(attr)))
1359 			break;
1360 		r = kvm_arm_vcpu_get_attr(vcpu, &attr);
1361 		break;
1362 	}
1363 	case KVM_HAS_DEVICE_ATTR: {
1364 		r = -EFAULT;
1365 		if (copy_from_user(&attr, argp, sizeof(attr)))
1366 			break;
1367 		r = kvm_arm_vcpu_has_attr(vcpu, &attr);
1368 		break;
1369 	}
1370 	case KVM_GET_VCPU_EVENTS: {
1371 		struct kvm_vcpu_events events;
1372 
1373 		if (kvm_arm_vcpu_get_events(vcpu, &events))
1374 			return -EINVAL;
1375 
1376 		if (copy_to_user(argp, &events, sizeof(events)))
1377 			return -EFAULT;
1378 
1379 		return 0;
1380 	}
1381 	case KVM_SET_VCPU_EVENTS: {
1382 		struct kvm_vcpu_events events;
1383 
1384 		if (copy_from_user(&events, argp, sizeof(events)))
1385 			return -EFAULT;
1386 
1387 		return kvm_arm_vcpu_set_events(vcpu, &events);
1388 	}
1389 	case KVM_ARM_VCPU_FINALIZE: {
1390 		int what;
1391 
1392 		if (!kvm_vcpu_initialized(vcpu))
1393 			return -ENOEXEC;
1394 
1395 		if (get_user(what, (const int __user *)argp))
1396 			return -EFAULT;
1397 
1398 		return kvm_arm_vcpu_finalize(vcpu, what);
1399 	}
1400 	default:
1401 		r = -EINVAL;
1402 	}
1403 
1404 	return r;
1405 }
1406 
kvm_arch_sync_dirty_log(struct kvm * kvm,struct kvm_memory_slot * memslot)1407 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
1408 {
1409 
1410 }
1411 
kvm_arch_flush_remote_tlbs_memslot(struct kvm * kvm,const struct kvm_memory_slot * memslot)1412 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
1413 					const struct kvm_memory_slot *memslot)
1414 {
1415 	kvm_flush_remote_tlbs(kvm);
1416 }
1417 
kvm_vm_ioctl_set_device_addr(struct kvm * kvm,struct kvm_arm_device_addr * dev_addr)1418 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
1419 					struct kvm_arm_device_addr *dev_addr)
1420 {
1421 	switch (FIELD_GET(KVM_ARM_DEVICE_ID_MASK, dev_addr->id)) {
1422 	case KVM_ARM_DEVICE_VGIC_V2:
1423 		if (!vgic_present)
1424 			return -ENXIO;
1425 		return kvm_set_legacy_vgic_v2_addr(kvm, dev_addr);
1426 	default:
1427 		return -ENODEV;
1428 	}
1429 }
1430 
kvm_arch_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)1431 long kvm_arch_vm_ioctl(struct file *filp,
1432 		       unsigned int ioctl, unsigned long arg)
1433 {
1434 	struct kvm *kvm = filp->private_data;
1435 	void __user *argp = (void __user *)arg;
1436 
1437 	switch (ioctl) {
1438 	case KVM_CREATE_IRQCHIP: {
1439 		int ret;
1440 		if (!vgic_present)
1441 			return -ENXIO;
1442 		mutex_lock(&kvm->lock);
1443 		ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
1444 		mutex_unlock(&kvm->lock);
1445 		return ret;
1446 	}
1447 	case KVM_ARM_SET_DEVICE_ADDR: {
1448 		struct kvm_arm_device_addr dev_addr;
1449 
1450 		if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
1451 			return -EFAULT;
1452 		return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
1453 	}
1454 	case KVM_ARM_PREFERRED_TARGET: {
1455 		struct kvm_vcpu_init init;
1456 
1457 		kvm_vcpu_preferred_target(&init);
1458 
1459 		if (copy_to_user(argp, &init, sizeof(init)))
1460 			return -EFAULT;
1461 
1462 		return 0;
1463 	}
1464 	case KVM_ARM_MTE_COPY_TAGS: {
1465 		struct kvm_arm_copy_mte_tags copy_tags;
1466 
1467 		if (copy_from_user(&copy_tags, argp, sizeof(copy_tags)))
1468 			return -EFAULT;
1469 		return kvm_vm_ioctl_mte_copy_tags(kvm, &copy_tags);
1470 	}
1471 	default:
1472 		return -EINVAL;
1473 	}
1474 }
1475 
nvhe_percpu_size(void)1476 static unsigned long nvhe_percpu_size(void)
1477 {
1478 	return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) -
1479 		(unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start);
1480 }
1481 
nvhe_percpu_order(void)1482 static unsigned long nvhe_percpu_order(void)
1483 {
1484 	unsigned long size = nvhe_percpu_size();
1485 
1486 	return size ? get_order(size) : 0;
1487 }
1488 
1489 /* A lookup table holding the hypervisor VA for each vector slot */
1490 static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS];
1491 
kvm_init_vector_slot(void * base,enum arm64_hyp_spectre_vector slot)1492 static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot)
1493 {
1494 	hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot);
1495 }
1496 
kvm_init_vector_slots(void)1497 static int kvm_init_vector_slots(void)
1498 {
1499 	int err;
1500 	void *base;
1501 
1502 	base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
1503 	kvm_init_vector_slot(base, HYP_VECTOR_DIRECT);
1504 
1505 	base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
1506 	kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
1507 
1508 	if (kvm_system_needs_idmapped_vectors() &&
1509 	    !is_protected_kvm_enabled()) {
1510 		err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),
1511 					       __BP_HARDEN_HYP_VECS_SZ, &base);
1512 		if (err)
1513 			return err;
1514 	}
1515 
1516 	kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT);
1517 	kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT);
1518 	return 0;
1519 }
1520 
cpu_prepare_hyp_mode(int cpu)1521 static void cpu_prepare_hyp_mode(int cpu)
1522 {
1523 	struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
1524 	unsigned long tcr;
1525 
1526 	/*
1527 	 * Calculate the raw per-cpu offset without a translation from the
1528 	 * kernel's mapping to the linear mapping, and store it in tpidr_el2
1529 	 * so that we can use adr_l to access per-cpu variables in EL2.
1530 	 * Also drop the KASAN tag which gets in the way...
1531 	 */
1532 	params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) -
1533 			    (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
1534 
1535 	params->mair_el2 = read_sysreg(mair_el1);
1536 
1537 	/*
1538 	 * The ID map may be configured to use an extended virtual address
1539 	 * range. This is only the case if system RAM is out of range for the
1540 	 * currently configured page size and VA_BITS, in which case we will
1541 	 * also need the extended virtual range for the HYP ID map, or we won't
1542 	 * be able to enable the EL2 MMU.
1543 	 *
1544 	 * However, at EL2, there is only one TTBR register, and we can't switch
1545 	 * between translation tables *and* update TCR_EL2.T0SZ at the same
1546 	 * time. Bottom line: we need to use the extended range with *both* our
1547 	 * translation tables.
1548 	 *
1549 	 * So use the same T0SZ value we use for the ID map.
1550 	 */
1551 	tcr = (read_sysreg(tcr_el1) & TCR_EL2_MASK) | TCR_EL2_RES1;
1552 	tcr &= ~TCR_T0SZ_MASK;
1553 	tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET;
1554 	params->tcr_el2 = tcr;
1555 
1556 	params->pgd_pa = kvm_mmu_get_httbr();
1557 	if (is_protected_kvm_enabled())
1558 		params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
1559 	else
1560 		params->hcr_el2 = HCR_HOST_NVHE_FLAGS;
1561 	params->vttbr = params->vtcr = 0;
1562 
1563 	/*
1564 	 * Flush the init params from the data cache because the struct will
1565 	 * be read while the MMU is off.
1566 	 */
1567 	kvm_flush_dcache_to_poc(params, sizeof(*params));
1568 }
1569 
hyp_install_host_vector(void)1570 static void hyp_install_host_vector(void)
1571 {
1572 	struct kvm_nvhe_init_params *params;
1573 	struct arm_smccc_res res;
1574 
1575 	/* Switch from the HYP stub to our own HYP init vector */
1576 	__hyp_set_vectors(kvm_get_idmap_vector());
1577 
1578 	/*
1579 	 * Call initialization code, and switch to the full blown HYP code.
1580 	 * If the cpucaps haven't been finalized yet, something has gone very
1581 	 * wrong, and hyp will crash and burn when it uses any
1582 	 * cpus_have_const_cap() wrapper.
1583 	 */
1584 	BUG_ON(!system_capabilities_finalized());
1585 	params = this_cpu_ptr_nvhe_sym(kvm_init_params);
1586 	arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res);
1587 	WARN_ON(res.a0 != SMCCC_RET_SUCCESS);
1588 }
1589 
cpu_init_hyp_mode(void)1590 static void cpu_init_hyp_mode(void)
1591 {
1592 	hyp_install_host_vector();
1593 
1594 	/*
1595 	 * Disabling SSBD on a non-VHE system requires us to enable SSBS
1596 	 * at EL2.
1597 	 */
1598 	if (this_cpu_has_cap(ARM64_SSBS) &&
1599 	    arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) {
1600 		kvm_call_hyp_nvhe(__kvm_enable_ssbs);
1601 	}
1602 }
1603 
cpu_hyp_reset(void)1604 static void cpu_hyp_reset(void)
1605 {
1606 	if (!is_kernel_in_hyp_mode())
1607 		__hyp_reset_vectors();
1608 }
1609 
1610 /*
1611  * EL2 vectors can be mapped and rerouted in a number of ways,
1612  * depending on the kernel configuration and CPU present:
1613  *
1614  * - If the CPU is affected by Spectre-v2, the hardening sequence is
1615  *   placed in one of the vector slots, which is executed before jumping
1616  *   to the real vectors.
1617  *
1618  * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot
1619  *   containing the hardening sequence is mapped next to the idmap page,
1620  *   and executed before jumping to the real vectors.
1621  *
1622  * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an
1623  *   empty slot is selected, mapped next to the idmap page, and
1624  *   executed before jumping to the real vectors.
1625  *
1626  * Note that ARM64_SPECTRE_V3A is somewhat incompatible with
1627  * VHE, as we don't have hypervisor-specific mappings. If the system
1628  * is VHE and yet selects this capability, it will be ignored.
1629  */
cpu_set_hyp_vector(void)1630 static void cpu_set_hyp_vector(void)
1631 {
1632 	struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
1633 	void *vector = hyp_spectre_vector_selector[data->slot];
1634 
1635 	if (!is_protected_kvm_enabled())
1636 		*this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector;
1637 	else
1638 		kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot);
1639 }
1640 
cpu_hyp_init_context(void)1641 static void cpu_hyp_init_context(void)
1642 {
1643 	kvm_init_host_cpu_context(&this_cpu_ptr_hyp_sym(kvm_host_data)->host_ctxt);
1644 
1645 	if (!is_kernel_in_hyp_mode())
1646 		cpu_init_hyp_mode();
1647 }
1648 
cpu_hyp_init_features(void)1649 static void cpu_hyp_init_features(void)
1650 {
1651 	cpu_set_hyp_vector();
1652 	kvm_arm_init_debug();
1653 
1654 	if (is_kernel_in_hyp_mode())
1655 		kvm_timer_init_vhe();
1656 
1657 	if (vgic_present)
1658 		kvm_vgic_init_cpu_hardware();
1659 }
1660 
cpu_hyp_reinit(void)1661 static void cpu_hyp_reinit(void)
1662 {
1663 	cpu_hyp_reset();
1664 	cpu_hyp_init_context();
1665 	cpu_hyp_init_features();
1666 }
1667 
_kvm_arch_hardware_enable(void * discard)1668 static void _kvm_arch_hardware_enable(void *discard)
1669 {
1670 	if (!__this_cpu_read(kvm_arm_hardware_enabled)) {
1671 		cpu_hyp_reinit();
1672 		__this_cpu_write(kvm_arm_hardware_enabled, 1);
1673 	}
1674 }
1675 
kvm_arch_hardware_enable(void)1676 int kvm_arch_hardware_enable(void)
1677 {
1678 	_kvm_arch_hardware_enable(NULL);
1679 	return 0;
1680 }
1681 
_kvm_arch_hardware_disable(void * discard)1682 static void _kvm_arch_hardware_disable(void *discard)
1683 {
1684 	if (__this_cpu_read(kvm_arm_hardware_enabled)) {
1685 		cpu_hyp_reset();
1686 		__this_cpu_write(kvm_arm_hardware_enabled, 0);
1687 	}
1688 }
1689 
kvm_arch_hardware_disable(void)1690 void kvm_arch_hardware_disable(void)
1691 {
1692 	if (!is_protected_kvm_enabled())
1693 		_kvm_arch_hardware_disable(NULL);
1694 }
1695 
1696 #ifdef CONFIG_CPU_PM
hyp_init_cpu_pm_notifier(struct notifier_block * self,unsigned long cmd,void * v)1697 static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
1698 				    unsigned long cmd,
1699 				    void *v)
1700 {
1701 	/*
1702 	 * kvm_arm_hardware_enabled is left with its old value over
1703 	 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should
1704 	 * re-enable hyp.
1705 	 */
1706 	switch (cmd) {
1707 	case CPU_PM_ENTER:
1708 		if (__this_cpu_read(kvm_arm_hardware_enabled))
1709 			/*
1710 			 * don't update kvm_arm_hardware_enabled here
1711 			 * so that the hardware will be re-enabled
1712 			 * when we resume. See below.
1713 			 */
1714 			cpu_hyp_reset();
1715 
1716 		return NOTIFY_OK;
1717 	case CPU_PM_ENTER_FAILED:
1718 	case CPU_PM_EXIT:
1719 		if (__this_cpu_read(kvm_arm_hardware_enabled))
1720 			/* The hardware was enabled before suspend. */
1721 			cpu_hyp_reinit();
1722 
1723 		return NOTIFY_OK;
1724 
1725 	default:
1726 		return NOTIFY_DONE;
1727 	}
1728 }
1729 
1730 static struct notifier_block hyp_init_cpu_pm_nb = {
1731 	.notifier_call = hyp_init_cpu_pm_notifier,
1732 };
1733 
hyp_cpu_pm_init(void)1734 static void hyp_cpu_pm_init(void)
1735 {
1736 	if (!is_protected_kvm_enabled())
1737 		cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
1738 }
hyp_cpu_pm_exit(void)1739 static void hyp_cpu_pm_exit(void)
1740 {
1741 	if (!is_protected_kvm_enabled())
1742 		cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb);
1743 }
1744 #else
hyp_cpu_pm_init(void)1745 static inline void hyp_cpu_pm_init(void)
1746 {
1747 }
hyp_cpu_pm_exit(void)1748 static inline void hyp_cpu_pm_exit(void)
1749 {
1750 }
1751 #endif
1752 
init_cpu_logical_map(void)1753 static void init_cpu_logical_map(void)
1754 {
1755 	unsigned int cpu;
1756 
1757 	/*
1758 	 * Copy the MPIDR <-> logical CPU ID mapping to hyp.
1759 	 * Only copy the set of online CPUs whose features have been checked
1760 	 * against the finalized system capabilities. The hypervisor will not
1761 	 * allow any other CPUs from the `possible` set to boot.
1762 	 */
1763 	for_each_online_cpu(cpu)
1764 		hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu);
1765 }
1766 
1767 #define init_psci_0_1_impl_state(config, what)	\
1768 	config.psci_0_1_ ## what ## _implemented = psci_ops.what
1769 
init_psci_relay(void)1770 static bool init_psci_relay(void)
1771 {
1772 	/*
1773 	 * If PSCI has not been initialized, protected KVM cannot install
1774 	 * itself on newly booted CPUs.
1775 	 */
1776 	if (!psci_ops.get_version) {
1777 		kvm_err("Cannot initialize protected mode without PSCI\n");
1778 		return false;
1779 	}
1780 
1781 	kvm_host_psci_config.version = psci_ops.get_version();
1782 
1783 	if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) {
1784 		kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids();
1785 		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend);
1786 		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on);
1787 		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off);
1788 		init_psci_0_1_impl_state(kvm_host_psci_config, migrate);
1789 	}
1790 	return true;
1791 }
1792 
init_subsystems(void)1793 static int init_subsystems(void)
1794 {
1795 	int err = 0;
1796 
1797 	/*
1798 	 * Enable hardware so that subsystem initialisation can access EL2.
1799 	 */
1800 	on_each_cpu(_kvm_arch_hardware_enable, NULL, 1);
1801 
1802 	/*
1803 	 * Register CPU lower-power notifier
1804 	 */
1805 	hyp_cpu_pm_init();
1806 
1807 	/*
1808 	 * Init HYP view of VGIC
1809 	 */
1810 	err = kvm_vgic_hyp_init();
1811 	switch (err) {
1812 	case 0:
1813 		vgic_present = true;
1814 		break;
1815 	case -ENODEV:
1816 	case -ENXIO:
1817 		vgic_present = false;
1818 		err = 0;
1819 		break;
1820 	default:
1821 		goto out;
1822 	}
1823 
1824 	/*
1825 	 * Init HYP architected timer support
1826 	 */
1827 	err = kvm_timer_hyp_init(vgic_present);
1828 	if (err)
1829 		goto out;
1830 
1831 	kvm_register_perf_callbacks(NULL);
1832 
1833 out:
1834 	if (err || !is_protected_kvm_enabled())
1835 		on_each_cpu(_kvm_arch_hardware_disable, NULL, 1);
1836 
1837 	return err;
1838 }
1839 
teardown_hyp_mode(void)1840 static void teardown_hyp_mode(void)
1841 {
1842 	int cpu;
1843 
1844 	free_hyp_pgds();
1845 	for_each_possible_cpu(cpu) {
1846 		free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
1847 		free_pages(kvm_arm_hyp_percpu_base[cpu], nvhe_percpu_order());
1848 	}
1849 }
1850 
do_pkvm_init(u32 hyp_va_bits)1851 static int do_pkvm_init(u32 hyp_va_bits)
1852 {
1853 	void *per_cpu_base = kvm_ksym_ref(kvm_arm_hyp_percpu_base);
1854 	int ret;
1855 
1856 	preempt_disable();
1857 	cpu_hyp_init_context();
1858 	ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size,
1859 				num_possible_cpus(), kern_hyp_va(per_cpu_base),
1860 				hyp_va_bits);
1861 	cpu_hyp_init_features();
1862 
1863 	/*
1864 	 * The stub hypercalls are now disabled, so set our local flag to
1865 	 * prevent a later re-init attempt in kvm_arch_hardware_enable().
1866 	 */
1867 	__this_cpu_write(kvm_arm_hardware_enabled, 1);
1868 	preempt_enable();
1869 
1870 	return ret;
1871 }
1872 
kvm_hyp_init_protection(u32 hyp_va_bits)1873 static int kvm_hyp_init_protection(u32 hyp_va_bits)
1874 {
1875 	void *addr = phys_to_virt(hyp_mem_base);
1876 	int ret;
1877 
1878 	kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1879 	kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
1880 	kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1);
1881 	kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
1882 	kvm_nvhe_sym(id_aa64isar2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
1883 	kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
1884 	kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
1885 	kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1);
1886 
1887 	ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
1888 	if (ret)
1889 		return ret;
1890 
1891 	ret = do_pkvm_init(hyp_va_bits);
1892 	if (ret)
1893 		return ret;
1894 
1895 	free_hyp_pgds();
1896 
1897 	return 0;
1898 }
1899 
1900 /**
1901  * Inits Hyp-mode on all online CPUs
1902  */
init_hyp_mode(void)1903 static int init_hyp_mode(void)
1904 {
1905 	u32 hyp_va_bits;
1906 	int cpu;
1907 	int err = -ENOMEM;
1908 
1909 	/*
1910 	 * The protected Hyp-mode cannot be initialized if the memory pool
1911 	 * allocation has failed.
1912 	 */
1913 	if (is_protected_kvm_enabled() && !hyp_mem_base)
1914 		goto out_err;
1915 
1916 	/*
1917 	 * Allocate Hyp PGD and setup Hyp identity mapping
1918 	 */
1919 	err = kvm_mmu_init(&hyp_va_bits);
1920 	if (err)
1921 		goto out_err;
1922 
1923 	/*
1924 	 * Allocate stack pages for Hypervisor-mode
1925 	 */
1926 	for_each_possible_cpu(cpu) {
1927 		unsigned long stack_page;
1928 
1929 		stack_page = __get_free_page(GFP_KERNEL);
1930 		if (!stack_page) {
1931 			err = -ENOMEM;
1932 			goto out_err;
1933 		}
1934 
1935 		per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page;
1936 	}
1937 
1938 	/*
1939 	 * Allocate and initialize pages for Hypervisor-mode percpu regions.
1940 	 */
1941 	for_each_possible_cpu(cpu) {
1942 		struct page *page;
1943 		void *page_addr;
1944 
1945 		page = alloc_pages(GFP_KERNEL, nvhe_percpu_order());
1946 		if (!page) {
1947 			err = -ENOMEM;
1948 			goto out_err;
1949 		}
1950 
1951 		page_addr = page_address(page);
1952 		memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size());
1953 		kvm_arm_hyp_percpu_base[cpu] = (unsigned long)page_addr;
1954 	}
1955 
1956 	/*
1957 	 * Map the Hyp-code called directly from the host
1958 	 */
1959 	err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
1960 				  kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
1961 	if (err) {
1962 		kvm_err("Cannot map world-switch code\n");
1963 		goto out_err;
1964 	}
1965 
1966 	err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
1967 				  kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
1968 	if (err) {
1969 		kvm_err("Cannot map .hyp.rodata section\n");
1970 		goto out_err;
1971 	}
1972 
1973 	err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
1974 				  kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
1975 	if (err) {
1976 		kvm_err("Cannot map rodata section\n");
1977 		goto out_err;
1978 	}
1979 
1980 	/*
1981 	 * .hyp.bss is guaranteed to be placed at the beginning of the .bss
1982 	 * section thanks to an assertion in the linker script. Map it RW and
1983 	 * the rest of .bss RO.
1984 	 */
1985 	err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start),
1986 				  kvm_ksym_ref(__hyp_bss_end), PAGE_HYP);
1987 	if (err) {
1988 		kvm_err("Cannot map hyp bss section: %d\n", err);
1989 		goto out_err;
1990 	}
1991 
1992 	err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end),
1993 				  kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
1994 	if (err) {
1995 		kvm_err("Cannot map bss section\n");
1996 		goto out_err;
1997 	}
1998 
1999 	/*
2000 	 * Map the Hyp stack pages
2001 	 */
2002 	for_each_possible_cpu(cpu) {
2003 		struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
2004 		char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
2005 		unsigned long hyp_addr;
2006 
2007 		/*
2008 		 * Allocate a contiguous HYP private VA range for the stack
2009 		 * and guard page. The allocation is also aligned based on
2010 		 * the order of its size.
2011 		 */
2012 		err = hyp_alloc_private_va_range(PAGE_SIZE * 2, &hyp_addr);
2013 		if (err) {
2014 			kvm_err("Cannot allocate hyp stack guard page\n");
2015 			goto out_err;
2016 		}
2017 
2018 		/*
2019 		 * Since the stack grows downwards, map the stack to the page
2020 		 * at the higher address and leave the lower guard page
2021 		 * unbacked.
2022 		 *
2023 		 * Any valid stack address now has the PAGE_SHIFT bit as 1
2024 		 * and addresses corresponding to the guard page have the
2025 		 * PAGE_SHIFT bit as 0 - this is used for overflow detection.
2026 		 */
2027 		err = __create_hyp_mappings(hyp_addr + PAGE_SIZE, PAGE_SIZE,
2028 					    __pa(stack_page), PAGE_HYP);
2029 		if (err) {
2030 			kvm_err("Cannot map hyp stack\n");
2031 			goto out_err;
2032 		}
2033 
2034 		/*
2035 		 * Save the stack PA in nvhe_init_params. This will be needed
2036 		 * to recreate the stack mapping in protected nVHE mode.
2037 		 * __hyp_pa() won't do the right thing there, since the stack
2038 		 * has been mapped in the flexible private VA space.
2039 		 */
2040 		params->stack_pa = __pa(stack_page);
2041 
2042 		params->stack_hyp_va = hyp_addr + (2 * PAGE_SIZE);
2043 	}
2044 
2045 	for_each_possible_cpu(cpu) {
2046 		char *percpu_begin = (char *)kvm_arm_hyp_percpu_base[cpu];
2047 		char *percpu_end = percpu_begin + nvhe_percpu_size();
2048 
2049 		/* Map Hyp percpu pages */
2050 		err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP);
2051 		if (err) {
2052 			kvm_err("Cannot map hyp percpu region\n");
2053 			goto out_err;
2054 		}
2055 
2056 		/* Prepare the CPU initialization parameters */
2057 		cpu_prepare_hyp_mode(cpu);
2058 	}
2059 
2060 	if (is_protected_kvm_enabled()) {
2061 		init_cpu_logical_map();
2062 
2063 		if (!init_psci_relay()) {
2064 			err = -ENODEV;
2065 			goto out_err;
2066 		}
2067 	}
2068 
2069 	if (is_protected_kvm_enabled()) {
2070 		err = kvm_hyp_init_protection(hyp_va_bits);
2071 		if (err) {
2072 			kvm_err("Failed to init hyp memory protection\n");
2073 			goto out_err;
2074 		}
2075 	}
2076 
2077 	return 0;
2078 
2079 out_err:
2080 	teardown_hyp_mode();
2081 	kvm_err("error initializing Hyp mode: %d\n", err);
2082 	return err;
2083 }
2084 
_kvm_host_prot_finalize(void * arg)2085 static void _kvm_host_prot_finalize(void *arg)
2086 {
2087 	int *err = arg;
2088 
2089 	if (WARN_ON(kvm_call_hyp_nvhe(__pkvm_prot_finalize)))
2090 		WRITE_ONCE(*err, -EINVAL);
2091 }
2092 
pkvm_drop_host_privileges(void)2093 static int pkvm_drop_host_privileges(void)
2094 {
2095 	int ret = 0;
2096 
2097 	/*
2098 	 * Flip the static key upfront as that may no longer be possible
2099 	 * once the host stage 2 is installed.
2100 	 */
2101 	static_branch_enable(&kvm_protected_mode_initialized);
2102 	on_each_cpu(_kvm_host_prot_finalize, &ret, 1);
2103 	return ret;
2104 }
2105 
finalize_hyp_mode(void)2106 static int finalize_hyp_mode(void)
2107 {
2108 	if (!is_protected_kvm_enabled())
2109 		return 0;
2110 
2111 	/*
2112 	 * Exclude HYP sections from kmemleak so that they don't get peeked
2113 	 * at, which would end badly once inaccessible.
2114 	 */
2115 	kmemleak_free_part(__hyp_bss_start, __hyp_bss_end - __hyp_bss_start);
2116 	kmemleak_free_part_phys(hyp_mem_base, hyp_mem_size);
2117 	return pkvm_drop_host_privileges();
2118 }
2119 
kvm_mpidr_to_vcpu(struct kvm * kvm,unsigned long mpidr)2120 struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
2121 {
2122 	struct kvm_vcpu *vcpu;
2123 	unsigned long i;
2124 
2125 	mpidr &= MPIDR_HWID_BITMASK;
2126 	kvm_for_each_vcpu(i, vcpu, kvm) {
2127 		if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
2128 			return vcpu;
2129 	}
2130 	return NULL;
2131 }
2132 
kvm_arch_has_irq_bypass(void)2133 bool kvm_arch_has_irq_bypass(void)
2134 {
2135 	return true;
2136 }
2137 
kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)2138 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
2139 				      struct irq_bypass_producer *prod)
2140 {
2141 	struct kvm_kernel_irqfd *irqfd =
2142 		container_of(cons, struct kvm_kernel_irqfd, consumer);
2143 
2144 	return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq,
2145 					  &irqfd->irq_entry);
2146 }
kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)2147 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
2148 				      struct irq_bypass_producer *prod)
2149 {
2150 	struct kvm_kernel_irqfd *irqfd =
2151 		container_of(cons, struct kvm_kernel_irqfd, consumer);
2152 
2153 	kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq,
2154 				     &irqfd->irq_entry);
2155 }
2156 
kvm_arch_irq_bypass_stop(struct irq_bypass_consumer * cons)2157 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons)
2158 {
2159 	struct kvm_kernel_irqfd *irqfd =
2160 		container_of(cons, struct kvm_kernel_irqfd, consumer);
2161 
2162 	kvm_arm_halt_guest(irqfd->kvm);
2163 }
2164 
kvm_arch_irq_bypass_start(struct irq_bypass_consumer * cons)2165 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons)
2166 {
2167 	struct kvm_kernel_irqfd *irqfd =
2168 		container_of(cons, struct kvm_kernel_irqfd, consumer);
2169 
2170 	kvm_arm_resume_guest(irqfd->kvm);
2171 }
2172 
2173 /**
2174  * Initialize Hyp-mode and memory mappings on all CPUs.
2175  */
kvm_arch_init(void * opaque)2176 int kvm_arch_init(void *opaque)
2177 {
2178 	int err;
2179 	bool in_hyp_mode;
2180 
2181 	if (!is_hyp_mode_available()) {
2182 		kvm_info("HYP mode not available\n");
2183 		return -ENODEV;
2184 	}
2185 
2186 	if (kvm_get_mode() == KVM_MODE_NONE) {
2187 		kvm_info("KVM disabled from command line\n");
2188 		return -ENODEV;
2189 	}
2190 
2191 	err = kvm_sys_reg_table_init();
2192 	if (err) {
2193 		kvm_info("Error initializing system register tables");
2194 		return err;
2195 	}
2196 
2197 	in_hyp_mode = is_kernel_in_hyp_mode();
2198 
2199 	if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) ||
2200 	    cpus_have_final_cap(ARM64_WORKAROUND_1508412))
2201 		kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \
2202 			 "Only trusted guests should be used on this system.\n");
2203 
2204 	err = kvm_set_ipa_limit();
2205 	if (err)
2206 		return err;
2207 
2208 	err = kvm_arm_init_sve();
2209 	if (err)
2210 		return err;
2211 
2212 	err = kvm_arm_vmid_alloc_init();
2213 	if (err) {
2214 		kvm_err("Failed to initialize VMID allocator.\n");
2215 		return err;
2216 	}
2217 
2218 	if (!in_hyp_mode) {
2219 		err = init_hyp_mode();
2220 		if (err)
2221 			goto out_err;
2222 	}
2223 
2224 	err = kvm_init_vector_slots();
2225 	if (err) {
2226 		kvm_err("Cannot initialise vector slots\n");
2227 		goto out_err;
2228 	}
2229 
2230 	err = init_subsystems();
2231 	if (err)
2232 		goto out_hyp;
2233 
2234 	if (!in_hyp_mode) {
2235 		err = finalize_hyp_mode();
2236 		if (err) {
2237 			kvm_err("Failed to finalize Hyp protection\n");
2238 			goto out_hyp;
2239 		}
2240 	}
2241 
2242 	if (is_protected_kvm_enabled()) {
2243 		kvm_info("Protected nVHE mode initialized successfully\n");
2244 	} else if (in_hyp_mode) {
2245 		kvm_info("VHE mode initialized successfully\n");
2246 	} else {
2247 		kvm_info("Hyp mode initialized successfully\n");
2248 	}
2249 
2250 	return 0;
2251 
2252 out_hyp:
2253 	hyp_cpu_pm_exit();
2254 	if (!in_hyp_mode)
2255 		teardown_hyp_mode();
2256 out_err:
2257 	kvm_arm_vmid_alloc_free();
2258 	return err;
2259 }
2260 
2261 /* NOP: Compiling as a module not supported */
kvm_arch_exit(void)2262 void kvm_arch_exit(void)
2263 {
2264 	kvm_unregister_perf_callbacks();
2265 }
2266 
early_kvm_mode_cfg(char * arg)2267 static int __init early_kvm_mode_cfg(char *arg)
2268 {
2269 	if (!arg)
2270 		return -EINVAL;
2271 
2272 	if (strcmp(arg, "none") == 0) {
2273 		kvm_mode = KVM_MODE_NONE;
2274 		return 0;
2275 	}
2276 
2277 	if (!is_hyp_mode_available()) {
2278 		pr_warn_once("KVM is not available. Ignoring kvm-arm.mode\n");
2279 		return 0;
2280 	}
2281 
2282 	if (strcmp(arg, "protected") == 0) {
2283 		if (!is_kernel_in_hyp_mode())
2284 			kvm_mode = KVM_MODE_PROTECTED;
2285 		else
2286 			pr_warn_once("Protected KVM not available with VHE\n");
2287 
2288 		return 0;
2289 	}
2290 
2291 	if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) {
2292 		kvm_mode = KVM_MODE_DEFAULT;
2293 		return 0;
2294 	}
2295 
2296 	return -EINVAL;
2297 }
2298 early_param("kvm-arm.mode", early_kvm_mode_cfg);
2299 
kvm_get_mode(void)2300 enum kvm_mode kvm_get_mode(void)
2301 {
2302 	return kvm_mode;
2303 }
2304 
arm_init(void)2305 static int arm_init(void)
2306 {
2307 	int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2308 	return rc;
2309 }
2310 
2311 module_init(arm_init);
2312