1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012-2015 - ARM Ltd
4  * Author: Marc Zyngier <marc.zyngier@arm.com>
5  */
6 
7 #include <hyp/sysreg-sr.h>
8 
9 #include <linux/compiler.h>
10 #include <linux/kvm_host.h>
11 
12 #include <asm/kprobes.h>
13 #include <asm/kvm_asm.h>
14 #include <asm/kvm_emulate.h>
15 #include <asm/kvm_hyp.h>
16 
17 /*
18  * VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and
19  * pstate, which are handled as part of the el2 return state) on every
20  * switch (sp_el0 is being dealt with in the assembly code).
21  * tpidr_el0 and tpidrro_el0 only need to be switched when going
22  * to host userspace or a different VCPU.  EL1 registers only need to be
23  * switched when potentially going to run a different VCPU.  The latter two
24  * classes are handled as part of kvm_arch_vcpu_load and kvm_arch_vcpu_put.
25  */
26 
sysreg_save_host_state_vhe(struct kvm_cpu_context * ctxt)27 void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt)
28 {
29 	__sysreg_save_common_state(ctxt);
30 }
31 NOKPROBE_SYMBOL(sysreg_save_host_state_vhe);
32 
sysreg_save_guest_state_vhe(struct kvm_cpu_context * ctxt)33 void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt)
34 {
35 	__sysreg_save_common_state(ctxt);
36 	__sysreg_save_el2_return_state(ctxt);
37 }
38 NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe);
39 
sysreg_restore_host_state_vhe(struct kvm_cpu_context * ctxt)40 void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt)
41 {
42 	__sysreg_restore_common_state(ctxt);
43 }
44 NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe);
45 
sysreg_restore_guest_state_vhe(struct kvm_cpu_context * ctxt)46 void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt)
47 {
48 	__sysreg_restore_common_state(ctxt);
49 	__sysreg_restore_el2_return_state(ctxt);
50 }
51 NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe);
52 
53 /**
54  * kvm_vcpu_load_sysregs_vhe - Load guest system registers to the physical CPU
55  *
56  * @vcpu: The VCPU pointer
57  *
58  * Load system registers that do not affect the host's execution, for
59  * example EL1 system registers on a VHE system where the host kernel
60  * runs at EL2.  This function is called from KVM's vcpu_load() function
61  * and loading system register state early avoids having to load them on
62  * every entry to the VM.
63  */
kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu * vcpu)64 void kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu *vcpu)
65 {
66 	struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
67 	struct kvm_cpu_context *host_ctxt;
68 
69 	host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
70 	__sysreg_save_user_state(host_ctxt);
71 
72 	/*
73 	 * Load guest EL1 and user state
74 	 *
75 	 * We must restore the 32-bit state before the sysregs, thanks
76 	 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
77 	 */
78 	__sysreg32_restore_state(vcpu);
79 	__sysreg_restore_user_state(guest_ctxt);
80 	__sysreg_restore_el1_state(guest_ctxt);
81 
82 	vcpu->arch.sysregs_loaded_on_cpu = true;
83 
84 	activate_traps_vhe_load(vcpu);
85 }
86 
87 /**
88  * kvm_vcpu_put_sysregs_vhe - Restore host system registers to the physical CPU
89  *
90  * @vcpu: The VCPU pointer
91  *
92  * Save guest system registers that do not affect the host's execution, for
93  * example EL1 system registers on a VHE system where the host kernel
94  * runs at EL2.  This function is called from KVM's vcpu_put() function
95  * and deferring saving system register state until we're no longer running the
96  * VCPU avoids having to save them on every exit from the VM.
97  */
kvm_vcpu_put_sysregs_vhe(struct kvm_vcpu * vcpu)98 void kvm_vcpu_put_sysregs_vhe(struct kvm_vcpu *vcpu)
99 {
100 	struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
101 	struct kvm_cpu_context *host_ctxt;
102 
103 	host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
104 	deactivate_traps_vhe_put(vcpu);
105 
106 	__sysreg_save_el1_state(guest_ctxt);
107 	__sysreg_save_user_state(guest_ctxt);
108 	__sysreg32_save_state(vcpu);
109 
110 	/* Restore host user state */
111 	__sysreg_restore_user_state(host_ctxt);
112 
113 	vcpu->arch.sysregs_loaded_on_cpu = false;
114 }
115