1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/kernel.h>
3 #include <linux/kvm_host.h>
4 #include <asm/asm-prototypes.h>
5 #include <asm/dbell.h>
6 #include <asm/kvm_ppc.h>
7 #include <asm/pmc.h>
8 #include <asm/ppc-opcode.h>
9
10 #include "book3s_hv.h"
11
freeze_pmu(unsigned long mmcr0,unsigned long mmcra)12 static void freeze_pmu(unsigned long mmcr0, unsigned long mmcra)
13 {
14 if (!(mmcr0 & MMCR0_FC))
15 goto do_freeze;
16 if (mmcra & MMCRA_SAMPLE_ENABLE)
17 goto do_freeze;
18 if (cpu_has_feature(CPU_FTR_ARCH_31)) {
19 if (!(mmcr0 & MMCR0_PMCCEXT))
20 goto do_freeze;
21 if (!(mmcra & MMCRA_BHRB_DISABLE))
22 goto do_freeze;
23 }
24 return;
25
26 do_freeze:
27 mmcr0 = MMCR0_FC;
28 mmcra = 0;
29 if (cpu_has_feature(CPU_FTR_ARCH_31)) {
30 mmcr0 |= MMCR0_PMCCEXT;
31 mmcra = MMCRA_BHRB_DISABLE;
32 }
33
34 mtspr(SPRN_MMCR0, mmcr0);
35 mtspr(SPRN_MMCRA, mmcra);
36 isync();
37 }
38
switch_pmu_to_guest(struct kvm_vcpu * vcpu,struct p9_host_os_sprs * host_os_sprs)39 void switch_pmu_to_guest(struct kvm_vcpu *vcpu,
40 struct p9_host_os_sprs *host_os_sprs)
41 {
42 struct lppaca *lp;
43 int load_pmu = 1;
44
45 lp = vcpu->arch.vpa.pinned_addr;
46 if (lp)
47 load_pmu = lp->pmcregs_in_use;
48
49 /* Save host */
50 if (ppc_get_pmu_inuse()) {
51 /*
52 * It might be better to put PMU handling (at least for the
53 * host) in the perf subsystem because it knows more about what
54 * is being used.
55 */
56
57 /* POWER9, POWER10 do not implement HPMC or SPMC */
58
59 host_os_sprs->mmcr0 = mfspr(SPRN_MMCR0);
60 host_os_sprs->mmcra = mfspr(SPRN_MMCRA);
61
62 freeze_pmu(host_os_sprs->mmcr0, host_os_sprs->mmcra);
63
64 host_os_sprs->pmc1 = mfspr(SPRN_PMC1);
65 host_os_sprs->pmc2 = mfspr(SPRN_PMC2);
66 host_os_sprs->pmc3 = mfspr(SPRN_PMC3);
67 host_os_sprs->pmc4 = mfspr(SPRN_PMC4);
68 host_os_sprs->pmc5 = mfspr(SPRN_PMC5);
69 host_os_sprs->pmc6 = mfspr(SPRN_PMC6);
70 host_os_sprs->mmcr1 = mfspr(SPRN_MMCR1);
71 host_os_sprs->mmcr2 = mfspr(SPRN_MMCR2);
72 host_os_sprs->sdar = mfspr(SPRN_SDAR);
73 host_os_sprs->siar = mfspr(SPRN_SIAR);
74 host_os_sprs->sier1 = mfspr(SPRN_SIER);
75
76 if (cpu_has_feature(CPU_FTR_ARCH_31)) {
77 host_os_sprs->mmcr3 = mfspr(SPRN_MMCR3);
78 host_os_sprs->sier2 = mfspr(SPRN_SIER2);
79 host_os_sprs->sier3 = mfspr(SPRN_SIER3);
80 }
81 }
82
83 #ifdef CONFIG_PPC_PSERIES
84 /* After saving PMU, before loading guest PMU, flip pmcregs_in_use */
85 if (kvmhv_on_pseries()) {
86 barrier();
87 get_lppaca()->pmcregs_in_use = load_pmu;
88 barrier();
89 }
90 #endif
91
92 /*
93 * Load guest. If the VPA said the PMCs are not in use but the guest
94 * tried to access them anyway, HFSCR[PM] will be set by the HFAC
95 * fault so we can make forward progress.
96 */
97 if (load_pmu || (vcpu->arch.hfscr & HFSCR_PM)) {
98 mtspr(SPRN_PMC1, vcpu->arch.pmc[0]);
99 mtspr(SPRN_PMC2, vcpu->arch.pmc[1]);
100 mtspr(SPRN_PMC3, vcpu->arch.pmc[2]);
101 mtspr(SPRN_PMC4, vcpu->arch.pmc[3]);
102 mtspr(SPRN_PMC5, vcpu->arch.pmc[4]);
103 mtspr(SPRN_PMC6, vcpu->arch.pmc[5]);
104 mtspr(SPRN_MMCR1, vcpu->arch.mmcr[1]);
105 mtspr(SPRN_MMCR2, vcpu->arch.mmcr[2]);
106 mtspr(SPRN_SDAR, vcpu->arch.sdar);
107 mtspr(SPRN_SIAR, vcpu->arch.siar);
108 mtspr(SPRN_SIER, vcpu->arch.sier[0]);
109
110 if (cpu_has_feature(CPU_FTR_ARCH_31)) {
111 mtspr(SPRN_MMCR3, vcpu->arch.mmcr[3]);
112 mtspr(SPRN_SIER2, vcpu->arch.sier[1]);
113 mtspr(SPRN_SIER3, vcpu->arch.sier[2]);
114 }
115
116 /* Set MMCRA then MMCR0 last */
117 mtspr(SPRN_MMCRA, vcpu->arch.mmcra);
118 mtspr(SPRN_MMCR0, vcpu->arch.mmcr[0]);
119 /* No isync necessary because we're starting counters */
120
121 if (!vcpu->arch.nested &&
122 (vcpu->arch.hfscr_permitted & HFSCR_PM))
123 vcpu->arch.hfscr |= HFSCR_PM;
124 }
125 }
126 EXPORT_SYMBOL_GPL(switch_pmu_to_guest);
127
switch_pmu_to_host(struct kvm_vcpu * vcpu,struct p9_host_os_sprs * host_os_sprs)128 void switch_pmu_to_host(struct kvm_vcpu *vcpu,
129 struct p9_host_os_sprs *host_os_sprs)
130 {
131 struct lppaca *lp;
132 int save_pmu = 1;
133
134 lp = vcpu->arch.vpa.pinned_addr;
135 if (lp)
136 save_pmu = lp->pmcregs_in_use;
137 if (IS_ENABLED(CONFIG_KVM_BOOK3S_HV_NESTED_PMU_WORKAROUND)) {
138 /*
139 * Save pmu if this guest is capable of running nested guests.
140 * This is option is for old L1s that do not set their
141 * lppaca->pmcregs_in_use properly when entering their L2.
142 */
143 save_pmu |= nesting_enabled(vcpu->kvm);
144 }
145
146 if (save_pmu) {
147 vcpu->arch.mmcr[0] = mfspr(SPRN_MMCR0);
148 vcpu->arch.mmcra = mfspr(SPRN_MMCRA);
149
150 freeze_pmu(vcpu->arch.mmcr[0], vcpu->arch.mmcra);
151
152 vcpu->arch.pmc[0] = mfspr(SPRN_PMC1);
153 vcpu->arch.pmc[1] = mfspr(SPRN_PMC2);
154 vcpu->arch.pmc[2] = mfspr(SPRN_PMC3);
155 vcpu->arch.pmc[3] = mfspr(SPRN_PMC4);
156 vcpu->arch.pmc[4] = mfspr(SPRN_PMC5);
157 vcpu->arch.pmc[5] = mfspr(SPRN_PMC6);
158 vcpu->arch.mmcr[1] = mfspr(SPRN_MMCR1);
159 vcpu->arch.mmcr[2] = mfspr(SPRN_MMCR2);
160 vcpu->arch.sdar = mfspr(SPRN_SDAR);
161 vcpu->arch.siar = mfspr(SPRN_SIAR);
162 vcpu->arch.sier[0] = mfspr(SPRN_SIER);
163
164 if (cpu_has_feature(CPU_FTR_ARCH_31)) {
165 vcpu->arch.mmcr[3] = mfspr(SPRN_MMCR3);
166 vcpu->arch.sier[1] = mfspr(SPRN_SIER2);
167 vcpu->arch.sier[2] = mfspr(SPRN_SIER3);
168 }
169
170 } else if (vcpu->arch.hfscr & HFSCR_PM) {
171 /*
172 * The guest accessed PMC SPRs without specifying they should
173 * be preserved, or it cleared pmcregs_in_use after the last
174 * access. Just ensure they are frozen.
175 */
176 freeze_pmu(mfspr(SPRN_MMCR0), mfspr(SPRN_MMCRA));
177
178 /*
179 * Demand-fault PMU register access in the guest.
180 *
181 * This is used to grab the guest's VPA pmcregs_in_use value
182 * and reflect it into the host's VPA in the case of a nested
183 * hypervisor.
184 *
185 * It also avoids having to zero-out SPRs after each guest
186 * exit to avoid side-channels when.
187 *
188 * This is cleared here when we exit the guest, so later HFSCR
189 * interrupt handling can add it back to run the guest with
190 * PM enabled next time.
191 */
192 if (!vcpu->arch.nested)
193 vcpu->arch.hfscr &= ~HFSCR_PM;
194 } /* otherwise the PMU should still be frozen */
195
196 #ifdef CONFIG_PPC_PSERIES
197 if (kvmhv_on_pseries()) {
198 barrier();
199 get_lppaca()->pmcregs_in_use = ppc_get_pmu_inuse();
200 barrier();
201 }
202 #endif
203
204 if (ppc_get_pmu_inuse()) {
205 mtspr(SPRN_PMC1, host_os_sprs->pmc1);
206 mtspr(SPRN_PMC2, host_os_sprs->pmc2);
207 mtspr(SPRN_PMC3, host_os_sprs->pmc3);
208 mtspr(SPRN_PMC4, host_os_sprs->pmc4);
209 mtspr(SPRN_PMC5, host_os_sprs->pmc5);
210 mtspr(SPRN_PMC6, host_os_sprs->pmc6);
211 mtspr(SPRN_MMCR1, host_os_sprs->mmcr1);
212 mtspr(SPRN_MMCR2, host_os_sprs->mmcr2);
213 mtspr(SPRN_SDAR, host_os_sprs->sdar);
214 mtspr(SPRN_SIAR, host_os_sprs->siar);
215 mtspr(SPRN_SIER, host_os_sprs->sier1);
216
217 if (cpu_has_feature(CPU_FTR_ARCH_31)) {
218 mtspr(SPRN_MMCR3, host_os_sprs->mmcr3);
219 mtspr(SPRN_SIER2, host_os_sprs->sier2);
220 mtspr(SPRN_SIER3, host_os_sprs->sier3);
221 }
222
223 /* Set MMCRA then MMCR0 last */
224 mtspr(SPRN_MMCRA, host_os_sprs->mmcra);
225 mtspr(SPRN_MMCR0, host_os_sprs->mmcr0);
226 isync();
227 }
228 }
229 EXPORT_SYMBOL_GPL(switch_pmu_to_host);
230
load_spr_state(struct kvm_vcpu * vcpu,struct p9_host_os_sprs * host_os_sprs)231 static void load_spr_state(struct kvm_vcpu *vcpu,
232 struct p9_host_os_sprs *host_os_sprs)
233 {
234 /* TAR is very fast */
235 mtspr(SPRN_TAR, vcpu->arch.tar);
236
237 #ifdef CONFIG_ALTIVEC
238 if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
239 current->thread.vrsave != vcpu->arch.vrsave)
240 mtspr(SPRN_VRSAVE, vcpu->arch.vrsave);
241 #endif
242
243 if (vcpu->arch.hfscr & HFSCR_EBB) {
244 if (current->thread.ebbhr != vcpu->arch.ebbhr)
245 mtspr(SPRN_EBBHR, vcpu->arch.ebbhr);
246 if (current->thread.ebbrr != vcpu->arch.ebbrr)
247 mtspr(SPRN_EBBRR, vcpu->arch.ebbrr);
248 if (current->thread.bescr != vcpu->arch.bescr)
249 mtspr(SPRN_BESCR, vcpu->arch.bescr);
250 }
251
252 if (cpu_has_feature(CPU_FTR_P9_TIDR) &&
253 current->thread.tidr != vcpu->arch.tid)
254 mtspr(SPRN_TIDR, vcpu->arch.tid);
255 if (host_os_sprs->iamr != vcpu->arch.iamr)
256 mtspr(SPRN_IAMR, vcpu->arch.iamr);
257 if (host_os_sprs->amr != vcpu->arch.amr)
258 mtspr(SPRN_AMR, vcpu->arch.amr);
259 if (vcpu->arch.uamor != 0)
260 mtspr(SPRN_UAMOR, vcpu->arch.uamor);
261 if (current->thread.fscr != vcpu->arch.fscr)
262 mtspr(SPRN_FSCR, vcpu->arch.fscr);
263 if (current->thread.dscr != vcpu->arch.dscr)
264 mtspr(SPRN_DSCR, vcpu->arch.dscr);
265 if (vcpu->arch.pspb != 0)
266 mtspr(SPRN_PSPB, vcpu->arch.pspb);
267
268 /*
269 * DAR, DSISR, and for nested HV, SPRGs must be set with MSR[RI]
270 * clear (or hstate set appropriately to catch those registers
271 * being clobbered if we take a MCE or SRESET), so those are done
272 * later.
273 */
274
275 if (!(vcpu->arch.ctrl & 1))
276 mtspr(SPRN_CTRLT, 0);
277 }
278
store_spr_state(struct kvm_vcpu * vcpu)279 static void store_spr_state(struct kvm_vcpu *vcpu)
280 {
281 vcpu->arch.tar = mfspr(SPRN_TAR);
282
283 #ifdef CONFIG_ALTIVEC
284 if (cpu_has_feature(CPU_FTR_ALTIVEC))
285 vcpu->arch.vrsave = mfspr(SPRN_VRSAVE);
286 #endif
287
288 if (vcpu->arch.hfscr & HFSCR_EBB) {
289 vcpu->arch.ebbhr = mfspr(SPRN_EBBHR);
290 vcpu->arch.ebbrr = mfspr(SPRN_EBBRR);
291 vcpu->arch.bescr = mfspr(SPRN_BESCR);
292 }
293
294 if (cpu_has_feature(CPU_FTR_P9_TIDR))
295 vcpu->arch.tid = mfspr(SPRN_TIDR);
296 vcpu->arch.iamr = mfspr(SPRN_IAMR);
297 vcpu->arch.amr = mfspr(SPRN_AMR);
298 vcpu->arch.uamor = mfspr(SPRN_UAMOR);
299 vcpu->arch.fscr = mfspr(SPRN_FSCR);
300 vcpu->arch.dscr = mfspr(SPRN_DSCR);
301 vcpu->arch.pspb = mfspr(SPRN_PSPB);
302
303 vcpu->arch.ctrl = mfspr(SPRN_CTRLF);
304 }
305
306 /* Returns true if current MSR and/or guest MSR may have changed */
load_vcpu_state(struct kvm_vcpu * vcpu,struct p9_host_os_sprs * host_os_sprs)307 bool load_vcpu_state(struct kvm_vcpu *vcpu,
308 struct p9_host_os_sprs *host_os_sprs)
309 {
310 bool ret = false;
311
312 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
313 if (cpu_has_feature(CPU_FTR_TM) ||
314 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) {
315 unsigned long guest_msr = vcpu->arch.shregs.msr;
316 if (MSR_TM_ACTIVE(guest_msr)) {
317 kvmppc_restore_tm_hv(vcpu, guest_msr, true);
318 ret = true;
319 } else if (vcpu->arch.hfscr & HFSCR_TM) {
320 mtspr(SPRN_TEXASR, vcpu->arch.texasr);
321 mtspr(SPRN_TFHAR, vcpu->arch.tfhar);
322 mtspr(SPRN_TFIAR, vcpu->arch.tfiar);
323 }
324 }
325 #endif
326
327 load_spr_state(vcpu, host_os_sprs);
328
329 load_fp_state(&vcpu->arch.fp);
330 #ifdef CONFIG_ALTIVEC
331 load_vr_state(&vcpu->arch.vr);
332 #endif
333
334 return ret;
335 }
336 EXPORT_SYMBOL_GPL(load_vcpu_state);
337
store_vcpu_state(struct kvm_vcpu * vcpu)338 void store_vcpu_state(struct kvm_vcpu *vcpu)
339 {
340 store_spr_state(vcpu);
341
342 store_fp_state(&vcpu->arch.fp);
343 #ifdef CONFIG_ALTIVEC
344 store_vr_state(&vcpu->arch.vr);
345 #endif
346
347 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
348 if (cpu_has_feature(CPU_FTR_TM) ||
349 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) {
350 unsigned long guest_msr = vcpu->arch.shregs.msr;
351 if (MSR_TM_ACTIVE(guest_msr)) {
352 kvmppc_save_tm_hv(vcpu, guest_msr, true);
353 } else if (vcpu->arch.hfscr & HFSCR_TM) {
354 vcpu->arch.texasr = mfspr(SPRN_TEXASR);
355 vcpu->arch.tfhar = mfspr(SPRN_TFHAR);
356 vcpu->arch.tfiar = mfspr(SPRN_TFIAR);
357
358 if (!vcpu->arch.nested) {
359 vcpu->arch.load_tm++; /* see load_ebb comment */
360 if (!vcpu->arch.load_tm)
361 vcpu->arch.hfscr &= ~HFSCR_TM;
362 }
363 }
364 }
365 #endif
366 }
367 EXPORT_SYMBOL_GPL(store_vcpu_state);
368
save_p9_host_os_sprs(struct p9_host_os_sprs * host_os_sprs)369 void save_p9_host_os_sprs(struct p9_host_os_sprs *host_os_sprs)
370 {
371 host_os_sprs->iamr = mfspr(SPRN_IAMR);
372 host_os_sprs->amr = mfspr(SPRN_AMR);
373 }
374 EXPORT_SYMBOL_GPL(save_p9_host_os_sprs);
375
376 /* vcpu guest regs must already be saved */
restore_p9_host_os_sprs(struct kvm_vcpu * vcpu,struct p9_host_os_sprs * host_os_sprs)377 void restore_p9_host_os_sprs(struct kvm_vcpu *vcpu,
378 struct p9_host_os_sprs *host_os_sprs)
379 {
380 /*
381 * current->thread.xxx registers must all be restored to host
382 * values before a potential context switch, otherwise the context
383 * switch itself will overwrite current->thread.xxx with the values
384 * from the guest SPRs.
385 */
386
387 mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso);
388
389 if (cpu_has_feature(CPU_FTR_P9_TIDR) &&
390 current->thread.tidr != vcpu->arch.tid)
391 mtspr(SPRN_TIDR, current->thread.tidr);
392 if (host_os_sprs->iamr != vcpu->arch.iamr)
393 mtspr(SPRN_IAMR, host_os_sprs->iamr);
394 if (vcpu->arch.uamor != 0)
395 mtspr(SPRN_UAMOR, 0);
396 if (host_os_sprs->amr != vcpu->arch.amr)
397 mtspr(SPRN_AMR, host_os_sprs->amr);
398 if (current->thread.fscr != vcpu->arch.fscr)
399 mtspr(SPRN_FSCR, current->thread.fscr);
400 if (current->thread.dscr != vcpu->arch.dscr)
401 mtspr(SPRN_DSCR, current->thread.dscr);
402 if (vcpu->arch.pspb != 0)
403 mtspr(SPRN_PSPB, 0);
404
405 /* Save guest CTRL register, set runlatch to 1 */
406 if (!(vcpu->arch.ctrl & 1))
407 mtspr(SPRN_CTRLT, 1);
408
409 #ifdef CONFIG_ALTIVEC
410 if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
411 vcpu->arch.vrsave != current->thread.vrsave)
412 mtspr(SPRN_VRSAVE, current->thread.vrsave);
413 #endif
414 if (vcpu->arch.hfscr & HFSCR_EBB) {
415 if (vcpu->arch.bescr != current->thread.bescr)
416 mtspr(SPRN_BESCR, current->thread.bescr);
417 if (vcpu->arch.ebbhr != current->thread.ebbhr)
418 mtspr(SPRN_EBBHR, current->thread.ebbhr);
419 if (vcpu->arch.ebbrr != current->thread.ebbrr)
420 mtspr(SPRN_EBBRR, current->thread.ebbrr);
421
422 if (!vcpu->arch.nested) {
423 /*
424 * This is like load_fp in context switching, turn off
425 * the facility after it wraps the u8 to try avoiding
426 * saving and restoring the registers each partition
427 * switch.
428 */
429 vcpu->arch.load_ebb++;
430 if (!vcpu->arch.load_ebb)
431 vcpu->arch.hfscr &= ~HFSCR_EBB;
432 }
433 }
434
435 if (vcpu->arch.tar != current->thread.tar)
436 mtspr(SPRN_TAR, current->thread.tar);
437 }
438 EXPORT_SYMBOL_GPL(restore_p9_host_os_sprs);
439
440 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
__accumulate_time(struct kvm_vcpu * vcpu,struct kvmhv_tb_accumulator * next)441 static void __accumulate_time(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next)
442 {
443 struct kvmppc_vcore *vc = vcpu->arch.vcore;
444 struct kvmhv_tb_accumulator *curr;
445 u64 tb = mftb() - vc->tb_offset_applied;
446 u64 prev_tb;
447 u64 delta;
448 u64 seq;
449
450 curr = vcpu->arch.cur_activity;
451 vcpu->arch.cur_activity = next;
452 prev_tb = vcpu->arch.cur_tb_start;
453 vcpu->arch.cur_tb_start = tb;
454
455 if (!curr)
456 return;
457
458 delta = tb - prev_tb;
459
460 seq = curr->seqcount;
461 curr->seqcount = seq + 1;
462 smp_wmb();
463 curr->tb_total += delta;
464 if (seq == 0 || delta < curr->tb_min)
465 curr->tb_min = delta;
466 if (delta > curr->tb_max)
467 curr->tb_max = delta;
468 smp_wmb();
469 curr->seqcount = seq + 2;
470 }
471
472 #define start_timing(vcpu, next) __accumulate_time(vcpu, next)
473 #define end_timing(vcpu) __accumulate_time(vcpu, NULL)
474 #define accumulate_time(vcpu, next) __accumulate_time(vcpu, next)
475 #else
476 #define start_timing(vcpu, next) do {} while (0)
477 #define end_timing(vcpu) do {} while (0)
478 #define accumulate_time(vcpu, next) do {} while (0)
479 #endif
480
mfslbv(unsigned int idx)481 static inline u64 mfslbv(unsigned int idx)
482 {
483 u64 slbev;
484
485 asm volatile("slbmfev %0,%1" : "=r" (slbev) : "r" (idx));
486
487 return slbev;
488 }
489
mfslbe(unsigned int idx)490 static inline u64 mfslbe(unsigned int idx)
491 {
492 u64 slbee;
493
494 asm volatile("slbmfee %0,%1" : "=r" (slbee) : "r" (idx));
495
496 return slbee;
497 }
498
mtslb(u64 slbee,u64 slbev)499 static inline void mtslb(u64 slbee, u64 slbev)
500 {
501 asm volatile("slbmte %0,%1" :: "r" (slbev), "r" (slbee));
502 }
503
clear_slb_entry(unsigned int idx)504 static inline void clear_slb_entry(unsigned int idx)
505 {
506 mtslb(idx, 0);
507 }
508
slb_clear_invalidate_partition(void)509 static inline void slb_clear_invalidate_partition(void)
510 {
511 clear_slb_entry(0);
512 asm volatile(PPC_SLBIA(6));
513 }
514
515 /*
516 * Malicious or buggy radix guests may have inserted SLB entries
517 * (only 0..3 because radix always runs with UPRT=1), so these must
518 * be cleared here to avoid side-channels. slbmte is used rather
519 * than slbia, as it won't clear cached translations.
520 */
radix_clear_slb(void)521 static void radix_clear_slb(void)
522 {
523 int i;
524
525 for (i = 0; i < 4; i++)
526 clear_slb_entry(i);
527 }
528
switch_mmu_to_guest_radix(struct kvm * kvm,struct kvm_vcpu * vcpu,u64 lpcr)529 static void switch_mmu_to_guest_radix(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr)
530 {
531 struct kvm_nested_guest *nested = vcpu->arch.nested;
532 u32 lpid;
533 u32 pid;
534
535 lpid = nested ? nested->shadow_lpid : kvm->arch.lpid;
536 pid = vcpu->arch.pid;
537
538 /*
539 * Prior memory accesses to host PID Q3 must be completed before we
540 * start switching, and stores must be drained to avoid not-my-LPAR
541 * logic (see switch_mmu_to_host).
542 */
543 asm volatile("hwsync" ::: "memory");
544 isync();
545 mtspr(SPRN_LPID, lpid);
546 mtspr(SPRN_LPCR, lpcr);
547 mtspr(SPRN_PID, pid);
548 /*
549 * isync not required here because we are HRFID'ing to guest before
550 * any guest context access, which is context synchronising.
551 */
552 }
553
switch_mmu_to_guest_hpt(struct kvm * kvm,struct kvm_vcpu * vcpu,u64 lpcr)554 static void switch_mmu_to_guest_hpt(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr)
555 {
556 u32 lpid;
557 u32 pid;
558 int i;
559
560 lpid = kvm->arch.lpid;
561 pid = vcpu->arch.pid;
562
563 /*
564 * See switch_mmu_to_guest_radix. ptesync should not be required here
565 * even if the host is in HPT mode because speculative accesses would
566 * not cause RC updates (we are in real mode).
567 */
568 asm volatile("hwsync" ::: "memory");
569 isync();
570 mtspr(SPRN_LPID, lpid);
571 mtspr(SPRN_LPCR, lpcr);
572 mtspr(SPRN_PID, pid);
573
574 for (i = 0; i < vcpu->arch.slb_max; i++)
575 mtslb(vcpu->arch.slb[i].orige, vcpu->arch.slb[i].origv);
576 /*
577 * isync not required here, see switch_mmu_to_guest_radix.
578 */
579 }
580
switch_mmu_to_host(struct kvm * kvm,u32 pid)581 static void switch_mmu_to_host(struct kvm *kvm, u32 pid)
582 {
583 u32 lpid = kvm->arch.host_lpid;
584 u64 lpcr = kvm->arch.host_lpcr;
585
586 /*
587 * The guest has exited, so guest MMU context is no longer being
588 * non-speculatively accessed, but a hwsync is needed before the
589 * mtLPIDR / mtPIDR switch, in order to ensure all stores are drained,
590 * so the not-my-LPAR tlbie logic does not overlook them.
591 */
592 asm volatile("hwsync" ::: "memory");
593 isync();
594 mtspr(SPRN_PID, pid);
595 mtspr(SPRN_LPID, lpid);
596 mtspr(SPRN_LPCR, lpcr);
597 /*
598 * isync is not required after the switch, because mtmsrd with L=0
599 * is performed after this switch, which is context synchronising.
600 */
601
602 if (!radix_enabled())
603 slb_restore_bolted_realmode();
604 }
605
save_clear_host_mmu(struct kvm * kvm)606 static void save_clear_host_mmu(struct kvm *kvm)
607 {
608 if (!radix_enabled()) {
609 /*
610 * Hash host could save and restore host SLB entries to
611 * reduce SLB fault overheads of VM exits, but for now the
612 * existing code clears all entries and restores just the
613 * bolted ones when switching back to host.
614 */
615 slb_clear_invalidate_partition();
616 }
617 }
618
save_clear_guest_mmu(struct kvm * kvm,struct kvm_vcpu * vcpu)619 static void save_clear_guest_mmu(struct kvm *kvm, struct kvm_vcpu *vcpu)
620 {
621 if (kvm_is_radix(kvm)) {
622 radix_clear_slb();
623 } else {
624 int i;
625 int nr = 0;
626
627 /*
628 * This must run before switching to host (radix host can't
629 * access all SLBs).
630 */
631 for (i = 0; i < vcpu->arch.slb_nr; i++) {
632 u64 slbee, slbev;
633
634 slbee = mfslbe(i);
635 if (slbee & SLB_ESID_V) {
636 slbev = mfslbv(i);
637 vcpu->arch.slb[nr].orige = slbee | i;
638 vcpu->arch.slb[nr].origv = slbev;
639 nr++;
640 }
641 }
642 vcpu->arch.slb_max = nr;
643 slb_clear_invalidate_partition();
644 }
645 }
646
flush_guest_tlb(struct kvm * kvm)647 static void flush_guest_tlb(struct kvm *kvm)
648 {
649 unsigned long rb, set;
650
651 rb = PPC_BIT(52); /* IS = 2 */
652 if (kvm_is_radix(kvm)) {
653 /* R=1 PRS=1 RIC=2 */
654 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
655 : : "r" (rb), "i" (1), "i" (1), "i" (2),
656 "r" (0) : "memory");
657 for (set = 1; set < kvm->arch.tlb_sets; ++set) {
658 rb += PPC_BIT(51); /* increment set number */
659 /* R=1 PRS=1 RIC=0 */
660 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
661 : : "r" (rb), "i" (1), "i" (1), "i" (0),
662 "r" (0) : "memory");
663 }
664 asm volatile("ptesync": : :"memory");
665 // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now.
666 asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory");
667 } else {
668 for (set = 0; set < kvm->arch.tlb_sets; ++set) {
669 /* R=0 PRS=0 RIC=0 */
670 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
671 : : "r" (rb), "i" (0), "i" (0), "i" (0),
672 "r" (0) : "memory");
673 rb += PPC_BIT(51); /* increment set number */
674 }
675 asm volatile("ptesync": : :"memory");
676 // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now.
677 asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory");
678 }
679 }
680
check_need_tlb_flush(struct kvm * kvm,int pcpu,struct kvm_nested_guest * nested)681 static void check_need_tlb_flush(struct kvm *kvm, int pcpu,
682 struct kvm_nested_guest *nested)
683 {
684 cpumask_t *need_tlb_flush;
685 bool all_set = true;
686 int i;
687
688 if (nested)
689 need_tlb_flush = &nested->need_tlb_flush;
690 else
691 need_tlb_flush = &kvm->arch.need_tlb_flush;
692
693 if (likely(!cpumask_test_cpu(pcpu, need_tlb_flush)))
694 return;
695
696 /*
697 * Individual threads can come in here, but the TLB is shared between
698 * the 4 threads in a core, hence invalidating on one thread
699 * invalidates for all, so only invalidate the first time (if all bits
700 * were set. The others must still execute a ptesync.
701 *
702 * If a race occurs and two threads do the TLB flush, that is not a
703 * problem, just sub-optimal.
704 */
705 for (i = cpu_first_tlb_thread_sibling(pcpu);
706 i <= cpu_last_tlb_thread_sibling(pcpu);
707 i += cpu_tlb_thread_sibling_step()) {
708 if (!cpumask_test_cpu(i, need_tlb_flush)) {
709 all_set = false;
710 break;
711 }
712 }
713 if (all_set)
714 flush_guest_tlb(kvm);
715 else
716 asm volatile("ptesync" ::: "memory");
717
718 /* Clear the bit after the TLB flush */
719 cpumask_clear_cpu(pcpu, need_tlb_flush);
720 }
721
kvmppc_msr_hard_disable_set_facilities(struct kvm_vcpu * vcpu,unsigned long msr)722 unsigned long kvmppc_msr_hard_disable_set_facilities(struct kvm_vcpu *vcpu, unsigned long msr)
723 {
724 unsigned long msr_needed = 0;
725
726 msr &= ~MSR_EE;
727
728 /* MSR bits may have been cleared by context switch so must recheck */
729 if (IS_ENABLED(CONFIG_PPC_FPU))
730 msr_needed |= MSR_FP;
731 if (cpu_has_feature(CPU_FTR_ALTIVEC))
732 msr_needed |= MSR_VEC;
733 if (cpu_has_feature(CPU_FTR_VSX))
734 msr_needed |= MSR_VSX;
735 if ((cpu_has_feature(CPU_FTR_TM) ||
736 cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) &&
737 (vcpu->arch.hfscr & HFSCR_TM))
738 msr_needed |= MSR_TM;
739
740 /*
741 * This could be combined with MSR[RI] clearing, but that expands
742 * the unrecoverable window. It would be better to cover unrecoverable
743 * with KVM bad interrupt handling rather than use MSR[RI] at all.
744 *
745 * Much more difficult and less worthwhile to combine with IR/DR
746 * disable.
747 */
748 if ((msr & msr_needed) != msr_needed) {
749 msr |= msr_needed;
750 __mtmsrd(msr, 0);
751 } else {
752 __hard_irq_disable();
753 }
754 local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
755
756 return msr;
757 }
758 EXPORT_SYMBOL_GPL(kvmppc_msr_hard_disable_set_facilities);
759
kvmhv_vcpu_entry_p9(struct kvm_vcpu * vcpu,u64 time_limit,unsigned long lpcr,u64 * tb)760 int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr, u64 *tb)
761 {
762 struct p9_host_os_sprs host_os_sprs;
763 struct kvm *kvm = vcpu->kvm;
764 struct kvm_nested_guest *nested = vcpu->arch.nested;
765 struct kvmppc_vcore *vc = vcpu->arch.vcore;
766 s64 hdec, dec;
767 u64 purr, spurr;
768 u64 *exsave;
769 int trap;
770 unsigned long msr;
771 unsigned long host_hfscr;
772 unsigned long host_ciabr;
773 unsigned long host_dawr0;
774 unsigned long host_dawrx0;
775 unsigned long host_psscr;
776 unsigned long host_hpsscr;
777 unsigned long host_pidr;
778 unsigned long host_dawr1;
779 unsigned long host_dawrx1;
780 unsigned long dpdes;
781
782 hdec = time_limit - *tb;
783 if (hdec < 0)
784 return BOOK3S_INTERRUPT_HV_DECREMENTER;
785
786 WARN_ON_ONCE(vcpu->arch.shregs.msr & MSR_HV);
787 WARN_ON_ONCE(!(vcpu->arch.shregs.msr & MSR_ME));
788
789 start_timing(vcpu, &vcpu->arch.rm_entry);
790
791 vcpu->arch.ceded = 0;
792
793 /* Save MSR for restore, with EE clear. */
794 msr = mfmsr() & ~MSR_EE;
795
796 host_hfscr = mfspr(SPRN_HFSCR);
797 host_ciabr = mfspr(SPRN_CIABR);
798 host_psscr = mfspr(SPRN_PSSCR_PR);
799 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
800 host_hpsscr = mfspr(SPRN_PSSCR);
801 host_pidr = mfspr(SPRN_PID);
802
803 if (dawr_enabled()) {
804 host_dawr0 = mfspr(SPRN_DAWR0);
805 host_dawrx0 = mfspr(SPRN_DAWRX0);
806 if (cpu_has_feature(CPU_FTR_DAWR1)) {
807 host_dawr1 = mfspr(SPRN_DAWR1);
808 host_dawrx1 = mfspr(SPRN_DAWRX1);
809 }
810 }
811
812 local_paca->kvm_hstate.host_purr = mfspr(SPRN_PURR);
813 local_paca->kvm_hstate.host_spurr = mfspr(SPRN_SPURR);
814
815 save_p9_host_os_sprs(&host_os_sprs);
816
817 msr = kvmppc_msr_hard_disable_set_facilities(vcpu, msr);
818 if (lazy_irq_pending()) {
819 trap = 0;
820 goto out;
821 }
822
823 if (unlikely(load_vcpu_state(vcpu, &host_os_sprs)))
824 msr = mfmsr(); /* MSR may have been updated */
825
826 if (vc->tb_offset) {
827 u64 new_tb = *tb + vc->tb_offset;
828 mtspr(SPRN_TBU40, new_tb);
829 if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) {
830 new_tb += 0x1000000;
831 mtspr(SPRN_TBU40, new_tb);
832 }
833 *tb = new_tb;
834 vc->tb_offset_applied = vc->tb_offset;
835 }
836
837 mtspr(SPRN_VTB, vc->vtb);
838 mtspr(SPRN_PURR, vcpu->arch.purr);
839 mtspr(SPRN_SPURR, vcpu->arch.spurr);
840
841 if (vc->pcr)
842 mtspr(SPRN_PCR, vc->pcr | PCR_MASK);
843 if (vcpu->arch.doorbell_request) {
844 vcpu->arch.doorbell_request = 0;
845 mtspr(SPRN_DPDES, 1);
846 }
847
848 if (dawr_enabled()) {
849 if (vcpu->arch.dawr0 != host_dawr0)
850 mtspr(SPRN_DAWR0, vcpu->arch.dawr0);
851 if (vcpu->arch.dawrx0 != host_dawrx0)
852 mtspr(SPRN_DAWRX0, vcpu->arch.dawrx0);
853 if (cpu_has_feature(CPU_FTR_DAWR1)) {
854 if (vcpu->arch.dawr1 != host_dawr1)
855 mtspr(SPRN_DAWR1, vcpu->arch.dawr1);
856 if (vcpu->arch.dawrx1 != host_dawrx1)
857 mtspr(SPRN_DAWRX1, vcpu->arch.dawrx1);
858 }
859 }
860 if (vcpu->arch.ciabr != host_ciabr)
861 mtspr(SPRN_CIABR, vcpu->arch.ciabr);
862
863
864 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) {
865 mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC |
866 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
867 } else {
868 if (vcpu->arch.psscr != host_psscr)
869 mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr);
870 }
871
872 mtspr(SPRN_HFSCR, vcpu->arch.hfscr);
873
874 mtspr(SPRN_HSRR0, vcpu->arch.regs.nip);
875 mtspr(SPRN_HSRR1, (vcpu->arch.shregs.msr & ~MSR_HV) | MSR_ME);
876
877 /*
878 * On POWER9 DD2.1 and below, sometimes on a Hypervisor Data Storage
879 * Interrupt (HDSI) the HDSISR is not be updated at all.
880 *
881 * To work around this we put a canary value into the HDSISR before
882 * returning to a guest and then check for this canary when we take a
883 * HDSI. If we find the canary on a HDSI, we know the hardware didn't
884 * update the HDSISR. In this case we return to the guest to retake the
885 * HDSI which should correctly update the HDSISR the second time HDSI
886 * entry.
887 *
888 * The "radix prefetch bug" test can be used to test for this bug, as
889 * it also exists fo DD2.1 and below.
890 */
891 if (cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
892 mtspr(SPRN_HDSISR, HDSISR_CANARY);
893
894 mtspr(SPRN_SPRG0, vcpu->arch.shregs.sprg0);
895 mtspr(SPRN_SPRG1, vcpu->arch.shregs.sprg1);
896 mtspr(SPRN_SPRG2, vcpu->arch.shregs.sprg2);
897 mtspr(SPRN_SPRG3, vcpu->arch.shregs.sprg3);
898
899 /*
900 * It might be preferable to load_vcpu_state here, in order to get the
901 * GPR/FP register loads executing in parallel with the previous mtSPR
902 * instructions, but for now that can't be done because the TM handling
903 * in load_vcpu_state can change some SPRs and vcpu state (nip, msr).
904 * But TM could be split out if this would be a significant benefit.
905 */
906
907 /*
908 * MSR[RI] does not need to be cleared (and is not, for radix guests
909 * with no prefetch bug), because in_guest is set. If we take a SRESET
910 * or MCE with in_guest set but still in HV mode, then
911 * kvmppc_p9_bad_interrupt handles the interrupt, which effectively
912 * clears MSR[RI] and doesn't return.
913 */
914 WRITE_ONCE(local_paca->kvm_hstate.in_guest, KVM_GUEST_MODE_HV_P9);
915 barrier(); /* Open in_guest critical section */
916
917 /*
918 * Hash host, hash guest, or radix guest with prefetch bug, all have
919 * to disable the MMU before switching to guest MMU state.
920 */
921 if (!radix_enabled() || !kvm_is_radix(kvm) ||
922 cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
923 __mtmsrd(msr & ~(MSR_IR|MSR_DR|MSR_RI), 0);
924
925 save_clear_host_mmu(kvm);
926
927 if (kvm_is_radix(kvm))
928 switch_mmu_to_guest_radix(kvm, vcpu, lpcr);
929 else
930 switch_mmu_to_guest_hpt(kvm, vcpu, lpcr);
931
932 /* TLBIEL uses LPID=LPIDR, so run this after setting guest LPID */
933 check_need_tlb_flush(kvm, vc->pcpu, nested);
934
935 /*
936 * P9 suppresses the HDEC exception when LPCR[HDICE] = 0,
937 * so set guest LPCR (with HDICE) before writing HDEC.
938 */
939 mtspr(SPRN_HDEC, hdec);
940
941 mtspr(SPRN_DEC, vcpu->arch.dec_expires - *tb);
942
943 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
944 tm_return_to_guest:
945 #endif
946 mtspr(SPRN_DAR, vcpu->arch.shregs.dar);
947 mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr);
948 mtspr(SPRN_SRR0, vcpu->arch.shregs.srr0);
949 mtspr(SPRN_SRR1, vcpu->arch.shregs.srr1);
950
951 accumulate_time(vcpu, &vcpu->arch.guest_time);
952
953 switch_pmu_to_guest(vcpu, &host_os_sprs);
954 kvmppc_p9_enter_guest(vcpu);
955 switch_pmu_to_host(vcpu, &host_os_sprs);
956
957 accumulate_time(vcpu, &vcpu->arch.rm_intr);
958
959 /* XXX: Could get these from r11/12 and paca exsave instead */
960 vcpu->arch.shregs.srr0 = mfspr(SPRN_SRR0);
961 vcpu->arch.shregs.srr1 = mfspr(SPRN_SRR1);
962 vcpu->arch.shregs.dar = mfspr(SPRN_DAR);
963 vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR);
964
965 /* 0x2 bit for HSRR is only used by PR and P7/8 HV paths, clear it */
966 trap = local_paca->kvm_hstate.scratch0 & ~0x2;
967
968 if (likely(trap > BOOK3S_INTERRUPT_MACHINE_CHECK))
969 exsave = local_paca->exgen;
970 else if (trap == BOOK3S_INTERRUPT_SYSTEM_RESET)
971 exsave = local_paca->exnmi;
972 else /* trap == 0x200 */
973 exsave = local_paca->exmc;
974
975 vcpu->arch.regs.gpr[1] = local_paca->kvm_hstate.scratch1;
976 vcpu->arch.regs.gpr[3] = local_paca->kvm_hstate.scratch2;
977
978 /*
979 * After reading machine check regs (DAR, DSISR, SRR0/1) and hstate
980 * scratch (which we need to move into exsave to make re-entrant vs
981 * SRESET/MCE), register state is protected from reentrancy. However
982 * timebase, MMU, among other state is still set to guest, so don't
983 * enable MSR[RI] here. It gets enabled at the end, after in_guest
984 * is cleared.
985 *
986 * It is possible an NMI could come in here, which is why it is
987 * important to save the above state early so it can be debugged.
988 */
989
990 vcpu->arch.regs.gpr[9] = exsave[EX_R9/sizeof(u64)];
991 vcpu->arch.regs.gpr[10] = exsave[EX_R10/sizeof(u64)];
992 vcpu->arch.regs.gpr[11] = exsave[EX_R11/sizeof(u64)];
993 vcpu->arch.regs.gpr[12] = exsave[EX_R12/sizeof(u64)];
994 vcpu->arch.regs.gpr[13] = exsave[EX_R13/sizeof(u64)];
995 vcpu->arch.ppr = exsave[EX_PPR/sizeof(u64)];
996 vcpu->arch.cfar = exsave[EX_CFAR/sizeof(u64)];
997 vcpu->arch.regs.ctr = exsave[EX_CTR/sizeof(u64)];
998
999 vcpu->arch.last_inst = KVM_INST_FETCH_FAILED;
1000
1001 if (unlikely(trap == BOOK3S_INTERRUPT_MACHINE_CHECK)) {
1002 vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)];
1003 vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)];
1004 kvmppc_realmode_machine_check(vcpu);
1005
1006 } else if (unlikely(trap == BOOK3S_INTERRUPT_HMI)) {
1007 kvmppc_p9_realmode_hmi_handler(vcpu);
1008
1009 } else if (trap == BOOK3S_INTERRUPT_H_EMUL_ASSIST) {
1010 vcpu->arch.emul_inst = mfspr(SPRN_HEIR);
1011
1012 } else if (trap == BOOK3S_INTERRUPT_H_DATA_STORAGE) {
1013 vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)];
1014 vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)];
1015 vcpu->arch.fault_gpa = mfspr(SPRN_ASDR);
1016
1017 } else if (trap == BOOK3S_INTERRUPT_H_INST_STORAGE) {
1018 vcpu->arch.fault_gpa = mfspr(SPRN_ASDR);
1019
1020 } else if (trap == BOOK3S_INTERRUPT_H_FAC_UNAVAIL) {
1021 vcpu->arch.hfscr = mfspr(SPRN_HFSCR);
1022
1023 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1024 /*
1025 * Softpatch interrupt for transactional memory emulation cases
1026 * on POWER9 DD2.2. This is early in the guest exit path - we
1027 * haven't saved registers or done a treclaim yet.
1028 */
1029 } else if (trap == BOOK3S_INTERRUPT_HV_SOFTPATCH) {
1030 vcpu->arch.emul_inst = mfspr(SPRN_HEIR);
1031
1032 /*
1033 * The cases we want to handle here are those where the guest
1034 * is in real suspend mode and is trying to transition to
1035 * transactional mode.
1036 */
1037 if (!local_paca->kvm_hstate.fake_suspend &&
1038 (vcpu->arch.shregs.msr & MSR_TS_S)) {
1039 if (kvmhv_p9_tm_emulation_early(vcpu)) {
1040 /*
1041 * Go straight back into the guest with the
1042 * new NIP/MSR as set by TM emulation.
1043 */
1044 mtspr(SPRN_HSRR0, vcpu->arch.regs.nip);
1045 mtspr(SPRN_HSRR1, vcpu->arch.shregs.msr);
1046 goto tm_return_to_guest;
1047 }
1048 }
1049 #endif
1050 }
1051
1052 accumulate_time(vcpu, &vcpu->arch.rm_exit);
1053
1054 /* Advance host PURR/SPURR by the amount used by guest */
1055 purr = mfspr(SPRN_PURR);
1056 spurr = mfspr(SPRN_SPURR);
1057 local_paca->kvm_hstate.host_purr += purr - vcpu->arch.purr;
1058 local_paca->kvm_hstate.host_spurr += spurr - vcpu->arch.spurr;
1059 vcpu->arch.purr = purr;
1060 vcpu->arch.spurr = spurr;
1061
1062 vcpu->arch.ic = mfspr(SPRN_IC);
1063 vcpu->arch.pid = mfspr(SPRN_PID);
1064 vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR);
1065
1066 vcpu->arch.shregs.sprg0 = mfspr(SPRN_SPRG0);
1067 vcpu->arch.shregs.sprg1 = mfspr(SPRN_SPRG1);
1068 vcpu->arch.shregs.sprg2 = mfspr(SPRN_SPRG2);
1069 vcpu->arch.shregs.sprg3 = mfspr(SPRN_SPRG3);
1070
1071 dpdes = mfspr(SPRN_DPDES);
1072 if (dpdes)
1073 vcpu->arch.doorbell_request = 1;
1074
1075 vc->vtb = mfspr(SPRN_VTB);
1076
1077 dec = mfspr(SPRN_DEC);
1078 if (!(lpcr & LPCR_LD)) /* Sign extend if not using large decrementer */
1079 dec = (s32) dec;
1080 *tb = mftb();
1081 vcpu->arch.dec_expires = dec + *tb;
1082
1083 if (vc->tb_offset_applied) {
1084 u64 new_tb = *tb - vc->tb_offset_applied;
1085 mtspr(SPRN_TBU40, new_tb);
1086 if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) {
1087 new_tb += 0x1000000;
1088 mtspr(SPRN_TBU40, new_tb);
1089 }
1090 *tb = new_tb;
1091 vc->tb_offset_applied = 0;
1092 }
1093
1094 save_clear_guest_mmu(kvm, vcpu);
1095 switch_mmu_to_host(kvm, host_pidr);
1096
1097 /*
1098 * Enable MSR here in order to have facilities enabled to save
1099 * guest registers. This enables MMU (if we were in realmode), so
1100 * only switch MMU on after the MMU is switched to host, to avoid
1101 * the P9_RADIX_PREFETCH_BUG or hash guest context.
1102 */
1103 if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
1104 vcpu->arch.shregs.msr & MSR_TS_MASK)
1105 msr |= MSR_TS_S;
1106 __mtmsrd(msr, 0);
1107
1108 store_vcpu_state(vcpu);
1109
1110 mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr);
1111 mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr);
1112
1113 if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) {
1114 /* Preserve PSSCR[FAKE_SUSPEND] until we've called kvmppc_save_tm_hv */
1115 mtspr(SPRN_PSSCR, host_hpsscr |
1116 (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
1117 }
1118
1119 mtspr(SPRN_HFSCR, host_hfscr);
1120 if (vcpu->arch.ciabr != host_ciabr)
1121 mtspr(SPRN_CIABR, host_ciabr);
1122
1123 if (dawr_enabled()) {
1124 if (vcpu->arch.dawr0 != host_dawr0)
1125 mtspr(SPRN_DAWR0, host_dawr0);
1126 if (vcpu->arch.dawrx0 != host_dawrx0)
1127 mtspr(SPRN_DAWRX0, host_dawrx0);
1128 if (cpu_has_feature(CPU_FTR_DAWR1)) {
1129 if (vcpu->arch.dawr1 != host_dawr1)
1130 mtspr(SPRN_DAWR1, host_dawr1);
1131 if (vcpu->arch.dawrx1 != host_dawrx1)
1132 mtspr(SPRN_DAWRX1, host_dawrx1);
1133 }
1134 }
1135
1136 if (dpdes)
1137 mtspr(SPRN_DPDES, 0);
1138 if (vc->pcr)
1139 mtspr(SPRN_PCR, PCR_MASK);
1140
1141 /* HDEC must be at least as large as DEC, so decrementer_max fits */
1142 mtspr(SPRN_HDEC, decrementer_max);
1143
1144 timer_rearm_host_dec(*tb);
1145
1146 restore_p9_host_os_sprs(vcpu, &host_os_sprs);
1147
1148 barrier(); /* Close in_guest critical section */
1149 WRITE_ONCE(local_paca->kvm_hstate.in_guest, KVM_GUEST_MODE_NONE);
1150 /* Interrupts are recoverable at this point */
1151
1152 /*
1153 * cp_abort is required if the processor supports local copy-paste
1154 * to clear the copy buffer that was under control of the guest.
1155 */
1156 if (cpu_has_feature(CPU_FTR_ARCH_31))
1157 asm volatile(PPC_CP_ABORT);
1158
1159 out:
1160 end_timing(vcpu);
1161
1162 return trap;
1163 }
1164 EXPORT_SYMBOL_GPL(kvmhv_vcpu_entry_p9);
1165