1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright IBM Corporation, 2018
4  * Authors Suraj Jitindar Singh <sjitindarsingh@gmail.com>
5  *	   Paul Mackerras <paulus@ozlabs.org>
6  *
7  * Description: KVM functions specific to running nested KVM-HV guests
8  * on Book3S processors (specifically POWER9 and later).
9  */
10 
11 #include <linux/kernel.h>
12 #include <linux/kvm_host.h>
13 #include <linux/llist.h>
14 #include <linux/pgtable.h>
15 
16 #include <asm/kvm_ppc.h>
17 #include <asm/kvm_book3s.h>
18 #include <asm/mmu.h>
19 #include <asm/pgalloc.h>
20 #include <asm/pte-walk.h>
21 #include <asm/reg.h>
22 #include <asm/plpar_wrappers.h>
23 
24 static struct patb_entry *pseries_partition_tb;
25 
26 static void kvmhv_update_ptbl_cache(struct kvm_nested_guest *gp);
27 static void kvmhv_free_memslot_nest_rmap(struct kvm_memory_slot *free);
28 
kvmhv_save_hv_regs(struct kvm_vcpu * vcpu,struct hv_guest_state * hr)29 void kvmhv_save_hv_regs(struct kvm_vcpu *vcpu, struct hv_guest_state *hr)
30 {
31 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
32 
33 	hr->pcr = vc->pcr | PCR_MASK;
34 	hr->dpdes = vc->dpdes;
35 	hr->hfscr = vcpu->arch.hfscr;
36 	hr->tb_offset = vc->tb_offset;
37 	hr->dawr0 = vcpu->arch.dawr0;
38 	hr->dawrx0 = vcpu->arch.dawrx0;
39 	hr->ciabr = vcpu->arch.ciabr;
40 	hr->purr = vcpu->arch.purr;
41 	hr->spurr = vcpu->arch.spurr;
42 	hr->ic = vcpu->arch.ic;
43 	hr->vtb = vc->vtb;
44 	hr->srr0 = vcpu->arch.shregs.srr0;
45 	hr->srr1 = vcpu->arch.shregs.srr1;
46 	hr->sprg[0] = vcpu->arch.shregs.sprg0;
47 	hr->sprg[1] = vcpu->arch.shregs.sprg1;
48 	hr->sprg[2] = vcpu->arch.shregs.sprg2;
49 	hr->sprg[3] = vcpu->arch.shregs.sprg3;
50 	hr->pidr = vcpu->arch.pid;
51 	hr->cfar = vcpu->arch.cfar;
52 	hr->ppr = vcpu->arch.ppr;
53 	hr->dawr1 = vcpu->arch.dawr1;
54 	hr->dawrx1 = vcpu->arch.dawrx1;
55 }
56 
57 /* Use noinline_for_stack due to https://bugs.llvm.org/show_bug.cgi?id=49610 */
byteswap_pt_regs(struct pt_regs * regs)58 static noinline_for_stack void byteswap_pt_regs(struct pt_regs *regs)
59 {
60 	unsigned long *addr = (unsigned long *) regs;
61 
62 	for (; addr < ((unsigned long *) (regs + 1)); addr++)
63 		*addr = swab64(*addr);
64 }
65 
byteswap_hv_regs(struct hv_guest_state * hr)66 static void byteswap_hv_regs(struct hv_guest_state *hr)
67 {
68 	hr->version = swab64(hr->version);
69 	hr->lpid = swab32(hr->lpid);
70 	hr->vcpu_token = swab32(hr->vcpu_token);
71 	hr->lpcr = swab64(hr->lpcr);
72 	hr->pcr = swab64(hr->pcr) | PCR_MASK;
73 	hr->amor = swab64(hr->amor);
74 	hr->dpdes = swab64(hr->dpdes);
75 	hr->hfscr = swab64(hr->hfscr);
76 	hr->tb_offset = swab64(hr->tb_offset);
77 	hr->dawr0 = swab64(hr->dawr0);
78 	hr->dawrx0 = swab64(hr->dawrx0);
79 	hr->ciabr = swab64(hr->ciabr);
80 	hr->hdec_expiry = swab64(hr->hdec_expiry);
81 	hr->purr = swab64(hr->purr);
82 	hr->spurr = swab64(hr->spurr);
83 	hr->ic = swab64(hr->ic);
84 	hr->vtb = swab64(hr->vtb);
85 	hr->hdar = swab64(hr->hdar);
86 	hr->hdsisr = swab64(hr->hdsisr);
87 	hr->heir = swab64(hr->heir);
88 	hr->asdr = swab64(hr->asdr);
89 	hr->srr0 = swab64(hr->srr0);
90 	hr->srr1 = swab64(hr->srr1);
91 	hr->sprg[0] = swab64(hr->sprg[0]);
92 	hr->sprg[1] = swab64(hr->sprg[1]);
93 	hr->sprg[2] = swab64(hr->sprg[2]);
94 	hr->sprg[3] = swab64(hr->sprg[3]);
95 	hr->pidr = swab64(hr->pidr);
96 	hr->cfar = swab64(hr->cfar);
97 	hr->ppr = swab64(hr->ppr);
98 	hr->dawr1 = swab64(hr->dawr1);
99 	hr->dawrx1 = swab64(hr->dawrx1);
100 }
101 
save_hv_return_state(struct kvm_vcpu * vcpu,struct hv_guest_state * hr)102 static void save_hv_return_state(struct kvm_vcpu *vcpu,
103 				 struct hv_guest_state *hr)
104 {
105 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
106 
107 	hr->dpdes = vc->dpdes;
108 	hr->purr = vcpu->arch.purr;
109 	hr->spurr = vcpu->arch.spurr;
110 	hr->ic = vcpu->arch.ic;
111 	hr->vtb = vc->vtb;
112 	hr->srr0 = vcpu->arch.shregs.srr0;
113 	hr->srr1 = vcpu->arch.shregs.srr1;
114 	hr->sprg[0] = vcpu->arch.shregs.sprg0;
115 	hr->sprg[1] = vcpu->arch.shregs.sprg1;
116 	hr->sprg[2] = vcpu->arch.shregs.sprg2;
117 	hr->sprg[3] = vcpu->arch.shregs.sprg3;
118 	hr->pidr = vcpu->arch.pid;
119 	hr->cfar = vcpu->arch.cfar;
120 	hr->ppr = vcpu->arch.ppr;
121 	switch (vcpu->arch.trap) {
122 	case BOOK3S_INTERRUPT_H_DATA_STORAGE:
123 		hr->hdar = vcpu->arch.fault_dar;
124 		hr->hdsisr = vcpu->arch.fault_dsisr;
125 		hr->asdr = vcpu->arch.fault_gpa;
126 		break;
127 	case BOOK3S_INTERRUPT_H_INST_STORAGE:
128 		hr->asdr = vcpu->arch.fault_gpa;
129 		break;
130 	case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
131 		hr->hfscr = ((~HFSCR_INTR_CAUSE & hr->hfscr) |
132 			     (HFSCR_INTR_CAUSE & vcpu->arch.hfscr));
133 		break;
134 	case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
135 		hr->heir = vcpu->arch.emul_inst;
136 		break;
137 	}
138 }
139 
restore_hv_regs(struct kvm_vcpu * vcpu,const struct hv_guest_state * hr)140 static void restore_hv_regs(struct kvm_vcpu *vcpu, const struct hv_guest_state *hr)
141 {
142 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
143 
144 	vc->pcr = hr->pcr | PCR_MASK;
145 	vc->dpdes = hr->dpdes;
146 	vcpu->arch.hfscr = hr->hfscr;
147 	vcpu->arch.dawr0 = hr->dawr0;
148 	vcpu->arch.dawrx0 = hr->dawrx0;
149 	vcpu->arch.ciabr = hr->ciabr;
150 	vcpu->arch.purr = hr->purr;
151 	vcpu->arch.spurr = hr->spurr;
152 	vcpu->arch.ic = hr->ic;
153 	vc->vtb = hr->vtb;
154 	vcpu->arch.shregs.srr0 = hr->srr0;
155 	vcpu->arch.shregs.srr1 = hr->srr1;
156 	vcpu->arch.shregs.sprg0 = hr->sprg[0];
157 	vcpu->arch.shregs.sprg1 = hr->sprg[1];
158 	vcpu->arch.shregs.sprg2 = hr->sprg[2];
159 	vcpu->arch.shregs.sprg3 = hr->sprg[3];
160 	vcpu->arch.pid = hr->pidr;
161 	vcpu->arch.cfar = hr->cfar;
162 	vcpu->arch.ppr = hr->ppr;
163 	vcpu->arch.dawr1 = hr->dawr1;
164 	vcpu->arch.dawrx1 = hr->dawrx1;
165 }
166 
kvmhv_restore_hv_return_state(struct kvm_vcpu * vcpu,struct hv_guest_state * hr)167 void kvmhv_restore_hv_return_state(struct kvm_vcpu *vcpu,
168 				   struct hv_guest_state *hr)
169 {
170 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
171 
172 	vc->dpdes = hr->dpdes;
173 	vcpu->arch.hfscr = hr->hfscr;
174 	vcpu->arch.purr = hr->purr;
175 	vcpu->arch.spurr = hr->spurr;
176 	vcpu->arch.ic = hr->ic;
177 	vc->vtb = hr->vtb;
178 	vcpu->arch.fault_dar = hr->hdar;
179 	vcpu->arch.fault_dsisr = hr->hdsisr;
180 	vcpu->arch.fault_gpa = hr->asdr;
181 	vcpu->arch.emul_inst = hr->heir;
182 	vcpu->arch.shregs.srr0 = hr->srr0;
183 	vcpu->arch.shregs.srr1 = hr->srr1;
184 	vcpu->arch.shregs.sprg0 = hr->sprg[0];
185 	vcpu->arch.shregs.sprg1 = hr->sprg[1];
186 	vcpu->arch.shregs.sprg2 = hr->sprg[2];
187 	vcpu->arch.shregs.sprg3 = hr->sprg[3];
188 	vcpu->arch.pid = hr->pidr;
189 	vcpu->arch.cfar = hr->cfar;
190 	vcpu->arch.ppr = hr->ppr;
191 }
192 
kvmhv_nested_mmio_needed(struct kvm_vcpu * vcpu,u64 regs_ptr)193 static void kvmhv_nested_mmio_needed(struct kvm_vcpu *vcpu, u64 regs_ptr)
194 {
195 	/* No need to reflect the page fault to L1, we've handled it */
196 	vcpu->arch.trap = 0;
197 
198 	/*
199 	 * Since the L2 gprs have already been written back into L1 memory when
200 	 * we complete the mmio, store the L1 memory location of the L2 gpr
201 	 * being loaded into by the mmio so that the loaded value can be
202 	 * written there in kvmppc_complete_mmio_load()
203 	 */
204 	if (((vcpu->arch.io_gpr & KVM_MMIO_REG_EXT_MASK) == KVM_MMIO_REG_GPR)
205 	    && (vcpu->mmio_is_write == 0)) {
206 		vcpu->arch.nested_io_gpr = (gpa_t) regs_ptr +
207 					   offsetof(struct pt_regs,
208 						    gpr[vcpu->arch.io_gpr]);
209 		vcpu->arch.io_gpr = KVM_MMIO_REG_NESTED_GPR;
210 	}
211 }
212 
kvmhv_read_guest_state_and_regs(struct kvm_vcpu * vcpu,struct hv_guest_state * l2_hv,struct pt_regs * l2_regs,u64 hv_ptr,u64 regs_ptr)213 static int kvmhv_read_guest_state_and_regs(struct kvm_vcpu *vcpu,
214 					   struct hv_guest_state *l2_hv,
215 					   struct pt_regs *l2_regs,
216 					   u64 hv_ptr, u64 regs_ptr)
217 {
218 	int size;
219 
220 	if (kvm_vcpu_read_guest(vcpu, hv_ptr, &l2_hv->version,
221 				sizeof(l2_hv->version)))
222 		return -1;
223 
224 	if (kvmppc_need_byteswap(vcpu))
225 		l2_hv->version = swab64(l2_hv->version);
226 
227 	size = hv_guest_state_size(l2_hv->version);
228 	if (size < 0)
229 		return -1;
230 
231 	return kvm_vcpu_read_guest(vcpu, hv_ptr, l2_hv, size) ||
232 		kvm_vcpu_read_guest(vcpu, regs_ptr, l2_regs,
233 				    sizeof(struct pt_regs));
234 }
235 
kvmhv_write_guest_state_and_regs(struct kvm_vcpu * vcpu,struct hv_guest_state * l2_hv,struct pt_regs * l2_regs,u64 hv_ptr,u64 regs_ptr)236 static int kvmhv_write_guest_state_and_regs(struct kvm_vcpu *vcpu,
237 					    struct hv_guest_state *l2_hv,
238 					    struct pt_regs *l2_regs,
239 					    u64 hv_ptr, u64 regs_ptr)
240 {
241 	int size;
242 
243 	size = hv_guest_state_size(l2_hv->version);
244 	if (size < 0)
245 		return -1;
246 
247 	return kvm_vcpu_write_guest(vcpu, hv_ptr, l2_hv, size) ||
248 		kvm_vcpu_write_guest(vcpu, regs_ptr, l2_regs,
249 				     sizeof(struct pt_regs));
250 }
251 
load_l2_hv_regs(struct kvm_vcpu * vcpu,const struct hv_guest_state * l2_hv,const struct hv_guest_state * l1_hv,u64 * lpcr)252 static void load_l2_hv_regs(struct kvm_vcpu *vcpu,
253 			    const struct hv_guest_state *l2_hv,
254 			    const struct hv_guest_state *l1_hv, u64 *lpcr)
255 {
256 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
257 	u64 mask;
258 
259 	restore_hv_regs(vcpu, l2_hv);
260 
261 	/*
262 	 * Don't let L1 change LPCR bits for the L2 except these:
263 	 */
264 	mask = LPCR_DPFD | LPCR_ILE | LPCR_TC | LPCR_AIL | LPCR_LD | LPCR_MER;
265 
266 	/*
267 	 * Additional filtering is required depending on hardware
268 	 * and configuration.
269 	 */
270 	*lpcr = kvmppc_filter_lpcr_hv(vcpu->kvm,
271 				      (vc->lpcr & ~mask) | (*lpcr & mask));
272 
273 	/*
274 	 * Don't let L1 enable features for L2 which we don't allow for L1,
275 	 * but preserve the interrupt cause field.
276 	 */
277 	vcpu->arch.hfscr = l2_hv->hfscr & (HFSCR_INTR_CAUSE | vcpu->arch.hfscr_permitted);
278 
279 	/* Don't let data address watchpoint match in hypervisor state */
280 	vcpu->arch.dawrx0 = l2_hv->dawrx0 & ~DAWRX_HYP;
281 	vcpu->arch.dawrx1 = l2_hv->dawrx1 & ~DAWRX_HYP;
282 
283 	/* Don't let completed instruction address breakpt match in HV state */
284 	if ((l2_hv->ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
285 		vcpu->arch.ciabr = l2_hv->ciabr & ~CIABR_PRIV;
286 }
287 
kvmhv_enter_nested_guest(struct kvm_vcpu * vcpu)288 long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
289 {
290 	long int err, r;
291 	struct kvm_nested_guest *l2;
292 	struct pt_regs l2_regs, saved_l1_regs;
293 	struct hv_guest_state l2_hv = {0}, saved_l1_hv;
294 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
295 	u64 hv_ptr, regs_ptr;
296 	u64 hdec_exp, lpcr;
297 	s64 delta_purr, delta_spurr, delta_ic, delta_vtb;
298 
299 	if (vcpu->kvm->arch.l1_ptcr == 0)
300 		return H_NOT_AVAILABLE;
301 
302 	if (MSR_TM_TRANSACTIONAL(vcpu->arch.shregs.msr))
303 		return H_BAD_MODE;
304 
305 	/* copy parameters in */
306 	hv_ptr = kvmppc_get_gpr(vcpu, 4);
307 	regs_ptr = kvmppc_get_gpr(vcpu, 5);
308 	kvm_vcpu_srcu_read_lock(vcpu);
309 	err = kvmhv_read_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
310 					      hv_ptr, regs_ptr);
311 	kvm_vcpu_srcu_read_unlock(vcpu);
312 	if (err)
313 		return H_PARAMETER;
314 
315 	if (kvmppc_need_byteswap(vcpu))
316 		byteswap_hv_regs(&l2_hv);
317 	if (l2_hv.version > HV_GUEST_STATE_VERSION)
318 		return H_P2;
319 
320 	if (kvmppc_need_byteswap(vcpu))
321 		byteswap_pt_regs(&l2_regs);
322 	if (l2_hv.vcpu_token >= NR_CPUS)
323 		return H_PARAMETER;
324 
325 	/*
326 	 * L1 must have set up a suspended state to enter the L2 in a
327 	 * transactional state, and only in that case. These have to be
328 	 * filtered out here to prevent causing a TM Bad Thing in the
329 	 * host HRFID. We could synthesize a TM Bad Thing back to the L1
330 	 * here but there doesn't seem like much point.
331 	 */
332 	if (MSR_TM_SUSPENDED(vcpu->arch.shregs.msr)) {
333 		if (!MSR_TM_ACTIVE(l2_regs.msr))
334 			return H_BAD_MODE;
335 	} else {
336 		if (l2_regs.msr & MSR_TS_MASK)
337 			return H_BAD_MODE;
338 		if (WARN_ON_ONCE(vcpu->arch.shregs.msr & MSR_TS_MASK))
339 			return H_BAD_MODE;
340 	}
341 
342 	/* translate lpid */
343 	l2 = kvmhv_get_nested(vcpu->kvm, l2_hv.lpid, true);
344 	if (!l2)
345 		return H_PARAMETER;
346 	if (!l2->l1_gr_to_hr) {
347 		mutex_lock(&l2->tlb_lock);
348 		kvmhv_update_ptbl_cache(l2);
349 		mutex_unlock(&l2->tlb_lock);
350 	}
351 
352 	/* save l1 values of things */
353 	vcpu->arch.regs.msr = vcpu->arch.shregs.msr;
354 	saved_l1_regs = vcpu->arch.regs;
355 	kvmhv_save_hv_regs(vcpu, &saved_l1_hv);
356 
357 	/* convert TB values/offsets to host (L0) values */
358 	hdec_exp = l2_hv.hdec_expiry - vc->tb_offset;
359 	vc->tb_offset += l2_hv.tb_offset;
360 	vcpu->arch.dec_expires += l2_hv.tb_offset;
361 
362 	/* set L1 state to L2 state */
363 	vcpu->arch.nested = l2;
364 	vcpu->arch.nested_vcpu_id = l2_hv.vcpu_token;
365 	vcpu->arch.nested_hfscr = l2_hv.hfscr;
366 	vcpu->arch.regs = l2_regs;
367 
368 	/* Guest must always run with ME enabled, HV disabled. */
369 	vcpu->arch.shregs.msr = (vcpu->arch.regs.msr | MSR_ME) & ~MSR_HV;
370 
371 	lpcr = l2_hv.lpcr;
372 	load_l2_hv_regs(vcpu, &l2_hv, &saved_l1_hv, &lpcr);
373 
374 	vcpu->arch.ret = RESUME_GUEST;
375 	vcpu->arch.trap = 0;
376 	do {
377 		r = kvmhv_run_single_vcpu(vcpu, hdec_exp, lpcr);
378 	} while (is_kvmppc_resume_guest(r));
379 
380 	/* save L2 state for return */
381 	l2_regs = vcpu->arch.regs;
382 	l2_regs.msr = vcpu->arch.shregs.msr;
383 	delta_purr = vcpu->arch.purr - l2_hv.purr;
384 	delta_spurr = vcpu->arch.spurr - l2_hv.spurr;
385 	delta_ic = vcpu->arch.ic - l2_hv.ic;
386 	delta_vtb = vc->vtb - l2_hv.vtb;
387 	save_hv_return_state(vcpu, &l2_hv);
388 
389 	/* restore L1 state */
390 	vcpu->arch.nested = NULL;
391 	vcpu->arch.regs = saved_l1_regs;
392 	vcpu->arch.shregs.msr = saved_l1_regs.msr & ~MSR_TS_MASK;
393 	/* set L1 MSR TS field according to L2 transaction state */
394 	if (l2_regs.msr & MSR_TS_MASK)
395 		vcpu->arch.shregs.msr |= MSR_TS_S;
396 	vc->tb_offset = saved_l1_hv.tb_offset;
397 	/* XXX: is this always the same delta as saved_l1_hv.tb_offset? */
398 	vcpu->arch.dec_expires -= l2_hv.tb_offset;
399 	restore_hv_regs(vcpu, &saved_l1_hv);
400 	vcpu->arch.purr += delta_purr;
401 	vcpu->arch.spurr += delta_spurr;
402 	vcpu->arch.ic += delta_ic;
403 	vc->vtb += delta_vtb;
404 
405 	kvmhv_put_nested(l2);
406 
407 	/* copy l2_hv_state and regs back to guest */
408 	if (kvmppc_need_byteswap(vcpu)) {
409 		byteswap_hv_regs(&l2_hv);
410 		byteswap_pt_regs(&l2_regs);
411 	}
412 	kvm_vcpu_srcu_read_lock(vcpu);
413 	err = kvmhv_write_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
414 					       hv_ptr, regs_ptr);
415 	kvm_vcpu_srcu_read_unlock(vcpu);
416 	if (err)
417 		return H_AUTHORITY;
418 
419 	if (r == -EINTR)
420 		return H_INTERRUPT;
421 
422 	if (vcpu->mmio_needed) {
423 		kvmhv_nested_mmio_needed(vcpu, regs_ptr);
424 		return H_TOO_HARD;
425 	}
426 
427 	return vcpu->arch.trap;
428 }
429 
kvmhv_nested_init(void)430 long kvmhv_nested_init(void)
431 {
432 	long int ptb_order;
433 	unsigned long ptcr;
434 	long rc;
435 
436 	if (!kvmhv_on_pseries())
437 		return 0;
438 	if (!radix_enabled())
439 		return -ENODEV;
440 
441 	/* Partition table entry is 1<<4 bytes in size, hence the 4. */
442 	ptb_order = KVM_MAX_NESTED_GUESTS_SHIFT + 4;
443 	/* Minimum partition table size is 1<<12 bytes */
444 	if (ptb_order < 12)
445 		ptb_order = 12;
446 	pseries_partition_tb = kmalloc(sizeof(struct patb_entry) << ptb_order,
447 				       GFP_KERNEL);
448 	if (!pseries_partition_tb) {
449 		pr_err("kvm-hv: failed to allocated nested partition table\n");
450 		return -ENOMEM;
451 	}
452 
453 	ptcr = __pa(pseries_partition_tb) | (ptb_order - 12);
454 	rc = plpar_hcall_norets(H_SET_PARTITION_TABLE, ptcr);
455 	if (rc != H_SUCCESS) {
456 		pr_err("kvm-hv: Parent hypervisor does not support nesting (rc=%ld)\n",
457 		       rc);
458 		kfree(pseries_partition_tb);
459 		pseries_partition_tb = NULL;
460 		return -ENODEV;
461 	}
462 
463 	return 0;
464 }
465 
kvmhv_nested_exit(void)466 void kvmhv_nested_exit(void)
467 {
468 	/*
469 	 * N.B. the kvmhv_on_pseries() test is there because it enables
470 	 * the compiler to remove the call to plpar_hcall_norets()
471 	 * when CONFIG_PPC_PSERIES=n.
472 	 */
473 	if (kvmhv_on_pseries() && pseries_partition_tb) {
474 		plpar_hcall_norets(H_SET_PARTITION_TABLE, 0);
475 		kfree(pseries_partition_tb);
476 		pseries_partition_tb = NULL;
477 	}
478 }
479 
kvmhv_flush_lpid(unsigned int lpid)480 static void kvmhv_flush_lpid(unsigned int lpid)
481 {
482 	long rc;
483 
484 	if (!kvmhv_on_pseries()) {
485 		radix__flush_all_lpid(lpid);
486 		return;
487 	}
488 
489 	if (!firmware_has_feature(FW_FEATURE_RPT_INVALIDATE))
490 		rc = plpar_hcall_norets(H_TLB_INVALIDATE, H_TLBIE_P1_ENC(2, 0, 1),
491 					lpid, TLBIEL_INVAL_SET_LPID);
492 	else
493 		rc = pseries_rpt_invalidate(lpid, H_RPTI_TARGET_CMMU,
494 					    H_RPTI_TYPE_NESTED |
495 					    H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
496 					    H_RPTI_TYPE_PAT,
497 					    H_RPTI_PAGE_ALL, 0, -1UL);
498 	if (rc)
499 		pr_err("KVM: TLB LPID invalidation hcall failed, rc=%ld\n", rc);
500 }
501 
kvmhv_set_ptbl_entry(unsigned int lpid,u64 dw0,u64 dw1)502 void kvmhv_set_ptbl_entry(unsigned int lpid, u64 dw0, u64 dw1)
503 {
504 	if (!kvmhv_on_pseries()) {
505 		mmu_partition_table_set_entry(lpid, dw0, dw1, true);
506 		return;
507 	}
508 
509 	pseries_partition_tb[lpid].patb0 = cpu_to_be64(dw0);
510 	pseries_partition_tb[lpid].patb1 = cpu_to_be64(dw1);
511 	/* L0 will do the necessary barriers */
512 	kvmhv_flush_lpid(lpid);
513 }
514 
kvmhv_set_nested_ptbl(struct kvm_nested_guest * gp)515 static void kvmhv_set_nested_ptbl(struct kvm_nested_guest *gp)
516 {
517 	unsigned long dw0;
518 
519 	dw0 = PATB_HR | radix__get_tree_size() |
520 		__pa(gp->shadow_pgtable) | RADIX_PGD_INDEX_SIZE;
521 	kvmhv_set_ptbl_entry(gp->shadow_lpid, dw0, gp->process_table);
522 }
523 
524 /*
525  * Handle the H_SET_PARTITION_TABLE hcall.
526  * r4 = guest real address of partition table + log_2(size) - 12
527  * (formatted as for the PTCR).
528  */
kvmhv_set_partition_table(struct kvm_vcpu * vcpu)529 long kvmhv_set_partition_table(struct kvm_vcpu *vcpu)
530 {
531 	struct kvm *kvm = vcpu->kvm;
532 	unsigned long ptcr = kvmppc_get_gpr(vcpu, 4);
533 	int srcu_idx;
534 	long ret = H_SUCCESS;
535 
536 	srcu_idx = srcu_read_lock(&kvm->srcu);
537 	/* Check partition size and base address. */
538 	if ((ptcr & PRTS_MASK) + 12 - 4 > KVM_MAX_NESTED_GUESTS_SHIFT ||
539 	    !kvm_is_visible_gfn(vcpu->kvm, (ptcr & PRTB_MASK) >> PAGE_SHIFT))
540 		ret = H_PARAMETER;
541 	srcu_read_unlock(&kvm->srcu, srcu_idx);
542 	if (ret == H_SUCCESS)
543 		kvm->arch.l1_ptcr = ptcr;
544 
545 	return ret;
546 }
547 
548 /*
549  * Handle the H_COPY_TOFROM_GUEST hcall.
550  * r4 = L1 lpid of nested guest
551  * r5 = pid
552  * r6 = eaddr to access
553  * r7 = to buffer (L1 gpa)
554  * r8 = from buffer (L1 gpa)
555  * r9 = n bytes to copy
556  */
kvmhv_copy_tofrom_guest_nested(struct kvm_vcpu * vcpu)557 long kvmhv_copy_tofrom_guest_nested(struct kvm_vcpu *vcpu)
558 {
559 	struct kvm_nested_guest *gp;
560 	int l1_lpid = kvmppc_get_gpr(vcpu, 4);
561 	int pid = kvmppc_get_gpr(vcpu, 5);
562 	gva_t eaddr = kvmppc_get_gpr(vcpu, 6);
563 	gpa_t gp_to = (gpa_t) kvmppc_get_gpr(vcpu, 7);
564 	gpa_t gp_from = (gpa_t) kvmppc_get_gpr(vcpu, 8);
565 	void *buf;
566 	unsigned long n = kvmppc_get_gpr(vcpu, 9);
567 	bool is_load = !!gp_to;
568 	long rc;
569 
570 	if (gp_to && gp_from) /* One must be NULL to determine the direction */
571 		return H_PARAMETER;
572 
573 	if (eaddr & (0xFFFUL << 52))
574 		return H_PARAMETER;
575 
576 	buf = kzalloc(n, GFP_KERNEL | __GFP_NOWARN);
577 	if (!buf)
578 		return H_NO_MEM;
579 
580 	gp = kvmhv_get_nested(vcpu->kvm, l1_lpid, false);
581 	if (!gp) {
582 		rc = H_PARAMETER;
583 		goto out_free;
584 	}
585 
586 	mutex_lock(&gp->tlb_lock);
587 
588 	if (is_load) {
589 		/* Load from the nested guest into our buffer */
590 		rc = __kvmhv_copy_tofrom_guest_radix(gp->shadow_lpid, pid,
591 						     eaddr, buf, NULL, n);
592 		if (rc)
593 			goto not_found;
594 
595 		/* Write what was loaded into our buffer back to the L1 guest */
596 		kvm_vcpu_srcu_read_lock(vcpu);
597 		rc = kvm_vcpu_write_guest(vcpu, gp_to, buf, n);
598 		kvm_vcpu_srcu_read_unlock(vcpu);
599 		if (rc)
600 			goto not_found;
601 	} else {
602 		/* Load the data to be stored from the L1 guest into our buf */
603 		kvm_vcpu_srcu_read_lock(vcpu);
604 		rc = kvm_vcpu_read_guest(vcpu, gp_from, buf, n);
605 		kvm_vcpu_srcu_read_unlock(vcpu);
606 		if (rc)
607 			goto not_found;
608 
609 		/* Store from our buffer into the nested guest */
610 		rc = __kvmhv_copy_tofrom_guest_radix(gp->shadow_lpid, pid,
611 						     eaddr, NULL, buf, n);
612 		if (rc)
613 			goto not_found;
614 	}
615 
616 out_unlock:
617 	mutex_unlock(&gp->tlb_lock);
618 	kvmhv_put_nested(gp);
619 out_free:
620 	kfree(buf);
621 	return rc;
622 not_found:
623 	rc = H_NOT_FOUND;
624 	goto out_unlock;
625 }
626 
627 /*
628  * Reload the partition table entry for a guest.
629  * Caller must hold gp->tlb_lock.
630  */
kvmhv_update_ptbl_cache(struct kvm_nested_guest * gp)631 static void kvmhv_update_ptbl_cache(struct kvm_nested_guest *gp)
632 {
633 	int ret;
634 	struct patb_entry ptbl_entry;
635 	unsigned long ptbl_addr;
636 	struct kvm *kvm = gp->l1_host;
637 
638 	ret = -EFAULT;
639 	ptbl_addr = (kvm->arch.l1_ptcr & PRTB_MASK) + (gp->l1_lpid << 4);
640 	if (gp->l1_lpid < (1ul << ((kvm->arch.l1_ptcr & PRTS_MASK) + 12 - 4))) {
641 		int srcu_idx = srcu_read_lock(&kvm->srcu);
642 		ret = kvm_read_guest(kvm, ptbl_addr,
643 				     &ptbl_entry, sizeof(ptbl_entry));
644 		srcu_read_unlock(&kvm->srcu, srcu_idx);
645 	}
646 	if (ret) {
647 		gp->l1_gr_to_hr = 0;
648 		gp->process_table = 0;
649 	} else {
650 		gp->l1_gr_to_hr = be64_to_cpu(ptbl_entry.patb0);
651 		gp->process_table = be64_to_cpu(ptbl_entry.patb1);
652 	}
653 	kvmhv_set_nested_ptbl(gp);
654 }
655 
kvmhv_vm_nested_init(struct kvm * kvm)656 void kvmhv_vm_nested_init(struct kvm *kvm)
657 {
658 	idr_init(&kvm->arch.kvm_nested_guest_idr);
659 }
660 
__find_nested(struct kvm * kvm,int lpid)661 static struct kvm_nested_guest *__find_nested(struct kvm *kvm, int lpid)
662 {
663 	return idr_find(&kvm->arch.kvm_nested_guest_idr, lpid);
664 }
665 
__prealloc_nested(struct kvm * kvm,int lpid)666 static bool __prealloc_nested(struct kvm *kvm, int lpid)
667 {
668 	if (idr_alloc(&kvm->arch.kvm_nested_guest_idr,
669 				NULL, lpid, lpid + 1, GFP_KERNEL) != lpid)
670 		return false;
671 	return true;
672 }
673 
__add_nested(struct kvm * kvm,int lpid,struct kvm_nested_guest * gp)674 static void __add_nested(struct kvm *kvm, int lpid, struct kvm_nested_guest *gp)
675 {
676 	if (idr_replace(&kvm->arch.kvm_nested_guest_idr, gp, lpid))
677 		WARN_ON(1);
678 }
679 
__remove_nested(struct kvm * kvm,int lpid)680 static void __remove_nested(struct kvm *kvm, int lpid)
681 {
682 	idr_remove(&kvm->arch.kvm_nested_guest_idr, lpid);
683 }
684 
kvmhv_alloc_nested(struct kvm * kvm,unsigned int lpid)685 static struct kvm_nested_guest *kvmhv_alloc_nested(struct kvm *kvm, unsigned int lpid)
686 {
687 	struct kvm_nested_guest *gp;
688 	long shadow_lpid;
689 
690 	gp = kzalloc(sizeof(*gp), GFP_KERNEL);
691 	if (!gp)
692 		return NULL;
693 	gp->l1_host = kvm;
694 	gp->l1_lpid = lpid;
695 	mutex_init(&gp->tlb_lock);
696 	gp->shadow_pgtable = pgd_alloc(kvm->mm);
697 	if (!gp->shadow_pgtable)
698 		goto out_free;
699 	shadow_lpid = kvmppc_alloc_lpid();
700 	if (shadow_lpid < 0)
701 		goto out_free2;
702 	gp->shadow_lpid = shadow_lpid;
703 	gp->radix = 1;
704 
705 	memset(gp->prev_cpu, -1, sizeof(gp->prev_cpu));
706 
707 	return gp;
708 
709  out_free2:
710 	pgd_free(kvm->mm, gp->shadow_pgtable);
711  out_free:
712 	kfree(gp);
713 	return NULL;
714 }
715 
716 /*
717  * Free up any resources allocated for a nested guest.
718  */
kvmhv_release_nested(struct kvm_nested_guest * gp)719 static void kvmhv_release_nested(struct kvm_nested_guest *gp)
720 {
721 	struct kvm *kvm = gp->l1_host;
722 
723 	if (gp->shadow_pgtable) {
724 		/*
725 		 * No vcpu is using this struct and no call to
726 		 * kvmhv_get_nested can find this struct,
727 		 * so we don't need to hold kvm->mmu_lock.
728 		 */
729 		kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable,
730 					  gp->shadow_lpid);
731 		pgd_free(kvm->mm, gp->shadow_pgtable);
732 	}
733 	kvmhv_set_ptbl_entry(gp->shadow_lpid, 0, 0);
734 	kvmppc_free_lpid(gp->shadow_lpid);
735 	kfree(gp);
736 }
737 
kvmhv_remove_nested(struct kvm_nested_guest * gp)738 static void kvmhv_remove_nested(struct kvm_nested_guest *gp)
739 {
740 	struct kvm *kvm = gp->l1_host;
741 	int lpid = gp->l1_lpid;
742 	long ref;
743 
744 	spin_lock(&kvm->mmu_lock);
745 	if (gp == __find_nested(kvm, lpid)) {
746 		__remove_nested(kvm, lpid);
747 		--gp->refcnt;
748 	}
749 	ref = gp->refcnt;
750 	spin_unlock(&kvm->mmu_lock);
751 	if (ref == 0)
752 		kvmhv_release_nested(gp);
753 }
754 
755 /*
756  * Free up all nested resources allocated for this guest.
757  * This is called with no vcpus of the guest running, when
758  * switching the guest to HPT mode or when destroying the
759  * guest.
760  */
kvmhv_release_all_nested(struct kvm * kvm)761 void kvmhv_release_all_nested(struct kvm *kvm)
762 {
763 	int lpid;
764 	struct kvm_nested_guest *gp;
765 	struct kvm_nested_guest *freelist = NULL;
766 	struct kvm_memory_slot *memslot;
767 	int srcu_idx, bkt;
768 
769 	spin_lock(&kvm->mmu_lock);
770 	idr_for_each_entry(&kvm->arch.kvm_nested_guest_idr, gp, lpid) {
771 		__remove_nested(kvm, lpid);
772 		if (--gp->refcnt == 0) {
773 			gp->next = freelist;
774 			freelist = gp;
775 		}
776 	}
777 	idr_destroy(&kvm->arch.kvm_nested_guest_idr);
778 	/* idr is empty and may be reused at this point */
779 	spin_unlock(&kvm->mmu_lock);
780 	while ((gp = freelist) != NULL) {
781 		freelist = gp->next;
782 		kvmhv_release_nested(gp);
783 	}
784 
785 	srcu_idx = srcu_read_lock(&kvm->srcu);
786 	kvm_for_each_memslot(memslot, bkt, kvm_memslots(kvm))
787 		kvmhv_free_memslot_nest_rmap(memslot);
788 	srcu_read_unlock(&kvm->srcu, srcu_idx);
789 }
790 
791 /* caller must hold gp->tlb_lock */
kvmhv_flush_nested(struct kvm_nested_guest * gp)792 static void kvmhv_flush_nested(struct kvm_nested_guest *gp)
793 {
794 	struct kvm *kvm = gp->l1_host;
795 
796 	spin_lock(&kvm->mmu_lock);
797 	kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable, gp->shadow_lpid);
798 	spin_unlock(&kvm->mmu_lock);
799 	kvmhv_flush_lpid(gp->shadow_lpid);
800 	kvmhv_update_ptbl_cache(gp);
801 	if (gp->l1_gr_to_hr == 0)
802 		kvmhv_remove_nested(gp);
803 }
804 
kvmhv_get_nested(struct kvm * kvm,int l1_lpid,bool create)805 struct kvm_nested_guest *kvmhv_get_nested(struct kvm *kvm, int l1_lpid,
806 					  bool create)
807 {
808 	struct kvm_nested_guest *gp, *newgp;
809 
810 	if (l1_lpid >= (1ul << ((kvm->arch.l1_ptcr & PRTS_MASK) + 12 - 4)))
811 		return NULL;
812 
813 	spin_lock(&kvm->mmu_lock);
814 	gp = __find_nested(kvm, l1_lpid);
815 	if (gp)
816 		++gp->refcnt;
817 	spin_unlock(&kvm->mmu_lock);
818 
819 	if (gp || !create)
820 		return gp;
821 
822 	newgp = kvmhv_alloc_nested(kvm, l1_lpid);
823 	if (!newgp)
824 		return NULL;
825 
826 	if (!__prealloc_nested(kvm, l1_lpid)) {
827 		kvmhv_release_nested(newgp);
828 		return NULL;
829 	}
830 
831 	spin_lock(&kvm->mmu_lock);
832 	gp = __find_nested(kvm, l1_lpid);
833 	if (!gp) {
834 		__add_nested(kvm, l1_lpid, newgp);
835 		++newgp->refcnt;
836 		gp = newgp;
837 		newgp = NULL;
838 	}
839 	++gp->refcnt;
840 	spin_unlock(&kvm->mmu_lock);
841 
842 	if (newgp)
843 		kvmhv_release_nested(newgp);
844 
845 	return gp;
846 }
847 
kvmhv_put_nested(struct kvm_nested_guest * gp)848 void kvmhv_put_nested(struct kvm_nested_guest *gp)
849 {
850 	struct kvm *kvm = gp->l1_host;
851 	long ref;
852 
853 	spin_lock(&kvm->mmu_lock);
854 	ref = --gp->refcnt;
855 	spin_unlock(&kvm->mmu_lock);
856 	if (ref == 0)
857 		kvmhv_release_nested(gp);
858 }
859 
find_kvm_nested_guest_pte(struct kvm * kvm,unsigned long lpid,unsigned long ea,unsigned * hshift)860 pte_t *find_kvm_nested_guest_pte(struct kvm *kvm, unsigned long lpid,
861 				 unsigned long ea, unsigned *hshift)
862 {
863 	struct kvm_nested_guest *gp;
864 	pte_t *pte;
865 
866 	gp = __find_nested(kvm, lpid);
867 	if (!gp)
868 		return NULL;
869 
870 	VM_WARN(!spin_is_locked(&kvm->mmu_lock),
871 		"%s called with kvm mmu_lock not held \n", __func__);
872 	pte = __find_linux_pte(gp->shadow_pgtable, ea, NULL, hshift);
873 
874 	return pte;
875 }
876 
kvmhv_n_rmap_is_equal(u64 rmap_1,u64 rmap_2)877 static inline bool kvmhv_n_rmap_is_equal(u64 rmap_1, u64 rmap_2)
878 {
879 	return !((rmap_1 ^ rmap_2) & (RMAP_NESTED_LPID_MASK |
880 				       RMAP_NESTED_GPA_MASK));
881 }
882 
kvmhv_insert_nest_rmap(struct kvm * kvm,unsigned long * rmapp,struct rmap_nested ** n_rmap)883 void kvmhv_insert_nest_rmap(struct kvm *kvm, unsigned long *rmapp,
884 			    struct rmap_nested **n_rmap)
885 {
886 	struct llist_node *entry = ((struct llist_head *) rmapp)->first;
887 	struct rmap_nested *cursor;
888 	u64 rmap, new_rmap = (*n_rmap)->rmap;
889 
890 	/* Are there any existing entries? */
891 	if (!(*rmapp)) {
892 		/* No -> use the rmap as a single entry */
893 		*rmapp = new_rmap | RMAP_NESTED_IS_SINGLE_ENTRY;
894 		return;
895 	}
896 
897 	/* Do any entries match what we're trying to insert? */
898 	for_each_nest_rmap_safe(cursor, entry, &rmap) {
899 		if (kvmhv_n_rmap_is_equal(rmap, new_rmap))
900 			return;
901 	}
902 
903 	/* Do we need to create a list or just add the new entry? */
904 	rmap = *rmapp;
905 	if (rmap & RMAP_NESTED_IS_SINGLE_ENTRY) /* Not previously a list */
906 		*rmapp = 0UL;
907 	llist_add(&((*n_rmap)->list), (struct llist_head *) rmapp);
908 	if (rmap & RMAP_NESTED_IS_SINGLE_ENTRY) /* Not previously a list */
909 		(*n_rmap)->list.next = (struct llist_node *) rmap;
910 
911 	/* Set NULL so not freed by caller */
912 	*n_rmap = NULL;
913 }
914 
kvmhv_update_nest_rmap_rc(struct kvm * kvm,u64 n_rmap,unsigned long clr,unsigned long set,unsigned long hpa,unsigned long mask)915 static void kvmhv_update_nest_rmap_rc(struct kvm *kvm, u64 n_rmap,
916 				      unsigned long clr, unsigned long set,
917 				      unsigned long hpa, unsigned long mask)
918 {
919 	unsigned long gpa;
920 	unsigned int shift, lpid;
921 	pte_t *ptep;
922 
923 	gpa = n_rmap & RMAP_NESTED_GPA_MASK;
924 	lpid = (n_rmap & RMAP_NESTED_LPID_MASK) >> RMAP_NESTED_LPID_SHIFT;
925 
926 	/* Find the pte */
927 	ptep = find_kvm_nested_guest_pte(kvm, lpid, gpa, &shift);
928 	/*
929 	 * If the pte is present and the pfn is still the same, update the pte.
930 	 * If the pfn has changed then this is a stale rmap entry, the nested
931 	 * gpa actually points somewhere else now, and there is nothing to do.
932 	 * XXX A future optimisation would be to remove the rmap entry here.
933 	 */
934 	if (ptep && pte_present(*ptep) && ((pte_val(*ptep) & mask) == hpa)) {
935 		__radix_pte_update(ptep, clr, set);
936 		kvmppc_radix_tlbie_page(kvm, gpa, shift, lpid);
937 	}
938 }
939 
940 /*
941  * For a given list of rmap entries, update the rc bits in all ptes in shadow
942  * page tables for nested guests which are referenced by the rmap list.
943  */
kvmhv_update_nest_rmap_rc_list(struct kvm * kvm,unsigned long * rmapp,unsigned long clr,unsigned long set,unsigned long hpa,unsigned long nbytes)944 void kvmhv_update_nest_rmap_rc_list(struct kvm *kvm, unsigned long *rmapp,
945 				    unsigned long clr, unsigned long set,
946 				    unsigned long hpa, unsigned long nbytes)
947 {
948 	struct llist_node *entry = ((struct llist_head *) rmapp)->first;
949 	struct rmap_nested *cursor;
950 	unsigned long rmap, mask;
951 
952 	if ((clr | set) & ~(_PAGE_DIRTY | _PAGE_ACCESSED))
953 		return;
954 
955 	mask = PTE_RPN_MASK & ~(nbytes - 1);
956 	hpa &= mask;
957 
958 	for_each_nest_rmap_safe(cursor, entry, &rmap)
959 		kvmhv_update_nest_rmap_rc(kvm, rmap, clr, set, hpa, mask);
960 }
961 
kvmhv_remove_nest_rmap(struct kvm * kvm,u64 n_rmap,unsigned long hpa,unsigned long mask)962 static void kvmhv_remove_nest_rmap(struct kvm *kvm, u64 n_rmap,
963 				   unsigned long hpa, unsigned long mask)
964 {
965 	struct kvm_nested_guest *gp;
966 	unsigned long gpa;
967 	unsigned int shift, lpid;
968 	pte_t *ptep;
969 
970 	gpa = n_rmap & RMAP_NESTED_GPA_MASK;
971 	lpid = (n_rmap & RMAP_NESTED_LPID_MASK) >> RMAP_NESTED_LPID_SHIFT;
972 	gp = __find_nested(kvm, lpid);
973 	if (!gp)
974 		return;
975 
976 	/* Find and invalidate the pte */
977 	ptep = find_kvm_nested_guest_pte(kvm, lpid, gpa, &shift);
978 	/* Don't spuriously invalidate ptes if the pfn has changed */
979 	if (ptep && pte_present(*ptep) && ((pte_val(*ptep) & mask) == hpa))
980 		kvmppc_unmap_pte(kvm, ptep, gpa, shift, NULL, gp->shadow_lpid);
981 }
982 
kvmhv_remove_nest_rmap_list(struct kvm * kvm,unsigned long * rmapp,unsigned long hpa,unsigned long mask)983 static void kvmhv_remove_nest_rmap_list(struct kvm *kvm, unsigned long *rmapp,
984 					unsigned long hpa, unsigned long mask)
985 {
986 	struct llist_node *entry = llist_del_all((struct llist_head *) rmapp);
987 	struct rmap_nested *cursor;
988 	unsigned long rmap;
989 
990 	for_each_nest_rmap_safe(cursor, entry, &rmap) {
991 		kvmhv_remove_nest_rmap(kvm, rmap, hpa, mask);
992 		kfree(cursor);
993 	}
994 }
995 
996 /* called with kvm->mmu_lock held */
kvmhv_remove_nest_rmap_range(struct kvm * kvm,const struct kvm_memory_slot * memslot,unsigned long gpa,unsigned long hpa,unsigned long nbytes)997 void kvmhv_remove_nest_rmap_range(struct kvm *kvm,
998 				  const struct kvm_memory_slot *memslot,
999 				  unsigned long gpa, unsigned long hpa,
1000 				  unsigned long nbytes)
1001 {
1002 	unsigned long gfn, end_gfn;
1003 	unsigned long addr_mask;
1004 
1005 	if (!memslot)
1006 		return;
1007 	gfn = (gpa >> PAGE_SHIFT) - memslot->base_gfn;
1008 	end_gfn = gfn + (nbytes >> PAGE_SHIFT);
1009 
1010 	addr_mask = PTE_RPN_MASK & ~(nbytes - 1);
1011 	hpa &= addr_mask;
1012 
1013 	for (; gfn < end_gfn; gfn++) {
1014 		unsigned long *rmap = &memslot->arch.rmap[gfn];
1015 		kvmhv_remove_nest_rmap_list(kvm, rmap, hpa, addr_mask);
1016 	}
1017 }
1018 
kvmhv_free_memslot_nest_rmap(struct kvm_memory_slot * free)1019 static void kvmhv_free_memslot_nest_rmap(struct kvm_memory_slot *free)
1020 {
1021 	unsigned long page;
1022 
1023 	for (page = 0; page < free->npages; page++) {
1024 		unsigned long rmap, *rmapp = &free->arch.rmap[page];
1025 		struct rmap_nested *cursor;
1026 		struct llist_node *entry;
1027 
1028 		entry = llist_del_all((struct llist_head *) rmapp);
1029 		for_each_nest_rmap_safe(cursor, entry, &rmap)
1030 			kfree(cursor);
1031 	}
1032 }
1033 
kvmhv_invalidate_shadow_pte(struct kvm_vcpu * vcpu,struct kvm_nested_guest * gp,long gpa,int * shift_ret)1034 static bool kvmhv_invalidate_shadow_pte(struct kvm_vcpu *vcpu,
1035 					struct kvm_nested_guest *gp,
1036 					long gpa, int *shift_ret)
1037 {
1038 	struct kvm *kvm = vcpu->kvm;
1039 	bool ret = false;
1040 	pte_t *ptep;
1041 	int shift;
1042 
1043 	spin_lock(&kvm->mmu_lock);
1044 	ptep = find_kvm_nested_guest_pte(kvm, gp->l1_lpid, gpa, &shift);
1045 	if (!shift)
1046 		shift = PAGE_SHIFT;
1047 	if (ptep && pte_present(*ptep)) {
1048 		kvmppc_unmap_pte(kvm, ptep, gpa, shift, NULL, gp->shadow_lpid);
1049 		ret = true;
1050 	}
1051 	spin_unlock(&kvm->mmu_lock);
1052 
1053 	if (shift_ret)
1054 		*shift_ret = shift;
1055 	return ret;
1056 }
1057 
get_ric(unsigned int instr)1058 static inline int get_ric(unsigned int instr)
1059 {
1060 	return (instr >> 18) & 0x3;
1061 }
1062 
get_prs(unsigned int instr)1063 static inline int get_prs(unsigned int instr)
1064 {
1065 	return (instr >> 17) & 0x1;
1066 }
1067 
get_r(unsigned int instr)1068 static inline int get_r(unsigned int instr)
1069 {
1070 	return (instr >> 16) & 0x1;
1071 }
1072 
get_lpid(unsigned long r_val)1073 static inline int get_lpid(unsigned long r_val)
1074 {
1075 	return r_val & 0xffffffff;
1076 }
1077 
get_is(unsigned long r_val)1078 static inline int get_is(unsigned long r_val)
1079 {
1080 	return (r_val >> 10) & 0x3;
1081 }
1082 
get_ap(unsigned long r_val)1083 static inline int get_ap(unsigned long r_val)
1084 {
1085 	return (r_val >> 5) & 0x7;
1086 }
1087 
get_epn(unsigned long r_val)1088 static inline long get_epn(unsigned long r_val)
1089 {
1090 	return r_val >> 12;
1091 }
1092 
kvmhv_emulate_tlbie_tlb_addr(struct kvm_vcpu * vcpu,int lpid,int ap,long epn)1093 static int kvmhv_emulate_tlbie_tlb_addr(struct kvm_vcpu *vcpu, int lpid,
1094 					int ap, long epn)
1095 {
1096 	struct kvm *kvm = vcpu->kvm;
1097 	struct kvm_nested_guest *gp;
1098 	long npages;
1099 	int shift, shadow_shift;
1100 	unsigned long addr;
1101 
1102 	shift = ap_to_shift(ap);
1103 	addr = epn << 12;
1104 	if (shift < 0)
1105 		/* Invalid ap encoding */
1106 		return -EINVAL;
1107 
1108 	addr &= ~((1UL << shift) - 1);
1109 	npages = 1UL << (shift - PAGE_SHIFT);
1110 
1111 	gp = kvmhv_get_nested(kvm, lpid, false);
1112 	if (!gp) /* No such guest -> nothing to do */
1113 		return 0;
1114 	mutex_lock(&gp->tlb_lock);
1115 
1116 	/* There may be more than one host page backing this single guest pte */
1117 	do {
1118 		kvmhv_invalidate_shadow_pte(vcpu, gp, addr, &shadow_shift);
1119 
1120 		npages -= 1UL << (shadow_shift - PAGE_SHIFT);
1121 		addr += 1UL << shadow_shift;
1122 	} while (npages > 0);
1123 
1124 	mutex_unlock(&gp->tlb_lock);
1125 	kvmhv_put_nested(gp);
1126 	return 0;
1127 }
1128 
kvmhv_emulate_tlbie_lpid(struct kvm_vcpu * vcpu,struct kvm_nested_guest * gp,int ric)1129 static void kvmhv_emulate_tlbie_lpid(struct kvm_vcpu *vcpu,
1130 				     struct kvm_nested_guest *gp, int ric)
1131 {
1132 	struct kvm *kvm = vcpu->kvm;
1133 
1134 	mutex_lock(&gp->tlb_lock);
1135 	switch (ric) {
1136 	case 0:
1137 		/* Invalidate TLB */
1138 		spin_lock(&kvm->mmu_lock);
1139 		kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable,
1140 					  gp->shadow_lpid);
1141 		kvmhv_flush_lpid(gp->shadow_lpid);
1142 		spin_unlock(&kvm->mmu_lock);
1143 		break;
1144 	case 1:
1145 		/*
1146 		 * Invalidate PWC
1147 		 * We don't cache this -> nothing to do
1148 		 */
1149 		break;
1150 	case 2:
1151 		/* Invalidate TLB, PWC and caching of partition table entries */
1152 		kvmhv_flush_nested(gp);
1153 		break;
1154 	default:
1155 		break;
1156 	}
1157 	mutex_unlock(&gp->tlb_lock);
1158 }
1159 
kvmhv_emulate_tlbie_all_lpid(struct kvm_vcpu * vcpu,int ric)1160 static void kvmhv_emulate_tlbie_all_lpid(struct kvm_vcpu *vcpu, int ric)
1161 {
1162 	struct kvm *kvm = vcpu->kvm;
1163 	struct kvm_nested_guest *gp;
1164 	int lpid;
1165 
1166 	spin_lock(&kvm->mmu_lock);
1167 	idr_for_each_entry(&kvm->arch.kvm_nested_guest_idr, gp, lpid) {
1168 		spin_unlock(&kvm->mmu_lock);
1169 		kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);
1170 		spin_lock(&kvm->mmu_lock);
1171 	}
1172 	spin_unlock(&kvm->mmu_lock);
1173 }
1174 
kvmhv_emulate_priv_tlbie(struct kvm_vcpu * vcpu,unsigned int instr,unsigned long rsval,unsigned long rbval)1175 static int kvmhv_emulate_priv_tlbie(struct kvm_vcpu *vcpu, unsigned int instr,
1176 				    unsigned long rsval, unsigned long rbval)
1177 {
1178 	struct kvm *kvm = vcpu->kvm;
1179 	struct kvm_nested_guest *gp;
1180 	int r, ric, prs, is, ap;
1181 	int lpid;
1182 	long epn;
1183 	int ret = 0;
1184 
1185 	ric = get_ric(instr);
1186 	prs = get_prs(instr);
1187 	r = get_r(instr);
1188 	lpid = get_lpid(rsval);
1189 	is = get_is(rbval);
1190 
1191 	/*
1192 	 * These cases are invalid and are not handled:
1193 	 * r   != 1 -> Only radix supported
1194 	 * prs == 1 -> Not HV privileged
1195 	 * ric == 3 -> No cluster bombs for radix
1196 	 * is  == 1 -> Partition scoped translations not associated with pid
1197 	 * (!is) && (ric == 1 || ric == 2) -> Not supported by ISA
1198 	 */
1199 	if ((!r) || (prs) || (ric == 3) || (is == 1) ||
1200 	    ((!is) && (ric == 1 || ric == 2)))
1201 		return -EINVAL;
1202 
1203 	switch (is) {
1204 	case 0:
1205 		/*
1206 		 * We know ric == 0
1207 		 * Invalidate TLB for a given target address
1208 		 */
1209 		epn = get_epn(rbval);
1210 		ap = get_ap(rbval);
1211 		ret = kvmhv_emulate_tlbie_tlb_addr(vcpu, lpid, ap, epn);
1212 		break;
1213 	case 2:
1214 		/* Invalidate matching LPID */
1215 		gp = kvmhv_get_nested(kvm, lpid, false);
1216 		if (gp) {
1217 			kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);
1218 			kvmhv_put_nested(gp);
1219 		}
1220 		break;
1221 	case 3:
1222 		/* Invalidate ALL LPIDs */
1223 		kvmhv_emulate_tlbie_all_lpid(vcpu, ric);
1224 		break;
1225 	default:
1226 		ret = -EINVAL;
1227 		break;
1228 	}
1229 
1230 	return ret;
1231 }
1232 
1233 /*
1234  * This handles the H_TLB_INVALIDATE hcall.
1235  * Parameters are (r4) tlbie instruction code, (r5) rS contents,
1236  * (r6) rB contents.
1237  */
kvmhv_do_nested_tlbie(struct kvm_vcpu * vcpu)1238 long kvmhv_do_nested_tlbie(struct kvm_vcpu *vcpu)
1239 {
1240 	int ret;
1241 
1242 	ret = kvmhv_emulate_priv_tlbie(vcpu, kvmppc_get_gpr(vcpu, 4),
1243 			kvmppc_get_gpr(vcpu, 5), kvmppc_get_gpr(vcpu, 6));
1244 	if (ret)
1245 		return H_PARAMETER;
1246 	return H_SUCCESS;
1247 }
1248 
do_tlb_invalidate_nested_all(struct kvm_vcpu * vcpu,unsigned long lpid,unsigned long ric)1249 static long do_tlb_invalidate_nested_all(struct kvm_vcpu *vcpu,
1250 					 unsigned long lpid, unsigned long ric)
1251 {
1252 	struct kvm *kvm = vcpu->kvm;
1253 	struct kvm_nested_guest *gp;
1254 
1255 	gp = kvmhv_get_nested(kvm, lpid, false);
1256 	if (gp) {
1257 		kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);
1258 		kvmhv_put_nested(gp);
1259 	}
1260 	return H_SUCCESS;
1261 }
1262 
1263 /*
1264  * Number of pages above which we invalidate the entire LPID rather than
1265  * flush individual pages.
1266  */
1267 static unsigned long tlb_range_flush_page_ceiling __read_mostly = 33;
1268 
do_tlb_invalidate_nested_tlb(struct kvm_vcpu * vcpu,unsigned long lpid,unsigned long pg_sizes,unsigned long start,unsigned long end)1269 static long do_tlb_invalidate_nested_tlb(struct kvm_vcpu *vcpu,
1270 					 unsigned long lpid,
1271 					 unsigned long pg_sizes,
1272 					 unsigned long start,
1273 					 unsigned long end)
1274 {
1275 	int ret = H_P4;
1276 	unsigned long addr, nr_pages;
1277 	struct mmu_psize_def *def;
1278 	unsigned long psize, ap, page_size;
1279 	bool flush_lpid;
1280 
1281 	for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
1282 		def = &mmu_psize_defs[psize];
1283 		if (!(pg_sizes & def->h_rpt_pgsize))
1284 			continue;
1285 
1286 		nr_pages = (end - start) >> def->shift;
1287 		flush_lpid = nr_pages > tlb_range_flush_page_ceiling;
1288 		if (flush_lpid)
1289 			return do_tlb_invalidate_nested_all(vcpu, lpid,
1290 							RIC_FLUSH_TLB);
1291 		addr = start;
1292 		ap = mmu_get_ap(psize);
1293 		page_size = 1UL << def->shift;
1294 		do {
1295 			ret = kvmhv_emulate_tlbie_tlb_addr(vcpu, lpid, ap,
1296 						   get_epn(addr));
1297 			if (ret)
1298 				return H_P4;
1299 			addr += page_size;
1300 		} while (addr < end);
1301 	}
1302 	return ret;
1303 }
1304 
1305 /*
1306  * Performs partition-scoped invalidations for nested guests
1307  * as part of H_RPT_INVALIDATE hcall.
1308  */
do_h_rpt_invalidate_pat(struct kvm_vcpu * vcpu,unsigned long lpid,unsigned long type,unsigned long pg_sizes,unsigned long start,unsigned long end)1309 long do_h_rpt_invalidate_pat(struct kvm_vcpu *vcpu, unsigned long lpid,
1310 			     unsigned long type, unsigned long pg_sizes,
1311 			     unsigned long start, unsigned long end)
1312 {
1313 	/*
1314 	 * If L2 lpid isn't valid, we need to return H_PARAMETER.
1315 	 *
1316 	 * However, nested KVM issues a L2 lpid flush call when creating
1317 	 * partition table entries for L2. This happens even before the
1318 	 * corresponding shadow lpid is created in HV which happens in
1319 	 * H_ENTER_NESTED call. Since we can't differentiate this case from
1320 	 * the invalid case, we ignore such flush requests and return success.
1321 	 */
1322 	if (!__find_nested(vcpu->kvm, lpid))
1323 		return H_SUCCESS;
1324 
1325 	/*
1326 	 * A flush all request can be handled by a full lpid flush only.
1327 	 */
1328 	if ((type & H_RPTI_TYPE_NESTED_ALL) == H_RPTI_TYPE_NESTED_ALL)
1329 		return do_tlb_invalidate_nested_all(vcpu, lpid, RIC_FLUSH_ALL);
1330 
1331 	/*
1332 	 * We don't need to handle a PWC flush like process table here,
1333 	 * because intermediate partition scoped table in nested guest doesn't
1334 	 * really have PWC. Only level we have PWC is in L0 and for nested
1335 	 * invalidate at L0 we always do kvm_flush_lpid() which does
1336 	 * radix__flush_all_lpid(). For range invalidate at any level, we
1337 	 * are not removing the higher level page tables and hence there is
1338 	 * no PWC invalidate needed.
1339 	 *
1340 	 * if (type & H_RPTI_TYPE_PWC) {
1341 	 *	ret = do_tlb_invalidate_nested_all(vcpu, lpid, RIC_FLUSH_PWC);
1342 	 *	if (ret)
1343 	 *		return H_P4;
1344 	 * }
1345 	 */
1346 
1347 	if (start == 0 && end == -1)
1348 		return do_tlb_invalidate_nested_all(vcpu, lpid, RIC_FLUSH_TLB);
1349 
1350 	if (type & H_RPTI_TYPE_TLB)
1351 		return do_tlb_invalidate_nested_tlb(vcpu, lpid, pg_sizes,
1352 						    start, end);
1353 	return H_SUCCESS;
1354 }
1355 
1356 /* Used to convert a nested guest real address to a L1 guest real address */
kvmhv_translate_addr_nested(struct kvm_vcpu * vcpu,struct kvm_nested_guest * gp,unsigned long n_gpa,unsigned long dsisr,struct kvmppc_pte * gpte_p)1357 static int kvmhv_translate_addr_nested(struct kvm_vcpu *vcpu,
1358 				       struct kvm_nested_guest *gp,
1359 				       unsigned long n_gpa, unsigned long dsisr,
1360 				       struct kvmppc_pte *gpte_p)
1361 {
1362 	u64 fault_addr, flags = dsisr & DSISR_ISSTORE;
1363 	int ret;
1364 
1365 	ret = kvmppc_mmu_walk_radix_tree(vcpu, n_gpa, gpte_p, gp->l1_gr_to_hr,
1366 					 &fault_addr);
1367 
1368 	if (ret) {
1369 		/* We didn't find a pte */
1370 		if (ret == -EINVAL) {
1371 			/* Unsupported mmu config */
1372 			flags |= DSISR_UNSUPP_MMU;
1373 		} else if (ret == -ENOENT) {
1374 			/* No translation found */
1375 			flags |= DSISR_NOHPTE;
1376 		} else if (ret == -EFAULT) {
1377 			/* Couldn't access L1 real address */
1378 			flags |= DSISR_PRTABLE_FAULT;
1379 			vcpu->arch.fault_gpa = fault_addr;
1380 		} else {
1381 			/* Unknown error */
1382 			return ret;
1383 		}
1384 		goto forward_to_l1;
1385 	} else {
1386 		/* We found a pte -> check permissions */
1387 		if (dsisr & DSISR_ISSTORE) {
1388 			/* Can we write? */
1389 			if (!gpte_p->may_write) {
1390 				flags |= DSISR_PROTFAULT;
1391 				goto forward_to_l1;
1392 			}
1393 		} else if (vcpu->arch.trap == BOOK3S_INTERRUPT_H_INST_STORAGE) {
1394 			/* Can we execute? */
1395 			if (!gpte_p->may_execute) {
1396 				flags |= SRR1_ISI_N_G_OR_CIP;
1397 				goto forward_to_l1;
1398 			}
1399 		} else {
1400 			/* Can we read? */
1401 			if (!gpte_p->may_read && !gpte_p->may_write) {
1402 				flags |= DSISR_PROTFAULT;
1403 				goto forward_to_l1;
1404 			}
1405 		}
1406 	}
1407 
1408 	return 0;
1409 
1410 forward_to_l1:
1411 	vcpu->arch.fault_dsisr = flags;
1412 	if (vcpu->arch.trap == BOOK3S_INTERRUPT_H_INST_STORAGE) {
1413 		vcpu->arch.shregs.msr &= SRR1_MSR_BITS;
1414 		vcpu->arch.shregs.msr |= flags;
1415 	}
1416 	return RESUME_HOST;
1417 }
1418 
kvmhv_handle_nested_set_rc(struct kvm_vcpu * vcpu,struct kvm_nested_guest * gp,unsigned long n_gpa,struct kvmppc_pte gpte,unsigned long dsisr)1419 static long kvmhv_handle_nested_set_rc(struct kvm_vcpu *vcpu,
1420 				       struct kvm_nested_guest *gp,
1421 				       unsigned long n_gpa,
1422 				       struct kvmppc_pte gpte,
1423 				       unsigned long dsisr)
1424 {
1425 	struct kvm *kvm = vcpu->kvm;
1426 	bool writing = !!(dsisr & DSISR_ISSTORE);
1427 	u64 pgflags;
1428 	long ret;
1429 
1430 	/* Are the rc bits set in the L1 partition scoped pte? */
1431 	pgflags = _PAGE_ACCESSED;
1432 	if (writing)
1433 		pgflags |= _PAGE_DIRTY;
1434 	if (pgflags & ~gpte.rc)
1435 		return RESUME_HOST;
1436 
1437 	spin_lock(&kvm->mmu_lock);
1438 	/* Set the rc bit in the pte of our (L0) pgtable for the L1 guest */
1439 	ret = kvmppc_hv_handle_set_rc(kvm, false, writing,
1440 				      gpte.raddr, kvm->arch.lpid);
1441 	if (!ret) {
1442 		ret = -EINVAL;
1443 		goto out_unlock;
1444 	}
1445 
1446 	/* Set the rc bit in the pte of the shadow_pgtable for the nest guest */
1447 	ret = kvmppc_hv_handle_set_rc(kvm, true, writing,
1448 				      n_gpa, gp->l1_lpid);
1449 	if (!ret)
1450 		ret = -EINVAL;
1451 	else
1452 		ret = 0;
1453 
1454 out_unlock:
1455 	spin_unlock(&kvm->mmu_lock);
1456 	return ret;
1457 }
1458 
kvmppc_radix_level_to_shift(int level)1459 static inline int kvmppc_radix_level_to_shift(int level)
1460 {
1461 	switch (level) {
1462 	case 2:
1463 		return PUD_SHIFT;
1464 	case 1:
1465 		return PMD_SHIFT;
1466 	default:
1467 		return PAGE_SHIFT;
1468 	}
1469 }
1470 
kvmppc_radix_shift_to_level(int shift)1471 static inline int kvmppc_radix_shift_to_level(int shift)
1472 {
1473 	if (shift == PUD_SHIFT)
1474 		return 2;
1475 	if (shift == PMD_SHIFT)
1476 		return 1;
1477 	if (shift == PAGE_SHIFT)
1478 		return 0;
1479 	WARN_ON_ONCE(1);
1480 	return 0;
1481 }
1482 
1483 /* called with gp->tlb_lock held */
__kvmhv_nested_page_fault(struct kvm_vcpu * vcpu,struct kvm_nested_guest * gp)1484 static long int __kvmhv_nested_page_fault(struct kvm_vcpu *vcpu,
1485 					  struct kvm_nested_guest *gp)
1486 {
1487 	struct kvm *kvm = vcpu->kvm;
1488 	struct kvm_memory_slot *memslot;
1489 	struct rmap_nested *n_rmap;
1490 	struct kvmppc_pte gpte;
1491 	pte_t pte, *pte_p;
1492 	unsigned long mmu_seq;
1493 	unsigned long dsisr = vcpu->arch.fault_dsisr;
1494 	unsigned long ea = vcpu->arch.fault_dar;
1495 	unsigned long *rmapp;
1496 	unsigned long n_gpa, gpa, gfn, perm = 0UL;
1497 	unsigned int shift, l1_shift, level;
1498 	bool writing = !!(dsisr & DSISR_ISSTORE);
1499 	bool kvm_ro = false;
1500 	long int ret;
1501 
1502 	if (!gp->l1_gr_to_hr) {
1503 		kvmhv_update_ptbl_cache(gp);
1504 		if (!gp->l1_gr_to_hr)
1505 			return RESUME_HOST;
1506 	}
1507 
1508 	/* Convert the nested guest real address into a L1 guest real address */
1509 
1510 	n_gpa = vcpu->arch.fault_gpa & ~0xF000000000000FFFULL;
1511 	if (!(dsisr & DSISR_PRTABLE_FAULT))
1512 		n_gpa |= ea & 0xFFF;
1513 	ret = kvmhv_translate_addr_nested(vcpu, gp, n_gpa, dsisr, &gpte);
1514 
1515 	/*
1516 	 * If the hardware found a translation but we don't now have a usable
1517 	 * translation in the l1 partition-scoped tree, remove the shadow pte
1518 	 * and let the guest retry.
1519 	 */
1520 	if (ret == RESUME_HOST &&
1521 	    (dsisr & (DSISR_PROTFAULT | DSISR_BADACCESS | DSISR_NOEXEC_OR_G |
1522 		      DSISR_BAD_COPYPASTE)))
1523 		goto inval;
1524 	if (ret)
1525 		return ret;
1526 
1527 	/* Failed to set the reference/change bits */
1528 	if (dsisr & DSISR_SET_RC) {
1529 		ret = kvmhv_handle_nested_set_rc(vcpu, gp, n_gpa, gpte, dsisr);
1530 		if (ret == RESUME_HOST)
1531 			return ret;
1532 		if (ret)
1533 			goto inval;
1534 		dsisr &= ~DSISR_SET_RC;
1535 		if (!(dsisr & (DSISR_BAD_FAULT_64S | DSISR_NOHPTE |
1536 			       DSISR_PROTFAULT)))
1537 			return RESUME_GUEST;
1538 	}
1539 
1540 	/*
1541 	 * We took an HISI or HDSI while we were running a nested guest which
1542 	 * means we have no partition scoped translation for that. This means
1543 	 * we need to insert a pte for the mapping into our shadow_pgtable.
1544 	 */
1545 
1546 	l1_shift = gpte.page_shift;
1547 	if (l1_shift < PAGE_SHIFT) {
1548 		/* We don't support l1 using a page size smaller than our own */
1549 		pr_err("KVM: L1 guest page shift (%d) less than our own (%d)\n",
1550 			l1_shift, PAGE_SHIFT);
1551 		return -EINVAL;
1552 	}
1553 	gpa = gpte.raddr;
1554 	gfn = gpa >> PAGE_SHIFT;
1555 
1556 	/* 1. Get the corresponding host memslot */
1557 
1558 	memslot = gfn_to_memslot(kvm, gfn);
1559 	if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) {
1560 		if (dsisr & (DSISR_PRTABLE_FAULT | DSISR_BADACCESS)) {
1561 			/* unusual error -> reflect to the guest as a DSI */
1562 			kvmppc_core_queue_data_storage(vcpu, ea, dsisr);
1563 			return RESUME_GUEST;
1564 		}
1565 
1566 		/* passthrough of emulated MMIO case */
1567 		return kvmppc_hv_emulate_mmio(vcpu, gpa, ea, writing);
1568 	}
1569 	if (memslot->flags & KVM_MEM_READONLY) {
1570 		if (writing) {
1571 			/* Give the guest a DSI */
1572 			kvmppc_core_queue_data_storage(vcpu, ea,
1573 					DSISR_ISSTORE | DSISR_PROTFAULT);
1574 			return RESUME_GUEST;
1575 		}
1576 		kvm_ro = true;
1577 	}
1578 
1579 	/* 2. Find the host pte for this L1 guest real address */
1580 
1581 	/* Used to check for invalidations in progress */
1582 	mmu_seq = kvm->mmu_notifier_seq;
1583 	smp_rmb();
1584 
1585 	/* See if can find translation in our partition scoped tables for L1 */
1586 	pte = __pte(0);
1587 	spin_lock(&kvm->mmu_lock);
1588 	pte_p = find_kvm_secondary_pte(kvm, gpa, &shift);
1589 	if (!shift)
1590 		shift = PAGE_SHIFT;
1591 	if (pte_p)
1592 		pte = *pte_p;
1593 	spin_unlock(&kvm->mmu_lock);
1594 
1595 	if (!pte_present(pte) || (writing && !(pte_val(pte) & _PAGE_WRITE))) {
1596 		/* No suitable pte found -> try to insert a mapping */
1597 		ret = kvmppc_book3s_instantiate_page(vcpu, gpa, memslot,
1598 					writing, kvm_ro, &pte, &level);
1599 		if (ret == -EAGAIN)
1600 			return RESUME_GUEST;
1601 		else if (ret)
1602 			return ret;
1603 		shift = kvmppc_radix_level_to_shift(level);
1604 	}
1605 	/* Align gfn to the start of the page */
1606 	gfn = (gpa & ~((1UL << shift) - 1)) >> PAGE_SHIFT;
1607 
1608 	/* 3. Compute the pte we need to insert for nest_gpa -> host r_addr */
1609 
1610 	/* The permissions is the combination of the host and l1 guest ptes */
1611 	perm |= gpte.may_read ? 0UL : _PAGE_READ;
1612 	perm |= gpte.may_write ? 0UL : _PAGE_WRITE;
1613 	perm |= gpte.may_execute ? 0UL : _PAGE_EXEC;
1614 	/* Only set accessed/dirty (rc) bits if set in host and l1 guest ptes */
1615 	perm |= (gpte.rc & _PAGE_ACCESSED) ? 0UL : _PAGE_ACCESSED;
1616 	perm |= ((gpte.rc & _PAGE_DIRTY) && writing) ? 0UL : _PAGE_DIRTY;
1617 	pte = __pte(pte_val(pte) & ~perm);
1618 
1619 	/* What size pte can we insert? */
1620 	if (shift > l1_shift) {
1621 		u64 mask;
1622 		unsigned int actual_shift = PAGE_SHIFT;
1623 		if (PMD_SHIFT < l1_shift)
1624 			actual_shift = PMD_SHIFT;
1625 		mask = (1UL << shift) - (1UL << actual_shift);
1626 		pte = __pte(pte_val(pte) | (gpa & mask));
1627 		shift = actual_shift;
1628 	}
1629 	level = kvmppc_radix_shift_to_level(shift);
1630 	n_gpa &= ~((1UL << shift) - 1);
1631 
1632 	/* 4. Insert the pte into our shadow_pgtable */
1633 
1634 	n_rmap = kzalloc(sizeof(*n_rmap), GFP_KERNEL);
1635 	if (!n_rmap)
1636 		return RESUME_GUEST; /* Let the guest try again */
1637 	n_rmap->rmap = (n_gpa & RMAP_NESTED_GPA_MASK) |
1638 		(((unsigned long) gp->l1_lpid) << RMAP_NESTED_LPID_SHIFT);
1639 	rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
1640 	ret = kvmppc_create_pte(kvm, gp->shadow_pgtable, pte, n_gpa, level,
1641 				mmu_seq, gp->shadow_lpid, rmapp, &n_rmap);
1642 	kfree(n_rmap);
1643 	if (ret == -EAGAIN)
1644 		ret = RESUME_GUEST;	/* Let the guest try again */
1645 
1646 	return ret;
1647 
1648  inval:
1649 	kvmhv_invalidate_shadow_pte(vcpu, gp, n_gpa, NULL);
1650 	return RESUME_GUEST;
1651 }
1652 
kvmhv_nested_page_fault(struct kvm_vcpu * vcpu)1653 long int kvmhv_nested_page_fault(struct kvm_vcpu *vcpu)
1654 {
1655 	struct kvm_nested_guest *gp = vcpu->arch.nested;
1656 	long int ret;
1657 
1658 	mutex_lock(&gp->tlb_lock);
1659 	ret = __kvmhv_nested_page_fault(vcpu, gp);
1660 	mutex_unlock(&gp->tlb_lock);
1661 	return ret;
1662 }
1663 
kvmhv_nested_next_lpid(struct kvm * kvm,int lpid)1664 int kvmhv_nested_next_lpid(struct kvm *kvm, int lpid)
1665 {
1666 	int ret = lpid + 1;
1667 
1668 	spin_lock(&kvm->mmu_lock);
1669 	if (!idr_get_next(&kvm->arch.kvm_nested_guest_idr, &ret))
1670 		ret = -1;
1671 	spin_unlock(&kvm->mmu_lock);
1672 
1673 	return ret;
1674 }
1675