1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
4  * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
5  *
6  * Authors:
7  *    Paul Mackerras <paulus@au1.ibm.com>
8  *    Alexander Graf <agraf@suse.de>
9  *    Kevin Wolf <mail@kevin-wolf.de>
10  *
11  * Description: KVM functions specific to running on Book 3S
12  * processors in hypervisor mode (specifically POWER7 and later).
13  *
14  * This file is derived from arch/powerpc/kvm/book3s.c,
15  * by Alexander Graf <agraf@suse.de>.
16  */
17 
18 #include <linux/kvm_host.h>
19 #include <linux/kernel.h>
20 #include <linux/err.h>
21 #include <linux/slab.h>
22 #include <linux/preempt.h>
23 #include <linux/sched/signal.h>
24 #include <linux/sched/stat.h>
25 #include <linux/delay.h>
26 #include <linux/export.h>
27 #include <linux/fs.h>
28 #include <linux/anon_inodes.h>
29 #include <linux/cpu.h>
30 #include <linux/cpumask.h>
31 #include <linux/spinlock.h>
32 #include <linux/page-flags.h>
33 #include <linux/srcu.h>
34 #include <linux/miscdevice.h>
35 #include <linux/debugfs.h>
36 #include <linux/gfp.h>
37 #include <linux/vmalloc.h>
38 #include <linux/highmem.h>
39 #include <linux/hugetlb.h>
40 #include <linux/kvm_irqfd.h>
41 #include <linux/irqbypass.h>
42 #include <linux/module.h>
43 #include <linux/compiler.h>
44 #include <linux/of.h>
45 #include <linux/irqdomain.h>
46 
47 #include <asm/ftrace.h>
48 #include <asm/reg.h>
49 #include <asm/ppc-opcode.h>
50 #include <asm/asm-prototypes.h>
51 #include <asm/archrandom.h>
52 #include <asm/debug.h>
53 #include <asm/disassemble.h>
54 #include <asm/cputable.h>
55 #include <asm/cacheflush.h>
56 #include <linux/uaccess.h>
57 #include <asm/interrupt.h>
58 #include <asm/io.h>
59 #include <asm/kvm_ppc.h>
60 #include <asm/kvm_book3s.h>
61 #include <asm/mmu_context.h>
62 #include <asm/lppaca.h>
63 #include <asm/pmc.h>
64 #include <asm/processor.h>
65 #include <asm/cputhreads.h>
66 #include <asm/page.h>
67 #include <asm/hvcall.h>
68 #include <asm/switch_to.h>
69 #include <asm/smp.h>
70 #include <asm/dbell.h>
71 #include <asm/hmi.h>
72 #include <asm/pnv-pci.h>
73 #include <asm/mmu.h>
74 #include <asm/opal.h>
75 #include <asm/xics.h>
76 #include <asm/xive.h>
77 #include <asm/hw_breakpoint.h>
78 #include <asm/kvm_book3s_uvmem.h>
79 #include <asm/ultravisor.h>
80 #include <asm/dtl.h>
81 #include <asm/plpar_wrappers.h>
82 
83 #include "book3s.h"
84 #include "book3s_hv.h"
85 
86 #define CREATE_TRACE_POINTS
87 #include "trace_hv.h"
88 
89 /* #define EXIT_DEBUG */
90 /* #define EXIT_DEBUG_SIMPLE */
91 /* #define EXIT_DEBUG_INT */
92 
93 /* Used to indicate that a guest page fault needs to be handled */
94 #define RESUME_PAGE_FAULT	(RESUME_GUEST | RESUME_FLAG_ARCH1)
95 /* Used to indicate that a guest passthrough interrupt needs to be handled */
96 #define RESUME_PASSTHROUGH	(RESUME_GUEST | RESUME_FLAG_ARCH2)
97 
98 /* Used as a "null" value for timebase values */
99 #define TB_NIL	(~(u64)0)
100 
101 static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);
102 
103 static int dynamic_mt_modes = 6;
104 module_param(dynamic_mt_modes, int, 0644);
105 MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)");
106 static int target_smt_mode;
107 module_param(target_smt_mode, int, 0644);
108 MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)");
109 
110 static bool one_vm_per_core;
111 module_param(one_vm_per_core, bool, S_IRUGO | S_IWUSR);
112 MODULE_PARM_DESC(one_vm_per_core, "Only run vCPUs from the same VM on a core (requires POWER8 or older)");
113 
114 #ifdef CONFIG_KVM_XICS
115 static const struct kernel_param_ops module_param_ops = {
116 	.set = param_set_int,
117 	.get = param_get_int,
118 };
119 
120 module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass, 0644);
121 MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization");
122 
123 module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, 0644);
124 MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core");
125 #endif
126 
127 /* If set, guests are allowed to create and control nested guests */
128 static bool nested = true;
129 module_param(nested, bool, S_IRUGO | S_IWUSR);
130 MODULE_PARM_DESC(nested, "Enable nested virtualization (only on POWER9)");
131 
132 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
133 
134 /*
135  * RWMR values for POWER8.  These control the rate at which PURR
136  * and SPURR count and should be set according to the number of
137  * online threads in the vcore being run.
138  */
139 #define RWMR_RPA_P8_1THREAD	0x164520C62609AECAUL
140 #define RWMR_RPA_P8_2THREAD	0x7FFF2908450D8DA9UL
141 #define RWMR_RPA_P8_3THREAD	0x164520C62609AECAUL
142 #define RWMR_RPA_P8_4THREAD	0x199A421245058DA9UL
143 #define RWMR_RPA_P8_5THREAD	0x164520C62609AECAUL
144 #define RWMR_RPA_P8_6THREAD	0x164520C62609AECAUL
145 #define RWMR_RPA_P8_7THREAD	0x164520C62609AECAUL
146 #define RWMR_RPA_P8_8THREAD	0x164520C62609AECAUL
147 
148 static unsigned long p8_rwmr_values[MAX_SMT_THREADS + 1] = {
149 	RWMR_RPA_P8_1THREAD,
150 	RWMR_RPA_P8_1THREAD,
151 	RWMR_RPA_P8_2THREAD,
152 	RWMR_RPA_P8_3THREAD,
153 	RWMR_RPA_P8_4THREAD,
154 	RWMR_RPA_P8_5THREAD,
155 	RWMR_RPA_P8_6THREAD,
156 	RWMR_RPA_P8_7THREAD,
157 	RWMR_RPA_P8_8THREAD,
158 };
159 
next_runnable_thread(struct kvmppc_vcore * vc,int * ip)160 static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc,
161 		int *ip)
162 {
163 	int i = *ip;
164 	struct kvm_vcpu *vcpu;
165 
166 	while (++i < MAX_SMT_THREADS) {
167 		vcpu = READ_ONCE(vc->runnable_threads[i]);
168 		if (vcpu) {
169 			*ip = i;
170 			return vcpu;
171 		}
172 	}
173 	return NULL;
174 }
175 
176 /* Used to traverse the list of runnable threads for a given vcore */
177 #define for_each_runnable_thread(i, vcpu, vc) \
178 	for (i = -1; (vcpu = next_runnable_thread(vc, &i)); )
179 
kvmppc_ipi_thread(int cpu)180 static bool kvmppc_ipi_thread(int cpu)
181 {
182 	unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
183 
184 	/* If we're a nested hypervisor, fall back to ordinary IPIs for now */
185 	if (kvmhv_on_pseries())
186 		return false;
187 
188 	/* On POWER9 we can use msgsnd to IPI any cpu */
189 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
190 		msg |= get_hard_smp_processor_id(cpu);
191 		smp_mb();
192 		__asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
193 		return true;
194 	}
195 
196 	/* On POWER8 for IPIs to threads in the same core, use msgsnd */
197 	if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
198 		preempt_disable();
199 		if (cpu_first_thread_sibling(cpu) ==
200 		    cpu_first_thread_sibling(smp_processor_id())) {
201 			msg |= cpu_thread_in_core(cpu);
202 			smp_mb();
203 			__asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
204 			preempt_enable();
205 			return true;
206 		}
207 		preempt_enable();
208 	}
209 
210 #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
211 	if (cpu >= 0 && cpu < nr_cpu_ids) {
212 		if (paca_ptrs[cpu]->kvm_hstate.xics_phys) {
213 			xics_wake_cpu(cpu);
214 			return true;
215 		}
216 		opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY);
217 		return true;
218 	}
219 #endif
220 
221 	return false;
222 }
223 
kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu * vcpu)224 static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
225 {
226 	int cpu;
227 	struct rcuwait *waitp;
228 
229 	/*
230 	 * rcuwait_wake_up contains smp_mb() which orders prior stores that
231 	 * create pending work vs below loads of cpu fields. The other side
232 	 * is the barrier in vcpu run that orders setting the cpu fields vs
233 	 * testing for pending work.
234 	 */
235 
236 	waitp = kvm_arch_vcpu_get_wait(vcpu);
237 	if (rcuwait_wake_up(waitp))
238 		++vcpu->stat.generic.halt_wakeup;
239 
240 	cpu = READ_ONCE(vcpu->arch.thread_cpu);
241 	if (cpu >= 0 && kvmppc_ipi_thread(cpu))
242 		return;
243 
244 	/* CPU points to the first thread of the core */
245 	cpu = vcpu->cpu;
246 	if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu))
247 		smp_send_reschedule(cpu);
248 }
249 
250 /*
251  * We use the vcpu_load/put functions to measure stolen time.
252  *
253  * Stolen time is counted as time when either the vcpu is able to
254  * run as part of a virtual core, but the task running the vcore
255  * is preempted or sleeping, or when the vcpu needs something done
256  * in the kernel by the task running the vcpu, but that task is
257  * preempted or sleeping.  Those two things have to be counted
258  * separately, since one of the vcpu tasks will take on the job
259  * of running the core, and the other vcpu tasks in the vcore will
260  * sleep waiting for it to do that, but that sleep shouldn't count
261  * as stolen time.
262  *
263  * Hence we accumulate stolen time when the vcpu can run as part of
264  * a vcore using vc->stolen_tb, and the stolen time when the vcpu
265  * needs its task to do other things in the kernel (for example,
266  * service a page fault) in busy_stolen.  We don't accumulate
267  * stolen time for a vcore when it is inactive, or for a vcpu
268  * when it is in state RUNNING or NOTREADY.  NOTREADY is a bit of
269  * a misnomer; it means that the vcpu task is not executing in
270  * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in
271  * the kernel.  We don't have any way of dividing up that time
272  * between time that the vcpu is genuinely stopped, time that
273  * the task is actively working on behalf of the vcpu, and time
274  * that the task is preempted, so we don't count any of it as
275  * stolen.
276  *
277  * Updates to busy_stolen are protected by arch.tbacct_lock;
278  * updates to vc->stolen_tb are protected by the vcore->stoltb_lock
279  * lock.  The stolen times are measured in units of timebase ticks.
280  * (Note that the != TB_NIL checks below are purely defensive;
281  * they should never fail.)
282  *
283  * The POWER9 path is simpler, one vcpu per virtual core so the
284  * former case does not exist. If a vcpu is preempted when it is
285  * BUSY_IN_HOST and not ceded or otherwise blocked, then accumulate
286  * the stolen cycles in busy_stolen. RUNNING is not a preemptible
287  * state in the P9 path.
288  */
289 
kvmppc_core_start_stolen(struct kvmppc_vcore * vc,u64 tb)290 static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc, u64 tb)
291 {
292 	unsigned long flags;
293 
294 	WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300));
295 
296 	spin_lock_irqsave(&vc->stoltb_lock, flags);
297 	vc->preempt_tb = tb;
298 	spin_unlock_irqrestore(&vc->stoltb_lock, flags);
299 }
300 
kvmppc_core_end_stolen(struct kvmppc_vcore * vc,u64 tb)301 static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc, u64 tb)
302 {
303 	unsigned long flags;
304 
305 	WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300));
306 
307 	spin_lock_irqsave(&vc->stoltb_lock, flags);
308 	if (vc->preempt_tb != TB_NIL) {
309 		vc->stolen_tb += tb - vc->preempt_tb;
310 		vc->preempt_tb = TB_NIL;
311 	}
312 	spin_unlock_irqrestore(&vc->stoltb_lock, flags);
313 }
314 
kvmppc_core_vcpu_load_hv(struct kvm_vcpu * vcpu,int cpu)315 static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu)
316 {
317 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
318 	unsigned long flags;
319 	u64 now;
320 
321 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
322 		if (vcpu->arch.busy_preempt != TB_NIL) {
323 			WARN_ON_ONCE(vcpu->arch.state != KVMPPC_VCPU_BUSY_IN_HOST);
324 			vc->stolen_tb += mftb() - vcpu->arch.busy_preempt;
325 			vcpu->arch.busy_preempt = TB_NIL;
326 		}
327 		return;
328 	}
329 
330 	now = mftb();
331 
332 	/*
333 	 * We can test vc->runner without taking the vcore lock,
334 	 * because only this task ever sets vc->runner to this
335 	 * vcpu, and once it is set to this vcpu, only this task
336 	 * ever sets it to NULL.
337 	 */
338 	if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
339 		kvmppc_core_end_stolen(vc, now);
340 
341 	spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
342 	if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST &&
343 	    vcpu->arch.busy_preempt != TB_NIL) {
344 		vcpu->arch.busy_stolen += now - vcpu->arch.busy_preempt;
345 		vcpu->arch.busy_preempt = TB_NIL;
346 	}
347 	spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
348 }
349 
kvmppc_core_vcpu_put_hv(struct kvm_vcpu * vcpu)350 static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu)
351 {
352 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
353 	unsigned long flags;
354 	u64 now;
355 
356 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
357 		/*
358 		 * In the P9 path, RUNNABLE is not preemptible
359 		 * (nor takes host interrupts)
360 		 */
361 		WARN_ON_ONCE(vcpu->arch.state == KVMPPC_VCPU_RUNNABLE);
362 		/*
363 		 * Account stolen time when preempted while the vcpu task is
364 		 * running in the kernel (but not in qemu, which is INACTIVE).
365 		 */
366 		if (task_is_running(current) &&
367 				vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
368 			vcpu->arch.busy_preempt = mftb();
369 		return;
370 	}
371 
372 	now = mftb();
373 
374 	if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING)
375 		kvmppc_core_start_stolen(vc, now);
376 
377 	spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
378 	if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST)
379 		vcpu->arch.busy_preempt = now;
380 	spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
381 }
382 
kvmppc_set_pvr_hv(struct kvm_vcpu * vcpu,u32 pvr)383 static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr)
384 {
385 	vcpu->arch.pvr = pvr;
386 }
387 
388 /* Dummy value used in computing PCR value below */
389 #define PCR_ARCH_31    (PCR_ARCH_300 << 1)
390 
kvmppc_set_arch_compat(struct kvm_vcpu * vcpu,u32 arch_compat)391 static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat)
392 {
393 	unsigned long host_pcr_bit = 0, guest_pcr_bit = 0;
394 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
395 
396 	/* We can (emulate) our own architecture version and anything older */
397 	if (cpu_has_feature(CPU_FTR_ARCH_31))
398 		host_pcr_bit = PCR_ARCH_31;
399 	else if (cpu_has_feature(CPU_FTR_ARCH_300))
400 		host_pcr_bit = PCR_ARCH_300;
401 	else if (cpu_has_feature(CPU_FTR_ARCH_207S))
402 		host_pcr_bit = PCR_ARCH_207;
403 	else if (cpu_has_feature(CPU_FTR_ARCH_206))
404 		host_pcr_bit = PCR_ARCH_206;
405 	else
406 		host_pcr_bit = PCR_ARCH_205;
407 
408 	/* Determine lowest PCR bit needed to run guest in given PVR level */
409 	guest_pcr_bit = host_pcr_bit;
410 	if (arch_compat) {
411 		switch (arch_compat) {
412 		case PVR_ARCH_205:
413 			guest_pcr_bit = PCR_ARCH_205;
414 			break;
415 		case PVR_ARCH_206:
416 		case PVR_ARCH_206p:
417 			guest_pcr_bit = PCR_ARCH_206;
418 			break;
419 		case PVR_ARCH_207:
420 			guest_pcr_bit = PCR_ARCH_207;
421 			break;
422 		case PVR_ARCH_300:
423 			guest_pcr_bit = PCR_ARCH_300;
424 			break;
425 		case PVR_ARCH_31:
426 			guest_pcr_bit = PCR_ARCH_31;
427 			break;
428 		default:
429 			return -EINVAL;
430 		}
431 	}
432 
433 	/* Check requested PCR bits don't exceed our capabilities */
434 	if (guest_pcr_bit > host_pcr_bit)
435 		return -EINVAL;
436 
437 	spin_lock(&vc->lock);
438 	vc->arch_compat = arch_compat;
439 	/*
440 	 * Set all PCR bits for which guest_pcr_bit <= bit < host_pcr_bit
441 	 * Also set all reserved PCR bits
442 	 */
443 	vc->pcr = (host_pcr_bit - guest_pcr_bit) | PCR_MASK;
444 	spin_unlock(&vc->lock);
445 
446 	return 0;
447 }
448 
kvmppc_dump_regs(struct kvm_vcpu * vcpu)449 static void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
450 {
451 	int r;
452 
453 	pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
454 	pr_err("pc  = %.16lx  msr = %.16llx  trap = %x\n",
455 	       vcpu->arch.regs.nip, vcpu->arch.shregs.msr, vcpu->arch.trap);
456 	for (r = 0; r < 16; ++r)
457 		pr_err("r%2d = %.16lx  r%d = %.16lx\n",
458 		       r, kvmppc_get_gpr(vcpu, r),
459 		       r+16, kvmppc_get_gpr(vcpu, r+16));
460 	pr_err("ctr = %.16lx  lr  = %.16lx\n",
461 	       vcpu->arch.regs.ctr, vcpu->arch.regs.link);
462 	pr_err("srr0 = %.16llx srr1 = %.16llx\n",
463 	       vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
464 	pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
465 	       vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
466 	pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
467 	       vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
468 	pr_err("cr = %.8lx  xer = %.16lx  dsisr = %.8x\n",
469 	       vcpu->arch.regs.ccr, vcpu->arch.regs.xer, vcpu->arch.shregs.dsisr);
470 	pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
471 	pr_err("fault dar = %.16lx dsisr = %.8x\n",
472 	       vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
473 	pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
474 	for (r = 0; r < vcpu->arch.slb_max; ++r)
475 		pr_err("  ESID = %.16llx VSID = %.16llx\n",
476 		       vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
477 	pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
478 	       vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1,
479 	       vcpu->arch.last_inst);
480 }
481 
kvmppc_find_vcpu(struct kvm * kvm,int id)482 static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
483 {
484 	return kvm_get_vcpu_by_id(kvm, id);
485 }
486 
init_vpa(struct kvm_vcpu * vcpu,struct lppaca * vpa)487 static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
488 {
489 	vpa->__old_status |= LPPACA_OLD_SHARED_PROC;
490 	vpa->yield_count = cpu_to_be32(1);
491 }
492 
set_vpa(struct kvm_vcpu * vcpu,struct kvmppc_vpa * v,unsigned long addr,unsigned long len)493 static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
494 		   unsigned long addr, unsigned long len)
495 {
496 	/* check address is cacheline aligned */
497 	if (addr & (L1_CACHE_BYTES - 1))
498 		return -EINVAL;
499 	spin_lock(&vcpu->arch.vpa_update_lock);
500 	if (v->next_gpa != addr || v->len != len) {
501 		v->next_gpa = addr;
502 		v->len = addr ? len : 0;
503 		v->update_pending = 1;
504 	}
505 	spin_unlock(&vcpu->arch.vpa_update_lock);
506 	return 0;
507 }
508 
509 /* Length for a per-processor buffer is passed in at offset 4 in the buffer */
510 struct reg_vpa {
511 	u32 dummy;
512 	union {
513 		__be16 hword;
514 		__be32 word;
515 	} length;
516 };
517 
vpa_is_registered(struct kvmppc_vpa * vpap)518 static int vpa_is_registered(struct kvmppc_vpa *vpap)
519 {
520 	if (vpap->update_pending)
521 		return vpap->next_gpa != 0;
522 	return vpap->pinned_addr != NULL;
523 }
524 
do_h_register_vpa(struct kvm_vcpu * vcpu,unsigned long flags,unsigned long vcpuid,unsigned long vpa)525 static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
526 				       unsigned long flags,
527 				       unsigned long vcpuid, unsigned long vpa)
528 {
529 	struct kvm *kvm = vcpu->kvm;
530 	unsigned long len, nb;
531 	void *va;
532 	struct kvm_vcpu *tvcpu;
533 	int err;
534 	int subfunc;
535 	struct kvmppc_vpa *vpap;
536 
537 	tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
538 	if (!tvcpu)
539 		return H_PARAMETER;
540 
541 	subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
542 	if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
543 	    subfunc == H_VPA_REG_SLB) {
544 		/* Registering new area - address must be cache-line aligned */
545 		if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
546 			return H_PARAMETER;
547 
548 		/* convert logical addr to kernel addr and read length */
549 		va = kvmppc_pin_guest_page(kvm, vpa, &nb);
550 		if (va == NULL)
551 			return H_PARAMETER;
552 		if (subfunc == H_VPA_REG_VPA)
553 			len = be16_to_cpu(((struct reg_vpa *)va)->length.hword);
554 		else
555 			len = be32_to_cpu(((struct reg_vpa *)va)->length.word);
556 		kvmppc_unpin_guest_page(kvm, va, vpa, false);
557 
558 		/* Check length */
559 		if (len > nb || len < sizeof(struct reg_vpa))
560 			return H_PARAMETER;
561 	} else {
562 		vpa = 0;
563 		len = 0;
564 	}
565 
566 	err = H_PARAMETER;
567 	vpap = NULL;
568 	spin_lock(&tvcpu->arch.vpa_update_lock);
569 
570 	switch (subfunc) {
571 	case H_VPA_REG_VPA:		/* register VPA */
572 		/*
573 		 * The size of our lppaca is 1kB because of the way we align
574 		 * it for the guest to avoid crossing a 4kB boundary. We only
575 		 * use 640 bytes of the structure though, so we should accept
576 		 * clients that set a size of 640.
577 		 */
578 		BUILD_BUG_ON(sizeof(struct lppaca) != 640);
579 		if (len < sizeof(struct lppaca))
580 			break;
581 		vpap = &tvcpu->arch.vpa;
582 		err = 0;
583 		break;
584 
585 	case H_VPA_REG_DTL:		/* register DTL */
586 		if (len < sizeof(struct dtl_entry))
587 			break;
588 		len -= len % sizeof(struct dtl_entry);
589 
590 		/* Check that they have previously registered a VPA */
591 		err = H_RESOURCE;
592 		if (!vpa_is_registered(&tvcpu->arch.vpa))
593 			break;
594 
595 		vpap = &tvcpu->arch.dtl;
596 		err = 0;
597 		break;
598 
599 	case H_VPA_REG_SLB:		/* register SLB shadow buffer */
600 		/* Check that they have previously registered a VPA */
601 		err = H_RESOURCE;
602 		if (!vpa_is_registered(&tvcpu->arch.vpa))
603 			break;
604 
605 		vpap = &tvcpu->arch.slb_shadow;
606 		err = 0;
607 		break;
608 
609 	case H_VPA_DEREG_VPA:		/* deregister VPA */
610 		/* Check they don't still have a DTL or SLB buf registered */
611 		err = H_RESOURCE;
612 		if (vpa_is_registered(&tvcpu->arch.dtl) ||
613 		    vpa_is_registered(&tvcpu->arch.slb_shadow))
614 			break;
615 
616 		vpap = &tvcpu->arch.vpa;
617 		err = 0;
618 		break;
619 
620 	case H_VPA_DEREG_DTL:		/* deregister DTL */
621 		vpap = &tvcpu->arch.dtl;
622 		err = 0;
623 		break;
624 
625 	case H_VPA_DEREG_SLB:		/* deregister SLB shadow buffer */
626 		vpap = &tvcpu->arch.slb_shadow;
627 		err = 0;
628 		break;
629 	}
630 
631 	if (vpap) {
632 		vpap->next_gpa = vpa;
633 		vpap->len = len;
634 		vpap->update_pending = 1;
635 	}
636 
637 	spin_unlock(&tvcpu->arch.vpa_update_lock);
638 
639 	return err;
640 }
641 
kvmppc_update_vpa(struct kvm_vcpu * vcpu,struct kvmppc_vpa * vpap)642 static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
643 {
644 	struct kvm *kvm = vcpu->kvm;
645 	void *va;
646 	unsigned long nb;
647 	unsigned long gpa;
648 
649 	/*
650 	 * We need to pin the page pointed to by vpap->next_gpa,
651 	 * but we can't call kvmppc_pin_guest_page under the lock
652 	 * as it does get_user_pages() and down_read().  So we
653 	 * have to drop the lock, pin the page, then get the lock
654 	 * again and check that a new area didn't get registered
655 	 * in the meantime.
656 	 */
657 	for (;;) {
658 		gpa = vpap->next_gpa;
659 		spin_unlock(&vcpu->arch.vpa_update_lock);
660 		va = NULL;
661 		nb = 0;
662 		if (gpa)
663 			va = kvmppc_pin_guest_page(kvm, gpa, &nb);
664 		spin_lock(&vcpu->arch.vpa_update_lock);
665 		if (gpa == vpap->next_gpa)
666 			break;
667 		/* sigh... unpin that one and try again */
668 		if (va)
669 			kvmppc_unpin_guest_page(kvm, va, gpa, false);
670 	}
671 
672 	vpap->update_pending = 0;
673 	if (va && nb < vpap->len) {
674 		/*
675 		 * If it's now too short, it must be that userspace
676 		 * has changed the mappings underlying guest memory,
677 		 * so unregister the region.
678 		 */
679 		kvmppc_unpin_guest_page(kvm, va, gpa, false);
680 		va = NULL;
681 	}
682 	if (vpap->pinned_addr)
683 		kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa,
684 					vpap->dirty);
685 	vpap->gpa = gpa;
686 	vpap->pinned_addr = va;
687 	vpap->dirty = false;
688 	if (va)
689 		vpap->pinned_end = va + vpap->len;
690 }
691 
kvmppc_update_vpas(struct kvm_vcpu * vcpu)692 static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
693 {
694 	if (!(vcpu->arch.vpa.update_pending ||
695 	      vcpu->arch.slb_shadow.update_pending ||
696 	      vcpu->arch.dtl.update_pending))
697 		return;
698 
699 	spin_lock(&vcpu->arch.vpa_update_lock);
700 	if (vcpu->arch.vpa.update_pending) {
701 		kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
702 		if (vcpu->arch.vpa.pinned_addr)
703 			init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
704 	}
705 	if (vcpu->arch.dtl.update_pending) {
706 		kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
707 		vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
708 		vcpu->arch.dtl_index = 0;
709 	}
710 	if (vcpu->arch.slb_shadow.update_pending)
711 		kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
712 	spin_unlock(&vcpu->arch.vpa_update_lock);
713 }
714 
715 /*
716  * Return the accumulated stolen time for the vcore up until `now'.
717  * The caller should hold the vcore lock.
718  */
vcore_stolen_time(struct kvmppc_vcore * vc,u64 now)719 static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now)
720 {
721 	u64 p;
722 	unsigned long flags;
723 
724 	WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300));
725 
726 	spin_lock_irqsave(&vc->stoltb_lock, flags);
727 	p = vc->stolen_tb;
728 	if (vc->vcore_state != VCORE_INACTIVE &&
729 	    vc->preempt_tb != TB_NIL)
730 		p += now - vc->preempt_tb;
731 	spin_unlock_irqrestore(&vc->stoltb_lock, flags);
732 	return p;
733 }
734 
__kvmppc_create_dtl_entry(struct kvm_vcpu * vcpu,struct lppaca * vpa,unsigned int pcpu,u64 now,unsigned long stolen)735 static void __kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
736 					struct lppaca *vpa,
737 					unsigned int pcpu, u64 now,
738 					unsigned long stolen)
739 {
740 	struct dtl_entry *dt;
741 
742 	dt = vcpu->arch.dtl_ptr;
743 
744 	if (!dt)
745 		return;
746 
747 	dt->dispatch_reason = 7;
748 	dt->preempt_reason = 0;
749 	dt->processor_id = cpu_to_be16(pcpu + vcpu->arch.ptid);
750 	dt->enqueue_to_dispatch_time = cpu_to_be32(stolen);
751 	dt->ready_to_enqueue_time = 0;
752 	dt->waiting_to_ready_time = 0;
753 	dt->timebase = cpu_to_be64(now);
754 	dt->fault_addr = 0;
755 	dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu));
756 	dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr);
757 
758 	++dt;
759 	if (dt == vcpu->arch.dtl.pinned_end)
760 		dt = vcpu->arch.dtl.pinned_addr;
761 	vcpu->arch.dtl_ptr = dt;
762 	/* order writing *dt vs. writing vpa->dtl_idx */
763 	smp_wmb();
764 	vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index);
765 
766 	/* vcpu->arch.dtl.dirty is set by the caller */
767 }
768 
kvmppc_update_vpa_dispatch(struct kvm_vcpu * vcpu,struct kvmppc_vcore * vc)769 static void kvmppc_update_vpa_dispatch(struct kvm_vcpu *vcpu,
770 				       struct kvmppc_vcore *vc)
771 {
772 	struct lppaca *vpa;
773 	unsigned long stolen;
774 	unsigned long core_stolen;
775 	u64 now;
776 	unsigned long flags;
777 
778 	vpa = vcpu->arch.vpa.pinned_addr;
779 	if (!vpa)
780 		return;
781 
782 	now = mftb();
783 
784 	core_stolen = vcore_stolen_time(vc, now);
785 	stolen = core_stolen - vcpu->arch.stolen_logged;
786 	vcpu->arch.stolen_logged = core_stolen;
787 	spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags);
788 	stolen += vcpu->arch.busy_stolen;
789 	vcpu->arch.busy_stolen = 0;
790 	spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags);
791 
792 	vpa->enqueue_dispatch_tb = cpu_to_be64(be64_to_cpu(vpa->enqueue_dispatch_tb) + stolen);
793 
794 	__kvmppc_create_dtl_entry(vcpu, vpa, vc->pcpu, now + vc->tb_offset, stolen);
795 
796 	vcpu->arch.vpa.dirty = true;
797 }
798 
kvmppc_update_vpa_dispatch_p9(struct kvm_vcpu * vcpu,struct kvmppc_vcore * vc,u64 now)799 static void kvmppc_update_vpa_dispatch_p9(struct kvm_vcpu *vcpu,
800 				       struct kvmppc_vcore *vc,
801 				       u64 now)
802 {
803 	struct lppaca *vpa;
804 	unsigned long stolen;
805 	unsigned long stolen_delta;
806 
807 	vpa = vcpu->arch.vpa.pinned_addr;
808 	if (!vpa)
809 		return;
810 
811 	stolen = vc->stolen_tb;
812 	stolen_delta = stolen - vcpu->arch.stolen_logged;
813 	vcpu->arch.stolen_logged = stolen;
814 
815 	vpa->enqueue_dispatch_tb = cpu_to_be64(stolen);
816 
817 	__kvmppc_create_dtl_entry(vcpu, vpa, vc->pcpu, now, stolen_delta);
818 
819 	vcpu->arch.vpa.dirty = true;
820 }
821 
822 /* See if there is a doorbell interrupt pending for a vcpu */
kvmppc_doorbell_pending(struct kvm_vcpu * vcpu)823 static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu)
824 {
825 	int thr;
826 	struct kvmppc_vcore *vc;
827 
828 	if (vcpu->arch.doorbell_request)
829 		return true;
830 	if (cpu_has_feature(CPU_FTR_ARCH_300))
831 		return false;
832 	/*
833 	 * Ensure that the read of vcore->dpdes comes after the read
834 	 * of vcpu->doorbell_request.  This barrier matches the
835 	 * smp_wmb() in kvmppc_guest_entry_inject().
836 	 */
837 	smp_rmb();
838 	vc = vcpu->arch.vcore;
839 	thr = vcpu->vcpu_id - vc->first_vcpuid;
840 	return !!(vc->dpdes & (1 << thr));
841 }
842 
kvmppc_power8_compatible(struct kvm_vcpu * vcpu)843 static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu)
844 {
845 	if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207)
846 		return true;
847 	if ((!vcpu->arch.vcore->arch_compat) &&
848 	    cpu_has_feature(CPU_FTR_ARCH_207S))
849 		return true;
850 	return false;
851 }
852 
kvmppc_h_set_mode(struct kvm_vcpu * vcpu,unsigned long mflags,unsigned long resource,unsigned long value1,unsigned long value2)853 static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags,
854 			     unsigned long resource, unsigned long value1,
855 			     unsigned long value2)
856 {
857 	switch (resource) {
858 	case H_SET_MODE_RESOURCE_SET_CIABR:
859 		if (!kvmppc_power8_compatible(vcpu))
860 			return H_P2;
861 		if (value2)
862 			return H_P4;
863 		if (mflags)
864 			return H_UNSUPPORTED_FLAG_START;
865 		/* Guests can't breakpoint the hypervisor */
866 		if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER)
867 			return H_P3;
868 		vcpu->arch.ciabr  = value1;
869 		return H_SUCCESS;
870 	case H_SET_MODE_RESOURCE_SET_DAWR0:
871 		if (!kvmppc_power8_compatible(vcpu))
872 			return H_P2;
873 		if (!ppc_breakpoint_available())
874 			return H_P2;
875 		if (mflags)
876 			return H_UNSUPPORTED_FLAG_START;
877 		if (value2 & DABRX_HYP)
878 			return H_P4;
879 		vcpu->arch.dawr0  = value1;
880 		vcpu->arch.dawrx0 = value2;
881 		return H_SUCCESS;
882 	case H_SET_MODE_RESOURCE_SET_DAWR1:
883 		if (!kvmppc_power8_compatible(vcpu))
884 			return H_P2;
885 		if (!ppc_breakpoint_available())
886 			return H_P2;
887 		if (!cpu_has_feature(CPU_FTR_DAWR1))
888 			return H_P2;
889 		if (!vcpu->kvm->arch.dawr1_enabled)
890 			return H_FUNCTION;
891 		if (mflags)
892 			return H_UNSUPPORTED_FLAG_START;
893 		if (value2 & DABRX_HYP)
894 			return H_P4;
895 		vcpu->arch.dawr1  = value1;
896 		vcpu->arch.dawrx1 = value2;
897 		return H_SUCCESS;
898 	case H_SET_MODE_RESOURCE_ADDR_TRANS_MODE:
899 		/*
900 		 * KVM does not support mflags=2 (AIL=2) and AIL=1 is reserved.
901 		 * Keep this in synch with kvmppc_filter_guest_lpcr_hv.
902 		 */
903 		if (cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG) &&
904 				kvmhv_vcpu_is_radix(vcpu) && mflags == 3)
905 			return H_UNSUPPORTED_FLAG_START;
906 		return H_TOO_HARD;
907 	default:
908 		return H_TOO_HARD;
909 	}
910 }
911 
912 /* Copy guest memory in place - must reside within a single memslot */
kvmppc_copy_guest(struct kvm * kvm,gpa_t to,gpa_t from,unsigned long len)913 static int kvmppc_copy_guest(struct kvm *kvm, gpa_t to, gpa_t from,
914 				  unsigned long len)
915 {
916 	struct kvm_memory_slot *to_memslot = NULL;
917 	struct kvm_memory_slot *from_memslot = NULL;
918 	unsigned long to_addr, from_addr;
919 	int r;
920 
921 	/* Get HPA for from address */
922 	from_memslot = gfn_to_memslot(kvm, from >> PAGE_SHIFT);
923 	if (!from_memslot)
924 		return -EFAULT;
925 	if ((from + len) >= ((from_memslot->base_gfn + from_memslot->npages)
926 			     << PAGE_SHIFT))
927 		return -EINVAL;
928 	from_addr = gfn_to_hva_memslot(from_memslot, from >> PAGE_SHIFT);
929 	if (kvm_is_error_hva(from_addr))
930 		return -EFAULT;
931 	from_addr |= (from & (PAGE_SIZE - 1));
932 
933 	/* Get HPA for to address */
934 	to_memslot = gfn_to_memslot(kvm, to >> PAGE_SHIFT);
935 	if (!to_memslot)
936 		return -EFAULT;
937 	if ((to + len) >= ((to_memslot->base_gfn + to_memslot->npages)
938 			   << PAGE_SHIFT))
939 		return -EINVAL;
940 	to_addr = gfn_to_hva_memslot(to_memslot, to >> PAGE_SHIFT);
941 	if (kvm_is_error_hva(to_addr))
942 		return -EFAULT;
943 	to_addr |= (to & (PAGE_SIZE - 1));
944 
945 	/* Perform copy */
946 	r = raw_copy_in_user((void __user *)to_addr, (void __user *)from_addr,
947 			     len);
948 	if (r)
949 		return -EFAULT;
950 	mark_page_dirty(kvm, to >> PAGE_SHIFT);
951 	return 0;
952 }
953 
kvmppc_h_page_init(struct kvm_vcpu * vcpu,unsigned long flags,unsigned long dest,unsigned long src)954 static long kvmppc_h_page_init(struct kvm_vcpu *vcpu, unsigned long flags,
955 			       unsigned long dest, unsigned long src)
956 {
957 	u64 pg_sz = SZ_4K;		/* 4K page size */
958 	u64 pg_mask = SZ_4K - 1;
959 	int ret;
960 
961 	/* Check for invalid flags (H_PAGE_SET_LOANED covers all CMO flags) */
962 	if (flags & ~(H_ICACHE_INVALIDATE | H_ICACHE_SYNCHRONIZE |
963 		      H_ZERO_PAGE | H_COPY_PAGE | H_PAGE_SET_LOANED))
964 		return H_PARAMETER;
965 
966 	/* dest (and src if copy_page flag set) must be page aligned */
967 	if ((dest & pg_mask) || ((flags & H_COPY_PAGE) && (src & pg_mask)))
968 		return H_PARAMETER;
969 
970 	/* zero and/or copy the page as determined by the flags */
971 	if (flags & H_COPY_PAGE) {
972 		ret = kvmppc_copy_guest(vcpu->kvm, dest, src, pg_sz);
973 		if (ret < 0)
974 			return H_PARAMETER;
975 	} else if (flags & H_ZERO_PAGE) {
976 		ret = kvm_clear_guest(vcpu->kvm, dest, pg_sz);
977 		if (ret < 0)
978 			return H_PARAMETER;
979 	}
980 
981 	/* We can ignore the remaining flags */
982 
983 	return H_SUCCESS;
984 }
985 
kvm_arch_vcpu_yield_to(struct kvm_vcpu * target)986 static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target)
987 {
988 	struct kvmppc_vcore *vcore = target->arch.vcore;
989 
990 	/*
991 	 * We expect to have been called by the real mode handler
992 	 * (kvmppc_rm_h_confer()) which would have directly returned
993 	 * H_SUCCESS if the source vcore wasn't idle (e.g. if it may
994 	 * have useful work to do and should not confer) so we don't
995 	 * recheck that here.
996 	 *
997 	 * In the case of the P9 single vcpu per vcore case, the real
998 	 * mode handler is not called but no other threads are in the
999 	 * source vcore.
1000 	 */
1001 	if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
1002 		spin_lock(&vcore->lock);
1003 		if (target->arch.state == KVMPPC_VCPU_RUNNABLE &&
1004 		    vcore->vcore_state != VCORE_INACTIVE &&
1005 		    vcore->runner)
1006 			target = vcore->runner;
1007 		spin_unlock(&vcore->lock);
1008 	}
1009 
1010 	return kvm_vcpu_yield_to(target);
1011 }
1012 
kvmppc_get_yield_count(struct kvm_vcpu * vcpu)1013 static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu)
1014 {
1015 	int yield_count = 0;
1016 	struct lppaca *lppaca;
1017 
1018 	spin_lock(&vcpu->arch.vpa_update_lock);
1019 	lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr;
1020 	if (lppaca)
1021 		yield_count = be32_to_cpu(lppaca->yield_count);
1022 	spin_unlock(&vcpu->arch.vpa_update_lock);
1023 	return yield_count;
1024 }
1025 
1026 /*
1027  * H_RPT_INVALIDATE hcall handler for nested guests.
1028  *
1029  * Handles only nested process-scoped invalidation requests in L0.
1030  */
kvmppc_nested_h_rpt_invalidate(struct kvm_vcpu * vcpu)1031 static int kvmppc_nested_h_rpt_invalidate(struct kvm_vcpu *vcpu)
1032 {
1033 	unsigned long type = kvmppc_get_gpr(vcpu, 6);
1034 	unsigned long pid, pg_sizes, start, end;
1035 
1036 	/*
1037 	 * The partition-scoped invalidations aren't handled here in L0.
1038 	 */
1039 	if (type & H_RPTI_TYPE_NESTED)
1040 		return RESUME_HOST;
1041 
1042 	pid = kvmppc_get_gpr(vcpu, 4);
1043 	pg_sizes = kvmppc_get_gpr(vcpu, 7);
1044 	start = kvmppc_get_gpr(vcpu, 8);
1045 	end = kvmppc_get_gpr(vcpu, 9);
1046 
1047 	do_h_rpt_invalidate_prt(pid, vcpu->arch.nested->shadow_lpid,
1048 				type, pg_sizes, start, end);
1049 
1050 	kvmppc_set_gpr(vcpu, 3, H_SUCCESS);
1051 	return RESUME_GUEST;
1052 }
1053 
kvmppc_h_rpt_invalidate(struct kvm_vcpu * vcpu,unsigned long id,unsigned long target,unsigned long type,unsigned long pg_sizes,unsigned long start,unsigned long end)1054 static long kvmppc_h_rpt_invalidate(struct kvm_vcpu *vcpu,
1055 				    unsigned long id, unsigned long target,
1056 				    unsigned long type, unsigned long pg_sizes,
1057 				    unsigned long start, unsigned long end)
1058 {
1059 	if (!kvm_is_radix(vcpu->kvm))
1060 		return H_UNSUPPORTED;
1061 
1062 	if (end < start)
1063 		return H_P5;
1064 
1065 	/*
1066 	 * Partition-scoped invalidation for nested guests.
1067 	 */
1068 	if (type & H_RPTI_TYPE_NESTED) {
1069 		if (!nesting_enabled(vcpu->kvm))
1070 			return H_FUNCTION;
1071 
1072 		/* Support only cores as target */
1073 		if (target != H_RPTI_TARGET_CMMU)
1074 			return H_P2;
1075 
1076 		return do_h_rpt_invalidate_pat(vcpu, id, type, pg_sizes,
1077 					       start, end);
1078 	}
1079 
1080 	/*
1081 	 * Process-scoped invalidation for L1 guests.
1082 	 */
1083 	do_h_rpt_invalidate_prt(id, vcpu->kvm->arch.lpid,
1084 				type, pg_sizes, start, end);
1085 	return H_SUCCESS;
1086 }
1087 
kvmppc_pseries_do_hcall(struct kvm_vcpu * vcpu)1088 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
1089 {
1090 	struct kvm *kvm = vcpu->kvm;
1091 	unsigned long req = kvmppc_get_gpr(vcpu, 3);
1092 	unsigned long target, ret = H_SUCCESS;
1093 	int yield_count;
1094 	struct kvm_vcpu *tvcpu;
1095 	int idx, rc;
1096 
1097 	if (req <= MAX_HCALL_OPCODE &&
1098 	    !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls))
1099 		return RESUME_HOST;
1100 
1101 	switch (req) {
1102 	case H_REMOVE:
1103 		ret = kvmppc_h_remove(vcpu, kvmppc_get_gpr(vcpu, 4),
1104 					kvmppc_get_gpr(vcpu, 5),
1105 					kvmppc_get_gpr(vcpu, 6));
1106 		if (ret == H_TOO_HARD)
1107 			return RESUME_HOST;
1108 		break;
1109 	case H_ENTER:
1110 		ret = kvmppc_h_enter(vcpu, kvmppc_get_gpr(vcpu, 4),
1111 					kvmppc_get_gpr(vcpu, 5),
1112 					kvmppc_get_gpr(vcpu, 6),
1113 					kvmppc_get_gpr(vcpu, 7));
1114 		if (ret == H_TOO_HARD)
1115 			return RESUME_HOST;
1116 		break;
1117 	case H_READ:
1118 		ret = kvmppc_h_read(vcpu, kvmppc_get_gpr(vcpu, 4),
1119 					kvmppc_get_gpr(vcpu, 5));
1120 		if (ret == H_TOO_HARD)
1121 			return RESUME_HOST;
1122 		break;
1123 	case H_CLEAR_MOD:
1124 		ret = kvmppc_h_clear_mod(vcpu, kvmppc_get_gpr(vcpu, 4),
1125 					kvmppc_get_gpr(vcpu, 5));
1126 		if (ret == H_TOO_HARD)
1127 			return RESUME_HOST;
1128 		break;
1129 	case H_CLEAR_REF:
1130 		ret = kvmppc_h_clear_ref(vcpu, kvmppc_get_gpr(vcpu, 4),
1131 					kvmppc_get_gpr(vcpu, 5));
1132 		if (ret == H_TOO_HARD)
1133 			return RESUME_HOST;
1134 		break;
1135 	case H_PROTECT:
1136 		ret = kvmppc_h_protect(vcpu, kvmppc_get_gpr(vcpu, 4),
1137 					kvmppc_get_gpr(vcpu, 5),
1138 					kvmppc_get_gpr(vcpu, 6));
1139 		if (ret == H_TOO_HARD)
1140 			return RESUME_HOST;
1141 		break;
1142 	case H_BULK_REMOVE:
1143 		ret = kvmppc_h_bulk_remove(vcpu);
1144 		if (ret == H_TOO_HARD)
1145 			return RESUME_HOST;
1146 		break;
1147 
1148 	case H_CEDE:
1149 		break;
1150 	case H_PROD:
1151 		target = kvmppc_get_gpr(vcpu, 4);
1152 		tvcpu = kvmppc_find_vcpu(kvm, target);
1153 		if (!tvcpu) {
1154 			ret = H_PARAMETER;
1155 			break;
1156 		}
1157 		tvcpu->arch.prodded = 1;
1158 		smp_mb(); /* This orders prodded store vs ceded load */
1159 		if (tvcpu->arch.ceded)
1160 			kvmppc_fast_vcpu_kick_hv(tvcpu);
1161 		break;
1162 	case H_CONFER:
1163 		target = kvmppc_get_gpr(vcpu, 4);
1164 		if (target == -1)
1165 			break;
1166 		tvcpu = kvmppc_find_vcpu(kvm, target);
1167 		if (!tvcpu) {
1168 			ret = H_PARAMETER;
1169 			break;
1170 		}
1171 		yield_count = kvmppc_get_gpr(vcpu, 5);
1172 		if (kvmppc_get_yield_count(tvcpu) != yield_count)
1173 			break;
1174 		kvm_arch_vcpu_yield_to(tvcpu);
1175 		break;
1176 	case H_REGISTER_VPA:
1177 		ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
1178 					kvmppc_get_gpr(vcpu, 5),
1179 					kvmppc_get_gpr(vcpu, 6));
1180 		break;
1181 	case H_RTAS:
1182 		if (list_empty(&kvm->arch.rtas_tokens))
1183 			return RESUME_HOST;
1184 
1185 		idx = srcu_read_lock(&kvm->srcu);
1186 		rc = kvmppc_rtas_hcall(vcpu);
1187 		srcu_read_unlock(&kvm->srcu, idx);
1188 
1189 		if (rc == -ENOENT)
1190 			return RESUME_HOST;
1191 		else if (rc == 0)
1192 			break;
1193 
1194 		/* Send the error out to userspace via KVM_RUN */
1195 		return rc;
1196 	case H_LOGICAL_CI_LOAD:
1197 		ret = kvmppc_h_logical_ci_load(vcpu);
1198 		if (ret == H_TOO_HARD)
1199 			return RESUME_HOST;
1200 		break;
1201 	case H_LOGICAL_CI_STORE:
1202 		ret = kvmppc_h_logical_ci_store(vcpu);
1203 		if (ret == H_TOO_HARD)
1204 			return RESUME_HOST;
1205 		break;
1206 	case H_SET_MODE:
1207 		ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4),
1208 					kvmppc_get_gpr(vcpu, 5),
1209 					kvmppc_get_gpr(vcpu, 6),
1210 					kvmppc_get_gpr(vcpu, 7));
1211 		if (ret == H_TOO_HARD)
1212 			return RESUME_HOST;
1213 		break;
1214 	case H_XIRR:
1215 	case H_CPPR:
1216 	case H_EOI:
1217 	case H_IPI:
1218 	case H_IPOLL:
1219 	case H_XIRR_X:
1220 		if (kvmppc_xics_enabled(vcpu)) {
1221 			if (xics_on_xive()) {
1222 				ret = H_NOT_AVAILABLE;
1223 				return RESUME_GUEST;
1224 			}
1225 			ret = kvmppc_xics_hcall(vcpu, req);
1226 			break;
1227 		}
1228 		return RESUME_HOST;
1229 	case H_SET_DABR:
1230 		ret = kvmppc_h_set_dabr(vcpu, kvmppc_get_gpr(vcpu, 4));
1231 		break;
1232 	case H_SET_XDABR:
1233 		ret = kvmppc_h_set_xdabr(vcpu, kvmppc_get_gpr(vcpu, 4),
1234 						kvmppc_get_gpr(vcpu, 5));
1235 		break;
1236 #ifdef CONFIG_SPAPR_TCE_IOMMU
1237 	case H_GET_TCE:
1238 		ret = kvmppc_h_get_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1239 						kvmppc_get_gpr(vcpu, 5));
1240 		if (ret == H_TOO_HARD)
1241 			return RESUME_HOST;
1242 		break;
1243 	case H_PUT_TCE:
1244 		ret = kvmppc_h_put_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1245 						kvmppc_get_gpr(vcpu, 5),
1246 						kvmppc_get_gpr(vcpu, 6));
1247 		if (ret == H_TOO_HARD)
1248 			return RESUME_HOST;
1249 		break;
1250 	case H_PUT_TCE_INDIRECT:
1251 		ret = kvmppc_h_put_tce_indirect(vcpu, kvmppc_get_gpr(vcpu, 4),
1252 						kvmppc_get_gpr(vcpu, 5),
1253 						kvmppc_get_gpr(vcpu, 6),
1254 						kvmppc_get_gpr(vcpu, 7));
1255 		if (ret == H_TOO_HARD)
1256 			return RESUME_HOST;
1257 		break;
1258 	case H_STUFF_TCE:
1259 		ret = kvmppc_h_stuff_tce(vcpu, kvmppc_get_gpr(vcpu, 4),
1260 						kvmppc_get_gpr(vcpu, 5),
1261 						kvmppc_get_gpr(vcpu, 6),
1262 						kvmppc_get_gpr(vcpu, 7));
1263 		if (ret == H_TOO_HARD)
1264 			return RESUME_HOST;
1265 		break;
1266 #endif
1267 	case H_RANDOM:
1268 		if (!arch_get_random_seed_longs(&vcpu->arch.regs.gpr[4], 1))
1269 			ret = H_HARDWARE;
1270 		break;
1271 	case H_RPT_INVALIDATE:
1272 		ret = kvmppc_h_rpt_invalidate(vcpu, kvmppc_get_gpr(vcpu, 4),
1273 					      kvmppc_get_gpr(vcpu, 5),
1274 					      kvmppc_get_gpr(vcpu, 6),
1275 					      kvmppc_get_gpr(vcpu, 7),
1276 					      kvmppc_get_gpr(vcpu, 8),
1277 					      kvmppc_get_gpr(vcpu, 9));
1278 		break;
1279 
1280 	case H_SET_PARTITION_TABLE:
1281 		ret = H_FUNCTION;
1282 		if (nesting_enabled(kvm))
1283 			ret = kvmhv_set_partition_table(vcpu);
1284 		break;
1285 	case H_ENTER_NESTED:
1286 		ret = H_FUNCTION;
1287 		if (!nesting_enabled(kvm))
1288 			break;
1289 		ret = kvmhv_enter_nested_guest(vcpu);
1290 		if (ret == H_INTERRUPT) {
1291 			kvmppc_set_gpr(vcpu, 3, 0);
1292 			vcpu->arch.hcall_needed = 0;
1293 			return -EINTR;
1294 		} else if (ret == H_TOO_HARD) {
1295 			kvmppc_set_gpr(vcpu, 3, 0);
1296 			vcpu->arch.hcall_needed = 0;
1297 			return RESUME_HOST;
1298 		}
1299 		break;
1300 	case H_TLB_INVALIDATE:
1301 		ret = H_FUNCTION;
1302 		if (nesting_enabled(kvm))
1303 			ret = kvmhv_do_nested_tlbie(vcpu);
1304 		break;
1305 	case H_COPY_TOFROM_GUEST:
1306 		ret = H_FUNCTION;
1307 		if (nesting_enabled(kvm))
1308 			ret = kvmhv_copy_tofrom_guest_nested(vcpu);
1309 		break;
1310 	case H_PAGE_INIT:
1311 		ret = kvmppc_h_page_init(vcpu, kvmppc_get_gpr(vcpu, 4),
1312 					 kvmppc_get_gpr(vcpu, 5),
1313 					 kvmppc_get_gpr(vcpu, 6));
1314 		break;
1315 	case H_SVM_PAGE_IN:
1316 		ret = H_UNSUPPORTED;
1317 		if (kvmppc_get_srr1(vcpu) & MSR_S)
1318 			ret = kvmppc_h_svm_page_in(kvm,
1319 						   kvmppc_get_gpr(vcpu, 4),
1320 						   kvmppc_get_gpr(vcpu, 5),
1321 						   kvmppc_get_gpr(vcpu, 6));
1322 		break;
1323 	case H_SVM_PAGE_OUT:
1324 		ret = H_UNSUPPORTED;
1325 		if (kvmppc_get_srr1(vcpu) & MSR_S)
1326 			ret = kvmppc_h_svm_page_out(kvm,
1327 						    kvmppc_get_gpr(vcpu, 4),
1328 						    kvmppc_get_gpr(vcpu, 5),
1329 						    kvmppc_get_gpr(vcpu, 6));
1330 		break;
1331 	case H_SVM_INIT_START:
1332 		ret = H_UNSUPPORTED;
1333 		if (kvmppc_get_srr1(vcpu) & MSR_S)
1334 			ret = kvmppc_h_svm_init_start(kvm);
1335 		break;
1336 	case H_SVM_INIT_DONE:
1337 		ret = H_UNSUPPORTED;
1338 		if (kvmppc_get_srr1(vcpu) & MSR_S)
1339 			ret = kvmppc_h_svm_init_done(kvm);
1340 		break;
1341 	case H_SVM_INIT_ABORT:
1342 		/*
1343 		 * Even if that call is made by the Ultravisor, the SSR1 value
1344 		 * is the guest context one, with the secure bit clear as it has
1345 		 * not yet been secured. So we can't check it here.
1346 		 * Instead the kvm->arch.secure_guest flag is checked inside
1347 		 * kvmppc_h_svm_init_abort().
1348 		 */
1349 		ret = kvmppc_h_svm_init_abort(kvm);
1350 		break;
1351 
1352 	default:
1353 		return RESUME_HOST;
1354 	}
1355 	WARN_ON_ONCE(ret == H_TOO_HARD);
1356 	kvmppc_set_gpr(vcpu, 3, ret);
1357 	vcpu->arch.hcall_needed = 0;
1358 	return RESUME_GUEST;
1359 }
1360 
1361 /*
1362  * Handle H_CEDE in the P9 path where we don't call the real-mode hcall
1363  * handlers in book3s_hv_rmhandlers.S.
1364  *
1365  * This has to be done early, not in kvmppc_pseries_do_hcall(), so
1366  * that the cede logic in kvmppc_run_single_vcpu() works properly.
1367  */
kvmppc_cede(struct kvm_vcpu * vcpu)1368 static void kvmppc_cede(struct kvm_vcpu *vcpu)
1369 {
1370 	vcpu->arch.shregs.msr |= MSR_EE;
1371 	vcpu->arch.ceded = 1;
1372 	smp_mb();
1373 	if (vcpu->arch.prodded) {
1374 		vcpu->arch.prodded = 0;
1375 		smp_mb();
1376 		vcpu->arch.ceded = 0;
1377 	}
1378 }
1379 
kvmppc_hcall_impl_hv(unsigned long cmd)1380 static int kvmppc_hcall_impl_hv(unsigned long cmd)
1381 {
1382 	switch (cmd) {
1383 	case H_CEDE:
1384 	case H_PROD:
1385 	case H_CONFER:
1386 	case H_REGISTER_VPA:
1387 	case H_SET_MODE:
1388 #ifdef CONFIG_SPAPR_TCE_IOMMU
1389 	case H_GET_TCE:
1390 	case H_PUT_TCE:
1391 	case H_PUT_TCE_INDIRECT:
1392 	case H_STUFF_TCE:
1393 #endif
1394 	case H_LOGICAL_CI_LOAD:
1395 	case H_LOGICAL_CI_STORE:
1396 #ifdef CONFIG_KVM_XICS
1397 	case H_XIRR:
1398 	case H_CPPR:
1399 	case H_EOI:
1400 	case H_IPI:
1401 	case H_IPOLL:
1402 	case H_XIRR_X:
1403 #endif
1404 	case H_PAGE_INIT:
1405 	case H_RPT_INVALIDATE:
1406 		return 1;
1407 	}
1408 
1409 	/* See if it's in the real-mode table */
1410 	return kvmppc_hcall_impl_hv_realmode(cmd);
1411 }
1412 
kvmppc_emulate_debug_inst(struct kvm_vcpu * vcpu)1413 static int kvmppc_emulate_debug_inst(struct kvm_vcpu *vcpu)
1414 {
1415 	u32 last_inst;
1416 
1417 	if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
1418 					EMULATE_DONE) {
1419 		/*
1420 		 * Fetch failed, so return to guest and
1421 		 * try executing it again.
1422 		 */
1423 		return RESUME_GUEST;
1424 	}
1425 
1426 	if (last_inst == KVMPPC_INST_SW_BREAKPOINT) {
1427 		vcpu->run->exit_reason = KVM_EXIT_DEBUG;
1428 		vcpu->run->debug.arch.address = kvmppc_get_pc(vcpu);
1429 		return RESUME_HOST;
1430 	} else {
1431 		kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1432 		return RESUME_GUEST;
1433 	}
1434 }
1435 
do_nothing(void * x)1436 static void do_nothing(void *x)
1437 {
1438 }
1439 
kvmppc_read_dpdes(struct kvm_vcpu * vcpu)1440 static unsigned long kvmppc_read_dpdes(struct kvm_vcpu *vcpu)
1441 {
1442 	int thr, cpu, pcpu, nthreads;
1443 	struct kvm_vcpu *v;
1444 	unsigned long dpdes;
1445 
1446 	nthreads = vcpu->kvm->arch.emul_smt_mode;
1447 	dpdes = 0;
1448 	cpu = vcpu->vcpu_id & ~(nthreads - 1);
1449 	for (thr = 0; thr < nthreads; ++thr, ++cpu) {
1450 		v = kvmppc_find_vcpu(vcpu->kvm, cpu);
1451 		if (!v)
1452 			continue;
1453 		/*
1454 		 * If the vcpu is currently running on a physical cpu thread,
1455 		 * interrupt it in order to pull it out of the guest briefly,
1456 		 * which will update its vcore->dpdes value.
1457 		 */
1458 		pcpu = READ_ONCE(v->cpu);
1459 		if (pcpu >= 0)
1460 			smp_call_function_single(pcpu, do_nothing, NULL, 1);
1461 		if (kvmppc_doorbell_pending(v))
1462 			dpdes |= 1 << thr;
1463 	}
1464 	return dpdes;
1465 }
1466 
1467 /*
1468  * On POWER9, emulate doorbell-related instructions in order to
1469  * give the guest the illusion of running on a multi-threaded core.
1470  * The instructions emulated are msgsndp, msgclrp, mfspr TIR,
1471  * and mfspr DPDES.
1472  */
kvmppc_emulate_doorbell_instr(struct kvm_vcpu * vcpu)1473 static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu)
1474 {
1475 	u32 inst, rb, thr;
1476 	unsigned long arg;
1477 	struct kvm *kvm = vcpu->kvm;
1478 	struct kvm_vcpu *tvcpu;
1479 
1480 	if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &inst) != EMULATE_DONE)
1481 		return RESUME_GUEST;
1482 	if (get_op(inst) != 31)
1483 		return EMULATE_FAIL;
1484 	rb = get_rb(inst);
1485 	thr = vcpu->vcpu_id & (kvm->arch.emul_smt_mode - 1);
1486 	switch (get_xop(inst)) {
1487 	case OP_31_XOP_MSGSNDP:
1488 		arg = kvmppc_get_gpr(vcpu, rb);
1489 		if (((arg >> 27) & 0x1f) != PPC_DBELL_SERVER)
1490 			break;
1491 		arg &= 0x7f;
1492 		if (arg >= kvm->arch.emul_smt_mode)
1493 			break;
1494 		tvcpu = kvmppc_find_vcpu(kvm, vcpu->vcpu_id - thr + arg);
1495 		if (!tvcpu)
1496 			break;
1497 		if (!tvcpu->arch.doorbell_request) {
1498 			tvcpu->arch.doorbell_request = 1;
1499 			kvmppc_fast_vcpu_kick_hv(tvcpu);
1500 		}
1501 		break;
1502 	case OP_31_XOP_MSGCLRP:
1503 		arg = kvmppc_get_gpr(vcpu, rb);
1504 		if (((arg >> 27) & 0x1f) != PPC_DBELL_SERVER)
1505 			break;
1506 		vcpu->arch.vcore->dpdes = 0;
1507 		vcpu->arch.doorbell_request = 0;
1508 		break;
1509 	case OP_31_XOP_MFSPR:
1510 		switch (get_sprn(inst)) {
1511 		case SPRN_TIR:
1512 			arg = thr;
1513 			break;
1514 		case SPRN_DPDES:
1515 			arg = kvmppc_read_dpdes(vcpu);
1516 			break;
1517 		default:
1518 			return EMULATE_FAIL;
1519 		}
1520 		kvmppc_set_gpr(vcpu, get_rt(inst), arg);
1521 		break;
1522 	default:
1523 		return EMULATE_FAIL;
1524 	}
1525 	kvmppc_set_pc(vcpu, kvmppc_get_pc(vcpu) + 4);
1526 	return RESUME_GUEST;
1527 }
1528 
1529 /*
1530  * If the lppaca had pmcregs_in_use clear when we exited the guest, then
1531  * HFSCR_PM is cleared for next entry. If the guest then tries to access
1532  * the PMU SPRs, we get this facility unavailable interrupt. Putting HFSCR_PM
1533  * back in the guest HFSCR will cause the next entry to load the PMU SPRs and
1534  * allow the guest access to continue.
1535  */
kvmppc_pmu_unavailable(struct kvm_vcpu * vcpu)1536 static int kvmppc_pmu_unavailable(struct kvm_vcpu *vcpu)
1537 {
1538 	if (!(vcpu->arch.hfscr_permitted & HFSCR_PM))
1539 		return EMULATE_FAIL;
1540 
1541 	vcpu->arch.hfscr |= HFSCR_PM;
1542 
1543 	return RESUME_GUEST;
1544 }
1545 
kvmppc_ebb_unavailable(struct kvm_vcpu * vcpu)1546 static int kvmppc_ebb_unavailable(struct kvm_vcpu *vcpu)
1547 {
1548 	if (!(vcpu->arch.hfscr_permitted & HFSCR_EBB))
1549 		return EMULATE_FAIL;
1550 
1551 	vcpu->arch.hfscr |= HFSCR_EBB;
1552 
1553 	return RESUME_GUEST;
1554 }
1555 
kvmppc_tm_unavailable(struct kvm_vcpu * vcpu)1556 static int kvmppc_tm_unavailable(struct kvm_vcpu *vcpu)
1557 {
1558 	if (!(vcpu->arch.hfscr_permitted & HFSCR_TM))
1559 		return EMULATE_FAIL;
1560 
1561 	vcpu->arch.hfscr |= HFSCR_TM;
1562 
1563 	return RESUME_GUEST;
1564 }
1565 
kvmppc_handle_exit_hv(struct kvm_vcpu * vcpu,struct task_struct * tsk)1566 static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu,
1567 				 struct task_struct *tsk)
1568 {
1569 	struct kvm_run *run = vcpu->run;
1570 	int r = RESUME_HOST;
1571 
1572 	vcpu->stat.sum_exits++;
1573 
1574 	/*
1575 	 * This can happen if an interrupt occurs in the last stages
1576 	 * of guest entry or the first stages of guest exit (i.e. after
1577 	 * setting paca->kvm_hstate.in_guest to KVM_GUEST_MODE_GUEST_HV
1578 	 * and before setting it to KVM_GUEST_MODE_HOST_HV).
1579 	 * That can happen due to a bug, or due to a machine check
1580 	 * occurring at just the wrong time.
1581 	 */
1582 	if (vcpu->arch.shregs.msr & MSR_HV) {
1583 		printk(KERN_EMERG "KVM trap in HV mode!\n");
1584 		printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1585 			vcpu->arch.trap, kvmppc_get_pc(vcpu),
1586 			vcpu->arch.shregs.msr);
1587 		kvmppc_dump_regs(vcpu);
1588 		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1589 		run->hw.hardware_exit_reason = vcpu->arch.trap;
1590 		return RESUME_HOST;
1591 	}
1592 	run->exit_reason = KVM_EXIT_UNKNOWN;
1593 	run->ready_for_interrupt_injection = 1;
1594 	switch (vcpu->arch.trap) {
1595 	/* We're good on these - the host merely wanted to get our attention */
1596 	case BOOK3S_INTERRUPT_NESTED_HV_DECREMENTER:
1597 		WARN_ON_ONCE(1); /* Should never happen */
1598 		vcpu->arch.trap = BOOK3S_INTERRUPT_HV_DECREMENTER;
1599 		fallthrough;
1600 	case BOOK3S_INTERRUPT_HV_DECREMENTER:
1601 		vcpu->stat.dec_exits++;
1602 		r = RESUME_GUEST;
1603 		break;
1604 	case BOOK3S_INTERRUPT_EXTERNAL:
1605 	case BOOK3S_INTERRUPT_H_DOORBELL:
1606 	case BOOK3S_INTERRUPT_H_VIRT:
1607 		vcpu->stat.ext_intr_exits++;
1608 		r = RESUME_GUEST;
1609 		break;
1610 	/* SR/HMI/PMI are HV interrupts that host has handled. Resume guest.*/
1611 	case BOOK3S_INTERRUPT_HMI:
1612 	case BOOK3S_INTERRUPT_PERFMON:
1613 	case BOOK3S_INTERRUPT_SYSTEM_RESET:
1614 		r = RESUME_GUEST;
1615 		break;
1616 	case BOOK3S_INTERRUPT_MACHINE_CHECK: {
1617 		static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
1618 					      DEFAULT_RATELIMIT_BURST);
1619 		/*
1620 		 * Print the MCE event to host console. Ratelimit so the guest
1621 		 * can't flood the host log.
1622 		 */
1623 		if (__ratelimit(&rs))
1624 			machine_check_print_event_info(&vcpu->arch.mce_evt,false, true);
1625 
1626 		/*
1627 		 * If the guest can do FWNMI, exit to userspace so it can
1628 		 * deliver a FWNMI to the guest.
1629 		 * Otherwise we synthesize a machine check for the guest
1630 		 * so that it knows that the machine check occurred.
1631 		 */
1632 		if (!vcpu->kvm->arch.fwnmi_enabled) {
1633 			ulong flags = vcpu->arch.shregs.msr & 0x083c0000;
1634 			kvmppc_core_queue_machine_check(vcpu, flags);
1635 			r = RESUME_GUEST;
1636 			break;
1637 		}
1638 
1639 		/* Exit to guest with KVM_EXIT_NMI as exit reason */
1640 		run->exit_reason = KVM_EXIT_NMI;
1641 		run->hw.hardware_exit_reason = vcpu->arch.trap;
1642 		/* Clear out the old NMI status from run->flags */
1643 		run->flags &= ~KVM_RUN_PPC_NMI_DISP_MASK;
1644 		/* Now set the NMI status */
1645 		if (vcpu->arch.mce_evt.disposition == MCE_DISPOSITION_RECOVERED)
1646 			run->flags |= KVM_RUN_PPC_NMI_DISP_FULLY_RECOV;
1647 		else
1648 			run->flags |= KVM_RUN_PPC_NMI_DISP_NOT_RECOV;
1649 
1650 		r = RESUME_HOST;
1651 		break;
1652 	}
1653 	case BOOK3S_INTERRUPT_PROGRAM:
1654 	{
1655 		ulong flags;
1656 		/*
1657 		 * Normally program interrupts are delivered directly
1658 		 * to the guest by the hardware, but we can get here
1659 		 * as a result of a hypervisor emulation interrupt
1660 		 * (e40) getting turned into a 700 by BML RTAS.
1661 		 */
1662 		flags = vcpu->arch.shregs.msr & 0x1f0000ull;
1663 		kvmppc_core_queue_program(vcpu, flags);
1664 		r = RESUME_GUEST;
1665 		break;
1666 	}
1667 	case BOOK3S_INTERRUPT_SYSCALL:
1668 	{
1669 		int i;
1670 
1671 		if (unlikely(vcpu->arch.shregs.msr & MSR_PR)) {
1672 			/*
1673 			 * Guest userspace executed sc 1. This can only be
1674 			 * reached by the P9 path because the old path
1675 			 * handles this case in realmode hcall handlers.
1676 			 */
1677 			if (!kvmhv_vcpu_is_radix(vcpu)) {
1678 				/*
1679 				 * A guest could be running PR KVM, so this
1680 				 * may be a PR KVM hcall. It must be reflected
1681 				 * to the guest kernel as a sc interrupt.
1682 				 */
1683 				kvmppc_core_queue_syscall(vcpu);
1684 			} else {
1685 				/*
1686 				 * Radix guests can not run PR KVM or nested HV
1687 				 * hash guests which might run PR KVM, so this
1688 				 * is always a privilege fault. Send a program
1689 				 * check to guest kernel.
1690 				 */
1691 				kvmppc_core_queue_program(vcpu, SRR1_PROGPRIV);
1692 			}
1693 			r = RESUME_GUEST;
1694 			break;
1695 		}
1696 
1697 		/*
1698 		 * hcall - gather args and set exit_reason. This will next be
1699 		 * handled by kvmppc_pseries_do_hcall which may be able to deal
1700 		 * with it and resume guest, or may punt to userspace.
1701 		 */
1702 		run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
1703 		for (i = 0; i < 9; ++i)
1704 			run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
1705 		run->exit_reason = KVM_EXIT_PAPR_HCALL;
1706 		vcpu->arch.hcall_needed = 1;
1707 		r = RESUME_HOST;
1708 		break;
1709 	}
1710 	/*
1711 	 * We get these next two if the guest accesses a page which it thinks
1712 	 * it has mapped but which is not actually present, either because
1713 	 * it is for an emulated I/O device or because the corresonding
1714 	 * host page has been paged out.
1715 	 *
1716 	 * Any other HDSI/HISI interrupts have been handled already for P7/8
1717 	 * guests. For POWER9 hash guests not using rmhandlers, basic hash
1718 	 * fault handling is done here.
1719 	 */
1720 	case BOOK3S_INTERRUPT_H_DATA_STORAGE: {
1721 		unsigned long vsid;
1722 		long err;
1723 
1724 		if (cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG) &&
1725 		    unlikely(vcpu->arch.fault_dsisr == HDSISR_CANARY)) {
1726 			r = RESUME_GUEST; /* Just retry if it's the canary */
1727 			break;
1728 		}
1729 
1730 		if (kvm_is_radix(vcpu->kvm) || !cpu_has_feature(CPU_FTR_ARCH_300)) {
1731 			/*
1732 			 * Radix doesn't require anything, and pre-ISAv3.0 hash
1733 			 * already attempted to handle this in rmhandlers. The
1734 			 * hash fault handling below is v3 only (it uses ASDR
1735 			 * via fault_gpa).
1736 			 */
1737 			r = RESUME_PAGE_FAULT;
1738 			break;
1739 		}
1740 
1741 		if (!(vcpu->arch.fault_dsisr & (DSISR_NOHPTE | DSISR_PROTFAULT))) {
1742 			kvmppc_core_queue_data_storage(vcpu,
1743 				vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
1744 			r = RESUME_GUEST;
1745 			break;
1746 		}
1747 
1748 		if (!(vcpu->arch.shregs.msr & MSR_DR))
1749 			vsid = vcpu->kvm->arch.vrma_slb_v;
1750 		else
1751 			vsid = vcpu->arch.fault_gpa;
1752 
1753 		err = kvmppc_hpte_hv_fault(vcpu, vcpu->arch.fault_dar,
1754 				vsid, vcpu->arch.fault_dsisr, true);
1755 		if (err == 0) {
1756 			r = RESUME_GUEST;
1757 		} else if (err == -1 || err == -2) {
1758 			r = RESUME_PAGE_FAULT;
1759 		} else {
1760 			kvmppc_core_queue_data_storage(vcpu,
1761 				vcpu->arch.fault_dar, err);
1762 			r = RESUME_GUEST;
1763 		}
1764 		break;
1765 	}
1766 	case BOOK3S_INTERRUPT_H_INST_STORAGE: {
1767 		unsigned long vsid;
1768 		long err;
1769 
1770 		vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1771 		vcpu->arch.fault_dsisr = vcpu->arch.shregs.msr &
1772 			DSISR_SRR1_MATCH_64S;
1773 		if (kvm_is_radix(vcpu->kvm) || !cpu_has_feature(CPU_FTR_ARCH_300)) {
1774 			/*
1775 			 * Radix doesn't require anything, and pre-ISAv3.0 hash
1776 			 * already attempted to handle this in rmhandlers. The
1777 			 * hash fault handling below is v3 only (it uses ASDR
1778 			 * via fault_gpa).
1779 			 */
1780 			if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
1781 				vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
1782 			r = RESUME_PAGE_FAULT;
1783 			break;
1784 		}
1785 
1786 		if (!(vcpu->arch.fault_dsisr & SRR1_ISI_NOPT)) {
1787 			kvmppc_core_queue_inst_storage(vcpu,
1788 				vcpu->arch.fault_dsisr);
1789 			r = RESUME_GUEST;
1790 			break;
1791 		}
1792 
1793 		if (!(vcpu->arch.shregs.msr & MSR_IR))
1794 			vsid = vcpu->kvm->arch.vrma_slb_v;
1795 		else
1796 			vsid = vcpu->arch.fault_gpa;
1797 
1798 		err = kvmppc_hpte_hv_fault(vcpu, vcpu->arch.fault_dar,
1799 				vsid, vcpu->arch.fault_dsisr, false);
1800 		if (err == 0) {
1801 			r = RESUME_GUEST;
1802 		} else if (err == -1) {
1803 			r = RESUME_PAGE_FAULT;
1804 		} else {
1805 			kvmppc_core_queue_inst_storage(vcpu, err);
1806 			r = RESUME_GUEST;
1807 		}
1808 		break;
1809 	}
1810 
1811 	/*
1812 	 * This occurs if the guest executes an illegal instruction.
1813 	 * If the guest debug is disabled, generate a program interrupt
1814 	 * to the guest. If guest debug is enabled, we need to check
1815 	 * whether the instruction is a software breakpoint instruction.
1816 	 * Accordingly return to Guest or Host.
1817 	 */
1818 	case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
1819 		if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED)
1820 			vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ?
1821 				swab32(vcpu->arch.emul_inst) :
1822 				vcpu->arch.emul_inst;
1823 		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
1824 			r = kvmppc_emulate_debug_inst(vcpu);
1825 		} else {
1826 			kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1827 			r = RESUME_GUEST;
1828 		}
1829 		break;
1830 
1831 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1832 	case BOOK3S_INTERRUPT_HV_SOFTPATCH:
1833 		/*
1834 		 * This occurs for various TM-related instructions that
1835 		 * we need to emulate on POWER9 DD2.2.  We have already
1836 		 * handled the cases where the guest was in real-suspend
1837 		 * mode and was transitioning to transactional state.
1838 		 */
1839 		r = kvmhv_p9_tm_emulation(vcpu);
1840 		if (r != -1)
1841 			break;
1842 		fallthrough; /* go to facility unavailable handler */
1843 #endif
1844 
1845 	/*
1846 	 * This occurs if the guest (kernel or userspace), does something that
1847 	 * is prohibited by HFSCR.
1848 	 * On POWER9, this could be a doorbell instruction that we need
1849 	 * to emulate.
1850 	 * Otherwise, we just generate a program interrupt to the guest.
1851 	 */
1852 	case BOOK3S_INTERRUPT_H_FAC_UNAVAIL: {
1853 		u64 cause = vcpu->arch.hfscr >> 56;
1854 
1855 		r = EMULATE_FAIL;
1856 		if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1857 			if (cause == FSCR_MSGP_LG)
1858 				r = kvmppc_emulate_doorbell_instr(vcpu);
1859 			if (cause == FSCR_PM_LG)
1860 				r = kvmppc_pmu_unavailable(vcpu);
1861 			if (cause == FSCR_EBB_LG)
1862 				r = kvmppc_ebb_unavailable(vcpu);
1863 			if (cause == FSCR_TM_LG)
1864 				r = kvmppc_tm_unavailable(vcpu);
1865 		}
1866 		if (r == EMULATE_FAIL) {
1867 			kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
1868 			r = RESUME_GUEST;
1869 		}
1870 		break;
1871 	}
1872 
1873 	case BOOK3S_INTERRUPT_HV_RM_HARD:
1874 		r = RESUME_PASSTHROUGH;
1875 		break;
1876 	default:
1877 		kvmppc_dump_regs(vcpu);
1878 		printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1879 			vcpu->arch.trap, kvmppc_get_pc(vcpu),
1880 			vcpu->arch.shregs.msr);
1881 		run->hw.hardware_exit_reason = vcpu->arch.trap;
1882 		r = RESUME_HOST;
1883 		break;
1884 	}
1885 
1886 	return r;
1887 }
1888 
kvmppc_handle_nested_exit(struct kvm_vcpu * vcpu)1889 static int kvmppc_handle_nested_exit(struct kvm_vcpu *vcpu)
1890 {
1891 	int r;
1892 	int srcu_idx;
1893 
1894 	vcpu->stat.sum_exits++;
1895 
1896 	/*
1897 	 * This can happen if an interrupt occurs in the last stages
1898 	 * of guest entry or the first stages of guest exit (i.e. after
1899 	 * setting paca->kvm_hstate.in_guest to KVM_GUEST_MODE_GUEST_HV
1900 	 * and before setting it to KVM_GUEST_MODE_HOST_HV).
1901 	 * That can happen due to a bug, or due to a machine check
1902 	 * occurring at just the wrong time.
1903 	 */
1904 	if (vcpu->arch.shregs.msr & MSR_HV) {
1905 		pr_emerg("KVM trap in HV mode while nested!\n");
1906 		pr_emerg("trap=0x%x | pc=0x%lx | msr=0x%llx\n",
1907 			 vcpu->arch.trap, kvmppc_get_pc(vcpu),
1908 			 vcpu->arch.shregs.msr);
1909 		kvmppc_dump_regs(vcpu);
1910 		return RESUME_HOST;
1911 	}
1912 	switch (vcpu->arch.trap) {
1913 	/* We're good on these - the host merely wanted to get our attention */
1914 	case BOOK3S_INTERRUPT_HV_DECREMENTER:
1915 		vcpu->stat.dec_exits++;
1916 		r = RESUME_GUEST;
1917 		break;
1918 	case BOOK3S_INTERRUPT_EXTERNAL:
1919 		vcpu->stat.ext_intr_exits++;
1920 		r = RESUME_HOST;
1921 		break;
1922 	case BOOK3S_INTERRUPT_H_DOORBELL:
1923 	case BOOK3S_INTERRUPT_H_VIRT:
1924 		vcpu->stat.ext_intr_exits++;
1925 		r = RESUME_GUEST;
1926 		break;
1927 	/* These need to go to the nested HV */
1928 	case BOOK3S_INTERRUPT_NESTED_HV_DECREMENTER:
1929 		vcpu->arch.trap = BOOK3S_INTERRUPT_HV_DECREMENTER;
1930 		vcpu->stat.dec_exits++;
1931 		r = RESUME_HOST;
1932 		break;
1933 	/* SR/HMI/PMI are HV interrupts that host has handled. Resume guest.*/
1934 	case BOOK3S_INTERRUPT_HMI:
1935 	case BOOK3S_INTERRUPT_PERFMON:
1936 	case BOOK3S_INTERRUPT_SYSTEM_RESET:
1937 		r = RESUME_GUEST;
1938 		break;
1939 	case BOOK3S_INTERRUPT_MACHINE_CHECK:
1940 	{
1941 		static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
1942 					      DEFAULT_RATELIMIT_BURST);
1943 		/* Pass the machine check to the L1 guest */
1944 		r = RESUME_HOST;
1945 		/* Print the MCE event to host console. */
1946 		if (__ratelimit(&rs))
1947 			machine_check_print_event_info(&vcpu->arch.mce_evt, false, true);
1948 		break;
1949 	}
1950 	/*
1951 	 * We get these next two if the guest accesses a page which it thinks
1952 	 * it has mapped but which is not actually present, either because
1953 	 * it is for an emulated I/O device or because the corresonding
1954 	 * host page has been paged out.
1955 	 */
1956 	case BOOK3S_INTERRUPT_H_DATA_STORAGE:
1957 		srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1958 		r = kvmhv_nested_page_fault(vcpu);
1959 		srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1960 		break;
1961 	case BOOK3S_INTERRUPT_H_INST_STORAGE:
1962 		vcpu->arch.fault_dar = kvmppc_get_pc(vcpu);
1963 		vcpu->arch.fault_dsisr = kvmppc_get_msr(vcpu) &
1964 					 DSISR_SRR1_MATCH_64S;
1965 		if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE)
1966 			vcpu->arch.fault_dsisr |= DSISR_ISSTORE;
1967 		srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1968 		r = kvmhv_nested_page_fault(vcpu);
1969 		srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
1970 		break;
1971 
1972 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1973 	case BOOK3S_INTERRUPT_HV_SOFTPATCH:
1974 		/*
1975 		 * This occurs for various TM-related instructions that
1976 		 * we need to emulate on POWER9 DD2.2.  We have already
1977 		 * handled the cases where the guest was in real-suspend
1978 		 * mode and was transitioning to transactional state.
1979 		 */
1980 		r = kvmhv_p9_tm_emulation(vcpu);
1981 		if (r != -1)
1982 			break;
1983 		fallthrough; /* go to facility unavailable handler */
1984 #endif
1985 
1986 	case BOOK3S_INTERRUPT_H_FAC_UNAVAIL: {
1987 		u64 cause = vcpu->arch.hfscr >> 56;
1988 
1989 		/*
1990 		 * Only pass HFU interrupts to the L1 if the facility is
1991 		 * permitted but disabled by the L1's HFSCR, otherwise
1992 		 * the interrupt does not make sense to the L1 so turn
1993 		 * it into a HEAI.
1994 		 */
1995 		if (!(vcpu->arch.hfscr_permitted & (1UL << cause)) ||
1996 				(vcpu->arch.nested_hfscr & (1UL << cause))) {
1997 			vcpu->arch.trap = BOOK3S_INTERRUPT_H_EMUL_ASSIST;
1998 
1999 			/*
2000 			 * If the fetch failed, return to guest and
2001 			 * try executing it again.
2002 			 */
2003 			r = kvmppc_get_last_inst(vcpu, INST_GENERIC,
2004 						 &vcpu->arch.emul_inst);
2005 			if (r != EMULATE_DONE)
2006 				r = RESUME_GUEST;
2007 			else
2008 				r = RESUME_HOST;
2009 		} else {
2010 			r = RESUME_HOST;
2011 		}
2012 
2013 		break;
2014 	}
2015 
2016 	case BOOK3S_INTERRUPT_HV_RM_HARD:
2017 		vcpu->arch.trap = 0;
2018 		r = RESUME_GUEST;
2019 		if (!xics_on_xive())
2020 			kvmppc_xics_rm_complete(vcpu, 0);
2021 		break;
2022 	case BOOK3S_INTERRUPT_SYSCALL:
2023 	{
2024 		unsigned long req = kvmppc_get_gpr(vcpu, 3);
2025 
2026 		/*
2027 		 * The H_RPT_INVALIDATE hcalls issued by nested
2028 		 * guests for process-scoped invalidations when
2029 		 * GTSE=0, are handled here in L0.
2030 		 */
2031 		if (req == H_RPT_INVALIDATE) {
2032 			r = kvmppc_nested_h_rpt_invalidate(vcpu);
2033 			break;
2034 		}
2035 
2036 		r = RESUME_HOST;
2037 		break;
2038 	}
2039 	default:
2040 		r = RESUME_HOST;
2041 		break;
2042 	}
2043 
2044 	return r;
2045 }
2046 
kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)2047 static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu,
2048 					    struct kvm_sregs *sregs)
2049 {
2050 	int i;
2051 
2052 	memset(sregs, 0, sizeof(struct kvm_sregs));
2053 	sregs->pvr = vcpu->arch.pvr;
2054 	for (i = 0; i < vcpu->arch.slb_max; i++) {
2055 		sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
2056 		sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
2057 	}
2058 
2059 	return 0;
2060 }
2061 
kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)2062 static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu,
2063 					    struct kvm_sregs *sregs)
2064 {
2065 	int i, j;
2066 
2067 	/* Only accept the same PVR as the host's, since we can't spoof it */
2068 	if (sregs->pvr != vcpu->arch.pvr)
2069 		return -EINVAL;
2070 
2071 	j = 0;
2072 	for (i = 0; i < vcpu->arch.slb_nr; i++) {
2073 		if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
2074 			vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
2075 			vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
2076 			++j;
2077 		}
2078 	}
2079 	vcpu->arch.slb_max = j;
2080 
2081 	return 0;
2082 }
2083 
2084 /*
2085  * Enforce limits on guest LPCR values based on hardware availability,
2086  * guest configuration, and possibly hypervisor support and security
2087  * concerns.
2088  */
kvmppc_filter_lpcr_hv(struct kvm * kvm,unsigned long lpcr)2089 unsigned long kvmppc_filter_lpcr_hv(struct kvm *kvm, unsigned long lpcr)
2090 {
2091 	/* LPCR_TC only applies to HPT guests */
2092 	if (kvm_is_radix(kvm))
2093 		lpcr &= ~LPCR_TC;
2094 
2095 	/* On POWER8 and above, userspace can modify AIL */
2096 	if (!cpu_has_feature(CPU_FTR_ARCH_207S))
2097 		lpcr &= ~LPCR_AIL;
2098 	if ((lpcr & LPCR_AIL) != LPCR_AIL_3)
2099 		lpcr &= ~LPCR_AIL; /* LPCR[AIL]=1/2 is disallowed */
2100 	/*
2101 	 * On some POWER9s we force AIL off for radix guests to prevent
2102 	 * executing in MSR[HV]=1 mode with the MMU enabled and PIDR set to
2103 	 * guest, which can result in Q0 translations with LPID=0 PID=PIDR to
2104 	 * be cached, which the host TLB management does not expect.
2105 	 */
2106 	if (kvm_is_radix(kvm) && cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
2107 		lpcr &= ~LPCR_AIL;
2108 
2109 	/*
2110 	 * On POWER9, allow userspace to enable large decrementer for the
2111 	 * guest, whether or not the host has it enabled.
2112 	 */
2113 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
2114 		lpcr &= ~LPCR_LD;
2115 
2116 	return lpcr;
2117 }
2118 
verify_lpcr(struct kvm * kvm,unsigned long lpcr)2119 static void verify_lpcr(struct kvm *kvm, unsigned long lpcr)
2120 {
2121 	if (lpcr != kvmppc_filter_lpcr_hv(kvm, lpcr)) {
2122 		WARN_ONCE(1, "lpcr 0x%lx differs from filtered 0x%lx\n",
2123 			  lpcr, kvmppc_filter_lpcr_hv(kvm, lpcr));
2124 	}
2125 }
2126 
kvmppc_set_lpcr(struct kvm_vcpu * vcpu,u64 new_lpcr,bool preserve_top32)2127 static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
2128 		bool preserve_top32)
2129 {
2130 	struct kvm *kvm = vcpu->kvm;
2131 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
2132 	u64 mask;
2133 
2134 	spin_lock(&vc->lock);
2135 
2136 	/*
2137 	 * Userspace can only modify
2138 	 * DPFD (default prefetch depth), ILE (interrupt little-endian),
2139 	 * TC (translation control), AIL (alternate interrupt location),
2140 	 * LD (large decrementer).
2141 	 * These are subject to restrictions from kvmppc_filter_lcpr_hv().
2142 	 */
2143 	mask = LPCR_DPFD | LPCR_ILE | LPCR_TC | LPCR_AIL | LPCR_LD;
2144 
2145 	/* Broken 32-bit version of LPCR must not clear top bits */
2146 	if (preserve_top32)
2147 		mask &= 0xFFFFFFFF;
2148 
2149 	new_lpcr = kvmppc_filter_lpcr_hv(kvm,
2150 			(vc->lpcr & ~mask) | (new_lpcr & mask));
2151 
2152 	/*
2153 	 * If ILE (interrupt little-endian) has changed, update the
2154 	 * MSR_LE bit in the intr_msr for each vcpu in this vcore.
2155 	 */
2156 	if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) {
2157 		struct kvm_vcpu *vcpu;
2158 		unsigned long i;
2159 
2160 		kvm_for_each_vcpu(i, vcpu, kvm) {
2161 			if (vcpu->arch.vcore != vc)
2162 				continue;
2163 			if (new_lpcr & LPCR_ILE)
2164 				vcpu->arch.intr_msr |= MSR_LE;
2165 			else
2166 				vcpu->arch.intr_msr &= ~MSR_LE;
2167 		}
2168 	}
2169 
2170 	vc->lpcr = new_lpcr;
2171 
2172 	spin_unlock(&vc->lock);
2173 }
2174 
kvmppc_get_one_reg_hv(struct kvm_vcpu * vcpu,u64 id,union kvmppc_one_reg * val)2175 static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
2176 				 union kvmppc_one_reg *val)
2177 {
2178 	int r = 0;
2179 	long int i;
2180 
2181 	switch (id) {
2182 	case KVM_REG_PPC_DEBUG_INST:
2183 		*val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
2184 		break;
2185 	case KVM_REG_PPC_HIOR:
2186 		*val = get_reg_val(id, 0);
2187 		break;
2188 	case KVM_REG_PPC_DABR:
2189 		*val = get_reg_val(id, vcpu->arch.dabr);
2190 		break;
2191 	case KVM_REG_PPC_DABRX:
2192 		*val = get_reg_val(id, vcpu->arch.dabrx);
2193 		break;
2194 	case KVM_REG_PPC_DSCR:
2195 		*val = get_reg_val(id, vcpu->arch.dscr);
2196 		break;
2197 	case KVM_REG_PPC_PURR:
2198 		*val = get_reg_val(id, vcpu->arch.purr);
2199 		break;
2200 	case KVM_REG_PPC_SPURR:
2201 		*val = get_reg_val(id, vcpu->arch.spurr);
2202 		break;
2203 	case KVM_REG_PPC_AMR:
2204 		*val = get_reg_val(id, vcpu->arch.amr);
2205 		break;
2206 	case KVM_REG_PPC_UAMOR:
2207 		*val = get_reg_val(id, vcpu->arch.uamor);
2208 		break;
2209 	case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCR1:
2210 		i = id - KVM_REG_PPC_MMCR0;
2211 		*val = get_reg_val(id, vcpu->arch.mmcr[i]);
2212 		break;
2213 	case KVM_REG_PPC_MMCR2:
2214 		*val = get_reg_val(id, vcpu->arch.mmcr[2]);
2215 		break;
2216 	case KVM_REG_PPC_MMCRA:
2217 		*val = get_reg_val(id, vcpu->arch.mmcra);
2218 		break;
2219 	case KVM_REG_PPC_MMCRS:
2220 		*val = get_reg_val(id, vcpu->arch.mmcrs);
2221 		break;
2222 	case KVM_REG_PPC_MMCR3:
2223 		*val = get_reg_val(id, vcpu->arch.mmcr[3]);
2224 		break;
2225 	case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
2226 		i = id - KVM_REG_PPC_PMC1;
2227 		*val = get_reg_val(id, vcpu->arch.pmc[i]);
2228 		break;
2229 	case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
2230 		i = id - KVM_REG_PPC_SPMC1;
2231 		*val = get_reg_val(id, vcpu->arch.spmc[i]);
2232 		break;
2233 	case KVM_REG_PPC_SIAR:
2234 		*val = get_reg_val(id, vcpu->arch.siar);
2235 		break;
2236 	case KVM_REG_PPC_SDAR:
2237 		*val = get_reg_val(id, vcpu->arch.sdar);
2238 		break;
2239 	case KVM_REG_PPC_SIER:
2240 		*val = get_reg_val(id, vcpu->arch.sier[0]);
2241 		break;
2242 	case KVM_REG_PPC_SIER2:
2243 		*val = get_reg_val(id, vcpu->arch.sier[1]);
2244 		break;
2245 	case KVM_REG_PPC_SIER3:
2246 		*val = get_reg_val(id, vcpu->arch.sier[2]);
2247 		break;
2248 	case KVM_REG_PPC_IAMR:
2249 		*val = get_reg_val(id, vcpu->arch.iamr);
2250 		break;
2251 	case KVM_REG_PPC_PSPB:
2252 		*val = get_reg_val(id, vcpu->arch.pspb);
2253 		break;
2254 	case KVM_REG_PPC_DPDES:
2255 		/*
2256 		 * On POWER9, where we are emulating msgsndp etc.,
2257 		 * we return 1 bit for each vcpu, which can come from
2258 		 * either vcore->dpdes or doorbell_request.
2259 		 * On POWER8, doorbell_request is 0.
2260 		 */
2261 		if (cpu_has_feature(CPU_FTR_ARCH_300))
2262 			*val = get_reg_val(id, vcpu->arch.doorbell_request);
2263 		else
2264 			*val = get_reg_val(id, vcpu->arch.vcore->dpdes);
2265 		break;
2266 	case KVM_REG_PPC_VTB:
2267 		*val = get_reg_val(id, vcpu->arch.vcore->vtb);
2268 		break;
2269 	case KVM_REG_PPC_DAWR:
2270 		*val = get_reg_val(id, vcpu->arch.dawr0);
2271 		break;
2272 	case KVM_REG_PPC_DAWRX:
2273 		*val = get_reg_val(id, vcpu->arch.dawrx0);
2274 		break;
2275 	case KVM_REG_PPC_DAWR1:
2276 		*val = get_reg_val(id, vcpu->arch.dawr1);
2277 		break;
2278 	case KVM_REG_PPC_DAWRX1:
2279 		*val = get_reg_val(id, vcpu->arch.dawrx1);
2280 		break;
2281 	case KVM_REG_PPC_CIABR:
2282 		*val = get_reg_val(id, vcpu->arch.ciabr);
2283 		break;
2284 	case KVM_REG_PPC_CSIGR:
2285 		*val = get_reg_val(id, vcpu->arch.csigr);
2286 		break;
2287 	case KVM_REG_PPC_TACR:
2288 		*val = get_reg_val(id, vcpu->arch.tacr);
2289 		break;
2290 	case KVM_REG_PPC_TCSCR:
2291 		*val = get_reg_val(id, vcpu->arch.tcscr);
2292 		break;
2293 	case KVM_REG_PPC_PID:
2294 		*val = get_reg_val(id, vcpu->arch.pid);
2295 		break;
2296 	case KVM_REG_PPC_ACOP:
2297 		*val = get_reg_val(id, vcpu->arch.acop);
2298 		break;
2299 	case KVM_REG_PPC_WORT:
2300 		*val = get_reg_val(id, vcpu->arch.wort);
2301 		break;
2302 	case KVM_REG_PPC_TIDR:
2303 		*val = get_reg_val(id, vcpu->arch.tid);
2304 		break;
2305 	case KVM_REG_PPC_PSSCR:
2306 		*val = get_reg_val(id, vcpu->arch.psscr);
2307 		break;
2308 	case KVM_REG_PPC_VPA_ADDR:
2309 		spin_lock(&vcpu->arch.vpa_update_lock);
2310 		*val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
2311 		spin_unlock(&vcpu->arch.vpa_update_lock);
2312 		break;
2313 	case KVM_REG_PPC_VPA_SLB:
2314 		spin_lock(&vcpu->arch.vpa_update_lock);
2315 		val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
2316 		val->vpaval.length = vcpu->arch.slb_shadow.len;
2317 		spin_unlock(&vcpu->arch.vpa_update_lock);
2318 		break;
2319 	case KVM_REG_PPC_VPA_DTL:
2320 		spin_lock(&vcpu->arch.vpa_update_lock);
2321 		val->vpaval.addr = vcpu->arch.dtl.next_gpa;
2322 		val->vpaval.length = vcpu->arch.dtl.len;
2323 		spin_unlock(&vcpu->arch.vpa_update_lock);
2324 		break;
2325 	case KVM_REG_PPC_TB_OFFSET:
2326 		*val = get_reg_val(id, vcpu->arch.vcore->tb_offset);
2327 		break;
2328 	case KVM_REG_PPC_LPCR:
2329 	case KVM_REG_PPC_LPCR_64:
2330 		*val = get_reg_val(id, vcpu->arch.vcore->lpcr);
2331 		break;
2332 	case KVM_REG_PPC_PPR:
2333 		*val = get_reg_val(id, vcpu->arch.ppr);
2334 		break;
2335 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2336 	case KVM_REG_PPC_TFHAR:
2337 		*val = get_reg_val(id, vcpu->arch.tfhar);
2338 		break;
2339 	case KVM_REG_PPC_TFIAR:
2340 		*val = get_reg_val(id, vcpu->arch.tfiar);
2341 		break;
2342 	case KVM_REG_PPC_TEXASR:
2343 		*val = get_reg_val(id, vcpu->arch.texasr);
2344 		break;
2345 	case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
2346 		i = id - KVM_REG_PPC_TM_GPR0;
2347 		*val = get_reg_val(id, vcpu->arch.gpr_tm[i]);
2348 		break;
2349 	case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
2350 	{
2351 		int j;
2352 		i = id - KVM_REG_PPC_TM_VSR0;
2353 		if (i < 32)
2354 			for (j = 0; j < TS_FPRWIDTH; j++)
2355 				val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j];
2356 		else {
2357 			if (cpu_has_feature(CPU_FTR_ALTIVEC))
2358 				val->vval = vcpu->arch.vr_tm.vr[i-32];
2359 			else
2360 				r = -ENXIO;
2361 		}
2362 		break;
2363 	}
2364 	case KVM_REG_PPC_TM_CR:
2365 		*val = get_reg_val(id, vcpu->arch.cr_tm);
2366 		break;
2367 	case KVM_REG_PPC_TM_XER:
2368 		*val = get_reg_val(id, vcpu->arch.xer_tm);
2369 		break;
2370 	case KVM_REG_PPC_TM_LR:
2371 		*val = get_reg_val(id, vcpu->arch.lr_tm);
2372 		break;
2373 	case KVM_REG_PPC_TM_CTR:
2374 		*val = get_reg_val(id, vcpu->arch.ctr_tm);
2375 		break;
2376 	case KVM_REG_PPC_TM_FPSCR:
2377 		*val = get_reg_val(id, vcpu->arch.fp_tm.fpscr);
2378 		break;
2379 	case KVM_REG_PPC_TM_AMR:
2380 		*val = get_reg_val(id, vcpu->arch.amr_tm);
2381 		break;
2382 	case KVM_REG_PPC_TM_PPR:
2383 		*val = get_reg_val(id, vcpu->arch.ppr_tm);
2384 		break;
2385 	case KVM_REG_PPC_TM_VRSAVE:
2386 		*val = get_reg_val(id, vcpu->arch.vrsave_tm);
2387 		break;
2388 	case KVM_REG_PPC_TM_VSCR:
2389 		if (cpu_has_feature(CPU_FTR_ALTIVEC))
2390 			*val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]);
2391 		else
2392 			r = -ENXIO;
2393 		break;
2394 	case KVM_REG_PPC_TM_DSCR:
2395 		*val = get_reg_val(id, vcpu->arch.dscr_tm);
2396 		break;
2397 	case KVM_REG_PPC_TM_TAR:
2398 		*val = get_reg_val(id, vcpu->arch.tar_tm);
2399 		break;
2400 #endif
2401 	case KVM_REG_PPC_ARCH_COMPAT:
2402 		*val = get_reg_val(id, vcpu->arch.vcore->arch_compat);
2403 		break;
2404 	case KVM_REG_PPC_DEC_EXPIRY:
2405 		*val = get_reg_val(id, vcpu->arch.dec_expires);
2406 		break;
2407 	case KVM_REG_PPC_ONLINE:
2408 		*val = get_reg_val(id, vcpu->arch.online);
2409 		break;
2410 	case KVM_REG_PPC_PTCR:
2411 		*val = get_reg_val(id, vcpu->kvm->arch.l1_ptcr);
2412 		break;
2413 	default:
2414 		r = -EINVAL;
2415 		break;
2416 	}
2417 
2418 	return r;
2419 }
2420 
kvmppc_set_one_reg_hv(struct kvm_vcpu * vcpu,u64 id,union kvmppc_one_reg * val)2421 static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
2422 				 union kvmppc_one_reg *val)
2423 {
2424 	int r = 0;
2425 	long int i;
2426 	unsigned long addr, len;
2427 
2428 	switch (id) {
2429 	case KVM_REG_PPC_HIOR:
2430 		/* Only allow this to be set to zero */
2431 		if (set_reg_val(id, *val))
2432 			r = -EINVAL;
2433 		break;
2434 	case KVM_REG_PPC_DABR:
2435 		vcpu->arch.dabr = set_reg_val(id, *val);
2436 		break;
2437 	case KVM_REG_PPC_DABRX:
2438 		vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP;
2439 		break;
2440 	case KVM_REG_PPC_DSCR:
2441 		vcpu->arch.dscr = set_reg_val(id, *val);
2442 		break;
2443 	case KVM_REG_PPC_PURR:
2444 		vcpu->arch.purr = set_reg_val(id, *val);
2445 		break;
2446 	case KVM_REG_PPC_SPURR:
2447 		vcpu->arch.spurr = set_reg_val(id, *val);
2448 		break;
2449 	case KVM_REG_PPC_AMR:
2450 		vcpu->arch.amr = set_reg_val(id, *val);
2451 		break;
2452 	case KVM_REG_PPC_UAMOR:
2453 		vcpu->arch.uamor = set_reg_val(id, *val);
2454 		break;
2455 	case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCR1:
2456 		i = id - KVM_REG_PPC_MMCR0;
2457 		vcpu->arch.mmcr[i] = set_reg_val(id, *val);
2458 		break;
2459 	case KVM_REG_PPC_MMCR2:
2460 		vcpu->arch.mmcr[2] = set_reg_val(id, *val);
2461 		break;
2462 	case KVM_REG_PPC_MMCRA:
2463 		vcpu->arch.mmcra = set_reg_val(id, *val);
2464 		break;
2465 	case KVM_REG_PPC_MMCRS:
2466 		vcpu->arch.mmcrs = set_reg_val(id, *val);
2467 		break;
2468 	case KVM_REG_PPC_MMCR3:
2469 		*val = get_reg_val(id, vcpu->arch.mmcr[3]);
2470 		break;
2471 	case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
2472 		i = id - KVM_REG_PPC_PMC1;
2473 		vcpu->arch.pmc[i] = set_reg_val(id, *val);
2474 		break;
2475 	case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2:
2476 		i = id - KVM_REG_PPC_SPMC1;
2477 		vcpu->arch.spmc[i] = set_reg_val(id, *val);
2478 		break;
2479 	case KVM_REG_PPC_SIAR:
2480 		vcpu->arch.siar = set_reg_val(id, *val);
2481 		break;
2482 	case KVM_REG_PPC_SDAR:
2483 		vcpu->arch.sdar = set_reg_val(id, *val);
2484 		break;
2485 	case KVM_REG_PPC_SIER:
2486 		vcpu->arch.sier[0] = set_reg_val(id, *val);
2487 		break;
2488 	case KVM_REG_PPC_SIER2:
2489 		vcpu->arch.sier[1] = set_reg_val(id, *val);
2490 		break;
2491 	case KVM_REG_PPC_SIER3:
2492 		vcpu->arch.sier[2] = set_reg_val(id, *val);
2493 		break;
2494 	case KVM_REG_PPC_IAMR:
2495 		vcpu->arch.iamr = set_reg_val(id, *val);
2496 		break;
2497 	case KVM_REG_PPC_PSPB:
2498 		vcpu->arch.pspb = set_reg_val(id, *val);
2499 		break;
2500 	case KVM_REG_PPC_DPDES:
2501 		if (cpu_has_feature(CPU_FTR_ARCH_300))
2502 			vcpu->arch.doorbell_request = set_reg_val(id, *val) & 1;
2503 		else
2504 			vcpu->arch.vcore->dpdes = set_reg_val(id, *val);
2505 		break;
2506 	case KVM_REG_PPC_VTB:
2507 		vcpu->arch.vcore->vtb = set_reg_val(id, *val);
2508 		break;
2509 	case KVM_REG_PPC_DAWR:
2510 		vcpu->arch.dawr0 = set_reg_val(id, *val);
2511 		break;
2512 	case KVM_REG_PPC_DAWRX:
2513 		vcpu->arch.dawrx0 = set_reg_val(id, *val) & ~DAWRX_HYP;
2514 		break;
2515 	case KVM_REG_PPC_DAWR1:
2516 		vcpu->arch.dawr1 = set_reg_val(id, *val);
2517 		break;
2518 	case KVM_REG_PPC_DAWRX1:
2519 		vcpu->arch.dawrx1 = set_reg_val(id, *val) & ~DAWRX_HYP;
2520 		break;
2521 	case KVM_REG_PPC_CIABR:
2522 		vcpu->arch.ciabr = set_reg_val(id, *val);
2523 		/* Don't allow setting breakpoints in hypervisor code */
2524 		if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
2525 			vcpu->arch.ciabr &= ~CIABR_PRIV;	/* disable */
2526 		break;
2527 	case KVM_REG_PPC_CSIGR:
2528 		vcpu->arch.csigr = set_reg_val(id, *val);
2529 		break;
2530 	case KVM_REG_PPC_TACR:
2531 		vcpu->arch.tacr = set_reg_val(id, *val);
2532 		break;
2533 	case KVM_REG_PPC_TCSCR:
2534 		vcpu->arch.tcscr = set_reg_val(id, *val);
2535 		break;
2536 	case KVM_REG_PPC_PID:
2537 		vcpu->arch.pid = set_reg_val(id, *val);
2538 		break;
2539 	case KVM_REG_PPC_ACOP:
2540 		vcpu->arch.acop = set_reg_val(id, *val);
2541 		break;
2542 	case KVM_REG_PPC_WORT:
2543 		vcpu->arch.wort = set_reg_val(id, *val);
2544 		break;
2545 	case KVM_REG_PPC_TIDR:
2546 		vcpu->arch.tid = set_reg_val(id, *val);
2547 		break;
2548 	case KVM_REG_PPC_PSSCR:
2549 		vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS;
2550 		break;
2551 	case KVM_REG_PPC_VPA_ADDR:
2552 		addr = set_reg_val(id, *val);
2553 		r = -EINVAL;
2554 		if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
2555 			      vcpu->arch.dtl.next_gpa))
2556 			break;
2557 		r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
2558 		break;
2559 	case KVM_REG_PPC_VPA_SLB:
2560 		addr = val->vpaval.addr;
2561 		len = val->vpaval.length;
2562 		r = -EINVAL;
2563 		if (addr && !vcpu->arch.vpa.next_gpa)
2564 			break;
2565 		r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
2566 		break;
2567 	case KVM_REG_PPC_VPA_DTL:
2568 		addr = val->vpaval.addr;
2569 		len = val->vpaval.length;
2570 		r = -EINVAL;
2571 		if (addr && (len < sizeof(struct dtl_entry) ||
2572 			     !vcpu->arch.vpa.next_gpa))
2573 			break;
2574 		len -= len % sizeof(struct dtl_entry);
2575 		r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
2576 		break;
2577 	case KVM_REG_PPC_TB_OFFSET:
2578 	{
2579 		/* round up to multiple of 2^24 */
2580 		u64 tb_offset = ALIGN(set_reg_val(id, *val), 1UL << 24);
2581 
2582 		/*
2583 		 * Now that we know the timebase offset, update the
2584 		 * decrementer expiry with a guest timebase value. If
2585 		 * the userspace does not set DEC_EXPIRY, this ensures
2586 		 * a migrated vcpu at least starts with an expired
2587 		 * decrementer, which is better than a large one that
2588 		 * causes a hang.
2589 		 */
2590 		if (!vcpu->arch.dec_expires && tb_offset)
2591 			vcpu->arch.dec_expires = get_tb() + tb_offset;
2592 
2593 		vcpu->arch.vcore->tb_offset = tb_offset;
2594 		break;
2595 	}
2596 	case KVM_REG_PPC_LPCR:
2597 		kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true);
2598 		break;
2599 	case KVM_REG_PPC_LPCR_64:
2600 		kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false);
2601 		break;
2602 	case KVM_REG_PPC_PPR:
2603 		vcpu->arch.ppr = set_reg_val(id, *val);
2604 		break;
2605 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2606 	case KVM_REG_PPC_TFHAR:
2607 		vcpu->arch.tfhar = set_reg_val(id, *val);
2608 		break;
2609 	case KVM_REG_PPC_TFIAR:
2610 		vcpu->arch.tfiar = set_reg_val(id, *val);
2611 		break;
2612 	case KVM_REG_PPC_TEXASR:
2613 		vcpu->arch.texasr = set_reg_val(id, *val);
2614 		break;
2615 	case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31:
2616 		i = id - KVM_REG_PPC_TM_GPR0;
2617 		vcpu->arch.gpr_tm[i] = set_reg_val(id, *val);
2618 		break;
2619 	case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63:
2620 	{
2621 		int j;
2622 		i = id - KVM_REG_PPC_TM_VSR0;
2623 		if (i < 32)
2624 			for (j = 0; j < TS_FPRWIDTH; j++)
2625 				vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j];
2626 		else
2627 			if (cpu_has_feature(CPU_FTR_ALTIVEC))
2628 				vcpu->arch.vr_tm.vr[i-32] = val->vval;
2629 			else
2630 				r = -ENXIO;
2631 		break;
2632 	}
2633 	case KVM_REG_PPC_TM_CR:
2634 		vcpu->arch.cr_tm = set_reg_val(id, *val);
2635 		break;
2636 	case KVM_REG_PPC_TM_XER:
2637 		vcpu->arch.xer_tm = set_reg_val(id, *val);
2638 		break;
2639 	case KVM_REG_PPC_TM_LR:
2640 		vcpu->arch.lr_tm = set_reg_val(id, *val);
2641 		break;
2642 	case KVM_REG_PPC_TM_CTR:
2643 		vcpu->arch.ctr_tm = set_reg_val(id, *val);
2644 		break;
2645 	case KVM_REG_PPC_TM_FPSCR:
2646 		vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val);
2647 		break;
2648 	case KVM_REG_PPC_TM_AMR:
2649 		vcpu->arch.amr_tm = set_reg_val(id, *val);
2650 		break;
2651 	case KVM_REG_PPC_TM_PPR:
2652 		vcpu->arch.ppr_tm = set_reg_val(id, *val);
2653 		break;
2654 	case KVM_REG_PPC_TM_VRSAVE:
2655 		vcpu->arch.vrsave_tm = set_reg_val(id, *val);
2656 		break;
2657 	case KVM_REG_PPC_TM_VSCR:
2658 		if (cpu_has_feature(CPU_FTR_ALTIVEC))
2659 			vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val);
2660 		else
2661 			r = - ENXIO;
2662 		break;
2663 	case KVM_REG_PPC_TM_DSCR:
2664 		vcpu->arch.dscr_tm = set_reg_val(id, *val);
2665 		break;
2666 	case KVM_REG_PPC_TM_TAR:
2667 		vcpu->arch.tar_tm = set_reg_val(id, *val);
2668 		break;
2669 #endif
2670 	case KVM_REG_PPC_ARCH_COMPAT:
2671 		r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val));
2672 		break;
2673 	case KVM_REG_PPC_DEC_EXPIRY:
2674 		vcpu->arch.dec_expires = set_reg_val(id, *val);
2675 		break;
2676 	case KVM_REG_PPC_ONLINE:
2677 		i = set_reg_val(id, *val);
2678 		if (i && !vcpu->arch.online)
2679 			atomic_inc(&vcpu->arch.vcore->online_count);
2680 		else if (!i && vcpu->arch.online)
2681 			atomic_dec(&vcpu->arch.vcore->online_count);
2682 		vcpu->arch.online = i;
2683 		break;
2684 	case KVM_REG_PPC_PTCR:
2685 		vcpu->kvm->arch.l1_ptcr = set_reg_val(id, *val);
2686 		break;
2687 	default:
2688 		r = -EINVAL;
2689 		break;
2690 	}
2691 
2692 	return r;
2693 }
2694 
2695 /*
2696  * On POWER9, threads are independent and can be in different partitions.
2697  * Therefore we consider each thread to be a subcore.
2698  * There is a restriction that all threads have to be in the same
2699  * MMU mode (radix or HPT), unfortunately, but since we only support
2700  * HPT guests on a HPT host so far, that isn't an impediment yet.
2701  */
threads_per_vcore(struct kvm * kvm)2702 static int threads_per_vcore(struct kvm *kvm)
2703 {
2704 	if (cpu_has_feature(CPU_FTR_ARCH_300))
2705 		return 1;
2706 	return threads_per_subcore;
2707 }
2708 
kvmppc_vcore_create(struct kvm * kvm,int id)2709 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int id)
2710 {
2711 	struct kvmppc_vcore *vcore;
2712 
2713 	vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
2714 
2715 	if (vcore == NULL)
2716 		return NULL;
2717 
2718 	spin_lock_init(&vcore->lock);
2719 	spin_lock_init(&vcore->stoltb_lock);
2720 	rcuwait_init(&vcore->wait);
2721 	vcore->preempt_tb = TB_NIL;
2722 	vcore->lpcr = kvm->arch.lpcr;
2723 	vcore->first_vcpuid = id;
2724 	vcore->kvm = kvm;
2725 	INIT_LIST_HEAD(&vcore->preempt_list);
2726 
2727 	return vcore;
2728 }
2729 
2730 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2731 static struct debugfs_timings_element {
2732 	const char *name;
2733 	size_t offset;
2734 } timings[] = {
2735 #ifdef CONFIG_KVM_BOOK3S_HV_P9_TIMING
2736 	{"vcpu_entry",	offsetof(struct kvm_vcpu, arch.vcpu_entry)},
2737 	{"guest_entry",	offsetof(struct kvm_vcpu, arch.guest_entry)},
2738 	{"in_guest",	offsetof(struct kvm_vcpu, arch.in_guest)},
2739 	{"guest_exit",	offsetof(struct kvm_vcpu, arch.guest_exit)},
2740 	{"vcpu_exit",	offsetof(struct kvm_vcpu, arch.vcpu_exit)},
2741 	{"hypercall",	offsetof(struct kvm_vcpu, arch.hcall)},
2742 	{"page_fault",	offsetof(struct kvm_vcpu, arch.pg_fault)},
2743 #else
2744 	{"rm_entry",	offsetof(struct kvm_vcpu, arch.rm_entry)},
2745 	{"rm_intr",	offsetof(struct kvm_vcpu, arch.rm_intr)},
2746 	{"rm_exit",	offsetof(struct kvm_vcpu, arch.rm_exit)},
2747 	{"guest",	offsetof(struct kvm_vcpu, arch.guest_time)},
2748 	{"cede",	offsetof(struct kvm_vcpu, arch.cede_time)},
2749 #endif
2750 };
2751 
2752 #define N_TIMINGS	(ARRAY_SIZE(timings))
2753 
2754 struct debugfs_timings_state {
2755 	struct kvm_vcpu	*vcpu;
2756 	unsigned int	buflen;
2757 	char		buf[N_TIMINGS * 100];
2758 };
2759 
debugfs_timings_open(struct inode * inode,struct file * file)2760 static int debugfs_timings_open(struct inode *inode, struct file *file)
2761 {
2762 	struct kvm_vcpu *vcpu = inode->i_private;
2763 	struct debugfs_timings_state *p;
2764 
2765 	p = kzalloc(sizeof(*p), GFP_KERNEL);
2766 	if (!p)
2767 		return -ENOMEM;
2768 
2769 	kvm_get_kvm(vcpu->kvm);
2770 	p->vcpu = vcpu;
2771 	file->private_data = p;
2772 
2773 	return nonseekable_open(inode, file);
2774 }
2775 
debugfs_timings_release(struct inode * inode,struct file * file)2776 static int debugfs_timings_release(struct inode *inode, struct file *file)
2777 {
2778 	struct debugfs_timings_state *p = file->private_data;
2779 
2780 	kvm_put_kvm(p->vcpu->kvm);
2781 	kfree(p);
2782 	return 0;
2783 }
2784 
debugfs_timings_read(struct file * file,char __user * buf,size_t len,loff_t * ppos)2785 static ssize_t debugfs_timings_read(struct file *file, char __user *buf,
2786 				    size_t len, loff_t *ppos)
2787 {
2788 	struct debugfs_timings_state *p = file->private_data;
2789 	struct kvm_vcpu *vcpu = p->vcpu;
2790 	char *s, *buf_end;
2791 	struct kvmhv_tb_accumulator tb;
2792 	u64 count;
2793 	loff_t pos;
2794 	ssize_t n;
2795 	int i, loops;
2796 	bool ok;
2797 
2798 	if (!p->buflen) {
2799 		s = p->buf;
2800 		buf_end = s + sizeof(p->buf);
2801 		for (i = 0; i < N_TIMINGS; ++i) {
2802 			struct kvmhv_tb_accumulator *acc;
2803 
2804 			acc = (struct kvmhv_tb_accumulator *)
2805 				((unsigned long)vcpu + timings[i].offset);
2806 			ok = false;
2807 			for (loops = 0; loops < 1000; ++loops) {
2808 				count = acc->seqcount;
2809 				if (!(count & 1)) {
2810 					smp_rmb();
2811 					tb = *acc;
2812 					smp_rmb();
2813 					if (count == acc->seqcount) {
2814 						ok = true;
2815 						break;
2816 					}
2817 				}
2818 				udelay(1);
2819 			}
2820 			if (!ok)
2821 				snprintf(s, buf_end - s, "%s: stuck\n",
2822 					timings[i].name);
2823 			else
2824 				snprintf(s, buf_end - s,
2825 					"%s: %llu %llu %llu %llu\n",
2826 					timings[i].name, count / 2,
2827 					tb_to_ns(tb.tb_total),
2828 					tb_to_ns(tb.tb_min),
2829 					tb_to_ns(tb.tb_max));
2830 			s += strlen(s);
2831 		}
2832 		p->buflen = s - p->buf;
2833 	}
2834 
2835 	pos = *ppos;
2836 	if (pos >= p->buflen)
2837 		return 0;
2838 	if (len > p->buflen - pos)
2839 		len = p->buflen - pos;
2840 	n = copy_to_user(buf, p->buf + pos, len);
2841 	if (n) {
2842 		if (n == len)
2843 			return -EFAULT;
2844 		len -= n;
2845 	}
2846 	*ppos = pos + len;
2847 	return len;
2848 }
2849 
debugfs_timings_write(struct file * file,const char __user * buf,size_t len,loff_t * ppos)2850 static ssize_t debugfs_timings_write(struct file *file, const char __user *buf,
2851 				     size_t len, loff_t *ppos)
2852 {
2853 	return -EACCES;
2854 }
2855 
2856 static const struct file_operations debugfs_timings_ops = {
2857 	.owner	 = THIS_MODULE,
2858 	.open	 = debugfs_timings_open,
2859 	.release = debugfs_timings_release,
2860 	.read	 = debugfs_timings_read,
2861 	.write	 = debugfs_timings_write,
2862 	.llseek	 = generic_file_llseek,
2863 };
2864 
2865 /* Create a debugfs directory for the vcpu */
kvmppc_arch_create_vcpu_debugfs_hv(struct kvm_vcpu * vcpu,struct dentry * debugfs_dentry)2866 static int kvmppc_arch_create_vcpu_debugfs_hv(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry)
2867 {
2868 	if (cpu_has_feature(CPU_FTR_ARCH_300) == IS_ENABLED(CONFIG_KVM_BOOK3S_HV_P9_TIMING))
2869 		debugfs_create_file("timings", 0444, debugfs_dentry, vcpu,
2870 				    &debugfs_timings_ops);
2871 	return 0;
2872 }
2873 
2874 #else /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */
kvmppc_arch_create_vcpu_debugfs_hv(struct kvm_vcpu * vcpu,struct dentry * debugfs_dentry)2875 static int kvmppc_arch_create_vcpu_debugfs_hv(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry)
2876 {
2877 	return 0;
2878 }
2879 #endif /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */
2880 
kvmppc_core_vcpu_create_hv(struct kvm_vcpu * vcpu)2881 static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu)
2882 {
2883 	int err;
2884 	int core;
2885 	struct kvmppc_vcore *vcore;
2886 	struct kvm *kvm;
2887 	unsigned int id;
2888 
2889 	kvm = vcpu->kvm;
2890 	id = vcpu->vcpu_id;
2891 
2892 	vcpu->arch.shared = &vcpu->arch.shregs;
2893 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2894 	/*
2895 	 * The shared struct is never shared on HV,
2896 	 * so we can always use host endianness
2897 	 */
2898 #ifdef __BIG_ENDIAN__
2899 	vcpu->arch.shared_big_endian = true;
2900 #else
2901 	vcpu->arch.shared_big_endian = false;
2902 #endif
2903 #endif
2904 	vcpu->arch.mmcr[0] = MMCR0_FC;
2905 	if (cpu_has_feature(CPU_FTR_ARCH_31)) {
2906 		vcpu->arch.mmcr[0] |= MMCR0_PMCCEXT;
2907 		vcpu->arch.mmcra = MMCRA_BHRB_DISABLE;
2908 	}
2909 
2910 	vcpu->arch.ctrl = CTRL_RUNLATCH;
2911 	/* default to host PVR, since we can't spoof it */
2912 	kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR));
2913 	spin_lock_init(&vcpu->arch.vpa_update_lock);
2914 	spin_lock_init(&vcpu->arch.tbacct_lock);
2915 	vcpu->arch.busy_preempt = TB_NIL;
2916 	vcpu->arch.shregs.msr = MSR_ME;
2917 	vcpu->arch.intr_msr = MSR_SF | MSR_ME;
2918 
2919 	/*
2920 	 * Set the default HFSCR for the guest from the host value.
2921 	 * This value is only used on POWER9.
2922 	 * On POWER9, we want to virtualize the doorbell facility, so we
2923 	 * don't set the HFSCR_MSGP bit, and that causes those instructions
2924 	 * to trap and then we emulate them.
2925 	 */
2926 	vcpu->arch.hfscr = HFSCR_TAR | HFSCR_EBB | HFSCR_PM | HFSCR_BHRB |
2927 		HFSCR_DSCR | HFSCR_VECVSX | HFSCR_FP;
2928 	if (cpu_has_feature(CPU_FTR_HVMODE)) {
2929 		vcpu->arch.hfscr &= mfspr(SPRN_HFSCR);
2930 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2931 		if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
2932 			vcpu->arch.hfscr |= HFSCR_TM;
2933 #endif
2934 	}
2935 	if (cpu_has_feature(CPU_FTR_TM_COMP))
2936 		vcpu->arch.hfscr |= HFSCR_TM;
2937 
2938 	vcpu->arch.hfscr_permitted = vcpu->arch.hfscr;
2939 
2940 	/*
2941 	 * PM, EBB, TM are demand-faulted so start with it clear.
2942 	 */
2943 	vcpu->arch.hfscr &= ~(HFSCR_PM | HFSCR_EBB | HFSCR_TM);
2944 
2945 	kvmppc_mmu_book3s_hv_init(vcpu);
2946 
2947 	vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
2948 
2949 	init_waitqueue_head(&vcpu->arch.cpu_run);
2950 
2951 	mutex_lock(&kvm->lock);
2952 	vcore = NULL;
2953 	err = -EINVAL;
2954 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
2955 		if (id >= (KVM_MAX_VCPUS * kvm->arch.emul_smt_mode)) {
2956 			pr_devel("KVM: VCPU ID too high\n");
2957 			core = KVM_MAX_VCORES;
2958 		} else {
2959 			BUG_ON(kvm->arch.smt_mode != 1);
2960 			core = kvmppc_pack_vcpu_id(kvm, id);
2961 		}
2962 	} else {
2963 		core = id / kvm->arch.smt_mode;
2964 	}
2965 	if (core < KVM_MAX_VCORES) {
2966 		vcore = kvm->arch.vcores[core];
2967 		if (vcore && cpu_has_feature(CPU_FTR_ARCH_300)) {
2968 			pr_devel("KVM: collision on id %u", id);
2969 			vcore = NULL;
2970 		} else if (!vcore) {
2971 			/*
2972 			 * Take mmu_setup_lock for mutual exclusion
2973 			 * with kvmppc_update_lpcr().
2974 			 */
2975 			err = -ENOMEM;
2976 			vcore = kvmppc_vcore_create(kvm,
2977 					id & ~(kvm->arch.smt_mode - 1));
2978 			mutex_lock(&kvm->arch.mmu_setup_lock);
2979 			kvm->arch.vcores[core] = vcore;
2980 			kvm->arch.online_vcores++;
2981 			mutex_unlock(&kvm->arch.mmu_setup_lock);
2982 		}
2983 	}
2984 	mutex_unlock(&kvm->lock);
2985 
2986 	if (!vcore)
2987 		return err;
2988 
2989 	spin_lock(&vcore->lock);
2990 	++vcore->num_threads;
2991 	spin_unlock(&vcore->lock);
2992 	vcpu->arch.vcore = vcore;
2993 	vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid;
2994 	vcpu->arch.thread_cpu = -1;
2995 	vcpu->arch.prev_cpu = -1;
2996 
2997 	vcpu->arch.cpu_type = KVM_CPU_3S_64;
2998 	kvmppc_sanity_check(vcpu);
2999 
3000 	return 0;
3001 }
3002 
kvmhv_set_smt_mode(struct kvm * kvm,unsigned long smt_mode,unsigned long flags)3003 static int kvmhv_set_smt_mode(struct kvm *kvm, unsigned long smt_mode,
3004 			      unsigned long flags)
3005 {
3006 	int err;
3007 	int esmt = 0;
3008 
3009 	if (flags)
3010 		return -EINVAL;
3011 	if (smt_mode > MAX_SMT_THREADS || !is_power_of_2(smt_mode))
3012 		return -EINVAL;
3013 	if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
3014 		/*
3015 		 * On POWER8 (or POWER7), the threading mode is "strict",
3016 		 * so we pack smt_mode vcpus per vcore.
3017 		 */
3018 		if (smt_mode > threads_per_subcore)
3019 			return -EINVAL;
3020 	} else {
3021 		/*
3022 		 * On POWER9, the threading mode is "loose",
3023 		 * so each vcpu gets its own vcore.
3024 		 */
3025 		esmt = smt_mode;
3026 		smt_mode = 1;
3027 	}
3028 	mutex_lock(&kvm->lock);
3029 	err = -EBUSY;
3030 	if (!kvm->arch.online_vcores) {
3031 		kvm->arch.smt_mode = smt_mode;
3032 		kvm->arch.emul_smt_mode = esmt;
3033 		err = 0;
3034 	}
3035 	mutex_unlock(&kvm->lock);
3036 
3037 	return err;
3038 }
3039 
unpin_vpa(struct kvm * kvm,struct kvmppc_vpa * vpa)3040 static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa)
3041 {
3042 	if (vpa->pinned_addr)
3043 		kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa,
3044 					vpa->dirty);
3045 }
3046 
kvmppc_core_vcpu_free_hv(struct kvm_vcpu * vcpu)3047 static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu)
3048 {
3049 	spin_lock(&vcpu->arch.vpa_update_lock);
3050 	unpin_vpa(vcpu->kvm, &vcpu->arch.dtl);
3051 	unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow);
3052 	unpin_vpa(vcpu->kvm, &vcpu->arch.vpa);
3053 	spin_unlock(&vcpu->arch.vpa_update_lock);
3054 }
3055 
kvmppc_core_check_requests_hv(struct kvm_vcpu * vcpu)3056 static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu)
3057 {
3058 	/* Indicate we want to get back into the guest */
3059 	return 1;
3060 }
3061 
kvmppc_set_timer(struct kvm_vcpu * vcpu)3062 static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
3063 {
3064 	unsigned long dec_nsec, now;
3065 
3066 	now = get_tb();
3067 	if (now > kvmppc_dec_expires_host_tb(vcpu)) {
3068 		/* decrementer has already gone negative */
3069 		kvmppc_core_queue_dec(vcpu);
3070 		kvmppc_core_prepare_to_enter(vcpu);
3071 		return;
3072 	}
3073 	dec_nsec = tb_to_ns(kvmppc_dec_expires_host_tb(vcpu) - now);
3074 	hrtimer_start(&vcpu->arch.dec_timer, dec_nsec, HRTIMER_MODE_REL);
3075 	vcpu->arch.timer_running = 1;
3076 }
3077 
3078 extern int __kvmppc_vcore_entry(void);
3079 
kvmppc_remove_runnable(struct kvmppc_vcore * vc,struct kvm_vcpu * vcpu,u64 tb)3080 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
3081 				   struct kvm_vcpu *vcpu, u64 tb)
3082 {
3083 	u64 now;
3084 
3085 	if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
3086 		return;
3087 	spin_lock_irq(&vcpu->arch.tbacct_lock);
3088 	now = tb;
3089 	vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) -
3090 		vcpu->arch.stolen_logged;
3091 	vcpu->arch.busy_preempt = now;
3092 	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
3093 	spin_unlock_irq(&vcpu->arch.tbacct_lock);
3094 	--vc->n_runnable;
3095 	WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], NULL);
3096 }
3097 
kvmppc_grab_hwthread(int cpu)3098 static int kvmppc_grab_hwthread(int cpu)
3099 {
3100 	struct paca_struct *tpaca;
3101 	long timeout = 10000;
3102 
3103 	tpaca = paca_ptrs[cpu];
3104 
3105 	/* Ensure the thread won't go into the kernel if it wakes */
3106 	tpaca->kvm_hstate.kvm_vcpu = NULL;
3107 	tpaca->kvm_hstate.kvm_vcore = NULL;
3108 	tpaca->kvm_hstate.napping = 0;
3109 	smp_wmb();
3110 	tpaca->kvm_hstate.hwthread_req = 1;
3111 
3112 	/*
3113 	 * If the thread is already executing in the kernel (e.g. handling
3114 	 * a stray interrupt), wait for it to get back to nap mode.
3115 	 * The smp_mb() is to ensure that our setting of hwthread_req
3116 	 * is visible before we look at hwthread_state, so if this
3117 	 * races with the code at system_reset_pSeries and the thread
3118 	 * misses our setting of hwthread_req, we are sure to see its
3119 	 * setting of hwthread_state, and vice versa.
3120 	 */
3121 	smp_mb();
3122 	while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
3123 		if (--timeout <= 0) {
3124 			pr_err("KVM: couldn't grab cpu %d\n", cpu);
3125 			return -EBUSY;
3126 		}
3127 		udelay(1);
3128 	}
3129 	return 0;
3130 }
3131 
kvmppc_release_hwthread(int cpu)3132 static void kvmppc_release_hwthread(int cpu)
3133 {
3134 	struct paca_struct *tpaca;
3135 
3136 	tpaca = paca_ptrs[cpu];
3137 	tpaca->kvm_hstate.hwthread_req = 0;
3138 	tpaca->kvm_hstate.kvm_vcpu = NULL;
3139 	tpaca->kvm_hstate.kvm_vcore = NULL;
3140 	tpaca->kvm_hstate.kvm_split_mode = NULL;
3141 }
3142 
3143 static DEFINE_PER_CPU(struct kvm *, cpu_in_guest);
3144 
radix_flush_cpu(struct kvm * kvm,int cpu,struct kvm_vcpu * vcpu)3145 static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
3146 {
3147 	struct kvm_nested_guest *nested = vcpu->arch.nested;
3148 	cpumask_t *need_tlb_flush;
3149 	int i;
3150 
3151 	if (nested)
3152 		need_tlb_flush = &nested->need_tlb_flush;
3153 	else
3154 		need_tlb_flush = &kvm->arch.need_tlb_flush;
3155 
3156 	cpu = cpu_first_tlb_thread_sibling(cpu);
3157 	for (i = cpu; i <= cpu_last_tlb_thread_sibling(cpu);
3158 					i += cpu_tlb_thread_sibling_step())
3159 		cpumask_set_cpu(i, need_tlb_flush);
3160 
3161 	/*
3162 	 * Make sure setting of bit in need_tlb_flush precedes testing of
3163 	 * cpu_in_guest. The matching barrier on the other side is hwsync
3164 	 * when switching to guest MMU mode, which happens between
3165 	 * cpu_in_guest being set to the guest kvm, and need_tlb_flush bit
3166 	 * being tested.
3167 	 */
3168 	smp_mb();
3169 
3170 	for (i = cpu; i <= cpu_last_tlb_thread_sibling(cpu);
3171 					i += cpu_tlb_thread_sibling_step()) {
3172 		struct kvm *running = *per_cpu_ptr(&cpu_in_guest, i);
3173 
3174 		if (running == kvm)
3175 			smp_call_function_single(i, do_nothing, NULL, 1);
3176 	}
3177 }
3178 
do_migrate_away_vcpu(void * arg)3179 static void do_migrate_away_vcpu(void *arg)
3180 {
3181 	struct kvm_vcpu *vcpu = arg;
3182 	struct kvm *kvm = vcpu->kvm;
3183 
3184 	/*
3185 	 * If the guest has GTSE, it may execute tlbie, so do a eieio; tlbsync;
3186 	 * ptesync sequence on the old CPU before migrating to a new one, in
3187 	 * case we interrupted the guest between a tlbie ; eieio ;
3188 	 * tlbsync; ptesync sequence.
3189 	 *
3190 	 * Otherwise, ptesync is sufficient for ordering tlbiel sequences.
3191 	 */
3192 	if (kvm->arch.lpcr & LPCR_GTSE)
3193 		asm volatile("eieio; tlbsync; ptesync");
3194 	else
3195 		asm volatile("ptesync");
3196 }
3197 
kvmppc_prepare_radix_vcpu(struct kvm_vcpu * vcpu,int pcpu)3198 static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
3199 {
3200 	struct kvm_nested_guest *nested = vcpu->arch.nested;
3201 	struct kvm *kvm = vcpu->kvm;
3202 	int prev_cpu;
3203 
3204 	if (!cpu_has_feature(CPU_FTR_HVMODE))
3205 		return;
3206 
3207 	if (nested)
3208 		prev_cpu = nested->prev_cpu[vcpu->arch.nested_vcpu_id];
3209 	else
3210 		prev_cpu = vcpu->arch.prev_cpu;
3211 
3212 	/*
3213 	 * With radix, the guest can do TLB invalidations itself,
3214 	 * and it could choose to use the local form (tlbiel) if
3215 	 * it is invalidating a translation that has only ever been
3216 	 * used on one vcpu.  However, that doesn't mean it has
3217 	 * only ever been used on one physical cpu, since vcpus
3218 	 * can move around between pcpus.  To cope with this, when
3219 	 * a vcpu moves from one pcpu to another, we need to tell
3220 	 * any vcpus running on the same core as this vcpu previously
3221 	 * ran to flush the TLB.
3222 	 */
3223 	if (prev_cpu != pcpu) {
3224 		if (prev_cpu >= 0) {
3225 			if (cpu_first_tlb_thread_sibling(prev_cpu) !=
3226 			    cpu_first_tlb_thread_sibling(pcpu))
3227 				radix_flush_cpu(kvm, prev_cpu, vcpu);
3228 
3229 			smp_call_function_single(prev_cpu,
3230 					do_migrate_away_vcpu, vcpu, 1);
3231 		}
3232 		if (nested)
3233 			nested->prev_cpu[vcpu->arch.nested_vcpu_id] = pcpu;
3234 		else
3235 			vcpu->arch.prev_cpu = pcpu;
3236 	}
3237 }
3238 
kvmppc_start_thread(struct kvm_vcpu * vcpu,struct kvmppc_vcore * vc)3239 static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc)
3240 {
3241 	int cpu;
3242 	struct paca_struct *tpaca;
3243 
3244 	cpu = vc->pcpu;
3245 	if (vcpu) {
3246 		if (vcpu->arch.timer_running) {
3247 			hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
3248 			vcpu->arch.timer_running = 0;
3249 		}
3250 		cpu += vcpu->arch.ptid;
3251 		vcpu->cpu = vc->pcpu;
3252 		vcpu->arch.thread_cpu = cpu;
3253 	}
3254 	tpaca = paca_ptrs[cpu];
3255 	tpaca->kvm_hstate.kvm_vcpu = vcpu;
3256 	tpaca->kvm_hstate.ptid = cpu - vc->pcpu;
3257 	tpaca->kvm_hstate.fake_suspend = 0;
3258 	/* Order stores to hstate.kvm_vcpu etc. before store to kvm_vcore */
3259 	smp_wmb();
3260 	tpaca->kvm_hstate.kvm_vcore = vc;
3261 	if (cpu != smp_processor_id())
3262 		kvmppc_ipi_thread(cpu);
3263 }
3264 
kvmppc_wait_for_nap(int n_threads)3265 static void kvmppc_wait_for_nap(int n_threads)
3266 {
3267 	int cpu = smp_processor_id();
3268 	int i, loops;
3269 
3270 	if (n_threads <= 1)
3271 		return;
3272 	for (loops = 0; loops < 1000000; ++loops) {
3273 		/*
3274 		 * Check if all threads are finished.
3275 		 * We set the vcore pointer when starting a thread
3276 		 * and the thread clears it when finished, so we look
3277 		 * for any threads that still have a non-NULL vcore ptr.
3278 		 */
3279 		for (i = 1; i < n_threads; ++i)
3280 			if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
3281 				break;
3282 		if (i == n_threads) {
3283 			HMT_medium();
3284 			return;
3285 		}
3286 		HMT_low();
3287 	}
3288 	HMT_medium();
3289 	for (i = 1; i < n_threads; ++i)
3290 		if (paca_ptrs[cpu + i]->kvm_hstate.kvm_vcore)
3291 			pr_err("KVM: CPU %d seems to be stuck\n", cpu + i);
3292 }
3293 
3294 /*
3295  * Check that we are on thread 0 and that any other threads in
3296  * this core are off-line.  Then grab the threads so they can't
3297  * enter the kernel.
3298  */
on_primary_thread(void)3299 static int on_primary_thread(void)
3300 {
3301 	int cpu = smp_processor_id();
3302 	int thr;
3303 
3304 	/* Are we on a primary subcore? */
3305 	if (cpu_thread_in_subcore(cpu))
3306 		return 0;
3307 
3308 	thr = 0;
3309 	while (++thr < threads_per_subcore)
3310 		if (cpu_online(cpu + thr))
3311 			return 0;
3312 
3313 	/* Grab all hw threads so they can't go into the kernel */
3314 	for (thr = 1; thr < threads_per_subcore; ++thr) {
3315 		if (kvmppc_grab_hwthread(cpu + thr)) {
3316 			/* Couldn't grab one; let the others go */
3317 			do {
3318 				kvmppc_release_hwthread(cpu + thr);
3319 			} while (--thr > 0);
3320 			return 0;
3321 		}
3322 	}
3323 	return 1;
3324 }
3325 
3326 /*
3327  * A list of virtual cores for each physical CPU.
3328  * These are vcores that could run but their runner VCPU tasks are
3329  * (or may be) preempted.
3330  */
3331 struct preempted_vcore_list {
3332 	struct list_head	list;
3333 	spinlock_t		lock;
3334 };
3335 
3336 static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores);
3337 
init_vcore_lists(void)3338 static void init_vcore_lists(void)
3339 {
3340 	int cpu;
3341 
3342 	for_each_possible_cpu(cpu) {
3343 		struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu);
3344 		spin_lock_init(&lp->lock);
3345 		INIT_LIST_HEAD(&lp->list);
3346 	}
3347 }
3348 
kvmppc_vcore_preempt(struct kvmppc_vcore * vc)3349 static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc)
3350 {
3351 	struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
3352 
3353 	WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300));
3354 
3355 	vc->vcore_state = VCORE_PREEMPT;
3356 	vc->pcpu = smp_processor_id();
3357 	if (vc->num_threads < threads_per_vcore(vc->kvm)) {
3358 		spin_lock(&lp->lock);
3359 		list_add_tail(&vc->preempt_list, &lp->list);
3360 		spin_unlock(&lp->lock);
3361 	}
3362 
3363 	/* Start accumulating stolen time */
3364 	kvmppc_core_start_stolen(vc, mftb());
3365 }
3366 
kvmppc_vcore_end_preempt(struct kvmppc_vcore * vc)3367 static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc)
3368 {
3369 	struct preempted_vcore_list *lp;
3370 
3371 	WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300));
3372 
3373 	kvmppc_core_end_stolen(vc, mftb());
3374 	if (!list_empty(&vc->preempt_list)) {
3375 		lp = &per_cpu(preempted_vcores, vc->pcpu);
3376 		spin_lock(&lp->lock);
3377 		list_del_init(&vc->preempt_list);
3378 		spin_unlock(&lp->lock);
3379 	}
3380 	vc->vcore_state = VCORE_INACTIVE;
3381 }
3382 
3383 /*
3384  * This stores information about the virtual cores currently
3385  * assigned to a physical core.
3386  */
3387 struct core_info {
3388 	int		n_subcores;
3389 	int		max_subcore_threads;
3390 	int		total_threads;
3391 	int		subcore_threads[MAX_SUBCORES];
3392 	struct kvmppc_vcore *vc[MAX_SUBCORES];
3393 };
3394 
3395 /*
3396  * This mapping means subcores 0 and 1 can use threads 0-3 and 4-7
3397  * respectively in 2-way micro-threading (split-core) mode on POWER8.
3398  */
3399 static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 };
3400 
init_core_info(struct core_info * cip,struct kvmppc_vcore * vc)3401 static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc)
3402 {
3403 	memset(cip, 0, sizeof(*cip));
3404 	cip->n_subcores = 1;
3405 	cip->max_subcore_threads = vc->num_threads;
3406 	cip->total_threads = vc->num_threads;
3407 	cip->subcore_threads[0] = vc->num_threads;
3408 	cip->vc[0] = vc;
3409 }
3410 
subcore_config_ok(int n_subcores,int n_threads)3411 static bool subcore_config_ok(int n_subcores, int n_threads)
3412 {
3413 	/*
3414 	 * POWER9 "SMT4" cores are permanently in what is effectively a 4-way
3415 	 * split-core mode, with one thread per subcore.
3416 	 */
3417 	if (cpu_has_feature(CPU_FTR_ARCH_300))
3418 		return n_subcores <= 4 && n_threads == 1;
3419 
3420 	/* On POWER8, can only dynamically split if unsplit to begin with */
3421 	if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS)
3422 		return false;
3423 	if (n_subcores > MAX_SUBCORES)
3424 		return false;
3425 	if (n_subcores > 1) {
3426 		if (!(dynamic_mt_modes & 2))
3427 			n_subcores = 4;
3428 		if (n_subcores > 2 && !(dynamic_mt_modes & 4))
3429 			return false;
3430 	}
3431 
3432 	return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS;
3433 }
3434 
init_vcore_to_run(struct kvmppc_vcore * vc)3435 static void init_vcore_to_run(struct kvmppc_vcore *vc)
3436 {
3437 	vc->entry_exit_map = 0;
3438 	vc->in_guest = 0;
3439 	vc->napping_threads = 0;
3440 	vc->conferring_threads = 0;
3441 	vc->tb_offset_applied = 0;
3442 }
3443 
can_dynamic_split(struct kvmppc_vcore * vc,struct core_info * cip)3444 static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip)
3445 {
3446 	int n_threads = vc->num_threads;
3447 	int sub;
3448 
3449 	if (!cpu_has_feature(CPU_FTR_ARCH_207S))
3450 		return false;
3451 
3452 	/* In one_vm_per_core mode, require all vcores to be from the same vm */
3453 	if (one_vm_per_core && vc->kvm != cip->vc[0]->kvm)
3454 		return false;
3455 
3456 	if (n_threads < cip->max_subcore_threads)
3457 		n_threads = cip->max_subcore_threads;
3458 	if (!subcore_config_ok(cip->n_subcores + 1, n_threads))
3459 		return false;
3460 	cip->max_subcore_threads = n_threads;
3461 
3462 	sub = cip->n_subcores;
3463 	++cip->n_subcores;
3464 	cip->total_threads += vc->num_threads;
3465 	cip->subcore_threads[sub] = vc->num_threads;
3466 	cip->vc[sub] = vc;
3467 	init_vcore_to_run(vc);
3468 	list_del_init(&vc->preempt_list);
3469 
3470 	return true;
3471 }
3472 
3473 /*
3474  * Work out whether it is possible to piggyback the execution of
3475  * vcore *pvc onto the execution of the other vcores described in *cip.
3476  */
can_piggyback(struct kvmppc_vcore * pvc,struct core_info * cip,int target_threads)3477 static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip,
3478 			  int target_threads)
3479 {
3480 	if (cip->total_threads + pvc->num_threads > target_threads)
3481 		return false;
3482 
3483 	return can_dynamic_split(pvc, cip);
3484 }
3485 
prepare_threads(struct kvmppc_vcore * vc)3486 static void prepare_threads(struct kvmppc_vcore *vc)
3487 {
3488 	int i;
3489 	struct kvm_vcpu *vcpu;
3490 
3491 	for_each_runnable_thread(i, vcpu, vc) {
3492 		if (signal_pending(vcpu->arch.run_task))
3493 			vcpu->arch.ret = -EINTR;
3494 		else if (vcpu->arch.vpa.update_pending ||
3495 			 vcpu->arch.slb_shadow.update_pending ||
3496 			 vcpu->arch.dtl.update_pending)
3497 			vcpu->arch.ret = RESUME_GUEST;
3498 		else
3499 			continue;
3500 		kvmppc_remove_runnable(vc, vcpu, mftb());
3501 		wake_up(&vcpu->arch.cpu_run);
3502 	}
3503 }
3504 
collect_piggybacks(struct core_info * cip,int target_threads)3505 static void collect_piggybacks(struct core_info *cip, int target_threads)
3506 {
3507 	struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores);
3508 	struct kvmppc_vcore *pvc, *vcnext;
3509 
3510 	spin_lock(&lp->lock);
3511 	list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) {
3512 		if (!spin_trylock(&pvc->lock))
3513 			continue;
3514 		prepare_threads(pvc);
3515 		if (!pvc->n_runnable || !pvc->kvm->arch.mmu_ready) {
3516 			list_del_init(&pvc->preempt_list);
3517 			if (pvc->runner == NULL) {
3518 				pvc->vcore_state = VCORE_INACTIVE;
3519 				kvmppc_core_end_stolen(pvc, mftb());
3520 			}
3521 			spin_unlock(&pvc->lock);
3522 			continue;
3523 		}
3524 		if (!can_piggyback(pvc, cip, target_threads)) {
3525 			spin_unlock(&pvc->lock);
3526 			continue;
3527 		}
3528 		kvmppc_core_end_stolen(pvc, mftb());
3529 		pvc->vcore_state = VCORE_PIGGYBACK;
3530 		if (cip->total_threads >= target_threads)
3531 			break;
3532 	}
3533 	spin_unlock(&lp->lock);
3534 }
3535 
recheck_signals_and_mmu(struct core_info * cip)3536 static bool recheck_signals_and_mmu(struct core_info *cip)
3537 {
3538 	int sub, i;
3539 	struct kvm_vcpu *vcpu;
3540 	struct kvmppc_vcore *vc;
3541 
3542 	for (sub = 0; sub < cip->n_subcores; ++sub) {
3543 		vc = cip->vc[sub];
3544 		if (!vc->kvm->arch.mmu_ready)
3545 			return true;
3546 		for_each_runnable_thread(i, vcpu, vc)
3547 			if (signal_pending(vcpu->arch.run_task))
3548 				return true;
3549 	}
3550 	return false;
3551 }
3552 
post_guest_process(struct kvmppc_vcore * vc,bool is_master)3553 static void post_guest_process(struct kvmppc_vcore *vc, bool is_master)
3554 {
3555 	int still_running = 0, i;
3556 	u64 now;
3557 	long ret;
3558 	struct kvm_vcpu *vcpu;
3559 
3560 	spin_lock(&vc->lock);
3561 	now = get_tb();
3562 	for_each_runnable_thread(i, vcpu, vc) {
3563 		/*
3564 		 * It's safe to unlock the vcore in the loop here, because
3565 		 * for_each_runnable_thread() is safe against removal of
3566 		 * the vcpu, and the vcore state is VCORE_EXITING here,
3567 		 * so any vcpus becoming runnable will have their arch.trap
3568 		 * set to zero and can't actually run in the guest.
3569 		 */
3570 		spin_unlock(&vc->lock);
3571 		/* cancel pending dec exception if dec is positive */
3572 		if (now < kvmppc_dec_expires_host_tb(vcpu) &&
3573 		    kvmppc_core_pending_dec(vcpu))
3574 			kvmppc_core_dequeue_dec(vcpu);
3575 
3576 		trace_kvm_guest_exit(vcpu);
3577 
3578 		ret = RESUME_GUEST;
3579 		if (vcpu->arch.trap)
3580 			ret = kvmppc_handle_exit_hv(vcpu,
3581 						    vcpu->arch.run_task);
3582 
3583 		vcpu->arch.ret = ret;
3584 		vcpu->arch.trap = 0;
3585 
3586 		spin_lock(&vc->lock);
3587 		if (is_kvmppc_resume_guest(vcpu->arch.ret)) {
3588 			if (vcpu->arch.pending_exceptions)
3589 				kvmppc_core_prepare_to_enter(vcpu);
3590 			if (vcpu->arch.ceded)
3591 				kvmppc_set_timer(vcpu);
3592 			else
3593 				++still_running;
3594 		} else {
3595 			kvmppc_remove_runnable(vc, vcpu, mftb());
3596 			wake_up(&vcpu->arch.cpu_run);
3597 		}
3598 	}
3599 	if (!is_master) {
3600 		if (still_running > 0) {
3601 			kvmppc_vcore_preempt(vc);
3602 		} else if (vc->runner) {
3603 			vc->vcore_state = VCORE_PREEMPT;
3604 			kvmppc_core_start_stolen(vc, mftb());
3605 		} else {
3606 			vc->vcore_state = VCORE_INACTIVE;
3607 		}
3608 		if (vc->n_runnable > 0 && vc->runner == NULL) {
3609 			/* make sure there's a candidate runner awake */
3610 			i = -1;
3611 			vcpu = next_runnable_thread(vc, &i);
3612 			wake_up(&vcpu->arch.cpu_run);
3613 		}
3614 	}
3615 	spin_unlock(&vc->lock);
3616 }
3617 
3618 /*
3619  * Clear core from the list of active host cores as we are about to
3620  * enter the guest. Only do this if it is the primary thread of the
3621  * core (not if a subcore) that is entering the guest.
3622  */
kvmppc_clear_host_core(unsigned int cpu)3623 static inline int kvmppc_clear_host_core(unsigned int cpu)
3624 {
3625 	int core;
3626 
3627 	if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
3628 		return 0;
3629 	/*
3630 	 * Memory barrier can be omitted here as we will do a smp_wmb()
3631 	 * later in kvmppc_start_thread and we need ensure that state is
3632 	 * visible to other CPUs only after we enter guest.
3633 	 */
3634 	core = cpu >> threads_shift;
3635 	kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0;
3636 	return 0;
3637 }
3638 
3639 /*
3640  * Advertise this core as an active host core since we exited the guest
3641  * Only need to do this if it is the primary thread of the core that is
3642  * exiting.
3643  */
kvmppc_set_host_core(unsigned int cpu)3644 static inline int kvmppc_set_host_core(unsigned int cpu)
3645 {
3646 	int core;
3647 
3648 	if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu))
3649 		return 0;
3650 
3651 	/*
3652 	 * Memory barrier can be omitted here because we do a spin_unlock
3653 	 * immediately after this which provides the memory barrier.
3654 	 */
3655 	core = cpu >> threads_shift;
3656 	kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1;
3657 	return 0;
3658 }
3659 
set_irq_happened(int trap)3660 static void set_irq_happened(int trap)
3661 {
3662 	switch (trap) {
3663 	case BOOK3S_INTERRUPT_EXTERNAL:
3664 		local_paca->irq_happened |= PACA_IRQ_EE;
3665 		break;
3666 	case BOOK3S_INTERRUPT_H_DOORBELL:
3667 		local_paca->irq_happened |= PACA_IRQ_DBELL;
3668 		break;
3669 	case BOOK3S_INTERRUPT_HMI:
3670 		local_paca->irq_happened |= PACA_IRQ_HMI;
3671 		break;
3672 	case BOOK3S_INTERRUPT_SYSTEM_RESET:
3673 		replay_system_reset();
3674 		break;
3675 	}
3676 }
3677 
3678 /*
3679  * Run a set of guest threads on a physical core.
3680  * Called with vc->lock held.
3681  */
kvmppc_run_core(struct kvmppc_vcore * vc)3682 static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
3683 {
3684 	struct kvm_vcpu *vcpu;
3685 	int i;
3686 	int srcu_idx;
3687 	struct core_info core_info;
3688 	struct kvmppc_vcore *pvc;
3689 	struct kvm_split_mode split_info, *sip;
3690 	int split, subcore_size, active;
3691 	int sub;
3692 	bool thr0_done;
3693 	unsigned long cmd_bit, stat_bit;
3694 	int pcpu, thr;
3695 	int target_threads;
3696 	int controlled_threads;
3697 	int trap;
3698 	bool is_power8;
3699 
3700 	if (WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300)))
3701 		return;
3702 
3703 	/*
3704 	 * Remove from the list any threads that have a signal pending
3705 	 * or need a VPA update done
3706 	 */
3707 	prepare_threads(vc);
3708 
3709 	/* if the runner is no longer runnable, let the caller pick a new one */
3710 	if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE)
3711 		return;
3712 
3713 	/*
3714 	 * Initialize *vc.
3715 	 */
3716 	init_vcore_to_run(vc);
3717 	vc->preempt_tb = TB_NIL;
3718 
3719 	/*
3720 	 * Number of threads that we will be controlling: the same as
3721 	 * the number of threads per subcore, except on POWER9,
3722 	 * where it's 1 because the threads are (mostly) independent.
3723 	 */
3724 	controlled_threads = threads_per_vcore(vc->kvm);
3725 
3726 	/*
3727 	 * Make sure we are running on primary threads, and that secondary
3728 	 * threads are offline.  Also check if the number of threads in this
3729 	 * guest are greater than the current system threads per guest.
3730 	 */
3731 	if ((controlled_threads > 1) &&
3732 	    ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) {
3733 		for_each_runnable_thread(i, vcpu, vc) {
3734 			vcpu->arch.ret = -EBUSY;
3735 			kvmppc_remove_runnable(vc, vcpu, mftb());
3736 			wake_up(&vcpu->arch.cpu_run);
3737 		}
3738 		goto out;
3739 	}
3740 
3741 	/*
3742 	 * See if we could run any other vcores on the physical core
3743 	 * along with this one.
3744 	 */
3745 	init_core_info(&core_info, vc);
3746 	pcpu = smp_processor_id();
3747 	target_threads = controlled_threads;
3748 	if (target_smt_mode && target_smt_mode < target_threads)
3749 		target_threads = target_smt_mode;
3750 	if (vc->num_threads < target_threads)
3751 		collect_piggybacks(&core_info, target_threads);
3752 
3753 	/*
3754 	 * Hard-disable interrupts, and check resched flag and signals.
3755 	 * If we need to reschedule or deliver a signal, clean up
3756 	 * and return without going into the guest(s).
3757 	 * If the mmu_ready flag has been cleared, don't go into the
3758 	 * guest because that means a HPT resize operation is in progress.
3759 	 */
3760 	local_irq_disable();
3761 	hard_irq_disable();
3762 	if (lazy_irq_pending() || need_resched() ||
3763 	    recheck_signals_and_mmu(&core_info)) {
3764 		local_irq_enable();
3765 		vc->vcore_state = VCORE_INACTIVE;
3766 		/* Unlock all except the primary vcore */
3767 		for (sub = 1; sub < core_info.n_subcores; ++sub) {
3768 			pvc = core_info.vc[sub];
3769 			/* Put back on to the preempted vcores list */
3770 			kvmppc_vcore_preempt(pvc);
3771 			spin_unlock(&pvc->lock);
3772 		}
3773 		for (i = 0; i < controlled_threads; ++i)
3774 			kvmppc_release_hwthread(pcpu + i);
3775 		return;
3776 	}
3777 
3778 	kvmppc_clear_host_core(pcpu);
3779 
3780 	/* Decide on micro-threading (split-core) mode */
3781 	subcore_size = threads_per_subcore;
3782 	cmd_bit = stat_bit = 0;
3783 	split = core_info.n_subcores;
3784 	sip = NULL;
3785 	is_power8 = cpu_has_feature(CPU_FTR_ARCH_207S);
3786 
3787 	if (split > 1) {
3788 		sip = &split_info;
3789 		memset(&split_info, 0, sizeof(split_info));
3790 		for (sub = 0; sub < core_info.n_subcores; ++sub)
3791 			split_info.vc[sub] = core_info.vc[sub];
3792 
3793 		if (is_power8) {
3794 			if (split == 2 && (dynamic_mt_modes & 2)) {
3795 				cmd_bit = HID0_POWER8_1TO2LPAR;
3796 				stat_bit = HID0_POWER8_2LPARMODE;
3797 			} else {
3798 				split = 4;
3799 				cmd_bit = HID0_POWER8_1TO4LPAR;
3800 				stat_bit = HID0_POWER8_4LPARMODE;
3801 			}
3802 			subcore_size = MAX_SMT_THREADS / split;
3803 			split_info.rpr = mfspr(SPRN_RPR);
3804 			split_info.pmmar = mfspr(SPRN_PMMAR);
3805 			split_info.ldbar = mfspr(SPRN_LDBAR);
3806 			split_info.subcore_size = subcore_size;
3807 		} else {
3808 			split_info.subcore_size = 1;
3809 		}
3810 
3811 		/* order writes to split_info before kvm_split_mode pointer */
3812 		smp_wmb();
3813 	}
3814 
3815 	for (thr = 0; thr < controlled_threads; ++thr) {
3816 		struct paca_struct *paca = paca_ptrs[pcpu + thr];
3817 
3818 		paca->kvm_hstate.napping = 0;
3819 		paca->kvm_hstate.kvm_split_mode = sip;
3820 	}
3821 
3822 	/* Initiate micro-threading (split-core) on POWER8 if required */
3823 	if (cmd_bit) {
3824 		unsigned long hid0 = mfspr(SPRN_HID0);
3825 
3826 		hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS;
3827 		mb();
3828 		mtspr(SPRN_HID0, hid0);
3829 		isync();
3830 		for (;;) {
3831 			hid0 = mfspr(SPRN_HID0);
3832 			if (hid0 & stat_bit)
3833 				break;
3834 			cpu_relax();
3835 		}
3836 	}
3837 
3838 	/*
3839 	 * On POWER8, set RWMR register.
3840 	 * Since it only affects PURR and SPURR, it doesn't affect
3841 	 * the host, so we don't save/restore the host value.
3842 	 */
3843 	if (is_power8) {
3844 		unsigned long rwmr_val = RWMR_RPA_P8_8THREAD;
3845 		int n_online = atomic_read(&vc->online_count);
3846 
3847 		/*
3848 		 * Use the 8-thread value if we're doing split-core
3849 		 * or if the vcore's online count looks bogus.
3850 		 */
3851 		if (split == 1 && threads_per_subcore == MAX_SMT_THREADS &&
3852 		    n_online >= 1 && n_online <= MAX_SMT_THREADS)
3853 			rwmr_val = p8_rwmr_values[n_online];
3854 		mtspr(SPRN_RWMR, rwmr_val);
3855 	}
3856 
3857 	/* Start all the threads */
3858 	active = 0;
3859 	for (sub = 0; sub < core_info.n_subcores; ++sub) {
3860 		thr = is_power8 ? subcore_thread_map[sub] : sub;
3861 		thr0_done = false;
3862 		active |= 1 << thr;
3863 		pvc = core_info.vc[sub];
3864 		pvc->pcpu = pcpu + thr;
3865 		for_each_runnable_thread(i, vcpu, pvc) {
3866 			/*
3867 			 * XXX: is kvmppc_start_thread called too late here?
3868 			 * It updates vcpu->cpu and vcpu->arch.thread_cpu
3869 			 * which are used by kvmppc_fast_vcpu_kick_hv(), but
3870 			 * kick is called after new exceptions become available
3871 			 * and exceptions are checked earlier than here, by
3872 			 * kvmppc_core_prepare_to_enter.
3873 			 */
3874 			kvmppc_start_thread(vcpu, pvc);
3875 			kvmppc_update_vpa_dispatch(vcpu, pvc);
3876 			trace_kvm_guest_enter(vcpu);
3877 			if (!vcpu->arch.ptid)
3878 				thr0_done = true;
3879 			active |= 1 << (thr + vcpu->arch.ptid);
3880 		}
3881 		/*
3882 		 * We need to start the first thread of each subcore
3883 		 * even if it doesn't have a vcpu.
3884 		 */
3885 		if (!thr0_done)
3886 			kvmppc_start_thread(NULL, pvc);
3887 	}
3888 
3889 	/*
3890 	 * Ensure that split_info.do_nap is set after setting
3891 	 * the vcore pointer in the PACA of the secondaries.
3892 	 */
3893 	smp_mb();
3894 
3895 	/*
3896 	 * When doing micro-threading, poke the inactive threads as well.
3897 	 * This gets them to the nap instruction after kvm_do_nap,
3898 	 * which reduces the time taken to unsplit later.
3899 	 */
3900 	if (cmd_bit) {
3901 		split_info.do_nap = 1;	/* ask secondaries to nap when done */
3902 		for (thr = 1; thr < threads_per_subcore; ++thr)
3903 			if (!(active & (1 << thr)))
3904 				kvmppc_ipi_thread(pcpu + thr);
3905 	}
3906 
3907 	vc->vcore_state = VCORE_RUNNING;
3908 	preempt_disable();
3909 
3910 	trace_kvmppc_run_core(vc, 0);
3911 
3912 	for (sub = 0; sub < core_info.n_subcores; ++sub)
3913 		spin_unlock(&core_info.vc[sub]->lock);
3914 
3915 	guest_timing_enter_irqoff();
3916 
3917 	srcu_idx = srcu_read_lock(&vc->kvm->srcu);
3918 
3919 	guest_state_enter_irqoff();
3920 	this_cpu_disable_ftrace();
3921 
3922 	trap = __kvmppc_vcore_entry();
3923 
3924 	this_cpu_enable_ftrace();
3925 	guest_state_exit_irqoff();
3926 
3927 	srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
3928 
3929 	set_irq_happened(trap);
3930 
3931 	spin_lock(&vc->lock);
3932 	/* prevent other vcpu threads from doing kvmppc_start_thread() now */
3933 	vc->vcore_state = VCORE_EXITING;
3934 
3935 	/* wait for secondary threads to finish writing their state to memory */
3936 	kvmppc_wait_for_nap(controlled_threads);
3937 
3938 	/* Return to whole-core mode if we split the core earlier */
3939 	if (cmd_bit) {
3940 		unsigned long hid0 = mfspr(SPRN_HID0);
3941 		unsigned long loops = 0;
3942 
3943 		hid0 &= ~HID0_POWER8_DYNLPARDIS;
3944 		stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE;
3945 		mb();
3946 		mtspr(SPRN_HID0, hid0);
3947 		isync();
3948 		for (;;) {
3949 			hid0 = mfspr(SPRN_HID0);
3950 			if (!(hid0 & stat_bit))
3951 				break;
3952 			cpu_relax();
3953 			++loops;
3954 		}
3955 		split_info.do_nap = 0;
3956 	}
3957 
3958 	kvmppc_set_host_core(pcpu);
3959 
3960 	if (!vtime_accounting_enabled_this_cpu()) {
3961 		local_irq_enable();
3962 		/*
3963 		 * Service IRQs here before guest_timing_exit_irqoff() so any
3964 		 * ticks that occurred while running the guest are accounted to
3965 		 * the guest. If vtime accounting is enabled, accounting uses
3966 		 * TB rather than ticks, so it can be done without enabling
3967 		 * interrupts here, which has the problem that it accounts
3968 		 * interrupt processing overhead to the host.
3969 		 */
3970 		local_irq_disable();
3971 	}
3972 	guest_timing_exit_irqoff();
3973 
3974 	local_irq_enable();
3975 
3976 	/* Let secondaries go back to the offline loop */
3977 	for (i = 0; i < controlled_threads; ++i) {
3978 		kvmppc_release_hwthread(pcpu + i);
3979 		if (sip && sip->napped[i])
3980 			kvmppc_ipi_thread(pcpu + i);
3981 	}
3982 
3983 	spin_unlock(&vc->lock);
3984 
3985 	/* make sure updates to secondary vcpu structs are visible now */
3986 	smp_mb();
3987 
3988 	preempt_enable();
3989 
3990 	for (sub = 0; sub < core_info.n_subcores; ++sub) {
3991 		pvc = core_info.vc[sub];
3992 		post_guest_process(pvc, pvc == vc);
3993 	}
3994 
3995 	spin_lock(&vc->lock);
3996 
3997  out:
3998 	vc->vcore_state = VCORE_INACTIVE;
3999 	trace_kvmppc_run_core(vc, 1);
4000 }
4001 
hcall_is_xics(unsigned long req)4002 static inline bool hcall_is_xics(unsigned long req)
4003 {
4004 	return req == H_EOI || req == H_CPPR || req == H_IPI ||
4005 		req == H_IPOLL || req == H_XIRR || req == H_XIRR_X;
4006 }
4007 
vcpu_vpa_increment_dispatch(struct kvm_vcpu * vcpu)4008 static void vcpu_vpa_increment_dispatch(struct kvm_vcpu *vcpu)
4009 {
4010 	struct lppaca *lp = vcpu->arch.vpa.pinned_addr;
4011 	if (lp) {
4012 		u32 yield_count = be32_to_cpu(lp->yield_count) + 1;
4013 		lp->yield_count = cpu_to_be32(yield_count);
4014 		vcpu->arch.vpa.dirty = 1;
4015 	}
4016 }
4017 
4018 /* call our hypervisor to load up HV regs and go */
kvmhv_vcpu_entry_p9_nested(struct kvm_vcpu * vcpu,u64 time_limit,unsigned long lpcr,u64 * tb)4019 static int kvmhv_vcpu_entry_p9_nested(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr, u64 *tb)
4020 {
4021 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
4022 	unsigned long host_psscr;
4023 	unsigned long msr;
4024 	struct hv_guest_state hvregs;
4025 	struct p9_host_os_sprs host_os_sprs;
4026 	s64 dec;
4027 	int trap;
4028 
4029 	msr = mfmsr();
4030 
4031 	save_p9_host_os_sprs(&host_os_sprs);
4032 
4033 	/*
4034 	 * We need to save and restore the guest visible part of the
4035 	 * psscr (i.e. using SPRN_PSSCR_PR) since the hypervisor
4036 	 * doesn't do this for us. Note only required if pseries since
4037 	 * this is done in kvmhv_vcpu_entry_p9() below otherwise.
4038 	 */
4039 	host_psscr = mfspr(SPRN_PSSCR_PR);
4040 
4041 	kvmppc_msr_hard_disable_set_facilities(vcpu, msr);
4042 	if (lazy_irq_pending())
4043 		return 0;
4044 
4045 	if (unlikely(load_vcpu_state(vcpu, &host_os_sprs)))
4046 		msr = mfmsr(); /* TM restore can update msr */
4047 
4048 	if (vcpu->arch.psscr != host_psscr)
4049 		mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr);
4050 
4051 	kvmhv_save_hv_regs(vcpu, &hvregs);
4052 	hvregs.lpcr = lpcr;
4053 	hvregs.amor = ~0;
4054 	vcpu->arch.regs.msr = vcpu->arch.shregs.msr;
4055 	hvregs.version = HV_GUEST_STATE_VERSION;
4056 	if (vcpu->arch.nested) {
4057 		hvregs.lpid = vcpu->arch.nested->shadow_lpid;
4058 		hvregs.vcpu_token = vcpu->arch.nested_vcpu_id;
4059 	} else {
4060 		hvregs.lpid = vcpu->kvm->arch.lpid;
4061 		hvregs.vcpu_token = vcpu->vcpu_id;
4062 	}
4063 	hvregs.hdec_expiry = time_limit;
4064 
4065 	/*
4066 	 * When setting DEC, we must always deal with irq_work_raise
4067 	 * via NMI vs setting DEC. The problem occurs right as we
4068 	 * switch into guest mode if a NMI hits and sets pending work
4069 	 * and sets DEC, then that will apply to the guest and not
4070 	 * bring us back to the host.
4071 	 *
4072 	 * irq_work_raise could check a flag (or possibly LPCR[HDICE]
4073 	 * for example) and set HDEC to 1? That wouldn't solve the
4074 	 * nested hv case which needs to abort the hcall or zero the
4075 	 * time limit.
4076 	 *
4077 	 * XXX: Another day's problem.
4078 	 */
4079 	mtspr(SPRN_DEC, kvmppc_dec_expires_host_tb(vcpu) - *tb);
4080 
4081 	mtspr(SPRN_DAR, vcpu->arch.shregs.dar);
4082 	mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr);
4083 	switch_pmu_to_guest(vcpu, &host_os_sprs);
4084 	accumulate_time(vcpu, &vcpu->arch.in_guest);
4085 	trap = plpar_hcall_norets(H_ENTER_NESTED, __pa(&hvregs),
4086 				  __pa(&vcpu->arch.regs));
4087 	accumulate_time(vcpu, &vcpu->arch.guest_exit);
4088 	kvmhv_restore_hv_return_state(vcpu, &hvregs);
4089 	switch_pmu_to_host(vcpu, &host_os_sprs);
4090 	vcpu->arch.shregs.msr = vcpu->arch.regs.msr;
4091 	vcpu->arch.shregs.dar = mfspr(SPRN_DAR);
4092 	vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR);
4093 	vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR);
4094 
4095 	store_vcpu_state(vcpu);
4096 
4097 	dec = mfspr(SPRN_DEC);
4098 	if (!(lpcr & LPCR_LD)) /* Sign extend if not using large decrementer */
4099 		dec = (s32) dec;
4100 	*tb = mftb();
4101 	vcpu->arch.dec_expires = dec + (*tb + vc->tb_offset);
4102 
4103 	timer_rearm_host_dec(*tb);
4104 
4105 	restore_p9_host_os_sprs(vcpu, &host_os_sprs);
4106 	if (vcpu->arch.psscr != host_psscr)
4107 		mtspr(SPRN_PSSCR_PR, host_psscr);
4108 
4109 	return trap;
4110 }
4111 
4112 /*
4113  * Guest entry for POWER9 and later CPUs.
4114  */
kvmhv_p9_guest_entry(struct kvm_vcpu * vcpu,u64 time_limit,unsigned long lpcr,u64 * tb)4115 static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit,
4116 			 unsigned long lpcr, u64 *tb)
4117 {
4118 	struct kvm *kvm = vcpu->kvm;
4119 	struct kvm_nested_guest *nested = vcpu->arch.nested;
4120 	u64 next_timer;
4121 	int trap;
4122 
4123 	next_timer = timer_get_next_tb();
4124 	if (*tb >= next_timer)
4125 		return BOOK3S_INTERRUPT_HV_DECREMENTER;
4126 	if (next_timer < time_limit)
4127 		time_limit = next_timer;
4128 	else if (*tb >= time_limit) /* nested time limit */
4129 		return BOOK3S_INTERRUPT_NESTED_HV_DECREMENTER;
4130 
4131 	vcpu->arch.ceded = 0;
4132 
4133 	vcpu_vpa_increment_dispatch(vcpu);
4134 
4135 	if (kvmhv_on_pseries()) {
4136 		trap = kvmhv_vcpu_entry_p9_nested(vcpu, time_limit, lpcr, tb);
4137 
4138 		/* H_CEDE has to be handled now, not later */
4139 		if (trap == BOOK3S_INTERRUPT_SYSCALL && !nested &&
4140 		    kvmppc_get_gpr(vcpu, 3) == H_CEDE) {
4141 			kvmppc_cede(vcpu);
4142 			kvmppc_set_gpr(vcpu, 3, 0);
4143 			trap = 0;
4144 		}
4145 
4146 	} else if (nested) {
4147 		__this_cpu_write(cpu_in_guest, kvm);
4148 		trap = kvmhv_vcpu_entry_p9(vcpu, time_limit, lpcr, tb);
4149 		__this_cpu_write(cpu_in_guest, NULL);
4150 
4151 	} else {
4152 		kvmppc_xive_push_vcpu(vcpu);
4153 
4154 		__this_cpu_write(cpu_in_guest, kvm);
4155 		trap = kvmhv_vcpu_entry_p9(vcpu, time_limit, lpcr, tb);
4156 		__this_cpu_write(cpu_in_guest, NULL);
4157 
4158 		if (trap == BOOK3S_INTERRUPT_SYSCALL &&
4159 		    !(vcpu->arch.shregs.msr & MSR_PR)) {
4160 			unsigned long req = kvmppc_get_gpr(vcpu, 3);
4161 
4162 			/*
4163 			 * XIVE rearm and XICS hcalls must be handled
4164 			 * before xive context is pulled (is this
4165 			 * true?)
4166 			 */
4167 			if (req == H_CEDE) {
4168 				/* H_CEDE has to be handled now */
4169 				kvmppc_cede(vcpu);
4170 				if (!kvmppc_xive_rearm_escalation(vcpu)) {
4171 					/*
4172 					 * Pending escalation so abort
4173 					 * the cede.
4174 					 */
4175 					vcpu->arch.ceded = 0;
4176 				}
4177 				kvmppc_set_gpr(vcpu, 3, 0);
4178 				trap = 0;
4179 
4180 			} else if (req == H_ENTER_NESTED) {
4181 				/*
4182 				 * L2 should not run with the L1
4183 				 * context so rearm and pull it.
4184 				 */
4185 				if (!kvmppc_xive_rearm_escalation(vcpu)) {
4186 					/*
4187 					 * Pending escalation so abort
4188 					 * H_ENTER_NESTED.
4189 					 */
4190 					kvmppc_set_gpr(vcpu, 3, 0);
4191 					trap = 0;
4192 				}
4193 
4194 			} else if (hcall_is_xics(req)) {
4195 				int ret;
4196 
4197 				ret = kvmppc_xive_xics_hcall(vcpu, req);
4198 				if (ret != H_TOO_HARD) {
4199 					kvmppc_set_gpr(vcpu, 3, ret);
4200 					trap = 0;
4201 				}
4202 			}
4203 		}
4204 		kvmppc_xive_pull_vcpu(vcpu);
4205 
4206 		if (kvm_is_radix(kvm))
4207 			vcpu->arch.slb_max = 0;
4208 	}
4209 
4210 	vcpu_vpa_increment_dispatch(vcpu);
4211 
4212 	return trap;
4213 }
4214 
4215 /*
4216  * Wait for some other vcpu thread to execute us, and
4217  * wake us up when we need to handle something in the host.
4218  */
kvmppc_wait_for_exec(struct kvmppc_vcore * vc,struct kvm_vcpu * vcpu,int wait_state)4219 static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc,
4220 				 struct kvm_vcpu *vcpu, int wait_state)
4221 {
4222 	DEFINE_WAIT(wait);
4223 
4224 	prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
4225 	if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
4226 		spin_unlock(&vc->lock);
4227 		schedule();
4228 		spin_lock(&vc->lock);
4229 	}
4230 	finish_wait(&vcpu->arch.cpu_run, &wait);
4231 }
4232 
grow_halt_poll_ns(struct kvmppc_vcore * vc)4233 static void grow_halt_poll_ns(struct kvmppc_vcore *vc)
4234 {
4235 	if (!halt_poll_ns_grow)
4236 		return;
4237 
4238 	vc->halt_poll_ns *= halt_poll_ns_grow;
4239 	if (vc->halt_poll_ns < halt_poll_ns_grow_start)
4240 		vc->halt_poll_ns = halt_poll_ns_grow_start;
4241 }
4242 
shrink_halt_poll_ns(struct kvmppc_vcore * vc)4243 static void shrink_halt_poll_ns(struct kvmppc_vcore *vc)
4244 {
4245 	if (halt_poll_ns_shrink == 0)
4246 		vc->halt_poll_ns = 0;
4247 	else
4248 		vc->halt_poll_ns /= halt_poll_ns_shrink;
4249 }
4250 
4251 #ifdef CONFIG_KVM_XICS
xive_interrupt_pending(struct kvm_vcpu * vcpu)4252 static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
4253 {
4254 	if (!xics_on_xive())
4255 		return false;
4256 	return vcpu->arch.irq_pending || vcpu->arch.xive_saved_state.pipr <
4257 		vcpu->arch.xive_saved_state.cppr;
4258 }
4259 #else
xive_interrupt_pending(struct kvm_vcpu * vcpu)4260 static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
4261 {
4262 	return false;
4263 }
4264 #endif /* CONFIG_KVM_XICS */
4265 
kvmppc_vcpu_woken(struct kvm_vcpu * vcpu)4266 static bool kvmppc_vcpu_woken(struct kvm_vcpu *vcpu)
4267 {
4268 	if (vcpu->arch.pending_exceptions || vcpu->arch.prodded ||
4269 	    kvmppc_doorbell_pending(vcpu) || xive_interrupt_pending(vcpu))
4270 		return true;
4271 
4272 	return false;
4273 }
4274 
kvmppc_vcpu_check_block(struct kvm_vcpu * vcpu)4275 static bool kvmppc_vcpu_check_block(struct kvm_vcpu *vcpu)
4276 {
4277 	if (!vcpu->arch.ceded || kvmppc_vcpu_woken(vcpu))
4278 		return true;
4279 	return false;
4280 }
4281 
4282 /*
4283  * Check to see if any of the runnable vcpus on the vcore have pending
4284  * exceptions or are no longer ceded
4285  */
kvmppc_vcore_check_block(struct kvmppc_vcore * vc)4286 static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc)
4287 {
4288 	struct kvm_vcpu *vcpu;
4289 	int i;
4290 
4291 	for_each_runnable_thread(i, vcpu, vc) {
4292 		if (kvmppc_vcpu_check_block(vcpu))
4293 			return 1;
4294 	}
4295 
4296 	return 0;
4297 }
4298 
4299 /*
4300  * All the vcpus in this vcore are idle, so wait for a decrementer
4301  * or external interrupt to one of the vcpus.  vc->lock is held.
4302  */
kvmppc_vcore_blocked(struct kvmppc_vcore * vc)4303 static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
4304 {
4305 	ktime_t cur, start_poll, start_wait;
4306 	int do_sleep = 1;
4307 	u64 block_ns;
4308 
4309 	WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300));
4310 
4311 	/* Poll for pending exceptions and ceded state */
4312 	cur = start_poll = ktime_get();
4313 	if (vc->halt_poll_ns) {
4314 		ktime_t stop = ktime_add_ns(start_poll, vc->halt_poll_ns);
4315 		++vc->runner->stat.generic.halt_attempted_poll;
4316 
4317 		vc->vcore_state = VCORE_POLLING;
4318 		spin_unlock(&vc->lock);
4319 
4320 		do {
4321 			if (kvmppc_vcore_check_block(vc)) {
4322 				do_sleep = 0;
4323 				break;
4324 			}
4325 			cur = ktime_get();
4326 		} while (kvm_vcpu_can_poll(cur, stop));
4327 
4328 		spin_lock(&vc->lock);
4329 		vc->vcore_state = VCORE_INACTIVE;
4330 
4331 		if (!do_sleep) {
4332 			++vc->runner->stat.generic.halt_successful_poll;
4333 			goto out;
4334 		}
4335 	}
4336 
4337 	prepare_to_rcuwait(&vc->wait);
4338 	set_current_state(TASK_INTERRUPTIBLE);
4339 	if (kvmppc_vcore_check_block(vc)) {
4340 		finish_rcuwait(&vc->wait);
4341 		do_sleep = 0;
4342 		/* If we polled, count this as a successful poll */
4343 		if (vc->halt_poll_ns)
4344 			++vc->runner->stat.generic.halt_successful_poll;
4345 		goto out;
4346 	}
4347 
4348 	start_wait = ktime_get();
4349 
4350 	vc->vcore_state = VCORE_SLEEPING;
4351 	trace_kvmppc_vcore_blocked(vc->runner, 0);
4352 	spin_unlock(&vc->lock);
4353 	schedule();
4354 	finish_rcuwait(&vc->wait);
4355 	spin_lock(&vc->lock);
4356 	vc->vcore_state = VCORE_INACTIVE;
4357 	trace_kvmppc_vcore_blocked(vc->runner, 1);
4358 	++vc->runner->stat.halt_successful_wait;
4359 
4360 	cur = ktime_get();
4361 
4362 out:
4363 	block_ns = ktime_to_ns(cur) - ktime_to_ns(start_poll);
4364 
4365 	/* Attribute wait time */
4366 	if (do_sleep) {
4367 		vc->runner->stat.generic.halt_wait_ns +=
4368 			ktime_to_ns(cur) - ktime_to_ns(start_wait);
4369 		KVM_STATS_LOG_HIST_UPDATE(
4370 				vc->runner->stat.generic.halt_wait_hist,
4371 				ktime_to_ns(cur) - ktime_to_ns(start_wait));
4372 		/* Attribute failed poll time */
4373 		if (vc->halt_poll_ns) {
4374 			vc->runner->stat.generic.halt_poll_fail_ns +=
4375 				ktime_to_ns(start_wait) -
4376 				ktime_to_ns(start_poll);
4377 			KVM_STATS_LOG_HIST_UPDATE(
4378 				vc->runner->stat.generic.halt_poll_fail_hist,
4379 				ktime_to_ns(start_wait) -
4380 				ktime_to_ns(start_poll));
4381 		}
4382 	} else {
4383 		/* Attribute successful poll time */
4384 		if (vc->halt_poll_ns) {
4385 			vc->runner->stat.generic.halt_poll_success_ns +=
4386 				ktime_to_ns(cur) -
4387 				ktime_to_ns(start_poll);
4388 			KVM_STATS_LOG_HIST_UPDATE(
4389 				vc->runner->stat.generic.halt_poll_success_hist,
4390 				ktime_to_ns(cur) - ktime_to_ns(start_poll));
4391 		}
4392 	}
4393 
4394 	/* Adjust poll time */
4395 	if (halt_poll_ns) {
4396 		if (block_ns <= vc->halt_poll_ns)
4397 			;
4398 		/* We slept and blocked for longer than the max halt time */
4399 		else if (vc->halt_poll_ns && block_ns > halt_poll_ns)
4400 			shrink_halt_poll_ns(vc);
4401 		/* We slept and our poll time is too small */
4402 		else if (vc->halt_poll_ns < halt_poll_ns &&
4403 				block_ns < halt_poll_ns)
4404 			grow_halt_poll_ns(vc);
4405 		if (vc->halt_poll_ns > halt_poll_ns)
4406 			vc->halt_poll_ns = halt_poll_ns;
4407 	} else
4408 		vc->halt_poll_ns = 0;
4409 
4410 	trace_kvmppc_vcore_wakeup(do_sleep, block_ns);
4411 }
4412 
4413 /*
4414  * This never fails for a radix guest, as none of the operations it does
4415  * for a radix guest can fail or have a way to report failure.
4416  */
kvmhv_setup_mmu(struct kvm_vcpu * vcpu)4417 static int kvmhv_setup_mmu(struct kvm_vcpu *vcpu)
4418 {
4419 	int r = 0;
4420 	struct kvm *kvm = vcpu->kvm;
4421 
4422 	mutex_lock(&kvm->arch.mmu_setup_lock);
4423 	if (!kvm->arch.mmu_ready) {
4424 		if (!kvm_is_radix(kvm))
4425 			r = kvmppc_hv_setup_htab_rma(vcpu);
4426 		if (!r) {
4427 			if (cpu_has_feature(CPU_FTR_ARCH_300))
4428 				kvmppc_setup_partition_table(kvm);
4429 			kvm->arch.mmu_ready = 1;
4430 		}
4431 	}
4432 	mutex_unlock(&kvm->arch.mmu_setup_lock);
4433 	return r;
4434 }
4435 
kvmppc_run_vcpu(struct kvm_vcpu * vcpu)4436 static int kvmppc_run_vcpu(struct kvm_vcpu *vcpu)
4437 {
4438 	struct kvm_run *run = vcpu->run;
4439 	int n_ceded, i, r;
4440 	struct kvmppc_vcore *vc;
4441 	struct kvm_vcpu *v;
4442 
4443 	trace_kvmppc_run_vcpu_enter(vcpu);
4444 
4445 	run->exit_reason = 0;
4446 	vcpu->arch.ret = RESUME_GUEST;
4447 	vcpu->arch.trap = 0;
4448 	kvmppc_update_vpas(vcpu);
4449 
4450 	/*
4451 	 * Synchronize with other threads in this virtual core
4452 	 */
4453 	vc = vcpu->arch.vcore;
4454 	spin_lock(&vc->lock);
4455 	vcpu->arch.ceded = 0;
4456 	vcpu->arch.run_task = current;
4457 	vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb());
4458 	vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
4459 	vcpu->arch.busy_preempt = TB_NIL;
4460 	WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], vcpu);
4461 	++vc->n_runnable;
4462 
4463 	/*
4464 	 * This happens the first time this is called for a vcpu.
4465 	 * If the vcore is already running, we may be able to start
4466 	 * this thread straight away and have it join in.
4467 	 */
4468 	if (!signal_pending(current)) {
4469 		if ((vc->vcore_state == VCORE_PIGGYBACK ||
4470 		     vc->vcore_state == VCORE_RUNNING) &&
4471 			   !VCORE_IS_EXITING(vc)) {
4472 			kvmppc_update_vpa_dispatch(vcpu, vc);
4473 			kvmppc_start_thread(vcpu, vc);
4474 			trace_kvm_guest_enter(vcpu);
4475 		} else if (vc->vcore_state == VCORE_SLEEPING) {
4476 		        rcuwait_wake_up(&vc->wait);
4477 		}
4478 
4479 	}
4480 
4481 	while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
4482 	       !signal_pending(current)) {
4483 		/* See if the MMU is ready to go */
4484 		if (!vcpu->kvm->arch.mmu_ready) {
4485 			spin_unlock(&vc->lock);
4486 			r = kvmhv_setup_mmu(vcpu);
4487 			spin_lock(&vc->lock);
4488 			if (r) {
4489 				run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4490 				run->fail_entry.
4491 					hardware_entry_failure_reason = 0;
4492 				vcpu->arch.ret = r;
4493 				break;
4494 			}
4495 		}
4496 
4497 		if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
4498 			kvmppc_vcore_end_preempt(vc);
4499 
4500 		if (vc->vcore_state != VCORE_INACTIVE) {
4501 			kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE);
4502 			continue;
4503 		}
4504 		for_each_runnable_thread(i, v, vc) {
4505 			kvmppc_core_prepare_to_enter(v);
4506 			if (signal_pending(v->arch.run_task)) {
4507 				kvmppc_remove_runnable(vc, v, mftb());
4508 				v->stat.signal_exits++;
4509 				v->run->exit_reason = KVM_EXIT_INTR;
4510 				v->arch.ret = -EINTR;
4511 				wake_up(&v->arch.cpu_run);
4512 			}
4513 		}
4514 		if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
4515 			break;
4516 		n_ceded = 0;
4517 		for_each_runnable_thread(i, v, vc) {
4518 			if (!kvmppc_vcpu_woken(v))
4519 				n_ceded += v->arch.ceded;
4520 			else
4521 				v->arch.ceded = 0;
4522 		}
4523 		vc->runner = vcpu;
4524 		if (n_ceded == vc->n_runnable) {
4525 			kvmppc_vcore_blocked(vc);
4526 		} else if (need_resched()) {
4527 			kvmppc_vcore_preempt(vc);
4528 			/* Let something else run */
4529 			cond_resched_lock(&vc->lock);
4530 			if (vc->vcore_state == VCORE_PREEMPT)
4531 				kvmppc_vcore_end_preempt(vc);
4532 		} else {
4533 			kvmppc_run_core(vc);
4534 		}
4535 		vc->runner = NULL;
4536 	}
4537 
4538 	while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
4539 	       (vc->vcore_state == VCORE_RUNNING ||
4540 		vc->vcore_state == VCORE_EXITING ||
4541 		vc->vcore_state == VCORE_PIGGYBACK))
4542 		kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE);
4543 
4544 	if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL)
4545 		kvmppc_vcore_end_preempt(vc);
4546 
4547 	if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
4548 		kvmppc_remove_runnable(vc, vcpu, mftb());
4549 		vcpu->stat.signal_exits++;
4550 		run->exit_reason = KVM_EXIT_INTR;
4551 		vcpu->arch.ret = -EINTR;
4552 	}
4553 
4554 	if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) {
4555 		/* Wake up some vcpu to run the core */
4556 		i = -1;
4557 		v = next_runnable_thread(vc, &i);
4558 		wake_up(&v->arch.cpu_run);
4559 	}
4560 
4561 	trace_kvmppc_run_vcpu_exit(vcpu);
4562 	spin_unlock(&vc->lock);
4563 	return vcpu->arch.ret;
4564 }
4565 
kvmhv_run_single_vcpu(struct kvm_vcpu * vcpu,u64 time_limit,unsigned long lpcr)4566 int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit,
4567 			  unsigned long lpcr)
4568 {
4569 	struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
4570 	struct kvm_run *run = vcpu->run;
4571 	int trap, r, pcpu;
4572 	int srcu_idx;
4573 	struct kvmppc_vcore *vc;
4574 	struct kvm *kvm = vcpu->kvm;
4575 	struct kvm_nested_guest *nested = vcpu->arch.nested;
4576 	unsigned long flags;
4577 	u64 tb;
4578 
4579 	trace_kvmppc_run_vcpu_enter(vcpu);
4580 
4581 	run->exit_reason = 0;
4582 	vcpu->arch.ret = RESUME_GUEST;
4583 	vcpu->arch.trap = 0;
4584 
4585 	vc = vcpu->arch.vcore;
4586 	vcpu->arch.ceded = 0;
4587 	vcpu->arch.run_task = current;
4588 	vcpu->arch.last_inst = KVM_INST_FETCH_FAILED;
4589 
4590 	/* See if the MMU is ready to go */
4591 	if (unlikely(!kvm->arch.mmu_ready)) {
4592 		r = kvmhv_setup_mmu(vcpu);
4593 		if (r) {
4594 			run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4595 			run->fail_entry.hardware_entry_failure_reason = 0;
4596 			vcpu->arch.ret = r;
4597 			return r;
4598 		}
4599 	}
4600 
4601 	if (need_resched())
4602 		cond_resched();
4603 
4604 	kvmppc_update_vpas(vcpu);
4605 
4606 	preempt_disable();
4607 	pcpu = smp_processor_id();
4608 	if (kvm_is_radix(kvm))
4609 		kvmppc_prepare_radix_vcpu(vcpu, pcpu);
4610 
4611 	/* flags save not required, but irq_pmu has no disable/enable API */
4612 	powerpc_local_irq_pmu_save(flags);
4613 
4614 	vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
4615 
4616 	if (signal_pending(current))
4617 		goto sigpend;
4618 	if (need_resched() || !kvm->arch.mmu_ready)
4619 		goto out;
4620 
4621 	vcpu->cpu = pcpu;
4622 	vcpu->arch.thread_cpu = pcpu;
4623 	vc->pcpu = pcpu;
4624 	local_paca->kvm_hstate.kvm_vcpu = vcpu;
4625 	local_paca->kvm_hstate.ptid = 0;
4626 	local_paca->kvm_hstate.fake_suspend = 0;
4627 
4628 	/*
4629 	 * Orders set cpu/thread_cpu vs testing for pending interrupts and
4630 	 * doorbells below. The other side is when these fields are set vs
4631 	 * kvmppc_fast_vcpu_kick_hv reading the cpu/thread_cpu fields to
4632 	 * kick a vCPU to notice the pending interrupt.
4633 	 */
4634 	smp_mb();
4635 
4636 	if (!nested) {
4637 		kvmppc_core_prepare_to_enter(vcpu);
4638 		if (vcpu->arch.shregs.msr & MSR_EE) {
4639 			if (xive_interrupt_pending(vcpu))
4640 				kvmppc_inject_interrupt_hv(vcpu,
4641 						BOOK3S_INTERRUPT_EXTERNAL, 0);
4642 		} else if (test_bit(BOOK3S_IRQPRIO_EXTERNAL,
4643 			     &vcpu->arch.pending_exceptions)) {
4644 			lpcr |= LPCR_MER;
4645 		}
4646 	} else if (vcpu->arch.pending_exceptions ||
4647 		   vcpu->arch.doorbell_request ||
4648 		   xive_interrupt_pending(vcpu)) {
4649 		vcpu->arch.ret = RESUME_HOST;
4650 		goto out;
4651 	}
4652 
4653 	if (vcpu->arch.timer_running) {
4654 		hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
4655 		vcpu->arch.timer_running = 0;
4656 	}
4657 
4658 	tb = mftb();
4659 
4660 	kvmppc_update_vpa_dispatch_p9(vcpu, vc, tb + vc->tb_offset);
4661 
4662 	trace_kvm_guest_enter(vcpu);
4663 
4664 	guest_timing_enter_irqoff();
4665 
4666 	srcu_idx = srcu_read_lock(&kvm->srcu);
4667 
4668 	guest_state_enter_irqoff();
4669 	this_cpu_disable_ftrace();
4670 
4671 	trap = kvmhv_p9_guest_entry(vcpu, time_limit, lpcr, &tb);
4672 	vcpu->arch.trap = trap;
4673 
4674 	this_cpu_enable_ftrace();
4675 	guest_state_exit_irqoff();
4676 
4677 	srcu_read_unlock(&kvm->srcu, srcu_idx);
4678 
4679 	set_irq_happened(trap);
4680 
4681 	vcpu->cpu = -1;
4682 	vcpu->arch.thread_cpu = -1;
4683 	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
4684 
4685 	if (!vtime_accounting_enabled_this_cpu()) {
4686 		powerpc_local_irq_pmu_restore(flags);
4687 		/*
4688 		 * Service IRQs here before guest_timing_exit_irqoff() so any
4689 		 * ticks that occurred while running the guest are accounted to
4690 		 * the guest. If vtime accounting is enabled, accounting uses
4691 		 * TB rather than ticks, so it can be done without enabling
4692 		 * interrupts here, which has the problem that it accounts
4693 		 * interrupt processing overhead to the host.
4694 		 */
4695 		powerpc_local_irq_pmu_save(flags);
4696 	}
4697 	guest_timing_exit_irqoff();
4698 
4699 	powerpc_local_irq_pmu_restore(flags);
4700 
4701 	preempt_enable();
4702 
4703 	/*
4704 	 * cancel pending decrementer exception if DEC is now positive, or if
4705 	 * entering a nested guest in which case the decrementer is now owned
4706 	 * by L2 and the L1 decrementer is provided in hdec_expires
4707 	 */
4708 	if (kvmppc_core_pending_dec(vcpu) &&
4709 			((tb < kvmppc_dec_expires_host_tb(vcpu)) ||
4710 			 (trap == BOOK3S_INTERRUPT_SYSCALL &&
4711 			  kvmppc_get_gpr(vcpu, 3) == H_ENTER_NESTED)))
4712 		kvmppc_core_dequeue_dec(vcpu);
4713 
4714 	trace_kvm_guest_exit(vcpu);
4715 	r = RESUME_GUEST;
4716 	if (trap) {
4717 		if (!nested)
4718 			r = kvmppc_handle_exit_hv(vcpu, current);
4719 		else
4720 			r = kvmppc_handle_nested_exit(vcpu);
4721 	}
4722 	vcpu->arch.ret = r;
4723 
4724 	if (is_kvmppc_resume_guest(r) && !kvmppc_vcpu_check_block(vcpu)) {
4725 		kvmppc_set_timer(vcpu);
4726 
4727 		prepare_to_rcuwait(wait);
4728 		for (;;) {
4729 			set_current_state(TASK_INTERRUPTIBLE);
4730 			if (signal_pending(current)) {
4731 				vcpu->stat.signal_exits++;
4732 				run->exit_reason = KVM_EXIT_INTR;
4733 				vcpu->arch.ret = -EINTR;
4734 				break;
4735 			}
4736 
4737 			if (kvmppc_vcpu_check_block(vcpu))
4738 				break;
4739 
4740 			trace_kvmppc_vcore_blocked(vcpu, 0);
4741 			schedule();
4742 			trace_kvmppc_vcore_blocked(vcpu, 1);
4743 		}
4744 		finish_rcuwait(wait);
4745 	}
4746 	vcpu->arch.ceded = 0;
4747 
4748  done:
4749 	trace_kvmppc_run_vcpu_exit(vcpu);
4750 
4751 	return vcpu->arch.ret;
4752 
4753  sigpend:
4754 	vcpu->stat.signal_exits++;
4755 	run->exit_reason = KVM_EXIT_INTR;
4756 	vcpu->arch.ret = -EINTR;
4757  out:
4758 	vcpu->cpu = -1;
4759 	vcpu->arch.thread_cpu = -1;
4760 	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
4761 	powerpc_local_irq_pmu_restore(flags);
4762 	preempt_enable();
4763 	goto done;
4764 }
4765 
kvmppc_vcpu_run_hv(struct kvm_vcpu * vcpu)4766 static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu)
4767 {
4768 	struct kvm_run *run = vcpu->run;
4769 	int r;
4770 	int srcu_idx;
4771 	struct kvm *kvm;
4772 	unsigned long msr;
4773 
4774 	start_timing(vcpu, &vcpu->arch.vcpu_entry);
4775 
4776 	if (!vcpu->arch.sane) {
4777 		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
4778 		return -EINVAL;
4779 	}
4780 
4781 	/* No need to go into the guest when all we'll do is come back out */
4782 	if (signal_pending(current)) {
4783 		run->exit_reason = KVM_EXIT_INTR;
4784 		return -EINTR;
4785 	}
4786 
4787 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
4788 	/*
4789 	 * Don't allow entry with a suspended transaction, because
4790 	 * the guest entry/exit code will lose it.
4791 	 */
4792 	if (cpu_has_feature(CPU_FTR_TM) && current->thread.regs &&
4793 	    (current->thread.regs->msr & MSR_TM)) {
4794 		if (MSR_TM_ACTIVE(current->thread.regs->msr)) {
4795 			run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4796 			run->fail_entry.hardware_entry_failure_reason = 0;
4797 			return -EINVAL;
4798 		}
4799 	}
4800 #endif
4801 
4802 	/*
4803 	 * Force online to 1 for the sake of old userspace which doesn't
4804 	 * set it.
4805 	 */
4806 	if (!vcpu->arch.online) {
4807 		atomic_inc(&vcpu->arch.vcore->online_count);
4808 		vcpu->arch.online = 1;
4809 	}
4810 
4811 	kvmppc_core_prepare_to_enter(vcpu);
4812 
4813 	kvm = vcpu->kvm;
4814 	atomic_inc(&kvm->arch.vcpus_running);
4815 	/* Order vcpus_running vs. mmu_ready, see kvmppc_alloc_reset_hpt */
4816 	smp_mb();
4817 
4818 	msr = 0;
4819 	if (IS_ENABLED(CONFIG_PPC_FPU))
4820 		msr |= MSR_FP;
4821 	if (cpu_has_feature(CPU_FTR_ALTIVEC))
4822 		msr |= MSR_VEC;
4823 	if (cpu_has_feature(CPU_FTR_VSX))
4824 		msr |= MSR_VSX;
4825 	if ((cpu_has_feature(CPU_FTR_TM) ||
4826 	    cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) &&
4827 			(vcpu->arch.hfscr & HFSCR_TM))
4828 		msr |= MSR_TM;
4829 	msr = msr_check_and_set(msr);
4830 
4831 	kvmppc_save_user_regs();
4832 
4833 	kvmppc_save_current_sprs();
4834 
4835 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
4836 		vcpu->arch.waitp = &vcpu->arch.vcore->wait;
4837 	vcpu->arch.pgdir = kvm->mm->pgd;
4838 	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
4839 
4840 	do {
4841 		accumulate_time(vcpu, &vcpu->arch.guest_entry);
4842 		if (cpu_has_feature(CPU_FTR_ARCH_300))
4843 			r = kvmhv_run_single_vcpu(vcpu, ~(u64)0,
4844 						  vcpu->arch.vcore->lpcr);
4845 		else
4846 			r = kvmppc_run_vcpu(vcpu);
4847 
4848 		if (run->exit_reason == KVM_EXIT_PAPR_HCALL) {
4849 			accumulate_time(vcpu, &vcpu->arch.hcall);
4850 
4851 			if (WARN_ON_ONCE(vcpu->arch.shregs.msr & MSR_PR)) {
4852 				/*
4853 				 * These should have been caught reflected
4854 				 * into the guest by now. Final sanity check:
4855 				 * don't allow userspace to execute hcalls in
4856 				 * the hypervisor.
4857 				 */
4858 				r = RESUME_GUEST;
4859 				continue;
4860 			}
4861 			trace_kvm_hcall_enter(vcpu);
4862 			r = kvmppc_pseries_do_hcall(vcpu);
4863 			trace_kvm_hcall_exit(vcpu, r);
4864 			kvmppc_core_prepare_to_enter(vcpu);
4865 		} else if (r == RESUME_PAGE_FAULT) {
4866 			accumulate_time(vcpu, &vcpu->arch.pg_fault);
4867 			srcu_idx = srcu_read_lock(&kvm->srcu);
4868 			r = kvmppc_book3s_hv_page_fault(vcpu,
4869 				vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
4870 			srcu_read_unlock(&kvm->srcu, srcu_idx);
4871 		} else if (r == RESUME_PASSTHROUGH) {
4872 			if (WARN_ON(xics_on_xive()))
4873 				r = H_SUCCESS;
4874 			else
4875 				r = kvmppc_xics_rm_complete(vcpu, 0);
4876 		}
4877 	} while (is_kvmppc_resume_guest(r));
4878 	accumulate_time(vcpu, &vcpu->arch.vcpu_exit);
4879 
4880 	vcpu->arch.state = KVMPPC_VCPU_NOTREADY;
4881 	atomic_dec(&kvm->arch.vcpus_running);
4882 
4883 	srr_regs_clobbered();
4884 
4885 	end_timing(vcpu);
4886 
4887 	return r;
4888 }
4889 
kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size ** sps,int shift,int sllp)4890 static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps,
4891 				     int shift, int sllp)
4892 {
4893 	(*sps)->page_shift = shift;
4894 	(*sps)->slb_enc = sllp;
4895 	(*sps)->enc[0].page_shift = shift;
4896 	(*sps)->enc[0].pte_enc = kvmppc_pgsize_lp_encoding(shift, shift);
4897 	/*
4898 	 * Add 16MB MPSS support (may get filtered out by userspace)
4899 	 */
4900 	if (shift != 24) {
4901 		int penc = kvmppc_pgsize_lp_encoding(shift, 24);
4902 		if (penc != -1) {
4903 			(*sps)->enc[1].page_shift = 24;
4904 			(*sps)->enc[1].pte_enc = penc;
4905 		}
4906 	}
4907 	(*sps)++;
4908 }
4909 
kvm_vm_ioctl_get_smmu_info_hv(struct kvm * kvm,struct kvm_ppc_smmu_info * info)4910 static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm,
4911 					 struct kvm_ppc_smmu_info *info)
4912 {
4913 	struct kvm_ppc_one_seg_page_size *sps;
4914 
4915 	/*
4916 	 * POWER7, POWER8 and POWER9 all support 32 storage keys for data.
4917 	 * POWER7 doesn't support keys for instruction accesses,
4918 	 * POWER8 and POWER9 do.
4919 	 */
4920 	info->data_keys = 32;
4921 	info->instr_keys = cpu_has_feature(CPU_FTR_ARCH_207S) ? 32 : 0;
4922 
4923 	/* POWER7, 8 and 9 all have 1T segments and 32-entry SLB */
4924 	info->flags = KVM_PPC_PAGE_SIZES_REAL | KVM_PPC_1T_SEGMENTS;
4925 	info->slb_size = 32;
4926 
4927 	/* We only support these sizes for now, and no muti-size segments */
4928 	sps = &info->sps[0];
4929 	kvmppc_add_seg_page_size(&sps, 12, 0);
4930 	kvmppc_add_seg_page_size(&sps, 16, SLB_VSID_L | SLB_VSID_LP_01);
4931 	kvmppc_add_seg_page_size(&sps, 24, SLB_VSID_L);
4932 
4933 	/* If running as a nested hypervisor, we don't support HPT guests */
4934 	if (kvmhv_on_pseries())
4935 		info->flags |= KVM_PPC_NO_HASH;
4936 
4937 	return 0;
4938 }
4939 
4940 /*
4941  * Get (and clear) the dirty memory log for a memory slot.
4942  */
kvm_vm_ioctl_get_dirty_log_hv(struct kvm * kvm,struct kvm_dirty_log * log)4943 static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
4944 					 struct kvm_dirty_log *log)
4945 {
4946 	struct kvm_memslots *slots;
4947 	struct kvm_memory_slot *memslot;
4948 	int r;
4949 	unsigned long n, i;
4950 	unsigned long *buf, *p;
4951 	struct kvm_vcpu *vcpu;
4952 
4953 	mutex_lock(&kvm->slots_lock);
4954 
4955 	r = -EINVAL;
4956 	if (log->slot >= KVM_USER_MEM_SLOTS)
4957 		goto out;
4958 
4959 	slots = kvm_memslots(kvm);
4960 	memslot = id_to_memslot(slots, log->slot);
4961 	r = -ENOENT;
4962 	if (!memslot || !memslot->dirty_bitmap)
4963 		goto out;
4964 
4965 	/*
4966 	 * Use second half of bitmap area because both HPT and radix
4967 	 * accumulate bits in the first half.
4968 	 */
4969 	n = kvm_dirty_bitmap_bytes(memslot);
4970 	buf = memslot->dirty_bitmap + n / sizeof(long);
4971 	memset(buf, 0, n);
4972 
4973 	if (kvm_is_radix(kvm))
4974 		r = kvmppc_hv_get_dirty_log_radix(kvm, memslot, buf);
4975 	else
4976 		r = kvmppc_hv_get_dirty_log_hpt(kvm, memslot, buf);
4977 	if (r)
4978 		goto out;
4979 
4980 	/*
4981 	 * We accumulate dirty bits in the first half of the
4982 	 * memslot's dirty_bitmap area, for when pages are paged
4983 	 * out or modified by the host directly.  Pick up these
4984 	 * bits and add them to the map.
4985 	 */
4986 	p = memslot->dirty_bitmap;
4987 	for (i = 0; i < n / sizeof(long); ++i)
4988 		buf[i] |= xchg(&p[i], 0);
4989 
4990 	/* Harvest dirty bits from VPA and DTL updates */
4991 	/* Note: we never modify the SLB shadow buffer areas */
4992 	kvm_for_each_vcpu(i, vcpu, kvm) {
4993 		spin_lock(&vcpu->arch.vpa_update_lock);
4994 		kvmppc_harvest_vpa_dirty(&vcpu->arch.vpa, memslot, buf);
4995 		kvmppc_harvest_vpa_dirty(&vcpu->arch.dtl, memslot, buf);
4996 		spin_unlock(&vcpu->arch.vpa_update_lock);
4997 	}
4998 
4999 	r = -EFAULT;
5000 	if (copy_to_user(log->dirty_bitmap, buf, n))
5001 		goto out;
5002 
5003 	r = 0;
5004 out:
5005 	mutex_unlock(&kvm->slots_lock);
5006 	return r;
5007 }
5008 
kvmppc_core_free_memslot_hv(struct kvm_memory_slot * slot)5009 static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *slot)
5010 {
5011 	vfree(slot->arch.rmap);
5012 	slot->arch.rmap = NULL;
5013 }
5014 
kvmppc_core_prepare_memory_region_hv(struct kvm * kvm,const struct kvm_memory_slot * old,struct kvm_memory_slot * new,enum kvm_mr_change change)5015 static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm,
5016 				const struct kvm_memory_slot *old,
5017 				struct kvm_memory_slot *new,
5018 				enum kvm_mr_change change)
5019 {
5020 	if (change == KVM_MR_CREATE) {
5021 		unsigned long size = array_size(new->npages, sizeof(*new->arch.rmap));
5022 
5023 		if ((size >> PAGE_SHIFT) > totalram_pages())
5024 			return -ENOMEM;
5025 
5026 		new->arch.rmap = vzalloc(size);
5027 		if (!new->arch.rmap)
5028 			return -ENOMEM;
5029 	} else if (change != KVM_MR_DELETE) {
5030 		new->arch.rmap = old->arch.rmap;
5031 	}
5032 
5033 	return 0;
5034 }
5035 
kvmppc_core_commit_memory_region_hv(struct kvm * kvm,struct kvm_memory_slot * old,const struct kvm_memory_slot * new,enum kvm_mr_change change)5036 static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
5037 				struct kvm_memory_slot *old,
5038 				const struct kvm_memory_slot *new,
5039 				enum kvm_mr_change change)
5040 {
5041 	/*
5042 	 * If we are creating or modifying a memslot, it might make
5043 	 * some address that was previously cached as emulated
5044 	 * MMIO be no longer emulated MMIO, so invalidate
5045 	 * all the caches of emulated MMIO translations.
5046 	 */
5047 	if (change != KVM_MR_DELETE)
5048 		atomic64_inc(&kvm->arch.mmio_update);
5049 
5050 	/*
5051 	 * For change == KVM_MR_MOVE or KVM_MR_DELETE, higher levels
5052 	 * have already called kvm_arch_flush_shadow_memslot() to
5053 	 * flush shadow mappings.  For KVM_MR_CREATE we have no
5054 	 * previous mappings.  So the only case to handle is
5055 	 * KVM_MR_FLAGS_ONLY when the KVM_MEM_LOG_DIRTY_PAGES bit
5056 	 * has been changed.
5057 	 * For radix guests, we flush on setting KVM_MEM_LOG_DIRTY_PAGES
5058 	 * to get rid of any THP PTEs in the partition-scoped page tables
5059 	 * so we can track dirtiness at the page level; we flush when
5060 	 * clearing KVM_MEM_LOG_DIRTY_PAGES so that we can go back to
5061 	 * using THP PTEs.
5062 	 */
5063 	if (change == KVM_MR_FLAGS_ONLY && kvm_is_radix(kvm) &&
5064 	    ((new->flags ^ old->flags) & KVM_MEM_LOG_DIRTY_PAGES))
5065 		kvmppc_radix_flush_memslot(kvm, old);
5066 	/*
5067 	 * If UV hasn't yet called H_SVM_INIT_START, don't register memslots.
5068 	 */
5069 	if (!kvm->arch.secure_guest)
5070 		return;
5071 
5072 	switch (change) {
5073 	case KVM_MR_CREATE:
5074 		/*
5075 		 * @TODO kvmppc_uvmem_memslot_create() can fail and
5076 		 * return error. Fix this.
5077 		 */
5078 		kvmppc_uvmem_memslot_create(kvm, new);
5079 		break;
5080 	case KVM_MR_DELETE:
5081 		kvmppc_uvmem_memslot_delete(kvm, old);
5082 		break;
5083 	default:
5084 		/* TODO: Handle KVM_MR_MOVE */
5085 		break;
5086 	}
5087 }
5088 
5089 /*
5090  * Update LPCR values in kvm->arch and in vcores.
5091  * Caller must hold kvm->arch.mmu_setup_lock (for mutual exclusion
5092  * of kvm->arch.lpcr update).
5093  */
kvmppc_update_lpcr(struct kvm * kvm,unsigned long lpcr,unsigned long mask)5094 void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask)
5095 {
5096 	long int i;
5097 	u32 cores_done = 0;
5098 
5099 	if ((kvm->arch.lpcr & mask) == lpcr)
5100 		return;
5101 
5102 	kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr;
5103 
5104 	for (i = 0; i < KVM_MAX_VCORES; ++i) {
5105 		struct kvmppc_vcore *vc = kvm->arch.vcores[i];
5106 		if (!vc)
5107 			continue;
5108 
5109 		spin_lock(&vc->lock);
5110 		vc->lpcr = (vc->lpcr & ~mask) | lpcr;
5111 		verify_lpcr(kvm, vc->lpcr);
5112 		spin_unlock(&vc->lock);
5113 		if (++cores_done >= kvm->arch.online_vcores)
5114 			break;
5115 	}
5116 }
5117 
kvmppc_setup_partition_table(struct kvm * kvm)5118 void kvmppc_setup_partition_table(struct kvm *kvm)
5119 {
5120 	unsigned long dw0, dw1;
5121 
5122 	if (!kvm_is_radix(kvm)) {
5123 		/* PS field - page size for VRMA */
5124 		dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) |
5125 			((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1);
5126 		/* HTABSIZE and HTABORG fields */
5127 		dw0 |= kvm->arch.sdr1;
5128 
5129 		/* Second dword as set by userspace */
5130 		dw1 = kvm->arch.process_table;
5131 	} else {
5132 		dw0 = PATB_HR | radix__get_tree_size() |
5133 			__pa(kvm->arch.pgtable) | RADIX_PGD_INDEX_SIZE;
5134 		dw1 = PATB_GR | kvm->arch.process_table;
5135 	}
5136 	kvmhv_set_ptbl_entry(kvm->arch.lpid, dw0, dw1);
5137 }
5138 
5139 /*
5140  * Set up HPT (hashed page table) and RMA (real-mode area).
5141  * Must be called with kvm->arch.mmu_setup_lock held.
5142  */
kvmppc_hv_setup_htab_rma(struct kvm_vcpu * vcpu)5143 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
5144 {
5145 	int err = 0;
5146 	struct kvm *kvm = vcpu->kvm;
5147 	unsigned long hva;
5148 	struct kvm_memory_slot *memslot;
5149 	struct vm_area_struct *vma;
5150 	unsigned long lpcr = 0, senc;
5151 	unsigned long psize, porder;
5152 	int srcu_idx;
5153 
5154 	/* Allocate hashed page table (if not done already) and reset it */
5155 	if (!kvm->arch.hpt.virt) {
5156 		int order = KVM_DEFAULT_HPT_ORDER;
5157 		struct kvm_hpt_info info;
5158 
5159 		err = kvmppc_allocate_hpt(&info, order);
5160 		/* If we get here, it means userspace didn't specify a
5161 		 * size explicitly.  So, try successively smaller
5162 		 * sizes if the default failed. */
5163 		while ((err == -ENOMEM) && --order >= PPC_MIN_HPT_ORDER)
5164 			err  = kvmppc_allocate_hpt(&info, order);
5165 
5166 		if (err < 0) {
5167 			pr_err("KVM: Couldn't alloc HPT\n");
5168 			goto out;
5169 		}
5170 
5171 		kvmppc_set_hpt(kvm, &info);
5172 	}
5173 
5174 	/* Look up the memslot for guest physical address 0 */
5175 	srcu_idx = srcu_read_lock(&kvm->srcu);
5176 	memslot = gfn_to_memslot(kvm, 0);
5177 
5178 	/* We must have some memory at 0 by now */
5179 	err = -EINVAL;
5180 	if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
5181 		goto out_srcu;
5182 
5183 	/* Look up the VMA for the start of this memory slot */
5184 	hva = memslot->userspace_addr;
5185 	mmap_read_lock(kvm->mm);
5186 	vma = vma_lookup(kvm->mm, hva);
5187 	if (!vma || (vma->vm_flags & VM_IO))
5188 		goto up_out;
5189 
5190 	psize = vma_kernel_pagesize(vma);
5191 
5192 	mmap_read_unlock(kvm->mm);
5193 
5194 	/* We can handle 4k, 64k or 16M pages in the VRMA */
5195 	if (psize >= 0x1000000)
5196 		psize = 0x1000000;
5197 	else if (psize >= 0x10000)
5198 		psize = 0x10000;
5199 	else
5200 		psize = 0x1000;
5201 	porder = __ilog2(psize);
5202 
5203 	senc = slb_pgsize_encoding(psize);
5204 	kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
5205 		(VRMA_VSID << SLB_VSID_SHIFT_1T);
5206 	/* Create HPTEs in the hash page table for the VRMA */
5207 	kvmppc_map_vrma(vcpu, memslot, porder);
5208 
5209 	/* Update VRMASD field in the LPCR */
5210 	if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
5211 		/* the -4 is to account for senc values starting at 0x10 */
5212 		lpcr = senc << (LPCR_VRMASD_SH - 4);
5213 		kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
5214 	}
5215 
5216 	/* Order updates to kvm->arch.lpcr etc. vs. mmu_ready */
5217 	smp_wmb();
5218 	err = 0;
5219  out_srcu:
5220 	srcu_read_unlock(&kvm->srcu, srcu_idx);
5221  out:
5222 	return err;
5223 
5224  up_out:
5225 	mmap_read_unlock(kvm->mm);
5226 	goto out_srcu;
5227 }
5228 
5229 /*
5230  * Must be called with kvm->arch.mmu_setup_lock held and
5231  * mmu_ready = 0 and no vcpus running.
5232  */
kvmppc_switch_mmu_to_hpt(struct kvm * kvm)5233 int kvmppc_switch_mmu_to_hpt(struct kvm *kvm)
5234 {
5235 	unsigned long lpcr, lpcr_mask;
5236 
5237 	if (nesting_enabled(kvm))
5238 		kvmhv_release_all_nested(kvm);
5239 	kvmppc_rmap_reset(kvm);
5240 	kvm->arch.process_table = 0;
5241 	/* Mutual exclusion with kvm_unmap_gfn_range etc. */
5242 	spin_lock(&kvm->mmu_lock);
5243 	kvm->arch.radix = 0;
5244 	spin_unlock(&kvm->mmu_lock);
5245 	kvmppc_free_radix(kvm);
5246 
5247 	lpcr = LPCR_VPM1;
5248 	lpcr_mask = LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR;
5249 	if (cpu_has_feature(CPU_FTR_ARCH_31))
5250 		lpcr_mask |= LPCR_HAIL;
5251 	kvmppc_update_lpcr(kvm, lpcr, lpcr_mask);
5252 
5253 	return 0;
5254 }
5255 
5256 /*
5257  * Must be called with kvm->arch.mmu_setup_lock held and
5258  * mmu_ready = 0 and no vcpus running.
5259  */
kvmppc_switch_mmu_to_radix(struct kvm * kvm)5260 int kvmppc_switch_mmu_to_radix(struct kvm *kvm)
5261 {
5262 	unsigned long lpcr, lpcr_mask;
5263 	int err;
5264 
5265 	err = kvmppc_init_vm_radix(kvm);
5266 	if (err)
5267 		return err;
5268 	kvmppc_rmap_reset(kvm);
5269 	/* Mutual exclusion with kvm_unmap_gfn_range etc. */
5270 	spin_lock(&kvm->mmu_lock);
5271 	kvm->arch.radix = 1;
5272 	spin_unlock(&kvm->mmu_lock);
5273 	kvmppc_free_hpt(&kvm->arch.hpt);
5274 
5275 	lpcr = LPCR_UPRT | LPCR_GTSE | LPCR_HR;
5276 	lpcr_mask = LPCR_VPM1 | LPCR_UPRT | LPCR_GTSE | LPCR_HR;
5277 	if (cpu_has_feature(CPU_FTR_ARCH_31)) {
5278 		lpcr_mask |= LPCR_HAIL;
5279 		if (cpu_has_feature(CPU_FTR_HVMODE) &&
5280 				(kvm->arch.host_lpcr & LPCR_HAIL))
5281 			lpcr |= LPCR_HAIL;
5282 	}
5283 	kvmppc_update_lpcr(kvm, lpcr, lpcr_mask);
5284 
5285 	return 0;
5286 }
5287 
5288 #ifdef CONFIG_KVM_XICS
5289 /*
5290  * Allocate a per-core structure for managing state about which cores are
5291  * running in the host versus the guest and for exchanging data between
5292  * real mode KVM and CPU running in the host.
5293  * This is only done for the first VM.
5294  * The allocated structure stays even if all VMs have stopped.
5295  * It is only freed when the kvm-hv module is unloaded.
5296  * It's OK for this routine to fail, we just don't support host
5297  * core operations like redirecting H_IPI wakeups.
5298  */
kvmppc_alloc_host_rm_ops(void)5299 void kvmppc_alloc_host_rm_ops(void)
5300 {
5301 	struct kvmppc_host_rm_ops *ops;
5302 	unsigned long l_ops;
5303 	int cpu, core;
5304 	int size;
5305 
5306 	if (cpu_has_feature(CPU_FTR_ARCH_300))
5307 		return;
5308 
5309 	/* Not the first time here ? */
5310 	if (kvmppc_host_rm_ops_hv != NULL)
5311 		return;
5312 
5313 	ops = kzalloc(sizeof(struct kvmppc_host_rm_ops), GFP_KERNEL);
5314 	if (!ops)
5315 		return;
5316 
5317 	size = cpu_nr_cores() * sizeof(struct kvmppc_host_rm_core);
5318 	ops->rm_core = kzalloc(size, GFP_KERNEL);
5319 
5320 	if (!ops->rm_core) {
5321 		kfree(ops);
5322 		return;
5323 	}
5324 
5325 	cpus_read_lock();
5326 
5327 	for (cpu = 0; cpu < nr_cpu_ids; cpu += threads_per_core) {
5328 		if (!cpu_online(cpu))
5329 			continue;
5330 
5331 		core = cpu >> threads_shift;
5332 		ops->rm_core[core].rm_state.in_host = 1;
5333 	}
5334 
5335 	ops->vcpu_kick = kvmppc_fast_vcpu_kick_hv;
5336 
5337 	/*
5338 	 * Make the contents of the kvmppc_host_rm_ops structure visible
5339 	 * to other CPUs before we assign it to the global variable.
5340 	 * Do an atomic assignment (no locks used here), but if someone
5341 	 * beats us to it, just free our copy and return.
5342 	 */
5343 	smp_wmb();
5344 	l_ops = (unsigned long) ops;
5345 
5346 	if (cmpxchg64((unsigned long *)&kvmppc_host_rm_ops_hv, 0, l_ops)) {
5347 		cpus_read_unlock();
5348 		kfree(ops->rm_core);
5349 		kfree(ops);
5350 		return;
5351 	}
5352 
5353 	cpuhp_setup_state_nocalls_cpuslocked(CPUHP_KVM_PPC_BOOK3S_PREPARE,
5354 					     "ppc/kvm_book3s:prepare",
5355 					     kvmppc_set_host_core,
5356 					     kvmppc_clear_host_core);
5357 	cpus_read_unlock();
5358 }
5359 
kvmppc_free_host_rm_ops(void)5360 void kvmppc_free_host_rm_ops(void)
5361 {
5362 	if (kvmppc_host_rm_ops_hv) {
5363 		cpuhp_remove_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE);
5364 		kfree(kvmppc_host_rm_ops_hv->rm_core);
5365 		kfree(kvmppc_host_rm_ops_hv);
5366 		kvmppc_host_rm_ops_hv = NULL;
5367 	}
5368 }
5369 #endif
5370 
kvmppc_core_init_vm_hv(struct kvm * kvm)5371 static int kvmppc_core_init_vm_hv(struct kvm *kvm)
5372 {
5373 	unsigned long lpcr, lpid;
5374 	int ret;
5375 
5376 	mutex_init(&kvm->arch.uvmem_lock);
5377 	INIT_LIST_HEAD(&kvm->arch.uvmem_pfns);
5378 	mutex_init(&kvm->arch.mmu_setup_lock);
5379 
5380 	/* Allocate the guest's logical partition ID */
5381 
5382 	lpid = kvmppc_alloc_lpid();
5383 	if ((long)lpid < 0)
5384 		return -ENOMEM;
5385 	kvm->arch.lpid = lpid;
5386 
5387 	kvmppc_alloc_host_rm_ops();
5388 
5389 	kvmhv_vm_nested_init(kvm);
5390 
5391 	/*
5392 	 * Since we don't flush the TLB when tearing down a VM,
5393 	 * and this lpid might have previously been used,
5394 	 * make sure we flush on each core before running the new VM.
5395 	 * On POWER9, the tlbie in mmu_partition_table_set_entry()
5396 	 * does this flush for us.
5397 	 */
5398 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
5399 		cpumask_setall(&kvm->arch.need_tlb_flush);
5400 
5401 	/* Start out with the default set of hcalls enabled */
5402 	memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,
5403 	       sizeof(kvm->arch.enabled_hcalls));
5404 
5405 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
5406 		kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
5407 
5408 	/* Init LPCR for virtual RMA mode */
5409 	if (cpu_has_feature(CPU_FTR_HVMODE)) {
5410 		kvm->arch.host_lpid = mfspr(SPRN_LPID);
5411 		kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
5412 		lpcr &= LPCR_PECE | LPCR_LPES;
5413 	} else {
5414 		/*
5415 		 * The L2 LPES mode will be set by the L0 according to whether
5416 		 * or not it needs to take external interrupts in HV mode.
5417 		 */
5418 		lpcr = 0;
5419 	}
5420 	lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
5421 		LPCR_VPM0 | LPCR_VPM1;
5422 	kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
5423 		(VRMA_VSID << SLB_VSID_SHIFT_1T);
5424 	/* On POWER8 turn on online bit to enable PURR/SPURR */
5425 	if (cpu_has_feature(CPU_FTR_ARCH_207S))
5426 		lpcr |= LPCR_ONL;
5427 	/*
5428 	 * On POWER9, VPM0 bit is reserved (VPM0=1 behaviour is assumed)
5429 	 * Set HVICE bit to enable hypervisor virtualization interrupts.
5430 	 * Set HEIC to prevent OS interrupts to go to hypervisor (should
5431 	 * be unnecessary but better safe than sorry in case we re-enable
5432 	 * EE in HV mode with this LPCR still set)
5433 	 */
5434 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
5435 		lpcr &= ~LPCR_VPM0;
5436 		lpcr |= LPCR_HVICE | LPCR_HEIC;
5437 
5438 		/*
5439 		 * If xive is enabled, we route 0x500 interrupts directly
5440 		 * to the guest.
5441 		 */
5442 		if (xics_on_xive())
5443 			lpcr |= LPCR_LPES;
5444 	}
5445 
5446 	/*
5447 	 * If the host uses radix, the guest starts out as radix.
5448 	 */
5449 	if (radix_enabled()) {
5450 		kvm->arch.radix = 1;
5451 		kvm->arch.mmu_ready = 1;
5452 		lpcr &= ~LPCR_VPM1;
5453 		lpcr |= LPCR_UPRT | LPCR_GTSE | LPCR_HR;
5454 		if (cpu_has_feature(CPU_FTR_HVMODE) &&
5455 		    cpu_has_feature(CPU_FTR_ARCH_31) &&
5456 		    (kvm->arch.host_lpcr & LPCR_HAIL))
5457 			lpcr |= LPCR_HAIL;
5458 		ret = kvmppc_init_vm_radix(kvm);
5459 		if (ret) {
5460 			kvmppc_free_lpid(kvm->arch.lpid);
5461 			return ret;
5462 		}
5463 		kvmppc_setup_partition_table(kvm);
5464 	}
5465 
5466 	verify_lpcr(kvm, lpcr);
5467 	kvm->arch.lpcr = lpcr;
5468 
5469 	/* Initialization for future HPT resizes */
5470 	kvm->arch.resize_hpt = NULL;
5471 
5472 	/*
5473 	 * Work out how many sets the TLB has, for the use of
5474 	 * the TLB invalidation loop in book3s_hv_rmhandlers.S.
5475 	 */
5476 	if (cpu_has_feature(CPU_FTR_ARCH_31)) {
5477 		/*
5478 		 * P10 will flush all the congruence class with a single tlbiel
5479 		 */
5480 		kvm->arch.tlb_sets = 1;
5481 	} else if (radix_enabled())
5482 		kvm->arch.tlb_sets = POWER9_TLB_SETS_RADIX;	/* 128 */
5483 	else if (cpu_has_feature(CPU_FTR_ARCH_300))
5484 		kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH;	/* 256 */
5485 	else if (cpu_has_feature(CPU_FTR_ARCH_207S))
5486 		kvm->arch.tlb_sets = POWER8_TLB_SETS;		/* 512 */
5487 	else
5488 		kvm->arch.tlb_sets = POWER7_TLB_SETS;		/* 128 */
5489 
5490 	/*
5491 	 * Track that we now have a HV mode VM active. This blocks secondary
5492 	 * CPU threads from coming online.
5493 	 */
5494 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
5495 		kvm_hv_vm_activated();
5496 
5497 	/*
5498 	 * Initialize smt_mode depending on processor.
5499 	 * POWER8 and earlier have to use "strict" threading, where
5500 	 * all vCPUs in a vcore have to run on the same (sub)core,
5501 	 * whereas on POWER9 the threads can each run a different
5502 	 * guest.
5503 	 */
5504 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
5505 		kvm->arch.smt_mode = threads_per_subcore;
5506 	else
5507 		kvm->arch.smt_mode = 1;
5508 	kvm->arch.emul_smt_mode = 1;
5509 
5510 	return 0;
5511 }
5512 
kvmppc_arch_create_vm_debugfs_hv(struct kvm * kvm)5513 static int kvmppc_arch_create_vm_debugfs_hv(struct kvm *kvm)
5514 {
5515 	kvmppc_mmu_debugfs_init(kvm);
5516 	if (radix_enabled())
5517 		kvmhv_radix_debugfs_init(kvm);
5518 	return 0;
5519 }
5520 
kvmppc_free_vcores(struct kvm * kvm)5521 static void kvmppc_free_vcores(struct kvm *kvm)
5522 {
5523 	long int i;
5524 
5525 	for (i = 0; i < KVM_MAX_VCORES; ++i)
5526 		kfree(kvm->arch.vcores[i]);
5527 	kvm->arch.online_vcores = 0;
5528 }
5529 
kvmppc_core_destroy_vm_hv(struct kvm * kvm)5530 static void kvmppc_core_destroy_vm_hv(struct kvm *kvm)
5531 {
5532 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
5533 		kvm_hv_vm_deactivated();
5534 
5535 	kvmppc_free_vcores(kvm);
5536 
5537 
5538 	if (kvm_is_radix(kvm))
5539 		kvmppc_free_radix(kvm);
5540 	else
5541 		kvmppc_free_hpt(&kvm->arch.hpt);
5542 
5543 	/* Perform global invalidation and return lpid to the pool */
5544 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
5545 		if (nesting_enabled(kvm))
5546 			kvmhv_release_all_nested(kvm);
5547 		kvm->arch.process_table = 0;
5548 		if (kvm->arch.secure_guest)
5549 			uv_svm_terminate(kvm->arch.lpid);
5550 		kvmhv_set_ptbl_entry(kvm->arch.lpid, 0, 0);
5551 	}
5552 
5553 	kvmppc_free_lpid(kvm->arch.lpid);
5554 
5555 	kvmppc_free_pimap(kvm);
5556 }
5557 
5558 /* We don't need to emulate any privileged instructions or dcbz */
kvmppc_core_emulate_op_hv(struct kvm_vcpu * vcpu,unsigned int inst,int * advance)5559 static int kvmppc_core_emulate_op_hv(struct kvm_vcpu *vcpu,
5560 				     unsigned int inst, int *advance)
5561 {
5562 	return EMULATE_FAIL;
5563 }
5564 
kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu * vcpu,int sprn,ulong spr_val)5565 static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn,
5566 					ulong spr_val)
5567 {
5568 	return EMULATE_FAIL;
5569 }
5570 
kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu * vcpu,int sprn,ulong * spr_val)5571 static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn,
5572 					ulong *spr_val)
5573 {
5574 	return EMULATE_FAIL;
5575 }
5576 
kvmppc_core_check_processor_compat_hv(void)5577 static int kvmppc_core_check_processor_compat_hv(void)
5578 {
5579 	if (cpu_has_feature(CPU_FTR_HVMODE) &&
5580 	    cpu_has_feature(CPU_FTR_ARCH_206))
5581 		return 0;
5582 
5583 	/* POWER9 in radix mode is capable of being a nested hypervisor. */
5584 	if (cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled())
5585 		return 0;
5586 
5587 	return -EIO;
5588 }
5589 
5590 #ifdef CONFIG_KVM_XICS
5591 
kvmppc_free_pimap(struct kvm * kvm)5592 void kvmppc_free_pimap(struct kvm *kvm)
5593 {
5594 	kfree(kvm->arch.pimap);
5595 }
5596 
kvmppc_alloc_pimap(void)5597 static struct kvmppc_passthru_irqmap *kvmppc_alloc_pimap(void)
5598 {
5599 	return kzalloc(sizeof(struct kvmppc_passthru_irqmap), GFP_KERNEL);
5600 }
5601 
kvmppc_set_passthru_irq(struct kvm * kvm,int host_irq,int guest_gsi)5602 static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
5603 {
5604 	struct irq_desc *desc;
5605 	struct kvmppc_irq_map *irq_map;
5606 	struct kvmppc_passthru_irqmap *pimap;
5607 	struct irq_chip *chip;
5608 	int i, rc = 0;
5609 	struct irq_data *host_data;
5610 
5611 	if (!kvm_irq_bypass)
5612 		return 1;
5613 
5614 	desc = irq_to_desc(host_irq);
5615 	if (!desc)
5616 		return -EIO;
5617 
5618 	mutex_lock(&kvm->lock);
5619 
5620 	pimap = kvm->arch.pimap;
5621 	if (pimap == NULL) {
5622 		/* First call, allocate structure to hold IRQ map */
5623 		pimap = kvmppc_alloc_pimap();
5624 		if (pimap == NULL) {
5625 			mutex_unlock(&kvm->lock);
5626 			return -ENOMEM;
5627 		}
5628 		kvm->arch.pimap = pimap;
5629 	}
5630 
5631 	/*
5632 	 * For now, we only support interrupts for which the EOI operation
5633 	 * is an OPAL call followed by a write to XIRR, since that's
5634 	 * what our real-mode EOI code does, or a XIVE interrupt
5635 	 */
5636 	chip = irq_data_get_irq_chip(&desc->irq_data);
5637 	if (!chip || !is_pnv_opal_msi(chip)) {
5638 		pr_warn("kvmppc_set_passthru_irq_hv: Could not assign IRQ map for (%d,%d)\n",
5639 			host_irq, guest_gsi);
5640 		mutex_unlock(&kvm->lock);
5641 		return -ENOENT;
5642 	}
5643 
5644 	/*
5645 	 * See if we already have an entry for this guest IRQ number.
5646 	 * If it's mapped to a hardware IRQ number, that's an error,
5647 	 * otherwise re-use this entry.
5648 	 */
5649 	for (i = 0; i < pimap->n_mapped; i++) {
5650 		if (guest_gsi == pimap->mapped[i].v_hwirq) {
5651 			if (pimap->mapped[i].r_hwirq) {
5652 				mutex_unlock(&kvm->lock);
5653 				return -EINVAL;
5654 			}
5655 			break;
5656 		}
5657 	}
5658 
5659 	if (i == KVMPPC_PIRQ_MAPPED) {
5660 		mutex_unlock(&kvm->lock);
5661 		return -EAGAIN;		/* table is full */
5662 	}
5663 
5664 	irq_map = &pimap->mapped[i];
5665 
5666 	irq_map->v_hwirq = guest_gsi;
5667 	irq_map->desc = desc;
5668 
5669 	/*
5670 	 * Order the above two stores before the next to serialize with
5671 	 * the KVM real mode handler.
5672 	 */
5673 	smp_wmb();
5674 
5675 	/*
5676 	 * The 'host_irq' number is mapped in the PCI-MSI domain but
5677 	 * the underlying calls, which will EOI the interrupt in real
5678 	 * mode, need an HW IRQ number mapped in the XICS IRQ domain.
5679 	 */
5680 	host_data = irq_domain_get_irq_data(irq_get_default_host(), host_irq);
5681 	irq_map->r_hwirq = (unsigned int)irqd_to_hwirq(host_data);
5682 
5683 	if (i == pimap->n_mapped)
5684 		pimap->n_mapped++;
5685 
5686 	if (xics_on_xive())
5687 		rc = kvmppc_xive_set_mapped(kvm, guest_gsi, host_irq);
5688 	else
5689 		kvmppc_xics_set_mapped(kvm, guest_gsi, irq_map->r_hwirq);
5690 	if (rc)
5691 		irq_map->r_hwirq = 0;
5692 
5693 	mutex_unlock(&kvm->lock);
5694 
5695 	return 0;
5696 }
5697 
kvmppc_clr_passthru_irq(struct kvm * kvm,int host_irq,int guest_gsi)5698 static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi)
5699 {
5700 	struct irq_desc *desc;
5701 	struct kvmppc_passthru_irqmap *pimap;
5702 	int i, rc = 0;
5703 
5704 	if (!kvm_irq_bypass)
5705 		return 0;
5706 
5707 	desc = irq_to_desc(host_irq);
5708 	if (!desc)
5709 		return -EIO;
5710 
5711 	mutex_lock(&kvm->lock);
5712 	if (!kvm->arch.pimap)
5713 		goto unlock;
5714 
5715 	pimap = kvm->arch.pimap;
5716 
5717 	for (i = 0; i < pimap->n_mapped; i++) {
5718 		if (guest_gsi == pimap->mapped[i].v_hwirq)
5719 			break;
5720 	}
5721 
5722 	if (i == pimap->n_mapped) {
5723 		mutex_unlock(&kvm->lock);
5724 		return -ENODEV;
5725 	}
5726 
5727 	if (xics_on_xive())
5728 		rc = kvmppc_xive_clr_mapped(kvm, guest_gsi, host_irq);
5729 	else
5730 		kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq);
5731 
5732 	/* invalidate the entry (what to do on error from the above ?) */
5733 	pimap->mapped[i].r_hwirq = 0;
5734 
5735 	/*
5736 	 * We don't free this structure even when the count goes to
5737 	 * zero. The structure is freed when we destroy the VM.
5738 	 */
5739  unlock:
5740 	mutex_unlock(&kvm->lock);
5741 	return rc;
5742 }
5743 
kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)5744 static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons,
5745 					     struct irq_bypass_producer *prod)
5746 {
5747 	int ret = 0;
5748 	struct kvm_kernel_irqfd *irqfd =
5749 		container_of(cons, struct kvm_kernel_irqfd, consumer);
5750 
5751 	irqfd->producer = prod;
5752 
5753 	ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
5754 	if (ret)
5755 		pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n",
5756 			prod->irq, irqfd->gsi, ret);
5757 
5758 	return ret;
5759 }
5760 
kvmppc_irq_bypass_del_producer_hv(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)5761 static void kvmppc_irq_bypass_del_producer_hv(struct irq_bypass_consumer *cons,
5762 					      struct irq_bypass_producer *prod)
5763 {
5764 	int ret;
5765 	struct kvm_kernel_irqfd *irqfd =
5766 		container_of(cons, struct kvm_kernel_irqfd, consumer);
5767 
5768 	irqfd->producer = NULL;
5769 
5770 	/*
5771 	 * When producer of consumer is unregistered, we change back to
5772 	 * default external interrupt handling mode - KVM real mode
5773 	 * will switch back to host.
5774 	 */
5775 	ret = kvmppc_clr_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi);
5776 	if (ret)
5777 		pr_warn("kvmppc_clr_passthru_irq (irq %d, gsi %d) fails: %d\n",
5778 			prod->irq, irqfd->gsi, ret);
5779 }
5780 #endif
5781 
kvm_arch_vm_ioctl_hv(struct file * filp,unsigned int ioctl,unsigned long arg)5782 static long kvm_arch_vm_ioctl_hv(struct file *filp,
5783 				 unsigned int ioctl, unsigned long arg)
5784 {
5785 	struct kvm *kvm __maybe_unused = filp->private_data;
5786 	void __user *argp = (void __user *)arg;
5787 	long r;
5788 
5789 	switch (ioctl) {
5790 
5791 	case KVM_PPC_ALLOCATE_HTAB: {
5792 		u32 htab_order;
5793 
5794 		/* If we're a nested hypervisor, we currently only support radix */
5795 		if (kvmhv_on_pseries()) {
5796 			r = -EOPNOTSUPP;
5797 			break;
5798 		}
5799 
5800 		r = -EFAULT;
5801 		if (get_user(htab_order, (u32 __user *)argp))
5802 			break;
5803 		r = kvmppc_alloc_reset_hpt(kvm, htab_order);
5804 		if (r)
5805 			break;
5806 		r = 0;
5807 		break;
5808 	}
5809 
5810 	case KVM_PPC_GET_HTAB_FD: {
5811 		struct kvm_get_htab_fd ghf;
5812 
5813 		r = -EFAULT;
5814 		if (copy_from_user(&ghf, argp, sizeof(ghf)))
5815 			break;
5816 		r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf);
5817 		break;
5818 	}
5819 
5820 	case KVM_PPC_RESIZE_HPT_PREPARE: {
5821 		struct kvm_ppc_resize_hpt rhpt;
5822 
5823 		r = -EFAULT;
5824 		if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
5825 			break;
5826 
5827 		r = kvm_vm_ioctl_resize_hpt_prepare(kvm, &rhpt);
5828 		break;
5829 	}
5830 
5831 	case KVM_PPC_RESIZE_HPT_COMMIT: {
5832 		struct kvm_ppc_resize_hpt rhpt;
5833 
5834 		r = -EFAULT;
5835 		if (copy_from_user(&rhpt, argp, sizeof(rhpt)))
5836 			break;
5837 
5838 		r = kvm_vm_ioctl_resize_hpt_commit(kvm, &rhpt);
5839 		break;
5840 	}
5841 
5842 	default:
5843 		r = -ENOTTY;
5844 	}
5845 
5846 	return r;
5847 }
5848 
5849 /*
5850  * List of hcall numbers to enable by default.
5851  * For compatibility with old userspace, we enable by default
5852  * all hcalls that were implemented before the hcall-enabling
5853  * facility was added.  Note this list should not include H_RTAS.
5854  */
5855 static unsigned int default_hcall_list[] = {
5856 	H_REMOVE,
5857 	H_ENTER,
5858 	H_READ,
5859 	H_PROTECT,
5860 	H_BULK_REMOVE,
5861 #ifdef CONFIG_SPAPR_TCE_IOMMU
5862 	H_GET_TCE,
5863 	H_PUT_TCE,
5864 #endif
5865 	H_SET_DABR,
5866 	H_SET_XDABR,
5867 	H_CEDE,
5868 	H_PROD,
5869 	H_CONFER,
5870 	H_REGISTER_VPA,
5871 #ifdef CONFIG_KVM_XICS
5872 	H_EOI,
5873 	H_CPPR,
5874 	H_IPI,
5875 	H_IPOLL,
5876 	H_XIRR,
5877 	H_XIRR_X,
5878 #endif
5879 	0
5880 };
5881 
init_default_hcalls(void)5882 static void init_default_hcalls(void)
5883 {
5884 	int i;
5885 	unsigned int hcall;
5886 
5887 	for (i = 0; default_hcall_list[i]; ++i) {
5888 		hcall = default_hcall_list[i];
5889 		WARN_ON(!kvmppc_hcall_impl_hv(hcall));
5890 		__set_bit(hcall / 4, default_enabled_hcalls);
5891 	}
5892 }
5893 
kvmhv_configure_mmu(struct kvm * kvm,struct kvm_ppc_mmuv3_cfg * cfg)5894 static int kvmhv_configure_mmu(struct kvm *kvm, struct kvm_ppc_mmuv3_cfg *cfg)
5895 {
5896 	unsigned long lpcr;
5897 	int radix;
5898 	int err;
5899 
5900 	/* If not on a POWER9, reject it */
5901 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
5902 		return -ENODEV;
5903 
5904 	/* If any unknown flags set, reject it */
5905 	if (cfg->flags & ~(KVM_PPC_MMUV3_RADIX | KVM_PPC_MMUV3_GTSE))
5906 		return -EINVAL;
5907 
5908 	/* GR (guest radix) bit in process_table field must match */
5909 	radix = !!(cfg->flags & KVM_PPC_MMUV3_RADIX);
5910 	if (!!(cfg->process_table & PATB_GR) != radix)
5911 		return -EINVAL;
5912 
5913 	/* Process table size field must be reasonable, i.e. <= 24 */
5914 	if ((cfg->process_table & PRTS_MASK) > 24)
5915 		return -EINVAL;
5916 
5917 	/* We can change a guest to/from radix now, if the host is radix */
5918 	if (radix && !radix_enabled())
5919 		return -EINVAL;
5920 
5921 	/* If we're a nested hypervisor, we currently only support radix */
5922 	if (kvmhv_on_pseries() && !radix)
5923 		return -EINVAL;
5924 
5925 	mutex_lock(&kvm->arch.mmu_setup_lock);
5926 	if (radix != kvm_is_radix(kvm)) {
5927 		if (kvm->arch.mmu_ready) {
5928 			kvm->arch.mmu_ready = 0;
5929 			/* order mmu_ready vs. vcpus_running */
5930 			smp_mb();
5931 			if (atomic_read(&kvm->arch.vcpus_running)) {
5932 				kvm->arch.mmu_ready = 1;
5933 				err = -EBUSY;
5934 				goto out_unlock;
5935 			}
5936 		}
5937 		if (radix)
5938 			err = kvmppc_switch_mmu_to_radix(kvm);
5939 		else
5940 			err = kvmppc_switch_mmu_to_hpt(kvm);
5941 		if (err)
5942 			goto out_unlock;
5943 	}
5944 
5945 	kvm->arch.process_table = cfg->process_table;
5946 	kvmppc_setup_partition_table(kvm);
5947 
5948 	lpcr = (cfg->flags & KVM_PPC_MMUV3_GTSE) ? LPCR_GTSE : 0;
5949 	kvmppc_update_lpcr(kvm, lpcr, LPCR_GTSE);
5950 	err = 0;
5951 
5952  out_unlock:
5953 	mutex_unlock(&kvm->arch.mmu_setup_lock);
5954 	return err;
5955 }
5956 
kvmhv_enable_nested(struct kvm * kvm)5957 static int kvmhv_enable_nested(struct kvm *kvm)
5958 {
5959 	if (!nested)
5960 		return -EPERM;
5961 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
5962 		return -ENODEV;
5963 	if (!radix_enabled())
5964 		return -ENODEV;
5965 
5966 	/* kvm == NULL means the caller is testing if the capability exists */
5967 	if (kvm)
5968 		kvm->arch.nested_enable = true;
5969 	return 0;
5970 }
5971 
kvmhv_load_from_eaddr(struct kvm_vcpu * vcpu,ulong * eaddr,void * ptr,int size)5972 static int kvmhv_load_from_eaddr(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr,
5973 				 int size)
5974 {
5975 	int rc = -EINVAL;
5976 
5977 	if (kvmhv_vcpu_is_radix(vcpu)) {
5978 		rc = kvmhv_copy_from_guest_radix(vcpu, *eaddr, ptr, size);
5979 
5980 		if (rc > 0)
5981 			rc = -EINVAL;
5982 	}
5983 
5984 	/* For now quadrants are the only way to access nested guest memory */
5985 	if (rc && vcpu->arch.nested)
5986 		rc = -EAGAIN;
5987 
5988 	return rc;
5989 }
5990 
kvmhv_store_to_eaddr(struct kvm_vcpu * vcpu,ulong * eaddr,void * ptr,int size)5991 static int kvmhv_store_to_eaddr(struct kvm_vcpu *vcpu, ulong *eaddr, void *ptr,
5992 				int size)
5993 {
5994 	int rc = -EINVAL;
5995 
5996 	if (kvmhv_vcpu_is_radix(vcpu)) {
5997 		rc = kvmhv_copy_to_guest_radix(vcpu, *eaddr, ptr, size);
5998 
5999 		if (rc > 0)
6000 			rc = -EINVAL;
6001 	}
6002 
6003 	/* For now quadrants are the only way to access nested guest memory */
6004 	if (rc && vcpu->arch.nested)
6005 		rc = -EAGAIN;
6006 
6007 	return rc;
6008 }
6009 
unpin_vpa_reset(struct kvm * kvm,struct kvmppc_vpa * vpa)6010 static void unpin_vpa_reset(struct kvm *kvm, struct kvmppc_vpa *vpa)
6011 {
6012 	unpin_vpa(kvm, vpa);
6013 	vpa->gpa = 0;
6014 	vpa->pinned_addr = NULL;
6015 	vpa->dirty = false;
6016 	vpa->update_pending = 0;
6017 }
6018 
6019 /*
6020  * Enable a guest to become a secure VM, or test whether
6021  * that could be enabled.
6022  * Called when the KVM_CAP_PPC_SECURE_GUEST capability is
6023  * tested (kvm == NULL) or enabled (kvm != NULL).
6024  */
kvmhv_enable_svm(struct kvm * kvm)6025 static int kvmhv_enable_svm(struct kvm *kvm)
6026 {
6027 	if (!kvmppc_uvmem_available())
6028 		return -EINVAL;
6029 	if (kvm)
6030 		kvm->arch.svm_enabled = 1;
6031 	return 0;
6032 }
6033 
6034 /*
6035  *  IOCTL handler to turn off secure mode of guest
6036  *
6037  * - Release all device pages
6038  * - Issue ucall to terminate the guest on the UV side
6039  * - Unpin the VPA pages.
6040  * - Reinit the partition scoped page tables
6041  */
kvmhv_svm_off(struct kvm * kvm)6042 static int kvmhv_svm_off(struct kvm *kvm)
6043 {
6044 	struct kvm_vcpu *vcpu;
6045 	int mmu_was_ready;
6046 	int srcu_idx;
6047 	int ret = 0;
6048 	unsigned long i;
6049 
6050 	if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
6051 		return ret;
6052 
6053 	mutex_lock(&kvm->arch.mmu_setup_lock);
6054 	mmu_was_ready = kvm->arch.mmu_ready;
6055 	if (kvm->arch.mmu_ready) {
6056 		kvm->arch.mmu_ready = 0;
6057 		/* order mmu_ready vs. vcpus_running */
6058 		smp_mb();
6059 		if (atomic_read(&kvm->arch.vcpus_running)) {
6060 			kvm->arch.mmu_ready = 1;
6061 			ret = -EBUSY;
6062 			goto out;
6063 		}
6064 	}
6065 
6066 	srcu_idx = srcu_read_lock(&kvm->srcu);
6067 	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
6068 		struct kvm_memory_slot *memslot;
6069 		struct kvm_memslots *slots = __kvm_memslots(kvm, i);
6070 		int bkt;
6071 
6072 		if (!slots)
6073 			continue;
6074 
6075 		kvm_for_each_memslot(memslot, bkt, slots) {
6076 			kvmppc_uvmem_drop_pages(memslot, kvm, true);
6077 			uv_unregister_mem_slot(kvm->arch.lpid, memslot->id);
6078 		}
6079 	}
6080 	srcu_read_unlock(&kvm->srcu, srcu_idx);
6081 
6082 	ret = uv_svm_terminate(kvm->arch.lpid);
6083 	if (ret != U_SUCCESS) {
6084 		ret = -EINVAL;
6085 		goto out;
6086 	}
6087 
6088 	/*
6089 	 * When secure guest is reset, all the guest pages are sent
6090 	 * to UV via UV_PAGE_IN before the non-boot vcpus get a
6091 	 * chance to run and unpin their VPA pages. Unpinning of all
6092 	 * VPA pages is done here explicitly so that VPA pages
6093 	 * can be migrated to the secure side.
6094 	 *
6095 	 * This is required to for the secure SMP guest to reboot
6096 	 * correctly.
6097 	 */
6098 	kvm_for_each_vcpu(i, vcpu, kvm) {
6099 		spin_lock(&vcpu->arch.vpa_update_lock);
6100 		unpin_vpa_reset(kvm, &vcpu->arch.dtl);
6101 		unpin_vpa_reset(kvm, &vcpu->arch.slb_shadow);
6102 		unpin_vpa_reset(kvm, &vcpu->arch.vpa);
6103 		spin_unlock(&vcpu->arch.vpa_update_lock);
6104 	}
6105 
6106 	kvmppc_setup_partition_table(kvm);
6107 	kvm->arch.secure_guest = 0;
6108 	kvm->arch.mmu_ready = mmu_was_ready;
6109 out:
6110 	mutex_unlock(&kvm->arch.mmu_setup_lock);
6111 	return ret;
6112 }
6113 
kvmhv_enable_dawr1(struct kvm * kvm)6114 static int kvmhv_enable_dawr1(struct kvm *kvm)
6115 {
6116 	if (!cpu_has_feature(CPU_FTR_DAWR1))
6117 		return -ENODEV;
6118 
6119 	/* kvm == NULL means the caller is testing if the capability exists */
6120 	if (kvm)
6121 		kvm->arch.dawr1_enabled = true;
6122 	return 0;
6123 }
6124 
kvmppc_hash_v3_possible(void)6125 static bool kvmppc_hash_v3_possible(void)
6126 {
6127 	if (!cpu_has_feature(CPU_FTR_ARCH_300))
6128 		return false;
6129 
6130 	if (!cpu_has_feature(CPU_FTR_HVMODE))
6131 		return false;
6132 
6133 	/*
6134 	 * POWER9 chips before version 2.02 can't have some threads in
6135 	 * HPT mode and some in radix mode on the same core.
6136 	 */
6137 	if (radix_enabled()) {
6138 		unsigned int pvr = mfspr(SPRN_PVR);
6139 		if ((pvr >> 16) == PVR_POWER9 &&
6140 		    (((pvr & 0xe000) == 0 && (pvr & 0xfff) < 0x202) ||
6141 		     ((pvr & 0xe000) == 0x2000 && (pvr & 0xfff) < 0x101)))
6142 			return false;
6143 	}
6144 
6145 	return true;
6146 }
6147 
6148 static struct kvmppc_ops kvm_ops_hv = {
6149 	.get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv,
6150 	.set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv,
6151 	.get_one_reg = kvmppc_get_one_reg_hv,
6152 	.set_one_reg = kvmppc_set_one_reg_hv,
6153 	.vcpu_load   = kvmppc_core_vcpu_load_hv,
6154 	.vcpu_put    = kvmppc_core_vcpu_put_hv,
6155 	.inject_interrupt = kvmppc_inject_interrupt_hv,
6156 	.set_msr     = kvmppc_set_msr_hv,
6157 	.vcpu_run    = kvmppc_vcpu_run_hv,
6158 	.vcpu_create = kvmppc_core_vcpu_create_hv,
6159 	.vcpu_free   = kvmppc_core_vcpu_free_hv,
6160 	.check_requests = kvmppc_core_check_requests_hv,
6161 	.get_dirty_log  = kvm_vm_ioctl_get_dirty_log_hv,
6162 	.flush_memslot  = kvmppc_core_flush_memslot_hv,
6163 	.prepare_memory_region = kvmppc_core_prepare_memory_region_hv,
6164 	.commit_memory_region  = kvmppc_core_commit_memory_region_hv,
6165 	.unmap_gfn_range = kvm_unmap_gfn_range_hv,
6166 	.age_gfn = kvm_age_gfn_hv,
6167 	.test_age_gfn = kvm_test_age_gfn_hv,
6168 	.set_spte_gfn = kvm_set_spte_gfn_hv,
6169 	.free_memslot = kvmppc_core_free_memslot_hv,
6170 	.init_vm =  kvmppc_core_init_vm_hv,
6171 	.destroy_vm = kvmppc_core_destroy_vm_hv,
6172 	.get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv,
6173 	.emulate_op = kvmppc_core_emulate_op_hv,
6174 	.emulate_mtspr = kvmppc_core_emulate_mtspr_hv,
6175 	.emulate_mfspr = kvmppc_core_emulate_mfspr_hv,
6176 	.fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv,
6177 	.arch_vm_ioctl  = kvm_arch_vm_ioctl_hv,
6178 	.hcall_implemented = kvmppc_hcall_impl_hv,
6179 #ifdef CONFIG_KVM_XICS
6180 	.irq_bypass_add_producer = kvmppc_irq_bypass_add_producer_hv,
6181 	.irq_bypass_del_producer = kvmppc_irq_bypass_del_producer_hv,
6182 #endif
6183 	.configure_mmu = kvmhv_configure_mmu,
6184 	.get_rmmu_info = kvmhv_get_rmmu_info,
6185 	.set_smt_mode = kvmhv_set_smt_mode,
6186 	.enable_nested = kvmhv_enable_nested,
6187 	.load_from_eaddr = kvmhv_load_from_eaddr,
6188 	.store_to_eaddr = kvmhv_store_to_eaddr,
6189 	.enable_svm = kvmhv_enable_svm,
6190 	.svm_off = kvmhv_svm_off,
6191 	.enable_dawr1 = kvmhv_enable_dawr1,
6192 	.hash_v3_possible = kvmppc_hash_v3_possible,
6193 	.create_vcpu_debugfs = kvmppc_arch_create_vcpu_debugfs_hv,
6194 	.create_vm_debugfs = kvmppc_arch_create_vm_debugfs_hv,
6195 };
6196 
kvm_init_subcore_bitmap(void)6197 static int kvm_init_subcore_bitmap(void)
6198 {
6199 	int i, j;
6200 	int nr_cores = cpu_nr_cores();
6201 	struct sibling_subcore_state *sibling_subcore_state;
6202 
6203 	for (i = 0; i < nr_cores; i++) {
6204 		int first_cpu = i * threads_per_core;
6205 		int node = cpu_to_node(first_cpu);
6206 
6207 		/* Ignore if it is already allocated. */
6208 		if (paca_ptrs[first_cpu]->sibling_subcore_state)
6209 			continue;
6210 
6211 		sibling_subcore_state =
6212 			kzalloc_node(sizeof(struct sibling_subcore_state),
6213 							GFP_KERNEL, node);
6214 		if (!sibling_subcore_state)
6215 			return -ENOMEM;
6216 
6217 
6218 		for (j = 0; j < threads_per_core; j++) {
6219 			int cpu = first_cpu + j;
6220 
6221 			paca_ptrs[cpu]->sibling_subcore_state =
6222 						sibling_subcore_state;
6223 		}
6224 	}
6225 	return 0;
6226 }
6227 
kvmppc_radix_possible(void)6228 static int kvmppc_radix_possible(void)
6229 {
6230 	return cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled();
6231 }
6232 
kvmppc_book3s_init_hv(void)6233 static int kvmppc_book3s_init_hv(void)
6234 {
6235 	int r;
6236 
6237 	if (!tlbie_capable) {
6238 		pr_err("KVM-HV: Host does not support TLBIE\n");
6239 		return -ENODEV;
6240 	}
6241 
6242 	/*
6243 	 * FIXME!! Do we need to check on all cpus ?
6244 	 */
6245 	r = kvmppc_core_check_processor_compat_hv();
6246 	if (r < 0)
6247 		return -ENODEV;
6248 
6249 	r = kvmhv_nested_init();
6250 	if (r)
6251 		return r;
6252 
6253 	if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
6254 		r = kvm_init_subcore_bitmap();
6255 		if (r)
6256 			goto err;
6257 	}
6258 
6259 	/*
6260 	 * We need a way of accessing the XICS interrupt controller,
6261 	 * either directly, via paca_ptrs[cpu]->kvm_hstate.xics_phys, or
6262 	 * indirectly, via OPAL.
6263 	 */
6264 #ifdef CONFIG_SMP
6265 	if (!xics_on_xive() && !kvmhv_on_pseries() &&
6266 	    !local_paca->kvm_hstate.xics_phys) {
6267 		struct device_node *np;
6268 
6269 		np = of_find_compatible_node(NULL, NULL, "ibm,opal-intc");
6270 		if (!np) {
6271 			pr_err("KVM-HV: Cannot determine method for accessing XICS\n");
6272 			r = -ENODEV;
6273 			goto err;
6274 		}
6275 		/* presence of intc confirmed - node can be dropped again */
6276 		of_node_put(np);
6277 	}
6278 #endif
6279 
6280 	init_default_hcalls();
6281 
6282 	init_vcore_lists();
6283 
6284 	r = kvmppc_mmu_hv_init();
6285 	if (r)
6286 		goto err;
6287 
6288 	if (kvmppc_radix_possible()) {
6289 		r = kvmppc_radix_init();
6290 		if (r)
6291 			goto err;
6292 	}
6293 
6294 	r = kvmppc_uvmem_init();
6295 	if (r < 0) {
6296 		pr_err("KVM-HV: kvmppc_uvmem_init failed %d\n", r);
6297 		return r;
6298 	}
6299 
6300 	kvm_ops_hv.owner = THIS_MODULE;
6301 	kvmppc_hv_ops = &kvm_ops_hv;
6302 
6303 	return 0;
6304 
6305 err:
6306 	kvmhv_nested_exit();
6307 	kvmppc_radix_exit();
6308 
6309 	return r;
6310 }
6311 
kvmppc_book3s_exit_hv(void)6312 static void kvmppc_book3s_exit_hv(void)
6313 {
6314 	kvmppc_uvmem_free();
6315 	kvmppc_free_host_rm_ops();
6316 	if (kvmppc_radix_possible())
6317 		kvmppc_radix_exit();
6318 	kvmppc_hv_ops = NULL;
6319 	kvmhv_nested_exit();
6320 }
6321 
6322 module_init(kvmppc_book3s_init_hv);
6323 module_exit(kvmppc_book3s_exit_hv);
6324 MODULE_LICENSE("GPL");
6325 MODULE_ALIAS_MISCDEV(KVM_MINOR);
6326 MODULE_ALIAS("devname:kvm");
6327