1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2015, 2016 ARM Ltd.
4  */
5 
6 #include <linux/uaccess.h>
7 #include <linux/interrupt.h>
8 #include <linux/cpu.h>
9 #include <linux/kvm_host.h>
10 #include <kvm/arm_vgic.h>
11 #include <asm/kvm_emulate.h>
12 #include <asm/kvm_mmu.h>
13 #include "vgic.h"
14 
15 /*
16  * Initialization rules: there are multiple stages to the vgic
17  * initialization, both for the distributor and the CPU interfaces.  The basic
18  * idea is that even though the VGIC is not functional or not requested from
19  * user space, the critical path of the run loop can still call VGIC functions
20  * that just won't do anything, without them having to check additional
21  * initialization flags to ensure they don't look at uninitialized data
22  * structures.
23  *
24  * Distributor:
25  *
26  * - kvm_vgic_early_init(): initialization of static data that doesn't
27  *   depend on any sizing information or emulation type. No allocation
28  *   is allowed there.
29  *
30  * - vgic_init(): allocation and initialization of the generic data
31  *   structures that depend on sizing information (number of CPUs,
32  *   number of interrupts). Also initializes the vcpu specific data
33  *   structures. Can be executed lazily for GICv2.
34  *
35  * CPU Interface:
36  *
37  * - kvm_vgic_vcpu_init(): initialization of static data that
38  *   doesn't depend on any sizing information or emulation type. No
39  *   allocation is allowed there.
40  */
41 
42 /* EARLY INIT */
43 
44 /**
45  * kvm_vgic_early_init() - Initialize static VGIC VCPU data structures
46  * @kvm: The VM whose VGIC districutor should be initialized
47  *
48  * Only do initialization of static structures that don't require any
49  * allocation or sizing information from userspace.  vgic_init() called
50  * kvm_vgic_dist_init() which takes care of the rest.
51  */
kvm_vgic_early_init(struct kvm * kvm)52 void kvm_vgic_early_init(struct kvm *kvm)
53 {
54 	struct vgic_dist *dist = &kvm->arch.vgic;
55 
56 	INIT_LIST_HEAD(&dist->lpi_list_head);
57 	INIT_LIST_HEAD(&dist->lpi_translation_cache);
58 	raw_spin_lock_init(&dist->lpi_list_lock);
59 }
60 
61 /* CREATION */
62 
63 /**
64  * kvm_vgic_create: triggered by the instantiation of the VGIC device by
65  * user space, either through the legacy KVM_CREATE_IRQCHIP ioctl (v2 only)
66  * or through the generic KVM_CREATE_DEVICE API ioctl.
67  * irqchip_in_kernel() tells you if this function succeeded or not.
68  * @kvm: kvm struct pointer
69  * @type: KVM_DEV_TYPE_ARM_VGIC_V[23]
70  */
kvm_vgic_create(struct kvm * kvm,u32 type)71 int kvm_vgic_create(struct kvm *kvm, u32 type)
72 {
73 	struct kvm_vcpu *vcpu;
74 	unsigned long i;
75 	int ret;
76 
77 	/*
78 	 * This function is also called by the KVM_CREATE_IRQCHIP handler,
79 	 * which had no chance yet to check the availability of the GICv2
80 	 * emulation. So check this here again. KVM_CREATE_DEVICE does
81 	 * the proper checks already.
82 	 */
83 	if (type == KVM_DEV_TYPE_ARM_VGIC_V2 &&
84 		!kvm_vgic_global_state.can_emulate_gicv2)
85 		return -ENODEV;
86 
87 	/* Must be held to avoid race with vCPU creation */
88 	lockdep_assert_held(&kvm->lock);
89 
90 	ret = -EBUSY;
91 	if (!lock_all_vcpus(kvm))
92 		return ret;
93 
94 	mutex_lock(&kvm->arch.config_lock);
95 
96 	if (irqchip_in_kernel(kvm)) {
97 		ret = -EEXIST;
98 		goto out_unlock;
99 	}
100 
101 	kvm_for_each_vcpu(i, vcpu, kvm) {
102 		if (vcpu_has_run_once(vcpu))
103 			goto out_unlock;
104 	}
105 	ret = 0;
106 
107 	if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
108 		kvm->max_vcpus = VGIC_V2_MAX_CPUS;
109 	else
110 		kvm->max_vcpus = VGIC_V3_MAX_CPUS;
111 
112 	if (atomic_read(&kvm->online_vcpus) > kvm->max_vcpus) {
113 		ret = -E2BIG;
114 		goto out_unlock;
115 	}
116 
117 	kvm->arch.vgic.in_kernel = true;
118 	kvm->arch.vgic.vgic_model = type;
119 
120 	kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
121 
122 	if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
123 		kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
124 	else
125 		INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
126 
127 out_unlock:
128 	mutex_unlock(&kvm->arch.config_lock);
129 	unlock_all_vcpus(kvm);
130 	return ret;
131 }
132 
133 /* INIT/DESTROY */
134 
135 /**
136  * kvm_vgic_dist_init: initialize the dist data structures
137  * @kvm: kvm struct pointer
138  * @nr_spis: number of spis, frozen by caller
139  */
kvm_vgic_dist_init(struct kvm * kvm,unsigned int nr_spis)140 static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis)
141 {
142 	struct vgic_dist *dist = &kvm->arch.vgic;
143 	struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
144 	int i;
145 
146 	dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), GFP_KERNEL_ACCOUNT);
147 	if (!dist->spis)
148 		return  -ENOMEM;
149 
150 	/*
151 	 * In the following code we do not take the irq struct lock since
152 	 * no other action on irq structs can happen while the VGIC is
153 	 * not initialized yet:
154 	 * If someone wants to inject an interrupt or does a MMIO access, we
155 	 * require prior initialization in case of a virtual GICv3 or trigger
156 	 * initialization when using a virtual GICv2.
157 	 */
158 	for (i = 0; i < nr_spis; i++) {
159 		struct vgic_irq *irq = &dist->spis[i];
160 
161 		irq->intid = i + VGIC_NR_PRIVATE_IRQS;
162 		INIT_LIST_HEAD(&irq->ap_list);
163 		raw_spin_lock_init(&irq->irq_lock);
164 		irq->vcpu = NULL;
165 		irq->target_vcpu = vcpu0;
166 		kref_init(&irq->refcount);
167 		switch (dist->vgic_model) {
168 		case KVM_DEV_TYPE_ARM_VGIC_V2:
169 			irq->targets = 0;
170 			irq->group = 0;
171 			break;
172 		case KVM_DEV_TYPE_ARM_VGIC_V3:
173 			irq->mpidr = 0;
174 			irq->group = 1;
175 			break;
176 		default:
177 			kfree(dist->spis);
178 			dist->spis = NULL;
179 			return -EINVAL;
180 		}
181 	}
182 	return 0;
183 }
184 
185 /**
186  * kvm_vgic_vcpu_init() - Initialize static VGIC VCPU data
187  * structures and register VCPU-specific KVM iodevs
188  *
189  * @vcpu: pointer to the VCPU being created and initialized
190  *
191  * Only do initialization, but do not actually enable the
192  * VGIC CPU interface
193  */
kvm_vgic_vcpu_init(struct kvm_vcpu * vcpu)194 int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
195 {
196 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
197 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
198 	int ret = 0;
199 	int i;
200 
201 	vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF;
202 
203 	INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
204 	raw_spin_lock_init(&vgic_cpu->ap_list_lock);
205 	atomic_set(&vgic_cpu->vgic_v3.its_vpe.vlpi_count, 0);
206 
207 	/*
208 	 * Enable and configure all SGIs to be edge-triggered and
209 	 * configure all PPIs as level-triggered.
210 	 */
211 	for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
212 		struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
213 
214 		INIT_LIST_HEAD(&irq->ap_list);
215 		raw_spin_lock_init(&irq->irq_lock);
216 		irq->intid = i;
217 		irq->vcpu = NULL;
218 		irq->target_vcpu = vcpu;
219 		kref_init(&irq->refcount);
220 		if (vgic_irq_is_sgi(i)) {
221 			/* SGIs */
222 			irq->enabled = 1;
223 			irq->config = VGIC_CONFIG_EDGE;
224 		} else {
225 			/* PPIs */
226 			irq->config = VGIC_CONFIG_LEVEL;
227 		}
228 	}
229 
230 	if (!irqchip_in_kernel(vcpu->kvm))
231 		return 0;
232 
233 	/*
234 	 * If we are creating a VCPU with a GICv3 we must also register the
235 	 * KVM io device for the redistributor that belongs to this VCPU.
236 	 */
237 	if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
238 		mutex_lock(&vcpu->kvm->slots_lock);
239 		ret = vgic_register_redist_iodev(vcpu);
240 		mutex_unlock(&vcpu->kvm->slots_lock);
241 	}
242 	return ret;
243 }
244 
kvm_vgic_vcpu_enable(struct kvm_vcpu * vcpu)245 static void kvm_vgic_vcpu_enable(struct kvm_vcpu *vcpu)
246 {
247 	if (kvm_vgic_global_state.type == VGIC_V2)
248 		vgic_v2_enable(vcpu);
249 	else
250 		vgic_v3_enable(vcpu);
251 }
252 
253 /*
254  * vgic_init: allocates and initializes dist and vcpu data structures
255  * depending on two dimensioning parameters:
256  * - the number of spis
257  * - the number of vcpus
258  * The function is generally called when nr_spis has been explicitly set
259  * by the guest through the KVM DEVICE API. If not nr_spis is set to 256.
260  * vgic_initialized() returns true when this function has succeeded.
261  */
vgic_init(struct kvm * kvm)262 int vgic_init(struct kvm *kvm)
263 {
264 	struct vgic_dist *dist = &kvm->arch.vgic;
265 	struct kvm_vcpu *vcpu;
266 	int ret = 0, i;
267 	unsigned long idx;
268 
269 	lockdep_assert_held(&kvm->arch.config_lock);
270 
271 	if (vgic_initialized(kvm))
272 		return 0;
273 
274 	/* Are we also in the middle of creating a VCPU? */
275 	if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
276 		return -EBUSY;
277 
278 	/* freeze the number of spis */
279 	if (!dist->nr_spis)
280 		dist->nr_spis = VGIC_NR_IRQS_LEGACY - VGIC_NR_PRIVATE_IRQS;
281 
282 	ret = kvm_vgic_dist_init(kvm, dist->nr_spis);
283 	if (ret)
284 		goto out;
285 
286 	/* Initialize groups on CPUs created before the VGIC type was known */
287 	kvm_for_each_vcpu(idx, vcpu, kvm) {
288 		struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
289 
290 		for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
291 			struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
292 			switch (dist->vgic_model) {
293 			case KVM_DEV_TYPE_ARM_VGIC_V3:
294 				irq->group = 1;
295 				irq->mpidr = kvm_vcpu_get_mpidr_aff(vcpu);
296 				break;
297 			case KVM_DEV_TYPE_ARM_VGIC_V2:
298 				irq->group = 0;
299 				irq->targets = 1U << idx;
300 				break;
301 			default:
302 				ret = -EINVAL;
303 				goto out;
304 			}
305 		}
306 	}
307 
308 	if (vgic_has_its(kvm))
309 		vgic_lpi_translation_cache_init(kvm);
310 
311 	/*
312 	 * If we have GICv4.1 enabled, unconditionnaly request enable the
313 	 * v4 support so that we get HW-accelerated vSGIs. Otherwise, only
314 	 * enable it if we present a virtual ITS to the guest.
315 	 */
316 	if (vgic_supports_direct_msis(kvm)) {
317 		ret = vgic_v4_init(kvm);
318 		if (ret)
319 			goto out;
320 	}
321 
322 	kvm_for_each_vcpu(idx, vcpu, kvm)
323 		kvm_vgic_vcpu_enable(vcpu);
324 
325 	ret = kvm_vgic_setup_default_irq_routing(kvm);
326 	if (ret)
327 		goto out;
328 
329 	vgic_debug_init(kvm);
330 
331 	/*
332 	 * If userspace didn't set the GIC implementation revision,
333 	 * default to the latest and greatest. You know want it.
334 	 */
335 	if (!dist->implementation_rev)
336 		dist->implementation_rev = KVM_VGIC_IMP_REV_LATEST;
337 	dist->initialized = true;
338 
339 out:
340 	return ret;
341 }
342 
kvm_vgic_dist_destroy(struct kvm * kvm)343 static void kvm_vgic_dist_destroy(struct kvm *kvm)
344 {
345 	struct vgic_dist *dist = &kvm->arch.vgic;
346 	struct vgic_redist_region *rdreg, *next;
347 
348 	dist->ready = false;
349 	dist->initialized = false;
350 
351 	kfree(dist->spis);
352 	dist->spis = NULL;
353 	dist->nr_spis = 0;
354 	dist->vgic_dist_base = VGIC_ADDR_UNDEF;
355 
356 	if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
357 		list_for_each_entry_safe(rdreg, next, &dist->rd_regions, list)
358 			vgic_v3_free_redist_region(rdreg);
359 		INIT_LIST_HEAD(&dist->rd_regions);
360 	} else {
361 		dist->vgic_cpu_base = VGIC_ADDR_UNDEF;
362 	}
363 
364 	if (vgic_has_its(kvm))
365 		vgic_lpi_translation_cache_destroy(kvm);
366 
367 	if (vgic_supports_direct_msis(kvm))
368 		vgic_v4_teardown(kvm);
369 }
370 
__kvm_vgic_vcpu_destroy(struct kvm_vcpu * vcpu)371 static void __kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
372 {
373 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
374 
375 	/*
376 	 * Retire all pending LPIs on this vcpu anyway as we're
377 	 * going to destroy it.
378 	 */
379 	vgic_flush_pending_lpis(vcpu);
380 
381 	INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
382 	if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
383 		vgic_unregister_redist_iodev(vcpu);
384 		vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF;
385 	}
386 }
387 
kvm_vgic_vcpu_destroy(struct kvm_vcpu * vcpu)388 void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
389 {
390 	struct kvm *kvm = vcpu->kvm;
391 
392 	mutex_lock(&kvm->slots_lock);
393 	__kvm_vgic_vcpu_destroy(vcpu);
394 	mutex_unlock(&kvm->slots_lock);
395 }
396 
kvm_vgic_destroy(struct kvm * kvm)397 void kvm_vgic_destroy(struct kvm *kvm)
398 {
399 	struct kvm_vcpu *vcpu;
400 	unsigned long i;
401 
402 	mutex_lock(&kvm->slots_lock);
403 
404 	vgic_debug_destroy(kvm);
405 
406 	kvm_for_each_vcpu(i, vcpu, kvm)
407 		__kvm_vgic_vcpu_destroy(vcpu);
408 
409 	mutex_lock(&kvm->arch.config_lock);
410 
411 	kvm_vgic_dist_destroy(kvm);
412 
413 	mutex_unlock(&kvm->arch.config_lock);
414 	mutex_unlock(&kvm->slots_lock);
415 }
416 
417 /**
418  * vgic_lazy_init: Lazy init is only allowed if the GIC exposed to the guest
419  * is a GICv2. A GICv3 must be explicitly initialized by userspace using the
420  * KVM_DEV_ARM_VGIC_GRP_CTRL KVM_DEVICE group.
421  * @kvm: kvm struct pointer
422  */
vgic_lazy_init(struct kvm * kvm)423 int vgic_lazy_init(struct kvm *kvm)
424 {
425 	int ret = 0;
426 
427 	if (unlikely(!vgic_initialized(kvm))) {
428 		/*
429 		 * We only provide the automatic initialization of the VGIC
430 		 * for the legacy case of a GICv2. Any other type must
431 		 * be explicitly initialized once setup with the respective
432 		 * KVM device call.
433 		 */
434 		if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2)
435 			return -EBUSY;
436 
437 		mutex_lock(&kvm->arch.config_lock);
438 		ret = vgic_init(kvm);
439 		mutex_unlock(&kvm->arch.config_lock);
440 	}
441 
442 	return ret;
443 }
444 
445 /* RESOURCE MAPPING */
446 
447 /**
448  * Map the MMIO regions depending on the VGIC model exposed to the guest
449  * called on the first VCPU run.
450  * Also map the virtual CPU interface into the VM.
451  * v2 calls vgic_init() if not already done.
452  * v3 and derivatives return an error if the VGIC is not initialized.
453  * vgic_ready() returns true if this function has succeeded.
454  * @kvm: kvm struct pointer
455  */
kvm_vgic_map_resources(struct kvm * kvm)456 int kvm_vgic_map_resources(struct kvm *kvm)
457 {
458 	struct vgic_dist *dist = &kvm->arch.vgic;
459 	enum vgic_type type;
460 	gpa_t dist_base;
461 	int ret = 0;
462 
463 	if (likely(vgic_ready(kvm)))
464 		return 0;
465 
466 	mutex_lock(&kvm->slots_lock);
467 	mutex_lock(&kvm->arch.config_lock);
468 	if (vgic_ready(kvm))
469 		goto out;
470 
471 	if (!irqchip_in_kernel(kvm))
472 		goto out;
473 
474 	if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2) {
475 		ret = vgic_v2_map_resources(kvm);
476 		type = VGIC_V2;
477 	} else {
478 		ret = vgic_v3_map_resources(kvm);
479 		type = VGIC_V3;
480 	}
481 
482 	if (ret)
483 		goto out;
484 
485 	dist->ready = true;
486 	dist_base = dist->vgic_dist_base;
487 	mutex_unlock(&kvm->arch.config_lock);
488 
489 	ret = vgic_register_dist_iodev(kvm, dist_base, type);
490 	if (ret)
491 		kvm_err("Unable to register VGIC dist MMIO regions\n");
492 
493 	goto out_slots;
494 out:
495 	mutex_unlock(&kvm->arch.config_lock);
496 out_slots:
497 	mutex_unlock(&kvm->slots_lock);
498 
499 	if (ret)
500 		kvm_vgic_destroy(kvm);
501 
502 	return ret;
503 }
504 
505 /* GENERIC PROBE */
506 
kvm_vgic_cpu_up(void)507 void kvm_vgic_cpu_up(void)
508 {
509 	enable_percpu_irq(kvm_vgic_global_state.maint_irq, 0);
510 }
511 
512 
kvm_vgic_cpu_down(void)513 void kvm_vgic_cpu_down(void)
514 {
515 	disable_percpu_irq(kvm_vgic_global_state.maint_irq);
516 }
517 
vgic_maintenance_handler(int irq,void * data)518 static irqreturn_t vgic_maintenance_handler(int irq, void *data)
519 {
520 	/*
521 	 * We cannot rely on the vgic maintenance interrupt to be
522 	 * delivered synchronously. This means we can only use it to
523 	 * exit the VM, and we perform the handling of EOIed
524 	 * interrupts on the exit path (see vgic_fold_lr_state).
525 	 */
526 	return IRQ_HANDLED;
527 }
528 
529 static struct gic_kvm_info *gic_kvm_info;
530 
vgic_set_kvm_info(const struct gic_kvm_info * info)531 void __init vgic_set_kvm_info(const struct gic_kvm_info *info)
532 {
533 	BUG_ON(gic_kvm_info != NULL);
534 	gic_kvm_info = kmalloc(sizeof(*info), GFP_KERNEL);
535 	if (gic_kvm_info)
536 		*gic_kvm_info = *info;
537 }
538 
539 /**
540  * kvm_vgic_init_cpu_hardware - initialize the GIC VE hardware
541  *
542  * For a specific CPU, initialize the GIC VE hardware.
543  */
kvm_vgic_init_cpu_hardware(void)544 void kvm_vgic_init_cpu_hardware(void)
545 {
546 	BUG_ON(preemptible());
547 
548 	/*
549 	 * We want to make sure the list registers start out clear so that we
550 	 * only have the program the used registers.
551 	 */
552 	if (kvm_vgic_global_state.type == VGIC_V2)
553 		vgic_v2_init_lrs();
554 	else
555 		kvm_call_hyp(__vgic_v3_init_lrs);
556 }
557 
558 /**
559  * kvm_vgic_hyp_init: populates the kvm_vgic_global_state variable
560  * according to the host GIC model. Accordingly calls either
561  * vgic_v2/v3_probe which registers the KVM_DEVICE that can be
562  * instantiated by a guest later on .
563  */
kvm_vgic_hyp_init(void)564 int kvm_vgic_hyp_init(void)
565 {
566 	bool has_mask;
567 	int ret;
568 
569 	if (!gic_kvm_info)
570 		return -ENODEV;
571 
572 	has_mask = !gic_kvm_info->no_maint_irq_mask;
573 
574 	if (has_mask && !gic_kvm_info->maint_irq) {
575 		kvm_err("No vgic maintenance irq\n");
576 		return -ENXIO;
577 	}
578 
579 	/*
580 	 * If we get one of these oddball non-GICs, taint the kernel,
581 	 * as we have no idea of how they *really* behave.
582 	 */
583 	if (gic_kvm_info->no_hw_deactivation) {
584 		kvm_info("Non-architectural vgic, tainting kernel\n");
585 		add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
586 		kvm_vgic_global_state.no_hw_deactivation = true;
587 	}
588 
589 	switch (gic_kvm_info->type) {
590 	case GIC_V2:
591 		ret = vgic_v2_probe(gic_kvm_info);
592 		break;
593 	case GIC_V3:
594 		ret = vgic_v3_probe(gic_kvm_info);
595 		if (!ret) {
596 			static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif);
597 			kvm_info("GIC system register CPU interface enabled\n");
598 		}
599 		break;
600 	default:
601 		ret = -ENODEV;
602 	}
603 
604 	kvm_vgic_global_state.maint_irq = gic_kvm_info->maint_irq;
605 
606 	kfree(gic_kvm_info);
607 	gic_kvm_info = NULL;
608 
609 	if (ret)
610 		return ret;
611 
612 	if (!has_mask && !kvm_vgic_global_state.maint_irq)
613 		return 0;
614 
615 	ret = request_percpu_irq(kvm_vgic_global_state.maint_irq,
616 				 vgic_maintenance_handler,
617 				 "vgic", kvm_get_running_vcpus());
618 	if (ret) {
619 		kvm_err("Cannot register interrupt %d\n",
620 			kvm_vgic_global_state.maint_irq);
621 		return ret;
622 	}
623 
624 	kvm_info("vgic interrupt IRQ%d\n", kvm_vgic_global_state.maint_irq);
625 	return 0;
626 }
627