1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VGICv3 MMIO handling functions
4  */
5 
6 #include <linux/bitfield.h>
7 #include <linux/irqchip/arm-gic-v3.h>
8 #include <linux/kvm.h>
9 #include <linux/kvm_host.h>
10 #include <linux/interrupt.h>
11 #include <kvm/iodev.h>
12 #include <kvm/arm_vgic.h>
13 
14 #include <asm/kvm_emulate.h>
15 #include <asm/kvm_arm.h>
16 #include <asm/kvm_mmu.h>
17 
18 #include "vgic.h"
19 #include "vgic-mmio.h"
20 
21 /* extract @num bytes at @offset bytes offset in data */
extract_bytes(u64 data,unsigned int offset,unsigned int num)22 unsigned long extract_bytes(u64 data, unsigned int offset,
23 			    unsigned int num)
24 {
25 	return (data >> (offset * 8)) & GENMASK_ULL(num * 8 - 1, 0);
26 }
27 
28 /* allows updates of any half of a 64-bit register (or the whole thing) */
update_64bit_reg(u64 reg,unsigned int offset,unsigned int len,unsigned long val)29 u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len,
30 		     unsigned long val)
31 {
32 	int lower = (offset & 4) * 8;
33 	int upper = lower + 8 * len - 1;
34 
35 	reg &= ~GENMASK_ULL(upper, lower);
36 	val &= GENMASK_ULL(len * 8 - 1, 0);
37 
38 	return reg | ((u64)val << lower);
39 }
40 
vgic_has_its(struct kvm * kvm)41 bool vgic_has_its(struct kvm *kvm)
42 {
43 	struct vgic_dist *dist = &kvm->arch.vgic;
44 
45 	if (dist->vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
46 		return false;
47 
48 	return dist->has_its;
49 }
50 
vgic_supports_direct_msis(struct kvm * kvm)51 bool vgic_supports_direct_msis(struct kvm *kvm)
52 {
53 	return (kvm_vgic_global_state.has_gicv4_1 ||
54 		(kvm_vgic_global_state.has_gicv4 && vgic_has_its(kvm)));
55 }
56 
57 /*
58  * The Revision field in the IIDR have the following meanings:
59  *
60  * Revision 2: Interrupt groups are guest-configurable and signaled using
61  * 	       their configured groups.
62  */
63 
vgic_mmio_read_v3_misc(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)64 static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu,
65 					    gpa_t addr, unsigned int len)
66 {
67 	struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
68 	u32 value = 0;
69 
70 	switch (addr & 0x0c) {
71 	case GICD_CTLR:
72 		if (vgic->enabled)
73 			value |= GICD_CTLR_ENABLE_SS_G1;
74 		value |= GICD_CTLR_ARE_NS | GICD_CTLR_DS;
75 		if (vgic->nassgireq)
76 			value |= GICD_CTLR_nASSGIreq;
77 		break;
78 	case GICD_TYPER:
79 		value = vgic->nr_spis + VGIC_NR_PRIVATE_IRQS;
80 		value = (value >> 5) - 1;
81 		if (vgic_has_its(vcpu->kvm)) {
82 			value |= (INTERRUPT_ID_BITS_ITS - 1) << 19;
83 			value |= GICD_TYPER_LPIS;
84 		} else {
85 			value |= (INTERRUPT_ID_BITS_SPIS - 1) << 19;
86 		}
87 		break;
88 	case GICD_TYPER2:
89 		if (kvm_vgic_global_state.has_gicv4_1 && gic_cpuif_has_vsgi())
90 			value = GICD_TYPER2_nASSGIcap;
91 		break;
92 	case GICD_IIDR:
93 		value = (PRODUCT_ID_KVM << GICD_IIDR_PRODUCT_ID_SHIFT) |
94 			(vgic->implementation_rev << GICD_IIDR_REVISION_SHIFT) |
95 			(IMPLEMENTER_ARM << GICD_IIDR_IMPLEMENTER_SHIFT);
96 		break;
97 	default:
98 		return 0;
99 	}
100 
101 	return value;
102 }
103 
vgic_mmio_write_v3_misc(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)104 static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu,
105 				    gpa_t addr, unsigned int len,
106 				    unsigned long val)
107 {
108 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
109 
110 	switch (addr & 0x0c) {
111 	case GICD_CTLR: {
112 		bool was_enabled, is_hwsgi;
113 
114 		mutex_lock(&vcpu->kvm->lock);
115 
116 		was_enabled = dist->enabled;
117 		is_hwsgi = dist->nassgireq;
118 
119 		dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;
120 
121 		/* Not a GICv4.1? No HW SGIs */
122 		if (!kvm_vgic_global_state.has_gicv4_1 || !gic_cpuif_has_vsgi())
123 			val &= ~GICD_CTLR_nASSGIreq;
124 
125 		/* Dist stays enabled? nASSGIreq is RO */
126 		if (was_enabled && dist->enabled) {
127 			val &= ~GICD_CTLR_nASSGIreq;
128 			val |= FIELD_PREP(GICD_CTLR_nASSGIreq, is_hwsgi);
129 		}
130 
131 		/* Switching HW SGIs? */
132 		dist->nassgireq = val & GICD_CTLR_nASSGIreq;
133 		if (is_hwsgi != dist->nassgireq)
134 			vgic_v4_configure_vsgis(vcpu->kvm);
135 
136 		if (kvm_vgic_global_state.has_gicv4_1 &&
137 		    was_enabled != dist->enabled)
138 			kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_RELOAD_GICv4);
139 		else if (!was_enabled && dist->enabled)
140 			vgic_kick_vcpus(vcpu->kvm);
141 
142 		mutex_unlock(&vcpu->kvm->lock);
143 		break;
144 	}
145 	case GICD_TYPER:
146 	case GICD_TYPER2:
147 	case GICD_IIDR:
148 		/* This is at best for documentation purposes... */
149 		return;
150 	}
151 }
152 
vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)153 static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu,
154 					   gpa_t addr, unsigned int len,
155 					   unsigned long val)
156 {
157 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
158 	u32 reg;
159 
160 	switch (addr & 0x0c) {
161 	case GICD_TYPER2:
162 		if (val != vgic_mmio_read_v3_misc(vcpu, addr, len))
163 			return -EINVAL;
164 		return 0;
165 	case GICD_IIDR:
166 		reg = vgic_mmio_read_v3_misc(vcpu, addr, len);
167 		if ((reg ^ val) & ~GICD_IIDR_REVISION_MASK)
168 			return -EINVAL;
169 
170 		reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg);
171 		switch (reg) {
172 		case KVM_VGIC_IMP_REV_2:
173 		case KVM_VGIC_IMP_REV_3:
174 			dist->implementation_rev = reg;
175 			return 0;
176 		default:
177 			return -EINVAL;
178 		}
179 	case GICD_CTLR:
180 		/* Not a GICv4.1? No HW SGIs */
181 		if (!kvm_vgic_global_state.has_gicv4_1)
182 			val &= ~GICD_CTLR_nASSGIreq;
183 
184 		dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;
185 		dist->nassgireq = val & GICD_CTLR_nASSGIreq;
186 		return 0;
187 	}
188 
189 	vgic_mmio_write_v3_misc(vcpu, addr, len, val);
190 	return 0;
191 }
192 
vgic_mmio_read_irouter(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)193 static unsigned long vgic_mmio_read_irouter(struct kvm_vcpu *vcpu,
194 					    gpa_t addr, unsigned int len)
195 {
196 	int intid = VGIC_ADDR_TO_INTID(addr, 64);
197 	struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, NULL, intid);
198 	unsigned long ret = 0;
199 
200 	if (!irq)
201 		return 0;
202 
203 	/* The upper word is RAZ for us. */
204 	if (!(addr & 4))
205 		ret = extract_bytes(READ_ONCE(irq->mpidr), addr & 7, len);
206 
207 	vgic_put_irq(vcpu->kvm, irq);
208 	return ret;
209 }
210 
vgic_mmio_write_irouter(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)211 static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu,
212 				    gpa_t addr, unsigned int len,
213 				    unsigned long val)
214 {
215 	int intid = VGIC_ADDR_TO_INTID(addr, 64);
216 	struct vgic_irq *irq;
217 	unsigned long flags;
218 
219 	/* The upper word is WI for us since we don't implement Aff3. */
220 	if (addr & 4)
221 		return;
222 
223 	irq = vgic_get_irq(vcpu->kvm, NULL, intid);
224 
225 	if (!irq)
226 		return;
227 
228 	raw_spin_lock_irqsave(&irq->irq_lock, flags);
229 
230 	/* We only care about and preserve Aff0, Aff1 and Aff2. */
231 	irq->mpidr = val & GENMASK(23, 0);
232 	irq->target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, irq->mpidr);
233 
234 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
235 	vgic_put_irq(vcpu->kvm, irq);
236 }
237 
vgic_lpis_enabled(struct kvm_vcpu * vcpu)238 bool vgic_lpis_enabled(struct kvm_vcpu *vcpu)
239 {
240 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
241 
242 	return atomic_read(&vgic_cpu->ctlr) == GICR_CTLR_ENABLE_LPIS;
243 }
244 
vgic_mmio_read_v3r_ctlr(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)245 static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu,
246 					     gpa_t addr, unsigned int len)
247 {
248 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
249 	unsigned long val;
250 
251 	val = atomic_read(&vgic_cpu->ctlr);
252 	if (vgic_get_implementation_rev(vcpu) >= KVM_VGIC_IMP_REV_3)
253 		val |= GICR_CTLR_IR | GICR_CTLR_CES;
254 
255 	return val;
256 }
257 
vgic_mmio_write_v3r_ctlr(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)258 static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu,
259 				     gpa_t addr, unsigned int len,
260 				     unsigned long val)
261 {
262 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
263 	u32 ctlr;
264 
265 	if (!vgic_has_its(vcpu->kvm))
266 		return;
267 
268 	if (!(val & GICR_CTLR_ENABLE_LPIS)) {
269 		/*
270 		 * Don't disable if RWP is set, as there already an
271 		 * ongoing disable. Funky guest...
272 		 */
273 		ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr,
274 					      GICR_CTLR_ENABLE_LPIS,
275 					      GICR_CTLR_RWP);
276 		if (ctlr != GICR_CTLR_ENABLE_LPIS)
277 			return;
278 
279 		vgic_flush_pending_lpis(vcpu);
280 		vgic_its_invalidate_cache(vcpu->kvm);
281 		atomic_set_release(&vgic_cpu->ctlr, 0);
282 	} else {
283 		ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr, 0,
284 					      GICR_CTLR_ENABLE_LPIS);
285 		if (ctlr != 0)
286 			return;
287 
288 		vgic_enable_lpis(vcpu);
289 	}
290 }
291 
vgic_mmio_vcpu_rdist_is_last(struct kvm_vcpu * vcpu)292 static bool vgic_mmio_vcpu_rdist_is_last(struct kvm_vcpu *vcpu)
293 {
294 	struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
295 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
296 	struct vgic_redist_region *iter, *rdreg = vgic_cpu->rdreg;
297 
298 	if (!rdreg)
299 		return false;
300 
301 	if (vgic_cpu->rdreg_index < rdreg->free_index - 1) {
302 		return false;
303 	} else if (rdreg->count && vgic_cpu->rdreg_index == (rdreg->count - 1)) {
304 		struct list_head *rd_regions = &vgic->rd_regions;
305 		gpa_t end = rdreg->base + rdreg->count * KVM_VGIC_V3_REDIST_SIZE;
306 
307 		/*
308 		 * the rdist is the last one of the redist region,
309 		 * check whether there is no other contiguous rdist region
310 		 */
311 		list_for_each_entry(iter, rd_regions, list) {
312 			if (iter->base == end && iter->free_index > 0)
313 				return false;
314 		}
315 	}
316 	return true;
317 }
318 
vgic_mmio_read_v3r_typer(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)319 static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu,
320 					      gpa_t addr, unsigned int len)
321 {
322 	unsigned long mpidr = kvm_vcpu_get_mpidr_aff(vcpu);
323 	int target_vcpu_id = vcpu->vcpu_id;
324 	u64 value;
325 
326 	value = (u64)(mpidr & GENMASK(23, 0)) << 32;
327 	value |= ((target_vcpu_id & 0xffff) << 8);
328 
329 	if (vgic_has_its(vcpu->kvm))
330 		value |= GICR_TYPER_PLPIS;
331 
332 	if (vgic_mmio_vcpu_rdist_is_last(vcpu))
333 		value |= GICR_TYPER_LAST;
334 
335 	return extract_bytes(value, addr & 7, len);
336 }
337 
vgic_mmio_read_v3r_iidr(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)338 static unsigned long vgic_mmio_read_v3r_iidr(struct kvm_vcpu *vcpu,
339 					     gpa_t addr, unsigned int len)
340 {
341 	return (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
342 }
343 
vgic_mmio_read_v3_idregs(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)344 static unsigned long vgic_mmio_read_v3_idregs(struct kvm_vcpu *vcpu,
345 					      gpa_t addr, unsigned int len)
346 {
347 	switch (addr & 0xffff) {
348 	case GICD_PIDR2:
349 		/* report a GICv3 compliant implementation */
350 		return 0x3b;
351 	}
352 
353 	return 0;
354 }
355 
vgic_v3_uaccess_write_pending(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)356 static int vgic_v3_uaccess_write_pending(struct kvm_vcpu *vcpu,
357 					 gpa_t addr, unsigned int len,
358 					 unsigned long val)
359 {
360 	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
361 	int i;
362 	unsigned long flags;
363 
364 	for (i = 0; i < len * 8; i++) {
365 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
366 
367 		raw_spin_lock_irqsave(&irq->irq_lock, flags);
368 		if (test_bit(i, &val)) {
369 			/*
370 			 * pending_latch is set irrespective of irq type
371 			 * (level or edge) to avoid dependency that VM should
372 			 * restore irq config before pending info.
373 			 */
374 			irq->pending_latch = true;
375 			vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
376 		} else {
377 			irq->pending_latch = false;
378 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
379 		}
380 
381 		vgic_put_irq(vcpu->kvm, irq);
382 	}
383 
384 	return 0;
385 }
386 
387 /* We want to avoid outer shareable. */
vgic_sanitise_shareability(u64 field)388 u64 vgic_sanitise_shareability(u64 field)
389 {
390 	switch (field) {
391 	case GIC_BASER_OuterShareable:
392 		return GIC_BASER_InnerShareable;
393 	default:
394 		return field;
395 	}
396 }
397 
398 /* Avoid any inner non-cacheable mapping. */
vgic_sanitise_inner_cacheability(u64 field)399 u64 vgic_sanitise_inner_cacheability(u64 field)
400 {
401 	switch (field) {
402 	case GIC_BASER_CACHE_nCnB:
403 	case GIC_BASER_CACHE_nC:
404 		return GIC_BASER_CACHE_RaWb;
405 	default:
406 		return field;
407 	}
408 }
409 
410 /* Non-cacheable or same-as-inner are OK. */
vgic_sanitise_outer_cacheability(u64 field)411 u64 vgic_sanitise_outer_cacheability(u64 field)
412 {
413 	switch (field) {
414 	case GIC_BASER_CACHE_SameAsInner:
415 	case GIC_BASER_CACHE_nC:
416 		return field;
417 	default:
418 		return GIC_BASER_CACHE_SameAsInner;
419 	}
420 }
421 
vgic_sanitise_field(u64 reg,u64 field_mask,int field_shift,u64 (* sanitise_fn)(u64))422 u64 vgic_sanitise_field(u64 reg, u64 field_mask, int field_shift,
423 			u64 (*sanitise_fn)(u64))
424 {
425 	u64 field = (reg & field_mask) >> field_shift;
426 
427 	field = sanitise_fn(field) << field_shift;
428 	return (reg & ~field_mask) | field;
429 }
430 
431 #define PROPBASER_RES0_MASK						\
432 	(GENMASK_ULL(63, 59) | GENMASK_ULL(55, 52) | GENMASK_ULL(6, 5))
433 #define PENDBASER_RES0_MASK						\
434 	(BIT_ULL(63) | GENMASK_ULL(61, 59) | GENMASK_ULL(55, 52) |	\
435 	 GENMASK_ULL(15, 12) | GENMASK_ULL(6, 0))
436 
vgic_sanitise_pendbaser(u64 reg)437 static u64 vgic_sanitise_pendbaser(u64 reg)
438 {
439 	reg = vgic_sanitise_field(reg, GICR_PENDBASER_SHAREABILITY_MASK,
440 				  GICR_PENDBASER_SHAREABILITY_SHIFT,
441 				  vgic_sanitise_shareability);
442 	reg = vgic_sanitise_field(reg, GICR_PENDBASER_INNER_CACHEABILITY_MASK,
443 				  GICR_PENDBASER_INNER_CACHEABILITY_SHIFT,
444 				  vgic_sanitise_inner_cacheability);
445 	reg = vgic_sanitise_field(reg, GICR_PENDBASER_OUTER_CACHEABILITY_MASK,
446 				  GICR_PENDBASER_OUTER_CACHEABILITY_SHIFT,
447 				  vgic_sanitise_outer_cacheability);
448 
449 	reg &= ~PENDBASER_RES0_MASK;
450 
451 	return reg;
452 }
453 
vgic_sanitise_propbaser(u64 reg)454 static u64 vgic_sanitise_propbaser(u64 reg)
455 {
456 	reg = vgic_sanitise_field(reg, GICR_PROPBASER_SHAREABILITY_MASK,
457 				  GICR_PROPBASER_SHAREABILITY_SHIFT,
458 				  vgic_sanitise_shareability);
459 	reg = vgic_sanitise_field(reg, GICR_PROPBASER_INNER_CACHEABILITY_MASK,
460 				  GICR_PROPBASER_INNER_CACHEABILITY_SHIFT,
461 				  vgic_sanitise_inner_cacheability);
462 	reg = vgic_sanitise_field(reg, GICR_PROPBASER_OUTER_CACHEABILITY_MASK,
463 				  GICR_PROPBASER_OUTER_CACHEABILITY_SHIFT,
464 				  vgic_sanitise_outer_cacheability);
465 
466 	reg &= ~PROPBASER_RES0_MASK;
467 	return reg;
468 }
469 
vgic_mmio_read_propbase(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)470 static unsigned long vgic_mmio_read_propbase(struct kvm_vcpu *vcpu,
471 					     gpa_t addr, unsigned int len)
472 {
473 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
474 
475 	return extract_bytes(dist->propbaser, addr & 7, len);
476 }
477 
vgic_mmio_write_propbase(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)478 static void vgic_mmio_write_propbase(struct kvm_vcpu *vcpu,
479 				     gpa_t addr, unsigned int len,
480 				     unsigned long val)
481 {
482 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
483 	u64 old_propbaser, propbaser;
484 
485 	/* Storing a value with LPIs already enabled is undefined */
486 	if (vgic_lpis_enabled(vcpu))
487 		return;
488 
489 	do {
490 		old_propbaser = READ_ONCE(dist->propbaser);
491 		propbaser = old_propbaser;
492 		propbaser = update_64bit_reg(propbaser, addr & 4, len, val);
493 		propbaser = vgic_sanitise_propbaser(propbaser);
494 	} while (cmpxchg64(&dist->propbaser, old_propbaser,
495 			   propbaser) != old_propbaser);
496 }
497 
vgic_mmio_read_pendbase(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)498 static unsigned long vgic_mmio_read_pendbase(struct kvm_vcpu *vcpu,
499 					     gpa_t addr, unsigned int len)
500 {
501 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
502 	u64 value = vgic_cpu->pendbaser;
503 
504 	value &= ~GICR_PENDBASER_PTZ;
505 
506 	return extract_bytes(value, addr & 7, len);
507 }
508 
vgic_mmio_write_pendbase(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)509 static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu,
510 				     gpa_t addr, unsigned int len,
511 				     unsigned long val)
512 {
513 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
514 	u64 old_pendbaser, pendbaser;
515 
516 	/* Storing a value with LPIs already enabled is undefined */
517 	if (vgic_lpis_enabled(vcpu))
518 		return;
519 
520 	do {
521 		old_pendbaser = READ_ONCE(vgic_cpu->pendbaser);
522 		pendbaser = old_pendbaser;
523 		pendbaser = update_64bit_reg(pendbaser, addr & 4, len, val);
524 		pendbaser = vgic_sanitise_pendbaser(pendbaser);
525 	} while (cmpxchg64(&vgic_cpu->pendbaser, old_pendbaser,
526 			   pendbaser) != old_pendbaser);
527 }
528 
vgic_mmio_read_sync(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)529 static unsigned long vgic_mmio_read_sync(struct kvm_vcpu *vcpu,
530 					 gpa_t addr, unsigned int len)
531 {
532 	return !!atomic_read(&vcpu->arch.vgic_cpu.syncr_busy);
533 }
534 
vgic_set_rdist_busy(struct kvm_vcpu * vcpu,bool busy)535 static void vgic_set_rdist_busy(struct kvm_vcpu *vcpu, bool busy)
536 {
537 	if (busy) {
538 		atomic_inc(&vcpu->arch.vgic_cpu.syncr_busy);
539 		smp_mb__after_atomic();
540 	} else {
541 		smp_mb__before_atomic();
542 		atomic_dec(&vcpu->arch.vgic_cpu.syncr_busy);
543 	}
544 }
545 
vgic_mmio_write_invlpi(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)546 static void vgic_mmio_write_invlpi(struct kvm_vcpu *vcpu,
547 				   gpa_t addr, unsigned int len,
548 				   unsigned long val)
549 {
550 	struct vgic_irq *irq;
551 
552 	/*
553 	 * If the guest wrote only to the upper 32bit part of the
554 	 * register, drop the write on the floor, as it is only for
555 	 * vPEs (which we don't support for obvious reasons).
556 	 *
557 	 * Also discard the access if LPIs are not enabled.
558 	 */
559 	if ((addr & 4) || !vgic_lpis_enabled(vcpu))
560 		return;
561 
562 	vgic_set_rdist_busy(vcpu, true);
563 
564 	irq = vgic_get_irq(vcpu->kvm, NULL, lower_32_bits(val));
565 	if (irq) {
566 		vgic_its_inv_lpi(vcpu->kvm, irq);
567 		vgic_put_irq(vcpu->kvm, irq);
568 	}
569 
570 	vgic_set_rdist_busy(vcpu, false);
571 }
572 
vgic_mmio_write_invall(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)573 static void vgic_mmio_write_invall(struct kvm_vcpu *vcpu,
574 				   gpa_t addr, unsigned int len,
575 				   unsigned long val)
576 {
577 	/* See vgic_mmio_write_invlpi() for the early return rationale */
578 	if ((addr & 4) || !vgic_lpis_enabled(vcpu))
579 		return;
580 
581 	vgic_set_rdist_busy(vcpu, true);
582 	vgic_its_invall(vcpu);
583 	vgic_set_rdist_busy(vcpu, false);
584 }
585 
586 /*
587  * The GICv3 per-IRQ registers are split to control PPIs and SGIs in the
588  * redistributors, while SPIs are covered by registers in the distributor
589  * block. Trying to set private IRQs in this block gets ignored.
590  * We take some special care here to fix the calculation of the register
591  * offset.
592  */
593 #define REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(off, rd, wr, ur, uw, bpi, acc) \
594 	{								\
595 		.reg_offset = off,					\
596 		.bits_per_irq = bpi,					\
597 		.len = (bpi * VGIC_NR_PRIVATE_IRQS) / 8,		\
598 		.access_flags = acc,					\
599 		.read = vgic_mmio_read_raz,				\
600 		.write = vgic_mmio_write_wi,				\
601 	}, {								\
602 		.reg_offset = off + (bpi * VGIC_NR_PRIVATE_IRQS) / 8,	\
603 		.bits_per_irq = bpi,					\
604 		.len = (bpi * (1024 - VGIC_NR_PRIVATE_IRQS)) / 8,	\
605 		.access_flags = acc,					\
606 		.read = rd,						\
607 		.write = wr,						\
608 		.uaccess_read = ur,					\
609 		.uaccess_write = uw,					\
610 	}
611 
612 static const struct vgic_register_region vgic_v3_dist_registers[] = {
613 	REGISTER_DESC_WITH_LENGTH_UACCESS(GICD_CTLR,
614 		vgic_mmio_read_v3_misc, vgic_mmio_write_v3_misc,
615 		NULL, vgic_mmio_uaccess_write_v3_misc,
616 		16, VGIC_ACCESS_32bit),
617 	REGISTER_DESC_WITH_LENGTH(GICD_STATUSR,
618 		vgic_mmio_read_rao, vgic_mmio_write_wi, 4,
619 		VGIC_ACCESS_32bit),
620 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGROUPR,
621 		vgic_mmio_read_group, vgic_mmio_write_group, NULL, NULL, 1,
622 		VGIC_ACCESS_32bit),
623 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISENABLER,
624 		vgic_mmio_read_enable, vgic_mmio_write_senable,
625 		NULL, vgic_uaccess_write_senable, 1,
626 		VGIC_ACCESS_32bit),
627 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICENABLER,
628 		vgic_mmio_read_enable, vgic_mmio_write_cenable,
629 	       NULL, vgic_uaccess_write_cenable, 1,
630 		VGIC_ACCESS_32bit),
631 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISPENDR,
632 		vgic_mmio_read_pending, vgic_mmio_write_spending,
633 		vgic_uaccess_read_pending, vgic_v3_uaccess_write_pending, 1,
634 		VGIC_ACCESS_32bit),
635 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICPENDR,
636 		vgic_mmio_read_pending, vgic_mmio_write_cpending,
637 		vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 1,
638 		VGIC_ACCESS_32bit),
639 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISACTIVER,
640 		vgic_mmio_read_active, vgic_mmio_write_sactive,
641 		vgic_uaccess_read_active, vgic_mmio_uaccess_write_sactive, 1,
642 		VGIC_ACCESS_32bit),
643 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICACTIVER,
644 		vgic_mmio_read_active, vgic_mmio_write_cactive,
645 		vgic_uaccess_read_active, vgic_mmio_uaccess_write_cactive,
646 		1, VGIC_ACCESS_32bit),
647 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IPRIORITYR,
648 		vgic_mmio_read_priority, vgic_mmio_write_priority, NULL, NULL,
649 		8, VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
650 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ITARGETSR,
651 		vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 8,
652 		VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
653 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICFGR,
654 		vgic_mmio_read_config, vgic_mmio_write_config, NULL, NULL, 2,
655 		VGIC_ACCESS_32bit),
656 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGRPMODR,
657 		vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 1,
658 		VGIC_ACCESS_32bit),
659 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IROUTER,
660 		vgic_mmio_read_irouter, vgic_mmio_write_irouter, NULL, NULL, 64,
661 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
662 	REGISTER_DESC_WITH_LENGTH(GICD_IDREGS,
663 		vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
664 		VGIC_ACCESS_32bit),
665 };
666 
667 static const struct vgic_register_region vgic_v3_rd_registers[] = {
668 	/* RD_base registers */
669 	REGISTER_DESC_WITH_LENGTH(GICR_CTLR,
670 		vgic_mmio_read_v3r_ctlr, vgic_mmio_write_v3r_ctlr, 4,
671 		VGIC_ACCESS_32bit),
672 	REGISTER_DESC_WITH_LENGTH(GICR_STATUSR,
673 		vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
674 		VGIC_ACCESS_32bit),
675 	REGISTER_DESC_WITH_LENGTH(GICR_IIDR,
676 		vgic_mmio_read_v3r_iidr, vgic_mmio_write_wi, 4,
677 		VGIC_ACCESS_32bit),
678 	REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_TYPER,
679 		vgic_mmio_read_v3r_typer, vgic_mmio_write_wi,
680 		NULL, vgic_mmio_uaccess_write_wi, 8,
681 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
682 	REGISTER_DESC_WITH_LENGTH(GICR_WAKER,
683 		vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
684 		VGIC_ACCESS_32bit),
685 	REGISTER_DESC_WITH_LENGTH(GICR_PROPBASER,
686 		vgic_mmio_read_propbase, vgic_mmio_write_propbase, 8,
687 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
688 	REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER,
689 		vgic_mmio_read_pendbase, vgic_mmio_write_pendbase, 8,
690 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
691 	REGISTER_DESC_WITH_LENGTH(GICR_INVLPIR,
692 		vgic_mmio_read_raz, vgic_mmio_write_invlpi, 8,
693 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
694 	REGISTER_DESC_WITH_LENGTH(GICR_INVALLR,
695 		vgic_mmio_read_raz, vgic_mmio_write_invall, 8,
696 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
697 	REGISTER_DESC_WITH_LENGTH(GICR_SYNCR,
698 		vgic_mmio_read_sync, vgic_mmio_write_wi, 4,
699 		VGIC_ACCESS_32bit),
700 	REGISTER_DESC_WITH_LENGTH(GICR_IDREGS,
701 		vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
702 		VGIC_ACCESS_32bit),
703 	/* SGI_base registers */
704 	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGROUPR0,
705 		vgic_mmio_read_group, vgic_mmio_write_group, 4,
706 		VGIC_ACCESS_32bit),
707 	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISENABLER0,
708 		vgic_mmio_read_enable, vgic_mmio_write_senable,
709 		NULL, vgic_uaccess_write_senable, 4,
710 		VGIC_ACCESS_32bit),
711 	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICENABLER0,
712 		vgic_mmio_read_enable, vgic_mmio_write_cenable,
713 		NULL, vgic_uaccess_write_cenable, 4,
714 		VGIC_ACCESS_32bit),
715 	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISPENDR0,
716 		vgic_mmio_read_pending, vgic_mmio_write_spending,
717 		vgic_uaccess_read_pending, vgic_v3_uaccess_write_pending, 4,
718 		VGIC_ACCESS_32bit),
719 	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICPENDR0,
720 		vgic_mmio_read_pending, vgic_mmio_write_cpending,
721 		vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 4,
722 		VGIC_ACCESS_32bit),
723 	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISACTIVER0,
724 		vgic_mmio_read_active, vgic_mmio_write_sactive,
725 		vgic_uaccess_read_active, vgic_mmio_uaccess_write_sactive, 4,
726 		VGIC_ACCESS_32bit),
727 	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICACTIVER0,
728 		vgic_mmio_read_active, vgic_mmio_write_cactive,
729 		vgic_uaccess_read_active, vgic_mmio_uaccess_write_cactive, 4,
730 		VGIC_ACCESS_32bit),
731 	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IPRIORITYR0,
732 		vgic_mmio_read_priority, vgic_mmio_write_priority, 32,
733 		VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
734 	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ICFGR0,
735 		vgic_mmio_read_config, vgic_mmio_write_config, 8,
736 		VGIC_ACCESS_32bit),
737 	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGRPMODR0,
738 		vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
739 		VGIC_ACCESS_32bit),
740 	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_NSACR,
741 		vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
742 		VGIC_ACCESS_32bit),
743 };
744 
vgic_v3_init_dist_iodev(struct vgic_io_device * dev)745 unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev)
746 {
747 	dev->regions = vgic_v3_dist_registers;
748 	dev->nr_regions = ARRAY_SIZE(vgic_v3_dist_registers);
749 
750 	kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops);
751 
752 	return SZ_64K;
753 }
754 
755 /**
756  * vgic_register_redist_iodev - register a single redist iodev
757  * @vcpu:    The VCPU to which the redistributor belongs
758  *
759  * Register a KVM iodev for this VCPU's redistributor using the address
760  * provided.
761  *
762  * Return 0 on success, -ERRNO otherwise.
763  */
vgic_register_redist_iodev(struct kvm_vcpu * vcpu)764 int vgic_register_redist_iodev(struct kvm_vcpu *vcpu)
765 {
766 	struct kvm *kvm = vcpu->kvm;
767 	struct vgic_dist *vgic = &kvm->arch.vgic;
768 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
769 	struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
770 	struct vgic_redist_region *rdreg;
771 	gpa_t rd_base;
772 	int ret;
773 
774 	if (!IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr))
775 		return 0;
776 
777 	/*
778 	 * We may be creating VCPUs before having set the base address for the
779 	 * redistributor region, in which case we will come back to this
780 	 * function for all VCPUs when the base address is set.  Just return
781 	 * without doing any work for now.
782 	 */
783 	rdreg = vgic_v3_rdist_free_slot(&vgic->rd_regions);
784 	if (!rdreg)
785 		return 0;
786 
787 	if (!vgic_v3_check_base(kvm))
788 		return -EINVAL;
789 
790 	vgic_cpu->rdreg = rdreg;
791 	vgic_cpu->rdreg_index = rdreg->free_index;
792 
793 	rd_base = rdreg->base + rdreg->free_index * KVM_VGIC_V3_REDIST_SIZE;
794 
795 	kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops);
796 	rd_dev->base_addr = rd_base;
797 	rd_dev->iodev_type = IODEV_REDIST;
798 	rd_dev->regions = vgic_v3_rd_registers;
799 	rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
800 	rd_dev->redist_vcpu = vcpu;
801 
802 	mutex_lock(&kvm->slots_lock);
803 	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base,
804 				      2 * SZ_64K, &rd_dev->dev);
805 	mutex_unlock(&kvm->slots_lock);
806 
807 	if (ret)
808 		return ret;
809 
810 	rdreg->free_index++;
811 	return 0;
812 }
813 
vgic_unregister_redist_iodev(struct kvm_vcpu * vcpu)814 static void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu)
815 {
816 	struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
817 
818 	kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev);
819 }
820 
vgic_register_all_redist_iodevs(struct kvm * kvm)821 static int vgic_register_all_redist_iodevs(struct kvm *kvm)
822 {
823 	struct kvm_vcpu *vcpu;
824 	unsigned long c;
825 	int ret = 0;
826 
827 	kvm_for_each_vcpu(c, vcpu, kvm) {
828 		ret = vgic_register_redist_iodev(vcpu);
829 		if (ret)
830 			break;
831 	}
832 
833 	if (ret) {
834 		/* The current c failed, so iterate over the previous ones. */
835 		int i;
836 
837 		mutex_lock(&kvm->slots_lock);
838 		for (i = 0; i < c; i++) {
839 			vcpu = kvm_get_vcpu(kvm, i);
840 			vgic_unregister_redist_iodev(vcpu);
841 		}
842 		mutex_unlock(&kvm->slots_lock);
843 	}
844 
845 	return ret;
846 }
847 
848 /**
849  * vgic_v3_alloc_redist_region - Allocate a new redistributor region
850  *
851  * Performs various checks before inserting the rdist region in the list.
852  * Those tests depend on whether the size of the rdist region is known
853  * (ie. count != 0). The list is sorted by rdist region index.
854  *
855  * @kvm: kvm handle
856  * @index: redist region index
857  * @base: base of the new rdist region
858  * @count: number of redistributors the region is made of (0 in the old style
859  * single region, whose size is induced from the number of vcpus)
860  *
861  * Return 0 on success, < 0 otherwise
862  */
vgic_v3_alloc_redist_region(struct kvm * kvm,uint32_t index,gpa_t base,uint32_t count)863 static int vgic_v3_alloc_redist_region(struct kvm *kvm, uint32_t index,
864 				       gpa_t base, uint32_t count)
865 {
866 	struct vgic_dist *d = &kvm->arch.vgic;
867 	struct vgic_redist_region *rdreg;
868 	struct list_head *rd_regions = &d->rd_regions;
869 	int nr_vcpus = atomic_read(&kvm->online_vcpus);
870 	size_t size = count ? count * KVM_VGIC_V3_REDIST_SIZE
871 			    : nr_vcpus * KVM_VGIC_V3_REDIST_SIZE;
872 	int ret;
873 
874 	/* cross the end of memory ? */
875 	if (base + size < base)
876 		return -EINVAL;
877 
878 	if (list_empty(rd_regions)) {
879 		if (index != 0)
880 			return -EINVAL;
881 	} else {
882 		rdreg = list_last_entry(rd_regions,
883 					struct vgic_redist_region, list);
884 
885 		/* Don't mix single region and discrete redist regions */
886 		if (!count && rdreg->count)
887 			return -EINVAL;
888 
889 		if (!count)
890 			return -EEXIST;
891 
892 		if (index != rdreg->index + 1)
893 			return -EINVAL;
894 	}
895 
896 	/*
897 	 * For legacy single-region redistributor regions (!count),
898 	 * check that the redistributor region does not overlap with the
899 	 * distributor's address space.
900 	 */
901 	if (!count && !IS_VGIC_ADDR_UNDEF(d->vgic_dist_base) &&
902 		vgic_dist_overlap(kvm, base, size))
903 		return -EINVAL;
904 
905 	/* collision with any other rdist region? */
906 	if (vgic_v3_rdist_overlap(kvm, base, size))
907 		return -EINVAL;
908 
909 	rdreg = kzalloc(sizeof(*rdreg), GFP_KERNEL_ACCOUNT);
910 	if (!rdreg)
911 		return -ENOMEM;
912 
913 	rdreg->base = VGIC_ADDR_UNDEF;
914 
915 	ret = vgic_check_iorange(kvm, rdreg->base, base, SZ_64K, size);
916 	if (ret)
917 		goto free;
918 
919 	rdreg->base = base;
920 	rdreg->count = count;
921 	rdreg->free_index = 0;
922 	rdreg->index = index;
923 
924 	list_add_tail(&rdreg->list, rd_regions);
925 	return 0;
926 free:
927 	kfree(rdreg);
928 	return ret;
929 }
930 
vgic_v3_free_redist_region(struct vgic_redist_region * rdreg)931 void vgic_v3_free_redist_region(struct vgic_redist_region *rdreg)
932 {
933 	list_del(&rdreg->list);
934 	kfree(rdreg);
935 }
936 
vgic_v3_set_redist_base(struct kvm * kvm,u32 index,u64 addr,u32 count)937 int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count)
938 {
939 	int ret;
940 
941 	ret = vgic_v3_alloc_redist_region(kvm, index, addr, count);
942 	if (ret)
943 		return ret;
944 
945 	/*
946 	 * Register iodevs for each existing VCPU.  Adding more VCPUs
947 	 * afterwards will register the iodevs when needed.
948 	 */
949 	ret = vgic_register_all_redist_iodevs(kvm);
950 	if (ret) {
951 		struct vgic_redist_region *rdreg;
952 
953 		rdreg = vgic_v3_rdist_region_from_index(kvm, index);
954 		vgic_v3_free_redist_region(rdreg);
955 		return ret;
956 	}
957 
958 	return 0;
959 }
960 
vgic_v3_has_attr_regs(struct kvm_device * dev,struct kvm_device_attr * attr)961 int vgic_v3_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr)
962 {
963 	const struct vgic_register_region *region;
964 	struct vgic_io_device iodev;
965 	struct vgic_reg_attr reg_attr;
966 	struct kvm_vcpu *vcpu;
967 	gpa_t addr;
968 	int ret;
969 
970 	ret = vgic_v3_parse_attr(dev, attr, &reg_attr);
971 	if (ret)
972 		return ret;
973 
974 	vcpu = reg_attr.vcpu;
975 	addr = reg_attr.addr;
976 
977 	switch (attr->group) {
978 	case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
979 		iodev.regions = vgic_v3_dist_registers;
980 		iodev.nr_regions = ARRAY_SIZE(vgic_v3_dist_registers);
981 		iodev.base_addr = 0;
982 		break;
983 	case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:{
984 		iodev.regions = vgic_v3_rd_registers;
985 		iodev.nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
986 		iodev.base_addr = 0;
987 		break;
988 	}
989 	case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: {
990 		u64 reg, id;
991 
992 		id = (attr->attr & KVM_DEV_ARM_VGIC_SYSREG_INSTR_MASK);
993 		return vgic_v3_has_cpu_sysregs_attr(vcpu, 0, id, &reg);
994 	}
995 	default:
996 		return -ENXIO;
997 	}
998 
999 	/* We only support aligned 32-bit accesses. */
1000 	if (addr & 3)
1001 		return -ENXIO;
1002 
1003 	region = vgic_get_mmio_region(vcpu, &iodev, addr, sizeof(u32));
1004 	if (!region)
1005 		return -ENXIO;
1006 
1007 	return 0;
1008 }
1009 /*
1010  * Compare a given affinity (level 1-3 and a level 0 mask, from the SGI
1011  * generation register ICC_SGI1R_EL1) with a given VCPU.
1012  * If the VCPU's MPIDR matches, return the level0 affinity, otherwise
1013  * return -1.
1014  */
match_mpidr(u64 sgi_aff,u16 sgi_cpu_mask,struct kvm_vcpu * vcpu)1015 static int match_mpidr(u64 sgi_aff, u16 sgi_cpu_mask, struct kvm_vcpu *vcpu)
1016 {
1017 	unsigned long affinity;
1018 	int level0;
1019 
1020 	/*
1021 	 * Split the current VCPU's MPIDR into affinity level 0 and the
1022 	 * rest as this is what we have to compare against.
1023 	 */
1024 	affinity = kvm_vcpu_get_mpidr_aff(vcpu);
1025 	level0 = MPIDR_AFFINITY_LEVEL(affinity, 0);
1026 	affinity &= ~MPIDR_LEVEL_MASK;
1027 
1028 	/* bail out if the upper three levels don't match */
1029 	if (sgi_aff != affinity)
1030 		return -1;
1031 
1032 	/* Is this VCPU's bit set in the mask ? */
1033 	if (!(sgi_cpu_mask & BIT(level0)))
1034 		return -1;
1035 
1036 	return level0;
1037 }
1038 
1039 /*
1040  * The ICC_SGI* registers encode the affinity differently from the MPIDR,
1041  * so provide a wrapper to use the existing defines to isolate a certain
1042  * affinity level.
1043  */
1044 #define SGI_AFFINITY_LEVEL(reg, level) \
1045 	((((reg) & ICC_SGI1R_AFFINITY_## level ##_MASK) \
1046 	>> ICC_SGI1R_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level))
1047 
1048 /**
1049  * vgic_v3_dispatch_sgi - handle SGI requests from VCPUs
1050  * @vcpu: The VCPU requesting a SGI
1051  * @reg: The value written into ICC_{ASGI1,SGI0,SGI1}R by that VCPU
1052  * @allow_group1: Does the sysreg access allow generation of G1 SGIs
1053  *
1054  * With GICv3 (and ARE=1) CPUs trigger SGIs by writing to a system register.
1055  * This will trap in sys_regs.c and call this function.
1056  * This ICC_SGI1R_EL1 register contains the upper three affinity levels of the
1057  * target processors as well as a bitmask of 16 Aff0 CPUs.
1058  * If the interrupt routing mode bit is not set, we iterate over all VCPUs to
1059  * check for matching ones. If this bit is set, we signal all, but not the
1060  * calling VCPU.
1061  */
vgic_v3_dispatch_sgi(struct kvm_vcpu * vcpu,u64 reg,bool allow_group1)1062 void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1)
1063 {
1064 	struct kvm *kvm = vcpu->kvm;
1065 	struct kvm_vcpu *c_vcpu;
1066 	u16 target_cpus;
1067 	u64 mpidr;
1068 	int sgi;
1069 	int vcpu_id = vcpu->vcpu_id;
1070 	bool broadcast;
1071 	unsigned long c, flags;
1072 
1073 	sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT;
1074 	broadcast = reg & BIT_ULL(ICC_SGI1R_IRQ_ROUTING_MODE_BIT);
1075 	target_cpus = (reg & ICC_SGI1R_TARGET_LIST_MASK) >> ICC_SGI1R_TARGET_LIST_SHIFT;
1076 	mpidr = SGI_AFFINITY_LEVEL(reg, 3);
1077 	mpidr |= SGI_AFFINITY_LEVEL(reg, 2);
1078 	mpidr |= SGI_AFFINITY_LEVEL(reg, 1);
1079 
1080 	/*
1081 	 * We iterate over all VCPUs to find the MPIDRs matching the request.
1082 	 * If we have handled one CPU, we clear its bit to detect early
1083 	 * if we are already finished. This avoids iterating through all
1084 	 * VCPUs when most of the times we just signal a single VCPU.
1085 	 */
1086 	kvm_for_each_vcpu(c, c_vcpu, kvm) {
1087 		struct vgic_irq *irq;
1088 
1089 		/* Exit early if we have dealt with all requested CPUs */
1090 		if (!broadcast && target_cpus == 0)
1091 			break;
1092 
1093 		/* Don't signal the calling VCPU */
1094 		if (broadcast && c == vcpu_id)
1095 			continue;
1096 
1097 		if (!broadcast) {
1098 			int level0;
1099 
1100 			level0 = match_mpidr(mpidr, target_cpus, c_vcpu);
1101 			if (level0 == -1)
1102 				continue;
1103 
1104 			/* remove this matching VCPU from the mask */
1105 			target_cpus &= ~BIT(level0);
1106 		}
1107 
1108 		irq = vgic_get_irq(vcpu->kvm, c_vcpu, sgi);
1109 
1110 		raw_spin_lock_irqsave(&irq->irq_lock, flags);
1111 
1112 		/*
1113 		 * An access targeting Group0 SGIs can only generate
1114 		 * those, while an access targeting Group1 SGIs can
1115 		 * generate interrupts of either group.
1116 		 */
1117 		if (!irq->group || allow_group1) {
1118 			if (!irq->hw) {
1119 				irq->pending_latch = true;
1120 				vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
1121 			} else {
1122 				/* HW SGI? Ask the GIC to inject it */
1123 				int err;
1124 				err = irq_set_irqchip_state(irq->host_irq,
1125 							    IRQCHIP_STATE_PENDING,
1126 							    true);
1127 				WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
1128 				raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
1129 			}
1130 		} else {
1131 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
1132 		}
1133 
1134 		vgic_put_irq(vcpu->kvm, irq);
1135 	}
1136 }
1137 
vgic_v3_dist_uaccess(struct kvm_vcpu * vcpu,bool is_write,int offset,u32 * val)1138 int vgic_v3_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
1139 			 int offset, u32 *val)
1140 {
1141 	struct vgic_io_device dev = {
1142 		.regions = vgic_v3_dist_registers,
1143 		.nr_regions = ARRAY_SIZE(vgic_v3_dist_registers),
1144 	};
1145 
1146 	return vgic_uaccess(vcpu, &dev, is_write, offset, val);
1147 }
1148 
vgic_v3_redist_uaccess(struct kvm_vcpu * vcpu,bool is_write,int offset,u32 * val)1149 int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
1150 			   int offset, u32 *val)
1151 {
1152 	struct vgic_io_device rd_dev = {
1153 		.regions = vgic_v3_rd_registers,
1154 		.nr_regions = ARRAY_SIZE(vgic_v3_rd_registers),
1155 	};
1156 
1157 	return vgic_uaccess(vcpu, &rd_dev, is_write, offset, val);
1158 }
1159 
vgic_v3_line_level_info_uaccess(struct kvm_vcpu * vcpu,bool is_write,u32 intid,u64 * val)1160 int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write,
1161 				    u32 intid, u64 *val)
1162 {
1163 	if (intid % 32)
1164 		return -EINVAL;
1165 
1166 	if (is_write)
1167 		vgic_write_irq_line_level_info(vcpu, intid, *val);
1168 	else
1169 		*val = vgic_read_irq_line_level_info(vcpu, intid);
1170 
1171 	return 0;
1172 }
1173