1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VGICv3 MMIO handling functions
4  */
5 
6 #include <linux/bitfield.h>
7 #include <linux/irqchip/arm-gic-v3.h>
8 #include <linux/kvm.h>
9 #include <linux/kvm_host.h>
10 #include <linux/interrupt.h>
11 #include <kvm/iodev.h>
12 #include <kvm/arm_vgic.h>
13 
14 #include <asm/kvm_emulate.h>
15 #include <asm/kvm_arm.h>
16 #include <asm/kvm_mmu.h>
17 
18 #include "vgic.h"
19 #include "vgic-mmio.h"
20 
21 /* extract @num bytes at @offset bytes offset in data */
extract_bytes(u64 data,unsigned int offset,unsigned int num)22 unsigned long extract_bytes(u64 data, unsigned int offset,
23 			    unsigned int num)
24 {
25 	return (data >> (offset * 8)) & GENMASK_ULL(num * 8 - 1, 0);
26 }
27 
28 /* allows updates of any half of a 64-bit register (or the whole thing) */
update_64bit_reg(u64 reg,unsigned int offset,unsigned int len,unsigned long val)29 u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len,
30 		     unsigned long val)
31 {
32 	int lower = (offset & 4) * 8;
33 	int upper = lower + 8 * len - 1;
34 
35 	reg &= ~GENMASK_ULL(upper, lower);
36 	val &= GENMASK_ULL(len * 8 - 1, 0);
37 
38 	return reg | ((u64)val << lower);
39 }
40 
vgic_has_its(struct kvm * kvm)41 bool vgic_has_its(struct kvm *kvm)
42 {
43 	struct vgic_dist *dist = &kvm->arch.vgic;
44 
45 	if (dist->vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
46 		return false;
47 
48 	return dist->has_its;
49 }
50 
vgic_supports_direct_msis(struct kvm * kvm)51 bool vgic_supports_direct_msis(struct kvm *kvm)
52 {
53 	return (kvm_vgic_global_state.has_gicv4_1 ||
54 		(kvm_vgic_global_state.has_gicv4 && vgic_has_its(kvm)));
55 }
56 
57 /*
58  * The Revision field in the IIDR have the following meanings:
59  *
60  * Revision 2: Interrupt groups are guest-configurable and signaled using
61  * 	       their configured groups.
62  */
63 
vgic_mmio_read_v3_misc(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)64 static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu,
65 					    gpa_t addr, unsigned int len)
66 {
67 	struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
68 	u32 value = 0;
69 
70 	switch (addr & 0x0c) {
71 	case GICD_CTLR:
72 		if (vgic->enabled)
73 			value |= GICD_CTLR_ENABLE_SS_G1;
74 		value |= GICD_CTLR_ARE_NS | GICD_CTLR_DS;
75 		if (vgic->nassgireq)
76 			value |= GICD_CTLR_nASSGIreq;
77 		break;
78 	case GICD_TYPER:
79 		value = vgic->nr_spis + VGIC_NR_PRIVATE_IRQS;
80 		value = (value >> 5) - 1;
81 		if (vgic_has_its(vcpu->kvm)) {
82 			value |= (INTERRUPT_ID_BITS_ITS - 1) << 19;
83 			value |= GICD_TYPER_LPIS;
84 		} else {
85 			value |= (INTERRUPT_ID_BITS_SPIS - 1) << 19;
86 		}
87 		break;
88 	case GICD_TYPER2:
89 		if (kvm_vgic_global_state.has_gicv4_1 && gic_cpuif_has_vsgi())
90 			value = GICD_TYPER2_nASSGIcap;
91 		break;
92 	case GICD_IIDR:
93 		value = (PRODUCT_ID_KVM << GICD_IIDR_PRODUCT_ID_SHIFT) |
94 			(vgic->implementation_rev << GICD_IIDR_REVISION_SHIFT) |
95 			(IMPLEMENTER_ARM << GICD_IIDR_IMPLEMENTER_SHIFT);
96 		break;
97 	default:
98 		return 0;
99 	}
100 
101 	return value;
102 }
103 
vgic_mmio_write_v3_misc(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)104 static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu,
105 				    gpa_t addr, unsigned int len,
106 				    unsigned long val)
107 {
108 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
109 
110 	switch (addr & 0x0c) {
111 	case GICD_CTLR: {
112 		bool was_enabled, is_hwsgi;
113 
114 		mutex_lock(&vcpu->kvm->arch.config_lock);
115 
116 		was_enabled = dist->enabled;
117 		is_hwsgi = dist->nassgireq;
118 
119 		dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;
120 
121 		/* Not a GICv4.1? No HW SGIs */
122 		if (!kvm_vgic_global_state.has_gicv4_1 || !gic_cpuif_has_vsgi())
123 			val &= ~GICD_CTLR_nASSGIreq;
124 
125 		/* Dist stays enabled? nASSGIreq is RO */
126 		if (was_enabled && dist->enabled) {
127 			val &= ~GICD_CTLR_nASSGIreq;
128 			val |= FIELD_PREP(GICD_CTLR_nASSGIreq, is_hwsgi);
129 		}
130 
131 		/* Switching HW SGIs? */
132 		dist->nassgireq = val & GICD_CTLR_nASSGIreq;
133 		if (is_hwsgi != dist->nassgireq)
134 			vgic_v4_configure_vsgis(vcpu->kvm);
135 
136 		if (kvm_vgic_global_state.has_gicv4_1 &&
137 		    was_enabled != dist->enabled)
138 			kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_RELOAD_GICv4);
139 		else if (!was_enabled && dist->enabled)
140 			vgic_kick_vcpus(vcpu->kvm);
141 
142 		mutex_unlock(&vcpu->kvm->arch.config_lock);
143 		break;
144 	}
145 	case GICD_TYPER:
146 	case GICD_TYPER2:
147 	case GICD_IIDR:
148 		/* This is at best for documentation purposes... */
149 		return;
150 	}
151 }
152 
vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)153 static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu,
154 					   gpa_t addr, unsigned int len,
155 					   unsigned long val)
156 {
157 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
158 	u32 reg;
159 
160 	switch (addr & 0x0c) {
161 	case GICD_TYPER2:
162 		if (val != vgic_mmio_read_v3_misc(vcpu, addr, len))
163 			return -EINVAL;
164 		return 0;
165 	case GICD_IIDR:
166 		reg = vgic_mmio_read_v3_misc(vcpu, addr, len);
167 		if ((reg ^ val) & ~GICD_IIDR_REVISION_MASK)
168 			return -EINVAL;
169 
170 		reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg);
171 		switch (reg) {
172 		case KVM_VGIC_IMP_REV_2:
173 		case KVM_VGIC_IMP_REV_3:
174 			dist->implementation_rev = reg;
175 			return 0;
176 		default:
177 			return -EINVAL;
178 		}
179 	case GICD_CTLR:
180 		/* Not a GICv4.1? No HW SGIs */
181 		if (!kvm_vgic_global_state.has_gicv4_1)
182 			val &= ~GICD_CTLR_nASSGIreq;
183 
184 		dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;
185 		dist->nassgireq = val & GICD_CTLR_nASSGIreq;
186 		return 0;
187 	}
188 
189 	vgic_mmio_write_v3_misc(vcpu, addr, len, val);
190 	return 0;
191 }
192 
vgic_mmio_read_irouter(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)193 static unsigned long vgic_mmio_read_irouter(struct kvm_vcpu *vcpu,
194 					    gpa_t addr, unsigned int len)
195 {
196 	int intid = VGIC_ADDR_TO_INTID(addr, 64);
197 	struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, NULL, intid);
198 	unsigned long ret = 0;
199 
200 	if (!irq)
201 		return 0;
202 
203 	/* The upper word is RAZ for us. */
204 	if (!(addr & 4))
205 		ret = extract_bytes(READ_ONCE(irq->mpidr), addr & 7, len);
206 
207 	vgic_put_irq(vcpu->kvm, irq);
208 	return ret;
209 }
210 
vgic_mmio_write_irouter(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)211 static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu,
212 				    gpa_t addr, unsigned int len,
213 				    unsigned long val)
214 {
215 	int intid = VGIC_ADDR_TO_INTID(addr, 64);
216 	struct vgic_irq *irq;
217 	unsigned long flags;
218 
219 	/* The upper word is WI for us since we don't implement Aff3. */
220 	if (addr & 4)
221 		return;
222 
223 	irq = vgic_get_irq(vcpu->kvm, NULL, intid);
224 
225 	if (!irq)
226 		return;
227 
228 	raw_spin_lock_irqsave(&irq->irq_lock, flags);
229 
230 	/* We only care about and preserve Aff0, Aff1 and Aff2. */
231 	irq->mpidr = val & GENMASK(23, 0);
232 	irq->target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, irq->mpidr);
233 
234 	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
235 	vgic_put_irq(vcpu->kvm, irq);
236 }
237 
vgic_lpis_enabled(struct kvm_vcpu * vcpu)238 bool vgic_lpis_enabled(struct kvm_vcpu *vcpu)
239 {
240 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
241 
242 	return atomic_read(&vgic_cpu->ctlr) == GICR_CTLR_ENABLE_LPIS;
243 }
244 
vgic_mmio_read_v3r_ctlr(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)245 static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu,
246 					     gpa_t addr, unsigned int len)
247 {
248 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
249 	unsigned long val;
250 
251 	val = atomic_read(&vgic_cpu->ctlr);
252 	if (vgic_get_implementation_rev(vcpu) >= KVM_VGIC_IMP_REV_3)
253 		val |= GICR_CTLR_IR | GICR_CTLR_CES;
254 
255 	return val;
256 }
257 
vgic_mmio_write_v3r_ctlr(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)258 static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu,
259 				     gpa_t addr, unsigned int len,
260 				     unsigned long val)
261 {
262 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
263 	u32 ctlr;
264 
265 	if (!vgic_has_its(vcpu->kvm))
266 		return;
267 
268 	if (!(val & GICR_CTLR_ENABLE_LPIS)) {
269 		/*
270 		 * Don't disable if RWP is set, as there already an
271 		 * ongoing disable. Funky guest...
272 		 */
273 		ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr,
274 					      GICR_CTLR_ENABLE_LPIS,
275 					      GICR_CTLR_RWP);
276 		if (ctlr != GICR_CTLR_ENABLE_LPIS)
277 			return;
278 
279 		vgic_flush_pending_lpis(vcpu);
280 		vgic_its_invalidate_cache(vcpu->kvm);
281 		atomic_set_release(&vgic_cpu->ctlr, 0);
282 	} else {
283 		ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr, 0,
284 					      GICR_CTLR_ENABLE_LPIS);
285 		if (ctlr != 0)
286 			return;
287 
288 		vgic_enable_lpis(vcpu);
289 	}
290 }
291 
vgic_mmio_vcpu_rdist_is_last(struct kvm_vcpu * vcpu)292 static bool vgic_mmio_vcpu_rdist_is_last(struct kvm_vcpu *vcpu)
293 {
294 	struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
295 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
296 	struct vgic_redist_region *iter, *rdreg = vgic_cpu->rdreg;
297 
298 	if (!rdreg)
299 		return false;
300 
301 	if (vgic_cpu->rdreg_index < rdreg->free_index - 1) {
302 		return false;
303 	} else if (rdreg->count && vgic_cpu->rdreg_index == (rdreg->count - 1)) {
304 		struct list_head *rd_regions = &vgic->rd_regions;
305 		gpa_t end = rdreg->base + rdreg->count * KVM_VGIC_V3_REDIST_SIZE;
306 
307 		/*
308 		 * the rdist is the last one of the redist region,
309 		 * check whether there is no other contiguous rdist region
310 		 */
311 		list_for_each_entry(iter, rd_regions, list) {
312 			if (iter->base == end && iter->free_index > 0)
313 				return false;
314 		}
315 	}
316 	return true;
317 }
318 
vgic_mmio_read_v3r_typer(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)319 static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu,
320 					      gpa_t addr, unsigned int len)
321 {
322 	unsigned long mpidr = kvm_vcpu_get_mpidr_aff(vcpu);
323 	int target_vcpu_id = vcpu->vcpu_id;
324 	u64 value;
325 
326 	value = (u64)(mpidr & GENMASK(23, 0)) << 32;
327 	value |= ((target_vcpu_id & 0xffff) << 8);
328 
329 	if (vgic_has_its(vcpu->kvm))
330 		value |= GICR_TYPER_PLPIS;
331 
332 	if (vgic_mmio_vcpu_rdist_is_last(vcpu))
333 		value |= GICR_TYPER_LAST;
334 
335 	return extract_bytes(value, addr & 7, len);
336 }
337 
vgic_mmio_read_v3r_iidr(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)338 static unsigned long vgic_mmio_read_v3r_iidr(struct kvm_vcpu *vcpu,
339 					     gpa_t addr, unsigned int len)
340 {
341 	return (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
342 }
343 
vgic_mmio_read_v3_idregs(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)344 static unsigned long vgic_mmio_read_v3_idregs(struct kvm_vcpu *vcpu,
345 					      gpa_t addr, unsigned int len)
346 {
347 	switch (addr & 0xffff) {
348 	case GICD_PIDR2:
349 		/* report a GICv3 compliant implementation */
350 		return 0x3b;
351 	}
352 
353 	return 0;
354 }
355 
vgic_v3_uaccess_write_pending(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)356 static int vgic_v3_uaccess_write_pending(struct kvm_vcpu *vcpu,
357 					 gpa_t addr, unsigned int len,
358 					 unsigned long val)
359 {
360 	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
361 	int i;
362 	unsigned long flags;
363 
364 	for (i = 0; i < len * 8; i++) {
365 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
366 
367 		raw_spin_lock_irqsave(&irq->irq_lock, flags);
368 
369 		/*
370 		 * pending_latch is set irrespective of irq type
371 		 * (level or edge) to avoid dependency that VM should
372 		 * restore irq config before pending info.
373 		 */
374 		irq->pending_latch = test_bit(i, &val);
375 
376 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
377 			irq_set_irqchip_state(irq->host_irq,
378 					      IRQCHIP_STATE_PENDING,
379 					      irq->pending_latch);
380 			irq->pending_latch = false;
381 		}
382 
383 		if (irq->pending_latch)
384 			vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
385 		else
386 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
387 
388 		vgic_put_irq(vcpu->kvm, irq);
389 	}
390 
391 	return 0;
392 }
393 
394 /* We want to avoid outer shareable. */
vgic_sanitise_shareability(u64 field)395 u64 vgic_sanitise_shareability(u64 field)
396 {
397 	switch (field) {
398 	case GIC_BASER_OuterShareable:
399 		return GIC_BASER_InnerShareable;
400 	default:
401 		return field;
402 	}
403 }
404 
405 /* Avoid any inner non-cacheable mapping. */
vgic_sanitise_inner_cacheability(u64 field)406 u64 vgic_sanitise_inner_cacheability(u64 field)
407 {
408 	switch (field) {
409 	case GIC_BASER_CACHE_nCnB:
410 	case GIC_BASER_CACHE_nC:
411 		return GIC_BASER_CACHE_RaWb;
412 	default:
413 		return field;
414 	}
415 }
416 
417 /* Non-cacheable or same-as-inner are OK. */
vgic_sanitise_outer_cacheability(u64 field)418 u64 vgic_sanitise_outer_cacheability(u64 field)
419 {
420 	switch (field) {
421 	case GIC_BASER_CACHE_SameAsInner:
422 	case GIC_BASER_CACHE_nC:
423 		return field;
424 	default:
425 		return GIC_BASER_CACHE_SameAsInner;
426 	}
427 }
428 
vgic_sanitise_field(u64 reg,u64 field_mask,int field_shift,u64 (* sanitise_fn)(u64))429 u64 vgic_sanitise_field(u64 reg, u64 field_mask, int field_shift,
430 			u64 (*sanitise_fn)(u64))
431 {
432 	u64 field = (reg & field_mask) >> field_shift;
433 
434 	field = sanitise_fn(field) << field_shift;
435 	return (reg & ~field_mask) | field;
436 }
437 
438 #define PROPBASER_RES0_MASK						\
439 	(GENMASK_ULL(63, 59) | GENMASK_ULL(55, 52) | GENMASK_ULL(6, 5))
440 #define PENDBASER_RES0_MASK						\
441 	(BIT_ULL(63) | GENMASK_ULL(61, 59) | GENMASK_ULL(55, 52) |	\
442 	 GENMASK_ULL(15, 12) | GENMASK_ULL(6, 0))
443 
vgic_sanitise_pendbaser(u64 reg)444 static u64 vgic_sanitise_pendbaser(u64 reg)
445 {
446 	reg = vgic_sanitise_field(reg, GICR_PENDBASER_SHAREABILITY_MASK,
447 				  GICR_PENDBASER_SHAREABILITY_SHIFT,
448 				  vgic_sanitise_shareability);
449 	reg = vgic_sanitise_field(reg, GICR_PENDBASER_INNER_CACHEABILITY_MASK,
450 				  GICR_PENDBASER_INNER_CACHEABILITY_SHIFT,
451 				  vgic_sanitise_inner_cacheability);
452 	reg = vgic_sanitise_field(reg, GICR_PENDBASER_OUTER_CACHEABILITY_MASK,
453 				  GICR_PENDBASER_OUTER_CACHEABILITY_SHIFT,
454 				  vgic_sanitise_outer_cacheability);
455 
456 	reg &= ~PENDBASER_RES0_MASK;
457 
458 	return reg;
459 }
460 
vgic_sanitise_propbaser(u64 reg)461 static u64 vgic_sanitise_propbaser(u64 reg)
462 {
463 	reg = vgic_sanitise_field(reg, GICR_PROPBASER_SHAREABILITY_MASK,
464 				  GICR_PROPBASER_SHAREABILITY_SHIFT,
465 				  vgic_sanitise_shareability);
466 	reg = vgic_sanitise_field(reg, GICR_PROPBASER_INNER_CACHEABILITY_MASK,
467 				  GICR_PROPBASER_INNER_CACHEABILITY_SHIFT,
468 				  vgic_sanitise_inner_cacheability);
469 	reg = vgic_sanitise_field(reg, GICR_PROPBASER_OUTER_CACHEABILITY_MASK,
470 				  GICR_PROPBASER_OUTER_CACHEABILITY_SHIFT,
471 				  vgic_sanitise_outer_cacheability);
472 
473 	reg &= ~PROPBASER_RES0_MASK;
474 	return reg;
475 }
476 
vgic_mmio_read_propbase(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)477 static unsigned long vgic_mmio_read_propbase(struct kvm_vcpu *vcpu,
478 					     gpa_t addr, unsigned int len)
479 {
480 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
481 
482 	return extract_bytes(dist->propbaser, addr & 7, len);
483 }
484 
vgic_mmio_write_propbase(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)485 static void vgic_mmio_write_propbase(struct kvm_vcpu *vcpu,
486 				     gpa_t addr, unsigned int len,
487 				     unsigned long val)
488 {
489 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
490 	u64 old_propbaser, propbaser;
491 
492 	/* Storing a value with LPIs already enabled is undefined */
493 	if (vgic_lpis_enabled(vcpu))
494 		return;
495 
496 	do {
497 		old_propbaser = READ_ONCE(dist->propbaser);
498 		propbaser = old_propbaser;
499 		propbaser = update_64bit_reg(propbaser, addr & 4, len, val);
500 		propbaser = vgic_sanitise_propbaser(propbaser);
501 	} while (cmpxchg64(&dist->propbaser, old_propbaser,
502 			   propbaser) != old_propbaser);
503 }
504 
vgic_mmio_read_pendbase(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)505 static unsigned long vgic_mmio_read_pendbase(struct kvm_vcpu *vcpu,
506 					     gpa_t addr, unsigned int len)
507 {
508 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
509 	u64 value = vgic_cpu->pendbaser;
510 
511 	value &= ~GICR_PENDBASER_PTZ;
512 
513 	return extract_bytes(value, addr & 7, len);
514 }
515 
vgic_mmio_write_pendbase(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)516 static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu,
517 				     gpa_t addr, unsigned int len,
518 				     unsigned long val)
519 {
520 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
521 	u64 old_pendbaser, pendbaser;
522 
523 	/* Storing a value with LPIs already enabled is undefined */
524 	if (vgic_lpis_enabled(vcpu))
525 		return;
526 
527 	do {
528 		old_pendbaser = READ_ONCE(vgic_cpu->pendbaser);
529 		pendbaser = old_pendbaser;
530 		pendbaser = update_64bit_reg(pendbaser, addr & 4, len, val);
531 		pendbaser = vgic_sanitise_pendbaser(pendbaser);
532 	} while (cmpxchg64(&vgic_cpu->pendbaser, old_pendbaser,
533 			   pendbaser) != old_pendbaser);
534 }
535 
vgic_mmio_read_sync(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len)536 static unsigned long vgic_mmio_read_sync(struct kvm_vcpu *vcpu,
537 					 gpa_t addr, unsigned int len)
538 {
539 	return !!atomic_read(&vcpu->arch.vgic_cpu.syncr_busy);
540 }
541 
vgic_set_rdist_busy(struct kvm_vcpu * vcpu,bool busy)542 static void vgic_set_rdist_busy(struct kvm_vcpu *vcpu, bool busy)
543 {
544 	if (busy) {
545 		atomic_inc(&vcpu->arch.vgic_cpu.syncr_busy);
546 		smp_mb__after_atomic();
547 	} else {
548 		smp_mb__before_atomic();
549 		atomic_dec(&vcpu->arch.vgic_cpu.syncr_busy);
550 	}
551 }
552 
vgic_mmio_write_invlpi(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)553 static void vgic_mmio_write_invlpi(struct kvm_vcpu *vcpu,
554 				   gpa_t addr, unsigned int len,
555 				   unsigned long val)
556 {
557 	struct vgic_irq *irq;
558 
559 	/*
560 	 * If the guest wrote only to the upper 32bit part of the
561 	 * register, drop the write on the floor, as it is only for
562 	 * vPEs (which we don't support for obvious reasons).
563 	 *
564 	 * Also discard the access if LPIs are not enabled.
565 	 */
566 	if ((addr & 4) || !vgic_lpis_enabled(vcpu))
567 		return;
568 
569 	vgic_set_rdist_busy(vcpu, true);
570 
571 	irq = vgic_get_irq(vcpu->kvm, NULL, lower_32_bits(val));
572 	if (irq) {
573 		vgic_its_inv_lpi(vcpu->kvm, irq);
574 		vgic_put_irq(vcpu->kvm, irq);
575 	}
576 
577 	vgic_set_rdist_busy(vcpu, false);
578 }
579 
vgic_mmio_write_invall(struct kvm_vcpu * vcpu,gpa_t addr,unsigned int len,unsigned long val)580 static void vgic_mmio_write_invall(struct kvm_vcpu *vcpu,
581 				   gpa_t addr, unsigned int len,
582 				   unsigned long val)
583 {
584 	/* See vgic_mmio_write_invlpi() for the early return rationale */
585 	if ((addr & 4) || !vgic_lpis_enabled(vcpu))
586 		return;
587 
588 	vgic_set_rdist_busy(vcpu, true);
589 	vgic_its_invall(vcpu);
590 	vgic_set_rdist_busy(vcpu, false);
591 }
592 
593 /*
594  * The GICv3 per-IRQ registers are split to control PPIs and SGIs in the
595  * redistributors, while SPIs are covered by registers in the distributor
596  * block. Trying to set private IRQs in this block gets ignored.
597  * We take some special care here to fix the calculation of the register
598  * offset.
599  */
600 #define REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(off, rd, wr, ur, uw, bpi, acc) \
601 	{								\
602 		.reg_offset = off,					\
603 		.bits_per_irq = bpi,					\
604 		.len = (bpi * VGIC_NR_PRIVATE_IRQS) / 8,		\
605 		.access_flags = acc,					\
606 		.read = vgic_mmio_read_raz,				\
607 		.write = vgic_mmio_write_wi,				\
608 	}, {								\
609 		.reg_offset = off + (bpi * VGIC_NR_PRIVATE_IRQS) / 8,	\
610 		.bits_per_irq = bpi,					\
611 		.len = (bpi * (1024 - VGIC_NR_PRIVATE_IRQS)) / 8,	\
612 		.access_flags = acc,					\
613 		.read = rd,						\
614 		.write = wr,						\
615 		.uaccess_read = ur,					\
616 		.uaccess_write = uw,					\
617 	}
618 
619 static const struct vgic_register_region vgic_v3_dist_registers[] = {
620 	REGISTER_DESC_WITH_LENGTH_UACCESS(GICD_CTLR,
621 		vgic_mmio_read_v3_misc, vgic_mmio_write_v3_misc,
622 		NULL, vgic_mmio_uaccess_write_v3_misc,
623 		16, VGIC_ACCESS_32bit),
624 	REGISTER_DESC_WITH_LENGTH(GICD_STATUSR,
625 		vgic_mmio_read_rao, vgic_mmio_write_wi, 4,
626 		VGIC_ACCESS_32bit),
627 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGROUPR,
628 		vgic_mmio_read_group, vgic_mmio_write_group, NULL, NULL, 1,
629 		VGIC_ACCESS_32bit),
630 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISENABLER,
631 		vgic_mmio_read_enable, vgic_mmio_write_senable,
632 		NULL, vgic_uaccess_write_senable, 1,
633 		VGIC_ACCESS_32bit),
634 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICENABLER,
635 		vgic_mmio_read_enable, vgic_mmio_write_cenable,
636 	       NULL, vgic_uaccess_write_cenable, 1,
637 		VGIC_ACCESS_32bit),
638 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISPENDR,
639 		vgic_mmio_read_pending, vgic_mmio_write_spending,
640 		vgic_uaccess_read_pending, vgic_v3_uaccess_write_pending, 1,
641 		VGIC_ACCESS_32bit),
642 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICPENDR,
643 		vgic_mmio_read_pending, vgic_mmio_write_cpending,
644 		vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 1,
645 		VGIC_ACCESS_32bit),
646 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISACTIVER,
647 		vgic_mmio_read_active, vgic_mmio_write_sactive,
648 		vgic_uaccess_read_active, vgic_mmio_uaccess_write_sactive, 1,
649 		VGIC_ACCESS_32bit),
650 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICACTIVER,
651 		vgic_mmio_read_active, vgic_mmio_write_cactive,
652 		vgic_uaccess_read_active, vgic_mmio_uaccess_write_cactive,
653 		1, VGIC_ACCESS_32bit),
654 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IPRIORITYR,
655 		vgic_mmio_read_priority, vgic_mmio_write_priority, NULL, NULL,
656 		8, VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
657 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ITARGETSR,
658 		vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 8,
659 		VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
660 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICFGR,
661 		vgic_mmio_read_config, vgic_mmio_write_config, NULL, NULL, 2,
662 		VGIC_ACCESS_32bit),
663 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGRPMODR,
664 		vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 1,
665 		VGIC_ACCESS_32bit),
666 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IROUTER,
667 		vgic_mmio_read_irouter, vgic_mmio_write_irouter, NULL, NULL, 64,
668 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
669 	REGISTER_DESC_WITH_LENGTH(GICD_IDREGS,
670 		vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
671 		VGIC_ACCESS_32bit),
672 };
673 
674 static const struct vgic_register_region vgic_v3_rd_registers[] = {
675 	/* RD_base registers */
676 	REGISTER_DESC_WITH_LENGTH(GICR_CTLR,
677 		vgic_mmio_read_v3r_ctlr, vgic_mmio_write_v3r_ctlr, 4,
678 		VGIC_ACCESS_32bit),
679 	REGISTER_DESC_WITH_LENGTH(GICR_STATUSR,
680 		vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
681 		VGIC_ACCESS_32bit),
682 	REGISTER_DESC_WITH_LENGTH(GICR_IIDR,
683 		vgic_mmio_read_v3r_iidr, vgic_mmio_write_wi, 4,
684 		VGIC_ACCESS_32bit),
685 	REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_TYPER,
686 		vgic_mmio_read_v3r_typer, vgic_mmio_write_wi,
687 		NULL, vgic_mmio_uaccess_write_wi, 8,
688 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
689 	REGISTER_DESC_WITH_LENGTH(GICR_WAKER,
690 		vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
691 		VGIC_ACCESS_32bit),
692 	REGISTER_DESC_WITH_LENGTH(GICR_PROPBASER,
693 		vgic_mmio_read_propbase, vgic_mmio_write_propbase, 8,
694 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
695 	REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER,
696 		vgic_mmio_read_pendbase, vgic_mmio_write_pendbase, 8,
697 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
698 	REGISTER_DESC_WITH_LENGTH(GICR_INVLPIR,
699 		vgic_mmio_read_raz, vgic_mmio_write_invlpi, 8,
700 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
701 	REGISTER_DESC_WITH_LENGTH(GICR_INVALLR,
702 		vgic_mmio_read_raz, vgic_mmio_write_invall, 8,
703 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
704 	REGISTER_DESC_WITH_LENGTH(GICR_SYNCR,
705 		vgic_mmio_read_sync, vgic_mmio_write_wi, 4,
706 		VGIC_ACCESS_32bit),
707 	REGISTER_DESC_WITH_LENGTH(GICR_IDREGS,
708 		vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
709 		VGIC_ACCESS_32bit),
710 	/* SGI_base registers */
711 	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGROUPR0,
712 		vgic_mmio_read_group, vgic_mmio_write_group, 4,
713 		VGIC_ACCESS_32bit),
714 	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISENABLER0,
715 		vgic_mmio_read_enable, vgic_mmio_write_senable,
716 		NULL, vgic_uaccess_write_senable, 4,
717 		VGIC_ACCESS_32bit),
718 	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICENABLER0,
719 		vgic_mmio_read_enable, vgic_mmio_write_cenable,
720 		NULL, vgic_uaccess_write_cenable, 4,
721 		VGIC_ACCESS_32bit),
722 	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISPENDR0,
723 		vgic_mmio_read_pending, vgic_mmio_write_spending,
724 		vgic_uaccess_read_pending, vgic_v3_uaccess_write_pending, 4,
725 		VGIC_ACCESS_32bit),
726 	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICPENDR0,
727 		vgic_mmio_read_pending, vgic_mmio_write_cpending,
728 		vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 4,
729 		VGIC_ACCESS_32bit),
730 	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISACTIVER0,
731 		vgic_mmio_read_active, vgic_mmio_write_sactive,
732 		vgic_uaccess_read_active, vgic_mmio_uaccess_write_sactive, 4,
733 		VGIC_ACCESS_32bit),
734 	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICACTIVER0,
735 		vgic_mmio_read_active, vgic_mmio_write_cactive,
736 		vgic_uaccess_read_active, vgic_mmio_uaccess_write_cactive, 4,
737 		VGIC_ACCESS_32bit),
738 	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IPRIORITYR0,
739 		vgic_mmio_read_priority, vgic_mmio_write_priority, 32,
740 		VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
741 	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ICFGR0,
742 		vgic_mmio_read_config, vgic_mmio_write_config, 8,
743 		VGIC_ACCESS_32bit),
744 	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGRPMODR0,
745 		vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
746 		VGIC_ACCESS_32bit),
747 	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_NSACR,
748 		vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
749 		VGIC_ACCESS_32bit),
750 };
751 
vgic_v3_init_dist_iodev(struct vgic_io_device * dev)752 unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev)
753 {
754 	dev->regions = vgic_v3_dist_registers;
755 	dev->nr_regions = ARRAY_SIZE(vgic_v3_dist_registers);
756 
757 	kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops);
758 
759 	return SZ_64K;
760 }
761 
762 /**
763  * vgic_register_redist_iodev - register a single redist iodev
764  * @vcpu:    The VCPU to which the redistributor belongs
765  *
766  * Register a KVM iodev for this VCPU's redistributor using the address
767  * provided.
768  *
769  * Return 0 on success, -ERRNO otherwise.
770  */
vgic_register_redist_iodev(struct kvm_vcpu * vcpu)771 int vgic_register_redist_iodev(struct kvm_vcpu *vcpu)
772 {
773 	struct kvm *kvm = vcpu->kvm;
774 	struct vgic_dist *vgic = &kvm->arch.vgic;
775 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
776 	struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
777 	struct vgic_redist_region *rdreg;
778 	gpa_t rd_base;
779 	int ret = 0;
780 
781 	lockdep_assert_held(&kvm->slots_lock);
782 	mutex_lock(&kvm->arch.config_lock);
783 
784 	if (!IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr))
785 		goto out_unlock;
786 
787 	/*
788 	 * We may be creating VCPUs before having set the base address for the
789 	 * redistributor region, in which case we will come back to this
790 	 * function for all VCPUs when the base address is set.  Just return
791 	 * without doing any work for now.
792 	 */
793 	rdreg = vgic_v3_rdist_free_slot(&vgic->rd_regions);
794 	if (!rdreg)
795 		goto out_unlock;
796 
797 	if (!vgic_v3_check_base(kvm)) {
798 		ret = -EINVAL;
799 		goto out_unlock;
800 	}
801 
802 	vgic_cpu->rdreg = rdreg;
803 	vgic_cpu->rdreg_index = rdreg->free_index;
804 
805 	rd_base = rdreg->base + rdreg->free_index * KVM_VGIC_V3_REDIST_SIZE;
806 
807 	kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops);
808 	rd_dev->base_addr = rd_base;
809 	rd_dev->iodev_type = IODEV_REDIST;
810 	rd_dev->regions = vgic_v3_rd_registers;
811 	rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
812 	rd_dev->redist_vcpu = vcpu;
813 
814 	mutex_unlock(&kvm->arch.config_lock);
815 
816 	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base,
817 				      2 * SZ_64K, &rd_dev->dev);
818 	if (ret)
819 		return ret;
820 
821 	/* Protected by slots_lock */
822 	rdreg->free_index++;
823 	return 0;
824 
825 out_unlock:
826 	mutex_unlock(&kvm->arch.config_lock);
827 	return ret;
828 }
829 
vgic_unregister_redist_iodev(struct kvm_vcpu * vcpu)830 void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu)
831 {
832 	struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
833 
834 	kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev);
835 }
836 
vgic_register_all_redist_iodevs(struct kvm * kvm)837 static int vgic_register_all_redist_iodevs(struct kvm *kvm)
838 {
839 	struct kvm_vcpu *vcpu;
840 	unsigned long c;
841 	int ret = 0;
842 
843 	kvm_for_each_vcpu(c, vcpu, kvm) {
844 		ret = vgic_register_redist_iodev(vcpu);
845 		if (ret)
846 			break;
847 	}
848 
849 	if (ret) {
850 		/* The current c failed, so iterate over the previous ones. */
851 		int i;
852 
853 		for (i = 0; i < c; i++) {
854 			vcpu = kvm_get_vcpu(kvm, i);
855 			vgic_unregister_redist_iodev(vcpu);
856 		}
857 	}
858 
859 	return ret;
860 }
861 
862 /**
863  * vgic_v3_alloc_redist_region - Allocate a new redistributor region
864  *
865  * Performs various checks before inserting the rdist region in the list.
866  * Those tests depend on whether the size of the rdist region is known
867  * (ie. count != 0). The list is sorted by rdist region index.
868  *
869  * @kvm: kvm handle
870  * @index: redist region index
871  * @base: base of the new rdist region
872  * @count: number of redistributors the region is made of (0 in the old style
873  * single region, whose size is induced from the number of vcpus)
874  *
875  * Return 0 on success, < 0 otherwise
876  */
vgic_v3_alloc_redist_region(struct kvm * kvm,uint32_t index,gpa_t base,uint32_t count)877 static int vgic_v3_alloc_redist_region(struct kvm *kvm, uint32_t index,
878 				       gpa_t base, uint32_t count)
879 {
880 	struct vgic_dist *d = &kvm->arch.vgic;
881 	struct vgic_redist_region *rdreg;
882 	struct list_head *rd_regions = &d->rd_regions;
883 	int nr_vcpus = atomic_read(&kvm->online_vcpus);
884 	size_t size = count ? count * KVM_VGIC_V3_REDIST_SIZE
885 			    : nr_vcpus * KVM_VGIC_V3_REDIST_SIZE;
886 	int ret;
887 
888 	/* cross the end of memory ? */
889 	if (base + size < base)
890 		return -EINVAL;
891 
892 	if (list_empty(rd_regions)) {
893 		if (index != 0)
894 			return -EINVAL;
895 	} else {
896 		rdreg = list_last_entry(rd_regions,
897 					struct vgic_redist_region, list);
898 
899 		/* Don't mix single region and discrete redist regions */
900 		if (!count && rdreg->count)
901 			return -EINVAL;
902 
903 		if (!count)
904 			return -EEXIST;
905 
906 		if (index != rdreg->index + 1)
907 			return -EINVAL;
908 	}
909 
910 	/*
911 	 * For legacy single-region redistributor regions (!count),
912 	 * check that the redistributor region does not overlap with the
913 	 * distributor's address space.
914 	 */
915 	if (!count && !IS_VGIC_ADDR_UNDEF(d->vgic_dist_base) &&
916 		vgic_dist_overlap(kvm, base, size))
917 		return -EINVAL;
918 
919 	/* collision with any other rdist region? */
920 	if (vgic_v3_rdist_overlap(kvm, base, size))
921 		return -EINVAL;
922 
923 	rdreg = kzalloc(sizeof(*rdreg), GFP_KERNEL_ACCOUNT);
924 	if (!rdreg)
925 		return -ENOMEM;
926 
927 	rdreg->base = VGIC_ADDR_UNDEF;
928 
929 	ret = vgic_check_iorange(kvm, rdreg->base, base, SZ_64K, size);
930 	if (ret)
931 		goto free;
932 
933 	rdreg->base = base;
934 	rdreg->count = count;
935 	rdreg->free_index = 0;
936 	rdreg->index = index;
937 
938 	list_add_tail(&rdreg->list, rd_regions);
939 	return 0;
940 free:
941 	kfree(rdreg);
942 	return ret;
943 }
944 
vgic_v3_free_redist_region(struct vgic_redist_region * rdreg)945 void vgic_v3_free_redist_region(struct vgic_redist_region *rdreg)
946 {
947 	list_del(&rdreg->list);
948 	kfree(rdreg);
949 }
950 
vgic_v3_set_redist_base(struct kvm * kvm,u32 index,u64 addr,u32 count)951 int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count)
952 {
953 	int ret;
954 
955 	mutex_lock(&kvm->arch.config_lock);
956 	ret = vgic_v3_alloc_redist_region(kvm, index, addr, count);
957 	mutex_unlock(&kvm->arch.config_lock);
958 	if (ret)
959 		return ret;
960 
961 	/*
962 	 * Register iodevs for each existing VCPU.  Adding more VCPUs
963 	 * afterwards will register the iodevs when needed.
964 	 */
965 	ret = vgic_register_all_redist_iodevs(kvm);
966 	if (ret) {
967 		struct vgic_redist_region *rdreg;
968 
969 		mutex_lock(&kvm->arch.config_lock);
970 		rdreg = vgic_v3_rdist_region_from_index(kvm, index);
971 		vgic_v3_free_redist_region(rdreg);
972 		mutex_unlock(&kvm->arch.config_lock);
973 		return ret;
974 	}
975 
976 	return 0;
977 }
978 
vgic_v3_has_attr_regs(struct kvm_device * dev,struct kvm_device_attr * attr)979 int vgic_v3_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr)
980 {
981 	const struct vgic_register_region *region;
982 	struct vgic_io_device iodev;
983 	struct vgic_reg_attr reg_attr;
984 	struct kvm_vcpu *vcpu;
985 	gpa_t addr;
986 	int ret;
987 
988 	ret = vgic_v3_parse_attr(dev, attr, &reg_attr);
989 	if (ret)
990 		return ret;
991 
992 	vcpu = reg_attr.vcpu;
993 	addr = reg_attr.addr;
994 
995 	switch (attr->group) {
996 	case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
997 		iodev.regions = vgic_v3_dist_registers;
998 		iodev.nr_regions = ARRAY_SIZE(vgic_v3_dist_registers);
999 		iodev.base_addr = 0;
1000 		break;
1001 	case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:{
1002 		iodev.regions = vgic_v3_rd_registers;
1003 		iodev.nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
1004 		iodev.base_addr = 0;
1005 		break;
1006 	}
1007 	case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
1008 		return vgic_v3_has_cpu_sysregs_attr(vcpu, attr);
1009 	default:
1010 		return -ENXIO;
1011 	}
1012 
1013 	/* We only support aligned 32-bit accesses. */
1014 	if (addr & 3)
1015 		return -ENXIO;
1016 
1017 	region = vgic_get_mmio_region(vcpu, &iodev, addr, sizeof(u32));
1018 	if (!region)
1019 		return -ENXIO;
1020 
1021 	return 0;
1022 }
1023 /*
1024  * Compare a given affinity (level 1-3 and a level 0 mask, from the SGI
1025  * generation register ICC_SGI1R_EL1) with a given VCPU.
1026  * If the VCPU's MPIDR matches, return the level0 affinity, otherwise
1027  * return -1.
1028  */
match_mpidr(u64 sgi_aff,u16 sgi_cpu_mask,struct kvm_vcpu * vcpu)1029 static int match_mpidr(u64 sgi_aff, u16 sgi_cpu_mask, struct kvm_vcpu *vcpu)
1030 {
1031 	unsigned long affinity;
1032 	int level0;
1033 
1034 	/*
1035 	 * Split the current VCPU's MPIDR into affinity level 0 and the
1036 	 * rest as this is what we have to compare against.
1037 	 */
1038 	affinity = kvm_vcpu_get_mpidr_aff(vcpu);
1039 	level0 = MPIDR_AFFINITY_LEVEL(affinity, 0);
1040 	affinity &= ~MPIDR_LEVEL_MASK;
1041 
1042 	/* bail out if the upper three levels don't match */
1043 	if (sgi_aff != affinity)
1044 		return -1;
1045 
1046 	/* Is this VCPU's bit set in the mask ? */
1047 	if (!(sgi_cpu_mask & BIT(level0)))
1048 		return -1;
1049 
1050 	return level0;
1051 }
1052 
1053 /*
1054  * The ICC_SGI* registers encode the affinity differently from the MPIDR,
1055  * so provide a wrapper to use the existing defines to isolate a certain
1056  * affinity level.
1057  */
1058 #define SGI_AFFINITY_LEVEL(reg, level) \
1059 	((((reg) & ICC_SGI1R_AFFINITY_## level ##_MASK) \
1060 	>> ICC_SGI1R_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level))
1061 
1062 /**
1063  * vgic_v3_dispatch_sgi - handle SGI requests from VCPUs
1064  * @vcpu: The VCPU requesting a SGI
1065  * @reg: The value written into ICC_{ASGI1,SGI0,SGI1}R by that VCPU
1066  * @allow_group1: Does the sysreg access allow generation of G1 SGIs
1067  *
1068  * With GICv3 (and ARE=1) CPUs trigger SGIs by writing to a system register.
1069  * This will trap in sys_regs.c and call this function.
1070  * This ICC_SGI1R_EL1 register contains the upper three affinity levels of the
1071  * target processors as well as a bitmask of 16 Aff0 CPUs.
1072  * If the interrupt routing mode bit is not set, we iterate over all VCPUs to
1073  * check for matching ones. If this bit is set, we signal all, but not the
1074  * calling VCPU.
1075  */
vgic_v3_dispatch_sgi(struct kvm_vcpu * vcpu,u64 reg,bool allow_group1)1076 void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1)
1077 {
1078 	struct kvm *kvm = vcpu->kvm;
1079 	struct kvm_vcpu *c_vcpu;
1080 	u16 target_cpus;
1081 	u64 mpidr;
1082 	int sgi;
1083 	int vcpu_id = vcpu->vcpu_id;
1084 	bool broadcast;
1085 	unsigned long c, flags;
1086 
1087 	sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT;
1088 	broadcast = reg & BIT_ULL(ICC_SGI1R_IRQ_ROUTING_MODE_BIT);
1089 	target_cpus = (reg & ICC_SGI1R_TARGET_LIST_MASK) >> ICC_SGI1R_TARGET_LIST_SHIFT;
1090 	mpidr = SGI_AFFINITY_LEVEL(reg, 3);
1091 	mpidr |= SGI_AFFINITY_LEVEL(reg, 2);
1092 	mpidr |= SGI_AFFINITY_LEVEL(reg, 1);
1093 
1094 	/*
1095 	 * We iterate over all VCPUs to find the MPIDRs matching the request.
1096 	 * If we have handled one CPU, we clear its bit to detect early
1097 	 * if we are already finished. This avoids iterating through all
1098 	 * VCPUs when most of the times we just signal a single VCPU.
1099 	 */
1100 	kvm_for_each_vcpu(c, c_vcpu, kvm) {
1101 		struct vgic_irq *irq;
1102 
1103 		/* Exit early if we have dealt with all requested CPUs */
1104 		if (!broadcast && target_cpus == 0)
1105 			break;
1106 
1107 		/* Don't signal the calling VCPU */
1108 		if (broadcast && c == vcpu_id)
1109 			continue;
1110 
1111 		if (!broadcast) {
1112 			int level0;
1113 
1114 			level0 = match_mpidr(mpidr, target_cpus, c_vcpu);
1115 			if (level0 == -1)
1116 				continue;
1117 
1118 			/* remove this matching VCPU from the mask */
1119 			target_cpus &= ~BIT(level0);
1120 		}
1121 
1122 		irq = vgic_get_irq(vcpu->kvm, c_vcpu, sgi);
1123 
1124 		raw_spin_lock_irqsave(&irq->irq_lock, flags);
1125 
1126 		/*
1127 		 * An access targeting Group0 SGIs can only generate
1128 		 * those, while an access targeting Group1 SGIs can
1129 		 * generate interrupts of either group.
1130 		 */
1131 		if (!irq->group || allow_group1) {
1132 			if (!irq->hw) {
1133 				irq->pending_latch = true;
1134 				vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
1135 			} else {
1136 				/* HW SGI? Ask the GIC to inject it */
1137 				int err;
1138 				err = irq_set_irqchip_state(irq->host_irq,
1139 							    IRQCHIP_STATE_PENDING,
1140 							    true);
1141 				WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
1142 				raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
1143 			}
1144 		} else {
1145 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
1146 		}
1147 
1148 		vgic_put_irq(vcpu->kvm, irq);
1149 	}
1150 }
1151 
vgic_v3_dist_uaccess(struct kvm_vcpu * vcpu,bool is_write,int offset,u32 * val)1152 int vgic_v3_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
1153 			 int offset, u32 *val)
1154 {
1155 	struct vgic_io_device dev = {
1156 		.regions = vgic_v3_dist_registers,
1157 		.nr_regions = ARRAY_SIZE(vgic_v3_dist_registers),
1158 	};
1159 
1160 	return vgic_uaccess(vcpu, &dev, is_write, offset, val);
1161 }
1162 
vgic_v3_redist_uaccess(struct kvm_vcpu * vcpu,bool is_write,int offset,u32 * val)1163 int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
1164 			   int offset, u32 *val)
1165 {
1166 	struct vgic_io_device rd_dev = {
1167 		.regions = vgic_v3_rd_registers,
1168 		.nr_regions = ARRAY_SIZE(vgic_v3_rd_registers),
1169 	};
1170 
1171 	return vgic_uaccess(vcpu, &rd_dev, is_write, offset, val);
1172 }
1173 
vgic_v3_line_level_info_uaccess(struct kvm_vcpu * vcpu,bool is_write,u32 intid,u32 * val)1174 int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write,
1175 				    u32 intid, u32 *val)
1176 {
1177 	if (intid % 32)
1178 		return -EINVAL;
1179 
1180 	if (is_write)
1181 		vgic_write_irq_line_level_info(vcpu, intid, *val);
1182 	else
1183 		*val = vgic_read_irq_line_level_info(vcpu, intid);
1184 
1185 	return 0;
1186 }
1187