53 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_read_group()  local
55 		if (irq->group)  in vgic_mmio_read_group()
58 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_read_group()
64 static void vgic_update_vsgi(struct vgic_irq *irq)  in vgic_update_vsgi()  argument
66 	WARN_ON(its_prop_update_vsgi(irq->host_irq, irq->priority, irq->group));  in vgic_update_vsgi()
77 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_group()  local
79 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_group()
80 		irq->group = !!(val & BIT(i));  in vgic_mmio_write_group()
81 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_write_group()
82 			vgic_update_vsgi(irq);  in vgic_mmio_write_group()
83 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_group()
85 			vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_mmio_write_group()
88 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_group()
105 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_read_enable()  local
107 		if (irq->enabled)  in vgic_mmio_read_enable()
110 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_read_enable()
125 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_senable()  local
127 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_senable()
128 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_write_senable()
129 			if (!irq->enabled) {  in vgic_mmio_write_senable()
132 				irq->enabled = true;  in vgic_mmio_write_senable()
133 				data = &irq_to_desc(irq->host_irq)->irq_data;  in vgic_mmio_write_senable()
135 					enable_irq(irq->host_irq);  in vgic_mmio_write_senable()
138 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_senable()
139 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_senable()
142 		} else if (vgic_irq_is_mapped_level(irq)) {  in vgic_mmio_write_senable()
143 			bool was_high = irq->line_level;  in vgic_mmio_write_senable()
150 			irq->line_level = vgic_get_phys_line_level(irq);  in vgic_mmio_write_senable()
155 			if (!irq->active && was_high && !irq->line_level)  in vgic_mmio_write_senable()
156 				vgic_irq_set_phys_active(irq, false);  in vgic_mmio_write_senable()
158 		irq->enabled = true;  in vgic_mmio_write_senable()
159 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_mmio_write_senable()
161 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_senable()
174 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_cenable()  local
176 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_cenable()
177 		if (irq->hw && vgic_irq_is_sgi(irq->intid) && irq->enabled)  in vgic_mmio_write_cenable()
178 			disable_irq_nosync(irq->host_irq);  in vgic_mmio_write_cenable()
180 		irq->enabled = false;  in vgic_mmio_write_cenable()
182 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_cenable()
183 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_cenable()
196 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_uaccess_write_senable()  local
198 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_uaccess_write_senable()
199 		irq->enabled = true;  in vgic_uaccess_write_senable()
200 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_uaccess_write_senable()
202 		vgic_put_irq(vcpu->kvm, irq);  in vgic_uaccess_write_senable()
217 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_uaccess_write_cenable()  local
219 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_uaccess_write_cenable()
220 		irq->enabled = false;  in vgic_uaccess_write_cenable()
221 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_uaccess_write_cenable()
223 		vgic_put_irq(vcpu->kvm, irq);  in vgic_uaccess_write_cenable()
239 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in __read_pending()  local
252 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in __read_pending()
253 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in __read_pending()
257 			err = irq_get_irqchip_state(irq->host_irq,  in __read_pending()
260 			WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);  in __read_pending()
261 		} else if (!is_user && vgic_irq_is_mapped_level(irq)) {  in __read_pending()
262 			val = vgic_get_phys_line_level(irq);  in __read_pending()
267 					val = irq->pending_latch;  in __read_pending()
272 				val = irq_is_pending(irq);  in __read_pending()
278 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in __read_pending()
280 		vgic_put_irq(vcpu->kvm, irq);  in __read_pending()
298 static bool is_vgic_v2_sgi(struct kvm_vcpu *vcpu, struct vgic_irq *irq)  in is_vgic_v2_sgi()  argument
300 	return (vgic_irq_is_sgi(irq->intid) &&  in is_vgic_v2_sgi()
313 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_spending()  local
316 		if (is_vgic_v2_sgi(vcpu, irq)) {  in vgic_mmio_write_spending()
317 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_spending()
321 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_spending()
323 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_write_spending()
326 			err = irq_set_irqchip_state(irq->host_irq,  in vgic_mmio_write_spending()
329 			WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);  in vgic_mmio_write_spending()
331 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_spending()
332 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_spending()
337 		irq->pending_latch = true;  in vgic_mmio_write_spending()
338 		if (irq->hw)  in vgic_mmio_write_spending()
339 			vgic_irq_set_phys_active(irq, true);  in vgic_mmio_write_spending()
341 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_mmio_write_spending()
342 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_spending()
355 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_uaccess_write_spending()  local
357 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_uaccess_write_spending()
358 		irq->pending_latch = true;  in vgic_uaccess_write_spending()
365 		if (is_vgic_v2_sgi(vcpu, irq))  in vgic_uaccess_write_spending()
366 			irq->source |= BIT(vcpu->vcpu_id);  in vgic_uaccess_write_spending()
368 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_uaccess_write_spending()
370 		vgic_put_irq(vcpu->kvm, irq);  in vgic_uaccess_write_spending()
377 static void vgic_hw_irq_cpending(struct kvm_vcpu *vcpu, struct vgic_irq *irq)  in vgic_hw_irq_cpending()  argument
379 	irq->pending_latch = false;  in vgic_hw_irq_cpending()
392 	vgic_irq_set_phys_pending(irq, false);  in vgic_hw_irq_cpending()
393 	if (!irq->active)  in vgic_hw_irq_cpending()
394 		vgic_irq_set_phys_active(irq, false);  in vgic_hw_irq_cpending()
406 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_cpending()  local
409 		if (is_vgic_v2_sgi(vcpu, irq)) {  in vgic_mmio_write_cpending()
410 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_cpending()
414 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_cpending()
416 		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_write_cpending()
419 			err = irq_set_irqchip_state(irq->host_irq,  in vgic_mmio_write_cpending()
422 			WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);  in vgic_mmio_write_cpending()
424 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_cpending()
425 			vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_cpending()
430 		if (irq->hw)  in vgic_mmio_write_cpending()
431 			vgic_hw_irq_cpending(vcpu, irq);  in vgic_mmio_write_cpending()
433 			irq->pending_latch = false;  in vgic_mmio_write_cpending()
435 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_cpending()
436 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_cpending()
449 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_uaccess_write_cpending()  local
451 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_uaccess_write_cpending()
457 		if (is_vgic_v2_sgi(vcpu, irq))  in vgic_uaccess_write_cpending()
458 			irq->source = 0;  in vgic_uaccess_write_cpending()
460 		irq->pending_latch = false;  in vgic_uaccess_write_cpending()
462 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_uaccess_write_cpending()
464 		vgic_put_irq(vcpu->kvm, irq);  in vgic_uaccess_write_cpending()
512 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in __vgic_mmio_read_active()  local
518 		if (irq->active)  in __vgic_mmio_read_active()
521 		vgic_put_irq(vcpu->kvm, irq);  in __vgic_mmio_read_active()
551 static void vgic_hw_irq_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,  in vgic_hw_irq_change_active()  argument
557 	irq->active = active;  in vgic_hw_irq_change_active()
558 	vgic_irq_set_phys_active(irq, active);  in vgic_hw_irq_change_active()
561 static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,  in vgic_mmio_change_active()  argument
567 	raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_change_active()
569 	if (irq->hw && !vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_change_active()
570 		vgic_hw_irq_change_active(vcpu, irq, active, !requester_vcpu);  in vgic_mmio_change_active()
571 	} else if (irq->hw && vgic_irq_is_sgi(irq->intid)) {  in vgic_mmio_change_active()
577 		irq->active = false;  in vgic_mmio_change_active()
582 		irq->active = active;  in vgic_mmio_change_active()
598 		    active && vgic_irq_is_sgi(irq->intid))  in vgic_mmio_change_active()
599 			irq->active_source = active_source;  in vgic_mmio_change_active()
602 	if (irq->active)  in vgic_mmio_change_active()
603 		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_mmio_change_active()
605 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_change_active()
616 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in __vgic_mmio_write_cactive()  local
617 		vgic_mmio_change_active(vcpu, irq, false);  in __vgic_mmio_write_cactive()
618 		vgic_put_irq(vcpu->kvm, irq);  in __vgic_mmio_write_cactive()
653 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in __vgic_mmio_write_sactive()  local
654 		vgic_mmio_change_active(vcpu, irq, true);  in __vgic_mmio_write_sactive()
655 		vgic_put_irq(vcpu->kvm, irq);  in __vgic_mmio_write_sactive()
690 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_read_priority()  local
692 		val |= (u64)irq->priority << (i * 8);  in vgic_mmio_read_priority()
694 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_read_priority()
716 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_priority()  local
718 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_priority()
720 		irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS);  in vgic_mmio_write_priority()
721 		if (irq->hw && vgic_irq_is_sgi(irq->intid))  in vgic_mmio_write_priority()
722 			vgic_update_vsgi(irq);  in vgic_mmio_write_priority()
723 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_priority()
725 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_priority()
737 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_read_config()  local
739 		if (irq->config == VGIC_CONFIG_EDGE)  in vgic_mmio_read_config()
742 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_read_config()
757 		struct vgic_irq *irq;  in vgic_mmio_write_config()  local
768 		irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_mmio_write_config()
769 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_mmio_write_config()
772 			irq->config = VGIC_CONFIG_EDGE;  in vgic_mmio_write_config()
774 			irq->config = VGIC_CONFIG_LEVEL;  in vgic_mmio_write_config()
776 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_mmio_write_config()
777 		vgic_put_irq(vcpu->kvm, irq);  in vgic_mmio_write_config()
788 		struct vgic_irq *irq;  in vgic_read_irq_line_level_info()  local
793 		irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_read_irq_line_level_info()
794 		if (irq->config == VGIC_CONFIG_LEVEL && irq->line_level)  in vgic_read_irq_line_level_info()
797 		vgic_put_irq(vcpu->kvm, irq);  in vgic_read_irq_line_level_info()
811 		struct vgic_irq *irq;  in vgic_write_irq_line_level_info()  local
817 		irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);  in vgic_write_irq_line_level_info()
825 		raw_spin_lock_irqsave(&irq->irq_lock, flags);  in vgic_write_irq_line_level_info()
826 		irq->line_level = new_level;  in vgic_write_irq_line_level_info()
828 			vgic_queue_irq_unlock(vcpu->kvm, irq, flags);  in vgic_write_irq_line_level_info()
830 			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);  in vgic_write_irq_line_level_info()
832 		vgic_put_irq(vcpu->kvm, irq);  in vgic_write_irq_line_level_info()