1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Xen event channels
4  *
5  * Xen models interrupts with abstract event channels.  Because each
6  * domain gets 1024 event channels, but NR_IRQ is not that large, we
7  * must dynamically map irqs<->event channels.  The event channels
8  * interface with the rest of the kernel by defining a xen interrupt
9  * chip.  When an event is received, it is mapped to an irq and sent
10  * through the normal interrupt processing path.
11  *
12  * There are four kinds of events which can be mapped to an event
13  * channel:
14  *
15  * 1. Inter-domain notifications.  This includes all the virtual
16  *    device events, since they're driven by front-ends in another domain
17  *    (typically dom0).
18  * 2. VIRQs, typically used for timers.  These are per-cpu events.
19  * 3. IPIs.
20  * 4. PIRQs - Hardware interrupts.
21  *
22  * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
23  */
24 
25 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
26 
27 #include <linux/linkage.h>
28 #include <linux/interrupt.h>
29 #include <linux/irq.h>
30 #include <linux/moduleparam.h>
31 #include <linux/string.h>
32 #include <linux/memblock.h>
33 #include <linux/slab.h>
34 #include <linux/irqnr.h>
35 #include <linux/pci.h>
36 #include <linux/rcupdate.h>
37 #include <linux/spinlock.h>
38 #include <linux/cpuhotplug.h>
39 #include <linux/atomic.h>
40 #include <linux/ktime.h>
41 
42 #ifdef CONFIG_X86
43 #include <asm/desc.h>
44 #include <asm/ptrace.h>
45 #include <asm/idtentry.h>
46 #include <asm/irq.h>
47 #include <asm/io_apic.h>
48 #include <asm/i8259.h>
49 #include <asm/xen/cpuid.h>
50 #include <asm/xen/pci.h>
51 #endif
52 #include <asm/sync_bitops.h>
53 #include <asm/xen/hypercall.h>
54 #include <asm/xen/hypervisor.h>
55 #include <xen/page.h>
56 
57 #include <xen/xen.h>
58 #include <xen/hvm.h>
59 #include <xen/xen-ops.h>
60 #include <xen/events.h>
61 #include <xen/interface/xen.h>
62 #include <xen/interface/event_channel.h>
63 #include <xen/interface/hvm/hvm_op.h>
64 #include <xen/interface/hvm/params.h>
65 #include <xen/interface/physdev.h>
66 #include <xen/interface/sched.h>
67 #include <xen/interface/vcpu.h>
68 #include <xen/xenbus.h>
69 #include <asm/hw_irq.h>
70 
71 #include "events_internal.h"
72 
73 #undef MODULE_PARAM_PREFIX
74 #define MODULE_PARAM_PREFIX "xen."
75 
76 /* Interrupt types. */
77 enum xen_irq_type {
78 	IRQT_UNBOUND = 0,
79 	IRQT_PIRQ,
80 	IRQT_VIRQ,
81 	IRQT_IPI,
82 	IRQT_EVTCHN
83 };
84 
85 /*
86  * Packed IRQ information:
87  * type - enum xen_irq_type
88  * event channel - irq->event channel mapping
89  * cpu - cpu this event channel is bound to
90  * index - type-specific information:
91  *    PIRQ - vector, with MSB being "needs EIO", or physical IRQ of the HVM
92  *           guest, or GSI (real passthrough IRQ) of the device.
93  *    VIRQ - virq number
94  *    IPI - IPI vector
95  *    EVTCHN -
96  */
97 struct irq_info {
98 	struct list_head list;
99 	struct list_head eoi_list;
100 	struct rcu_work rwork;
101 	short refcnt;
102 	u8 spurious_cnt;
103 	u8 is_accounted;
104 	short type;		/* type: IRQT_* */
105 	u8 mask_reason;		/* Why is event channel masked */
106 #define EVT_MASK_REASON_EXPLICIT	0x01
107 #define EVT_MASK_REASON_TEMPORARY	0x02
108 #define EVT_MASK_REASON_EOI_PENDING	0x04
109 	u8 is_active;		/* Is event just being handled? */
110 	unsigned irq;
111 	evtchn_port_t evtchn;   /* event channel */
112 	unsigned short cpu;     /* cpu bound */
113 	unsigned short eoi_cpu; /* EOI must happen on this cpu-1 */
114 	unsigned int irq_epoch; /* If eoi_cpu valid: irq_epoch of event */
115 	u64 eoi_time;           /* Time in jiffies when to EOI. */
116 	raw_spinlock_t lock;
117 	bool is_static;           /* Is event channel static */
118 
119 	union {
120 		unsigned short virq;
121 		enum ipi_vector ipi;
122 		struct {
123 			unsigned short pirq;
124 			unsigned short gsi;
125 			unsigned char vector;
126 			unsigned char flags;
127 			uint16_t domid;
128 		} pirq;
129 		struct xenbus_device *interdomain;
130 	} u;
131 };
132 
133 #define PIRQ_NEEDS_EOI	(1 << 0)
134 #define PIRQ_SHAREABLE	(1 << 1)
135 #define PIRQ_MSI_GROUP	(1 << 2)
136 
137 static uint __read_mostly event_loop_timeout = 2;
138 module_param(event_loop_timeout, uint, 0644);
139 
140 static uint __read_mostly event_eoi_delay = 10;
141 module_param(event_eoi_delay, uint, 0644);
142 
143 const struct evtchn_ops *evtchn_ops;
144 
145 /*
146  * This lock protects updates to the following mapping and reference-count
147  * arrays. The lock does not need to be acquired to read the mapping tables.
148  */
149 static DEFINE_MUTEX(irq_mapping_update_lock);
150 
151 /*
152  * Lock hierarchy:
153  *
154  * irq_mapping_update_lock
155  *   IRQ-desc lock
156  *     percpu eoi_list_lock
157  *       irq_info->lock
158  */
159 
160 static LIST_HEAD(xen_irq_list_head);
161 
162 /* IRQ <-> VIRQ mapping. */
163 static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
164 
165 /* IRQ <-> IPI mapping */
166 static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
167 /* Cache for IPI event channels - needed for hot cpu unplug (avoid RCU usage). */
168 static DEFINE_PER_CPU(evtchn_port_t [XEN_NR_IPIS], ipi_to_evtchn) = {[0 ... XEN_NR_IPIS-1] = 0};
169 
170 /* Event channel distribution data */
171 static atomic_t channels_on_cpu[NR_CPUS];
172 
173 static int **evtchn_to_irq;
174 #ifdef CONFIG_X86
175 static unsigned long *pirq_eoi_map;
176 #endif
177 static bool (*pirq_needs_eoi)(unsigned irq);
178 
179 #define EVTCHN_ROW(e)  (e / (PAGE_SIZE/sizeof(**evtchn_to_irq)))
180 #define EVTCHN_COL(e)  (e % (PAGE_SIZE/sizeof(**evtchn_to_irq)))
181 #define EVTCHN_PER_ROW (PAGE_SIZE / sizeof(**evtchn_to_irq))
182 
183 /* Xen will never allocate port zero for any purpose. */
184 #define VALID_EVTCHN(chn)	((chn) != 0)
185 
186 static struct irq_info *legacy_info_ptrs[NR_IRQS_LEGACY];
187 
188 static struct irq_chip xen_dynamic_chip;
189 static struct irq_chip xen_lateeoi_chip;
190 static struct irq_chip xen_percpu_chip;
191 static struct irq_chip xen_pirq_chip;
192 static void enable_dynirq(struct irq_data *data);
193 static void disable_dynirq(struct irq_data *data);
194 
195 static DEFINE_PER_CPU(unsigned int, irq_epoch);
196 
clear_evtchn_to_irq_row(int * evtchn_row)197 static void clear_evtchn_to_irq_row(int *evtchn_row)
198 {
199 	unsigned col;
200 
201 	for (col = 0; col < EVTCHN_PER_ROW; col++)
202 		WRITE_ONCE(evtchn_row[col], -1);
203 }
204 
clear_evtchn_to_irq_all(void)205 static void clear_evtchn_to_irq_all(void)
206 {
207 	unsigned row;
208 
209 	for (row = 0; row < EVTCHN_ROW(xen_evtchn_max_channels()); row++) {
210 		if (evtchn_to_irq[row] == NULL)
211 			continue;
212 		clear_evtchn_to_irq_row(evtchn_to_irq[row]);
213 	}
214 }
215 
set_evtchn_to_irq(evtchn_port_t evtchn,unsigned int irq)216 static int set_evtchn_to_irq(evtchn_port_t evtchn, unsigned int irq)
217 {
218 	unsigned row;
219 	unsigned col;
220 	int *evtchn_row;
221 
222 	if (evtchn >= xen_evtchn_max_channels())
223 		return -EINVAL;
224 
225 	row = EVTCHN_ROW(evtchn);
226 	col = EVTCHN_COL(evtchn);
227 
228 	if (evtchn_to_irq[row] == NULL) {
229 		/* Unallocated irq entries return -1 anyway */
230 		if (irq == -1)
231 			return 0;
232 
233 		evtchn_row = (int *) __get_free_pages(GFP_KERNEL, 0);
234 		if (evtchn_row == NULL)
235 			return -ENOMEM;
236 
237 		clear_evtchn_to_irq_row(evtchn_row);
238 
239 		/*
240 		 * We've prepared an empty row for the mapping. If a different
241 		 * thread was faster inserting it, we can drop ours.
242 		 */
243 		if (cmpxchg(&evtchn_to_irq[row], NULL, evtchn_row) != NULL)
244 			free_page((unsigned long) evtchn_row);
245 	}
246 
247 	WRITE_ONCE(evtchn_to_irq[row][col], irq);
248 	return 0;
249 }
250 
251 /* Get info for IRQ */
info_for_irq(unsigned irq)252 static struct irq_info *info_for_irq(unsigned irq)
253 {
254 	if (irq < nr_legacy_irqs())
255 		return legacy_info_ptrs[irq];
256 	else
257 		return irq_get_chip_data(irq);
258 }
259 
set_info_for_irq(unsigned int irq,struct irq_info * info)260 static void set_info_for_irq(unsigned int irq, struct irq_info *info)
261 {
262 	if (irq < nr_legacy_irqs())
263 		legacy_info_ptrs[irq] = info;
264 	else
265 		irq_set_chip_data(irq, info);
266 }
267 
evtchn_to_info(evtchn_port_t evtchn)268 static struct irq_info *evtchn_to_info(evtchn_port_t evtchn)
269 {
270 	int irq;
271 
272 	if (evtchn >= xen_evtchn_max_channels())
273 		return NULL;
274 	if (evtchn_to_irq[EVTCHN_ROW(evtchn)] == NULL)
275 		return NULL;
276 	irq = READ_ONCE(evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)]);
277 
278 	return (irq < 0) ? NULL : info_for_irq(irq);
279 }
280 
281 /* Per CPU channel accounting */
channels_on_cpu_dec(struct irq_info * info)282 static void channels_on_cpu_dec(struct irq_info *info)
283 {
284 	if (!info->is_accounted)
285 		return;
286 
287 	info->is_accounted = 0;
288 
289 	if (WARN_ON_ONCE(info->cpu >= nr_cpu_ids))
290 		return;
291 
292 	WARN_ON_ONCE(!atomic_add_unless(&channels_on_cpu[info->cpu], -1 , 0));
293 }
294 
channels_on_cpu_inc(struct irq_info * info)295 static void channels_on_cpu_inc(struct irq_info *info)
296 {
297 	if (WARN_ON_ONCE(info->cpu >= nr_cpu_ids))
298 		return;
299 
300 	if (WARN_ON_ONCE(!atomic_add_unless(&channels_on_cpu[info->cpu], 1,
301 					    INT_MAX)))
302 		return;
303 
304 	info->is_accounted = 1;
305 }
306 
xen_irq_free_desc(unsigned int irq)307 static void xen_irq_free_desc(unsigned int irq)
308 {
309 	/* Legacy IRQ descriptors are managed by the arch. */
310 	if (irq >= nr_legacy_irqs())
311 		irq_free_desc(irq);
312 }
313 
delayed_free_irq(struct work_struct * work)314 static void delayed_free_irq(struct work_struct *work)
315 {
316 	struct irq_info *info = container_of(to_rcu_work(work), struct irq_info,
317 					     rwork);
318 	unsigned int irq = info->irq;
319 
320 	/* Remove the info pointer only now, with no potential users left. */
321 	set_info_for_irq(irq, NULL);
322 
323 	kfree(info);
324 
325 	xen_irq_free_desc(irq);
326 }
327 
328 /* Constructors for packed IRQ information. */
xen_irq_info_common_setup(struct irq_info * info,enum xen_irq_type type,evtchn_port_t evtchn,unsigned short cpu)329 static int xen_irq_info_common_setup(struct irq_info *info,
330 				     enum xen_irq_type type,
331 				     evtchn_port_t evtchn,
332 				     unsigned short cpu)
333 {
334 	int ret;
335 
336 	BUG_ON(info->type != IRQT_UNBOUND && info->type != type);
337 
338 	info->type = type;
339 	info->evtchn = evtchn;
340 	info->cpu = cpu;
341 	info->mask_reason = EVT_MASK_REASON_EXPLICIT;
342 	raw_spin_lock_init(&info->lock);
343 
344 	ret = set_evtchn_to_irq(evtchn, info->irq);
345 	if (ret < 0)
346 		return ret;
347 
348 	irq_clear_status_flags(info->irq, IRQ_NOREQUEST | IRQ_NOAUTOEN);
349 
350 	return xen_evtchn_port_setup(evtchn);
351 }
352 
xen_irq_info_evtchn_setup(struct irq_info * info,evtchn_port_t evtchn,struct xenbus_device * dev)353 static int xen_irq_info_evtchn_setup(struct irq_info *info,
354 				     evtchn_port_t evtchn,
355 				     struct xenbus_device *dev)
356 {
357 	int ret;
358 
359 	ret = xen_irq_info_common_setup(info, IRQT_EVTCHN, evtchn, 0);
360 	info->u.interdomain = dev;
361 	if (dev)
362 		atomic_inc(&dev->event_channels);
363 
364 	return ret;
365 }
366 
xen_irq_info_ipi_setup(struct irq_info * info,unsigned int cpu,evtchn_port_t evtchn,enum ipi_vector ipi)367 static int xen_irq_info_ipi_setup(struct irq_info *info, unsigned int cpu,
368 				  evtchn_port_t evtchn, enum ipi_vector ipi)
369 {
370 	info->u.ipi = ipi;
371 
372 	per_cpu(ipi_to_irq, cpu)[ipi] = info->irq;
373 	per_cpu(ipi_to_evtchn, cpu)[ipi] = evtchn;
374 
375 	return xen_irq_info_common_setup(info, IRQT_IPI, evtchn, 0);
376 }
377 
xen_irq_info_virq_setup(struct irq_info * info,unsigned int cpu,evtchn_port_t evtchn,unsigned int virq)378 static int xen_irq_info_virq_setup(struct irq_info *info, unsigned int cpu,
379 				   evtchn_port_t evtchn, unsigned int virq)
380 {
381 	info->u.virq = virq;
382 
383 	per_cpu(virq_to_irq, cpu)[virq] = info->irq;
384 
385 	return xen_irq_info_common_setup(info, IRQT_VIRQ, evtchn, 0);
386 }
387 
xen_irq_info_pirq_setup(struct irq_info * info,evtchn_port_t evtchn,unsigned int pirq,unsigned int gsi,uint16_t domid,unsigned char flags)388 static int xen_irq_info_pirq_setup(struct irq_info *info, evtchn_port_t evtchn,
389 				   unsigned int pirq, unsigned int gsi,
390 				   uint16_t domid, unsigned char flags)
391 {
392 	info->u.pirq.pirq = pirq;
393 	info->u.pirq.gsi = gsi;
394 	info->u.pirq.domid = domid;
395 	info->u.pirq.flags = flags;
396 
397 	return xen_irq_info_common_setup(info, IRQT_PIRQ, evtchn, 0);
398 }
399 
xen_irq_info_cleanup(struct irq_info * info)400 static void xen_irq_info_cleanup(struct irq_info *info)
401 {
402 	set_evtchn_to_irq(info->evtchn, -1);
403 	xen_evtchn_port_remove(info->evtchn, info->cpu);
404 	info->evtchn = 0;
405 	channels_on_cpu_dec(info);
406 }
407 
408 /*
409  * Accessors for packed IRQ information.
410  */
evtchn_from_irq(unsigned int irq)411 static evtchn_port_t evtchn_from_irq(unsigned int irq)
412 {
413 	const struct irq_info *info = NULL;
414 
415 	if (likely(irq < nr_irqs))
416 		info = info_for_irq(irq);
417 	if (!info)
418 		return 0;
419 
420 	return info->evtchn;
421 }
422 
irq_from_evtchn(evtchn_port_t evtchn)423 unsigned int irq_from_evtchn(evtchn_port_t evtchn)
424 {
425 	struct irq_info *info = evtchn_to_info(evtchn);
426 
427 	return info ? info->irq : -1;
428 }
429 EXPORT_SYMBOL_GPL(irq_from_evtchn);
430 
irq_evtchn_from_virq(unsigned int cpu,unsigned int virq,evtchn_port_t * evtchn)431 int irq_evtchn_from_virq(unsigned int cpu, unsigned int virq,
432 			 evtchn_port_t *evtchn)
433 {
434 	int irq = per_cpu(virq_to_irq, cpu)[virq];
435 
436 	*evtchn = evtchn_from_irq(irq);
437 
438 	return irq;
439 }
440 
ipi_from_irq(struct irq_info * info)441 static enum ipi_vector ipi_from_irq(struct irq_info *info)
442 {
443 	BUG_ON(info == NULL);
444 	BUG_ON(info->type != IRQT_IPI);
445 
446 	return info->u.ipi;
447 }
448 
virq_from_irq(struct irq_info * info)449 static unsigned int virq_from_irq(struct irq_info *info)
450 {
451 	BUG_ON(info == NULL);
452 	BUG_ON(info->type != IRQT_VIRQ);
453 
454 	return info->u.virq;
455 }
456 
pirq_from_irq(unsigned irq)457 static unsigned pirq_from_irq(unsigned irq)
458 {
459 	struct irq_info *info = info_for_irq(irq);
460 
461 	BUG_ON(info == NULL);
462 	BUG_ON(info->type != IRQT_PIRQ);
463 
464 	return info->u.pirq.pirq;
465 }
466 
cpu_from_evtchn(evtchn_port_t evtchn)467 unsigned int cpu_from_evtchn(evtchn_port_t evtchn)
468 {
469 	struct irq_info *info = evtchn_to_info(evtchn);
470 
471 	return info ? info->cpu : 0;
472 }
473 
do_mask(struct irq_info * info,u8 reason)474 static void do_mask(struct irq_info *info, u8 reason)
475 {
476 	unsigned long flags;
477 
478 	raw_spin_lock_irqsave(&info->lock, flags);
479 
480 	if (!info->mask_reason)
481 		mask_evtchn(info->evtchn);
482 
483 	info->mask_reason |= reason;
484 
485 	raw_spin_unlock_irqrestore(&info->lock, flags);
486 }
487 
do_unmask(struct irq_info * info,u8 reason)488 static void do_unmask(struct irq_info *info, u8 reason)
489 {
490 	unsigned long flags;
491 
492 	raw_spin_lock_irqsave(&info->lock, flags);
493 
494 	info->mask_reason &= ~reason;
495 
496 	if (!info->mask_reason)
497 		unmask_evtchn(info->evtchn);
498 
499 	raw_spin_unlock_irqrestore(&info->lock, flags);
500 }
501 
502 #ifdef CONFIG_X86
pirq_check_eoi_map(unsigned irq)503 static bool pirq_check_eoi_map(unsigned irq)
504 {
505 	return test_bit(pirq_from_irq(irq), pirq_eoi_map);
506 }
507 #endif
508 
pirq_needs_eoi_flag(unsigned irq)509 static bool pirq_needs_eoi_flag(unsigned irq)
510 {
511 	struct irq_info *info = info_for_irq(irq);
512 	BUG_ON(info->type != IRQT_PIRQ);
513 
514 	return info->u.pirq.flags & PIRQ_NEEDS_EOI;
515 }
516 
bind_evtchn_to_cpu(struct irq_info * info,unsigned int cpu,bool force_affinity)517 static void bind_evtchn_to_cpu(struct irq_info *info, unsigned int cpu,
518 			       bool force_affinity)
519 {
520 	if (IS_ENABLED(CONFIG_SMP) && force_affinity) {
521 		struct irq_data *data = irq_get_irq_data(info->irq);
522 
523 		irq_data_update_affinity(data, cpumask_of(cpu));
524 		irq_data_update_effective_affinity(data, cpumask_of(cpu));
525 	}
526 
527 	xen_evtchn_port_bind_to_cpu(info->evtchn, cpu, info->cpu);
528 
529 	channels_on_cpu_dec(info);
530 	info->cpu = cpu;
531 	channels_on_cpu_inc(info);
532 }
533 
534 /**
535  * notify_remote_via_irq - send event to remote end of event channel via irq
536  * @irq: irq of event channel to send event to
537  *
538  * Unlike notify_remote_via_evtchn(), this is safe to use across
539  * save/restore. Notifications on a broken connection are silently
540  * dropped.
541  */
notify_remote_via_irq(int irq)542 void notify_remote_via_irq(int irq)
543 {
544 	evtchn_port_t evtchn = evtchn_from_irq(irq);
545 
546 	if (VALID_EVTCHN(evtchn))
547 		notify_remote_via_evtchn(evtchn);
548 }
549 EXPORT_SYMBOL_GPL(notify_remote_via_irq);
550 
551 struct lateeoi_work {
552 	struct delayed_work delayed;
553 	spinlock_t eoi_list_lock;
554 	struct list_head eoi_list;
555 };
556 
557 static DEFINE_PER_CPU(struct lateeoi_work, lateeoi);
558 
lateeoi_list_del(struct irq_info * info)559 static void lateeoi_list_del(struct irq_info *info)
560 {
561 	struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);
562 	unsigned long flags;
563 
564 	spin_lock_irqsave(&eoi->eoi_list_lock, flags);
565 	list_del_init(&info->eoi_list);
566 	spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
567 }
568 
lateeoi_list_add(struct irq_info * info)569 static void lateeoi_list_add(struct irq_info *info)
570 {
571 	struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);
572 	struct irq_info *elem;
573 	u64 now = get_jiffies_64();
574 	unsigned long delay;
575 	unsigned long flags;
576 
577 	if (now < info->eoi_time)
578 		delay = info->eoi_time - now;
579 	else
580 		delay = 1;
581 
582 	spin_lock_irqsave(&eoi->eoi_list_lock, flags);
583 
584 	elem = list_first_entry_or_null(&eoi->eoi_list, struct irq_info,
585 					eoi_list);
586 	if (!elem || info->eoi_time < elem->eoi_time) {
587 		list_add(&info->eoi_list, &eoi->eoi_list);
588 		mod_delayed_work_on(info->eoi_cpu, system_wq,
589 				    &eoi->delayed, delay);
590 	} else {
591 		list_for_each_entry_reverse(elem, &eoi->eoi_list, eoi_list) {
592 			if (elem->eoi_time <= info->eoi_time)
593 				break;
594 		}
595 		list_add(&info->eoi_list, &elem->eoi_list);
596 	}
597 
598 	spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
599 }
600 
xen_irq_lateeoi_locked(struct irq_info * info,bool spurious)601 static void xen_irq_lateeoi_locked(struct irq_info *info, bool spurious)
602 {
603 	evtchn_port_t evtchn;
604 	unsigned int cpu;
605 	unsigned int delay = 0;
606 
607 	evtchn = info->evtchn;
608 	if (!VALID_EVTCHN(evtchn) || !list_empty(&info->eoi_list))
609 		return;
610 
611 	if (spurious) {
612 		struct xenbus_device *dev = info->u.interdomain;
613 		unsigned int threshold = 1;
614 
615 		if (dev && dev->spurious_threshold)
616 			threshold = dev->spurious_threshold;
617 
618 		if ((1 << info->spurious_cnt) < (HZ << 2)) {
619 			if (info->spurious_cnt != 0xFF)
620 				info->spurious_cnt++;
621 		}
622 		if (info->spurious_cnt > threshold) {
623 			delay = 1 << (info->spurious_cnt - 1 - threshold);
624 			if (delay > HZ)
625 				delay = HZ;
626 			if (!info->eoi_time)
627 				info->eoi_cpu = smp_processor_id();
628 			info->eoi_time = get_jiffies_64() + delay;
629 			if (dev)
630 				atomic_add(delay, &dev->jiffies_eoi_delayed);
631 		}
632 		if (dev)
633 			atomic_inc(&dev->spurious_events);
634 	} else {
635 		info->spurious_cnt = 0;
636 	}
637 
638 	cpu = info->eoi_cpu;
639 	if (info->eoi_time &&
640 	    (info->irq_epoch == per_cpu(irq_epoch, cpu) || delay)) {
641 		lateeoi_list_add(info);
642 		return;
643 	}
644 
645 	info->eoi_time = 0;
646 
647 	/* is_active hasn't been reset yet, do it now. */
648 	smp_store_release(&info->is_active, 0);
649 	do_unmask(info, EVT_MASK_REASON_EOI_PENDING);
650 }
651 
xen_irq_lateeoi_worker(struct work_struct * work)652 static void xen_irq_lateeoi_worker(struct work_struct *work)
653 {
654 	struct lateeoi_work *eoi;
655 	struct irq_info *info;
656 	u64 now = get_jiffies_64();
657 	unsigned long flags;
658 
659 	eoi = container_of(to_delayed_work(work), struct lateeoi_work, delayed);
660 
661 	rcu_read_lock();
662 
663 	while (true) {
664 		spin_lock_irqsave(&eoi->eoi_list_lock, flags);
665 
666 		info = list_first_entry_or_null(&eoi->eoi_list, struct irq_info,
667 						eoi_list);
668 
669 		if (info == NULL)
670 			break;
671 
672 		if (now < info->eoi_time) {
673 			mod_delayed_work_on(info->eoi_cpu, system_wq,
674 					    &eoi->delayed,
675 					    info->eoi_time - now);
676 			break;
677 		}
678 
679 		list_del_init(&info->eoi_list);
680 
681 		spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
682 
683 		info->eoi_time = 0;
684 
685 		xen_irq_lateeoi_locked(info, false);
686 	}
687 
688 	spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
689 
690 	rcu_read_unlock();
691 }
692 
xen_cpu_init_eoi(unsigned int cpu)693 static void xen_cpu_init_eoi(unsigned int cpu)
694 {
695 	struct lateeoi_work *eoi = &per_cpu(lateeoi, cpu);
696 
697 	INIT_DELAYED_WORK(&eoi->delayed, xen_irq_lateeoi_worker);
698 	spin_lock_init(&eoi->eoi_list_lock);
699 	INIT_LIST_HEAD(&eoi->eoi_list);
700 }
701 
xen_irq_lateeoi(unsigned int irq,unsigned int eoi_flags)702 void xen_irq_lateeoi(unsigned int irq, unsigned int eoi_flags)
703 {
704 	struct irq_info *info;
705 
706 	rcu_read_lock();
707 
708 	info = info_for_irq(irq);
709 
710 	if (info)
711 		xen_irq_lateeoi_locked(info, eoi_flags & XEN_EOI_FLAG_SPURIOUS);
712 
713 	rcu_read_unlock();
714 }
715 EXPORT_SYMBOL_GPL(xen_irq_lateeoi);
716 
xen_irq_init(unsigned int irq)717 static struct irq_info *xen_irq_init(unsigned int irq)
718 {
719 	struct irq_info *info;
720 
721 	info = kzalloc(sizeof(*info), GFP_KERNEL);
722 	if (info) {
723 		info->irq = irq;
724 		info->type = IRQT_UNBOUND;
725 		info->refcnt = -1;
726 		INIT_RCU_WORK(&info->rwork, delayed_free_irq);
727 
728 		set_info_for_irq(irq, info);
729 		/*
730 		 * Interrupt affinity setting can be immediate. No point
731 		 * in delaying it until an interrupt is handled.
732 		 */
733 		irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
734 
735 		INIT_LIST_HEAD(&info->eoi_list);
736 		list_add_tail(&info->list, &xen_irq_list_head);
737 	}
738 
739 	return info;
740 }
741 
xen_allocate_irq_dynamic(void)742 static struct irq_info *xen_allocate_irq_dynamic(void)
743 {
744 	int irq = irq_alloc_desc_from(0, -1);
745 	struct irq_info *info = NULL;
746 
747 	if (irq >= 0) {
748 		info = xen_irq_init(irq);
749 		if (!info)
750 			xen_irq_free_desc(irq);
751 	}
752 
753 	return info;
754 }
755 
xen_allocate_irq_gsi(unsigned int gsi)756 static struct irq_info *xen_allocate_irq_gsi(unsigned int gsi)
757 {
758 	int irq;
759 	struct irq_info *info;
760 
761 	/*
762 	 * A PV guest has no concept of a GSI (since it has no ACPI
763 	 * nor access to/knowledge of the physical APICs). Therefore
764 	 * all IRQs are dynamically allocated from the entire IRQ
765 	 * space.
766 	 */
767 	if (xen_pv_domain() && !xen_initial_domain())
768 		return xen_allocate_irq_dynamic();
769 
770 	/* Legacy IRQ descriptors are already allocated by the arch. */
771 	if (gsi < nr_legacy_irqs())
772 		irq = gsi;
773 	else
774 		irq = irq_alloc_desc_at(gsi, -1);
775 
776 	info = xen_irq_init(irq);
777 	if (!info)
778 		xen_irq_free_desc(irq);
779 
780 	return info;
781 }
782 
xen_free_irq(struct irq_info * info)783 static void xen_free_irq(struct irq_info *info)
784 {
785 	if (WARN_ON(!info))
786 		return;
787 
788 	if (!list_empty(&info->eoi_list))
789 		lateeoi_list_del(info);
790 
791 	list_del(&info->list);
792 
793 	WARN_ON(info->refcnt > 0);
794 
795 	queue_rcu_work(system_wq, &info->rwork);
796 }
797 
798 /* Not called for lateeoi events. */
event_handler_exit(struct irq_info * info)799 static void event_handler_exit(struct irq_info *info)
800 {
801 	smp_store_release(&info->is_active, 0);
802 	clear_evtchn(info->evtchn);
803 }
804 
pirq_query_unmask(int irq)805 static void pirq_query_unmask(int irq)
806 {
807 	struct physdev_irq_status_query irq_status;
808 	struct irq_info *info = info_for_irq(irq);
809 
810 	BUG_ON(info->type != IRQT_PIRQ);
811 
812 	irq_status.irq = pirq_from_irq(irq);
813 	if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
814 		irq_status.flags = 0;
815 
816 	info->u.pirq.flags &= ~PIRQ_NEEDS_EOI;
817 	if (irq_status.flags & XENIRQSTAT_needs_eoi)
818 		info->u.pirq.flags |= PIRQ_NEEDS_EOI;
819 }
820 
eoi_pirq(struct irq_data * data)821 static void eoi_pirq(struct irq_data *data)
822 {
823 	struct irq_info *info = info_for_irq(data->irq);
824 	evtchn_port_t evtchn = info ? info->evtchn : 0;
825 	struct physdev_eoi eoi = { .irq = pirq_from_irq(data->irq) };
826 	int rc = 0;
827 
828 	if (!VALID_EVTCHN(evtchn))
829 		return;
830 
831 	event_handler_exit(info);
832 
833 	if (pirq_needs_eoi(data->irq)) {
834 		rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
835 		WARN_ON(rc);
836 	}
837 }
838 
mask_ack_pirq(struct irq_data * data)839 static void mask_ack_pirq(struct irq_data *data)
840 {
841 	disable_dynirq(data);
842 	eoi_pirq(data);
843 }
844 
__startup_pirq(unsigned int irq)845 static unsigned int __startup_pirq(unsigned int irq)
846 {
847 	struct evtchn_bind_pirq bind_pirq;
848 	struct irq_info *info = info_for_irq(irq);
849 	evtchn_port_t evtchn = evtchn_from_irq(irq);
850 	int rc;
851 
852 	BUG_ON(info->type != IRQT_PIRQ);
853 
854 	if (VALID_EVTCHN(evtchn))
855 		goto out;
856 
857 	bind_pirq.pirq = pirq_from_irq(irq);
858 	/* NB. We are happy to share unless we are probing. */
859 	bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ?
860 					BIND_PIRQ__WILL_SHARE : 0;
861 	rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
862 	if (rc != 0) {
863 		pr_warn("Failed to obtain physical IRQ %d\n", irq);
864 		return 0;
865 	}
866 	evtchn = bind_pirq.port;
867 
868 	pirq_query_unmask(irq);
869 
870 	rc = set_evtchn_to_irq(evtchn, irq);
871 	if (rc)
872 		goto err;
873 
874 	info->evtchn = evtchn;
875 	bind_evtchn_to_cpu(info, 0, false);
876 
877 	rc = xen_evtchn_port_setup(evtchn);
878 	if (rc)
879 		goto err;
880 
881 out:
882 	do_unmask(info, EVT_MASK_REASON_EXPLICIT);
883 
884 	eoi_pirq(irq_get_irq_data(irq));
885 
886 	return 0;
887 
888 err:
889 	pr_err("irq%d: Failed to set port to irq mapping (%d)\n", irq, rc);
890 	xen_evtchn_close(evtchn);
891 	return 0;
892 }
893 
startup_pirq(struct irq_data * data)894 static unsigned int startup_pirq(struct irq_data *data)
895 {
896 	return __startup_pirq(data->irq);
897 }
898 
shutdown_pirq(struct irq_data * data)899 static void shutdown_pirq(struct irq_data *data)
900 {
901 	unsigned int irq = data->irq;
902 	struct irq_info *info = info_for_irq(irq);
903 	evtchn_port_t evtchn = evtchn_from_irq(irq);
904 
905 	BUG_ON(info->type != IRQT_PIRQ);
906 
907 	if (!VALID_EVTCHN(evtchn))
908 		return;
909 
910 	do_mask(info, EVT_MASK_REASON_EXPLICIT);
911 	xen_irq_info_cleanup(info);
912 	xen_evtchn_close(evtchn);
913 }
914 
enable_pirq(struct irq_data * data)915 static void enable_pirq(struct irq_data *data)
916 {
917 	enable_dynirq(data);
918 }
919 
disable_pirq(struct irq_data * data)920 static void disable_pirq(struct irq_data *data)
921 {
922 	disable_dynirq(data);
923 }
924 
xen_irq_from_gsi(unsigned gsi)925 int xen_irq_from_gsi(unsigned gsi)
926 {
927 	struct irq_info *info;
928 
929 	list_for_each_entry(info, &xen_irq_list_head, list) {
930 		if (info->type != IRQT_PIRQ)
931 			continue;
932 
933 		if (info->u.pirq.gsi == gsi)
934 			return info->irq;
935 	}
936 
937 	return -1;
938 }
939 EXPORT_SYMBOL_GPL(xen_irq_from_gsi);
940 
__unbind_from_irq(struct irq_info * info,unsigned int irq)941 static void __unbind_from_irq(struct irq_info *info, unsigned int irq)
942 {
943 	evtchn_port_t evtchn;
944 	bool close_evtchn = false;
945 
946 	if (!info) {
947 		xen_irq_free_desc(irq);
948 		return;
949 	}
950 
951 	if (info->refcnt > 0) {
952 		info->refcnt--;
953 		if (info->refcnt != 0)
954 			return;
955 	}
956 
957 	evtchn = info->evtchn;
958 
959 	if (VALID_EVTCHN(evtchn)) {
960 		unsigned int cpu = info->cpu;
961 		struct xenbus_device *dev;
962 
963 		if (!info->is_static)
964 			close_evtchn = true;
965 
966 		switch (info->type) {
967 		case IRQT_VIRQ:
968 			per_cpu(virq_to_irq, cpu)[virq_from_irq(info)] = -1;
969 			break;
970 		case IRQT_IPI:
971 			per_cpu(ipi_to_irq, cpu)[ipi_from_irq(info)] = -1;
972 			per_cpu(ipi_to_evtchn, cpu)[ipi_from_irq(info)] = 0;
973 			break;
974 		case IRQT_EVTCHN:
975 			dev = info->u.interdomain;
976 			if (dev)
977 				atomic_dec(&dev->event_channels);
978 			break;
979 		default:
980 			break;
981 		}
982 
983 		xen_irq_info_cleanup(info);
984 
985 		if (close_evtchn)
986 			xen_evtchn_close(evtchn);
987 	}
988 
989 	xen_free_irq(info);
990 }
991 
992 /*
993  * Do not make any assumptions regarding the relationship between the
994  * IRQ number returned here and the Xen pirq argument.
995  *
996  * Note: We don't assign an event channel until the irq actually started
997  * up.  Return an existing irq if we've already got one for the gsi.
998  *
999  * Shareable implies level triggered, not shareable implies edge
1000  * triggered here.
1001  */
xen_bind_pirq_gsi_to_irq(unsigned gsi,unsigned pirq,int shareable,char * name)1002 int xen_bind_pirq_gsi_to_irq(unsigned gsi,
1003 			     unsigned pirq, int shareable, char *name)
1004 {
1005 	struct irq_info *info;
1006 	struct physdev_irq irq_op;
1007 	int ret;
1008 
1009 	mutex_lock(&irq_mapping_update_lock);
1010 
1011 	ret = xen_irq_from_gsi(gsi);
1012 	if (ret != -1) {
1013 		pr_info("%s: returning irq %d for gsi %u\n",
1014 			__func__, ret, gsi);
1015 		goto out;
1016 	}
1017 
1018 	info = xen_allocate_irq_gsi(gsi);
1019 	if (!info)
1020 		goto out;
1021 
1022 	irq_op.irq = info->irq;
1023 	irq_op.vector = 0;
1024 
1025 	/* Only the privileged domain can do this. For non-priv, the pcifront
1026 	 * driver provides a PCI bus that does the call to do exactly
1027 	 * this in the priv domain. */
1028 	if (xen_initial_domain() &&
1029 	    HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) {
1030 		xen_free_irq(info);
1031 		ret = -ENOSPC;
1032 		goto out;
1033 	}
1034 
1035 	ret = xen_irq_info_pirq_setup(info, 0, pirq, gsi, DOMID_SELF,
1036 			       shareable ? PIRQ_SHAREABLE : 0);
1037 	if (ret < 0) {
1038 		__unbind_from_irq(info, info->irq);
1039 		goto out;
1040 	}
1041 
1042 	pirq_query_unmask(info->irq);
1043 	/* We try to use the handler with the appropriate semantic for the
1044 	 * type of interrupt: if the interrupt is an edge triggered
1045 	 * interrupt we use handle_edge_irq.
1046 	 *
1047 	 * On the other hand if the interrupt is level triggered we use
1048 	 * handle_fasteoi_irq like the native code does for this kind of
1049 	 * interrupts.
1050 	 *
1051 	 * Depending on the Xen version, pirq_needs_eoi might return true
1052 	 * not only for level triggered interrupts but for edge triggered
1053 	 * interrupts too. In any case Xen always honors the eoi mechanism,
1054 	 * not injecting any more pirqs of the same kind if the first one
1055 	 * hasn't received an eoi yet. Therefore using the fasteoi handler
1056 	 * is the right choice either way.
1057 	 */
1058 	if (shareable)
1059 		irq_set_chip_and_handler_name(info->irq, &xen_pirq_chip,
1060 				handle_fasteoi_irq, name);
1061 	else
1062 		irq_set_chip_and_handler_name(info->irq, &xen_pirq_chip,
1063 				handle_edge_irq, name);
1064 
1065 	ret = info->irq;
1066 
1067 out:
1068 	mutex_unlock(&irq_mapping_update_lock);
1069 
1070 	return ret;
1071 }
1072 
1073 #ifdef CONFIG_PCI_MSI
xen_allocate_pirq_msi(struct pci_dev * dev,struct msi_desc * msidesc)1074 int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc)
1075 {
1076 	int rc;
1077 	struct physdev_get_free_pirq op_get_free_pirq;
1078 
1079 	op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI;
1080 	rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq);
1081 
1082 	WARN_ONCE(rc == -ENOSYS,
1083 		  "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n");
1084 
1085 	return rc ? -1 : op_get_free_pirq.pirq;
1086 }
1087 
xen_bind_pirq_msi_to_irq(struct pci_dev * dev,struct msi_desc * msidesc,int pirq,int nvec,const char * name,domid_t domid)1088 int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc,
1089 			     int pirq, int nvec, const char *name, domid_t domid)
1090 {
1091 	int i, irq, ret;
1092 	struct irq_info *info;
1093 
1094 	mutex_lock(&irq_mapping_update_lock);
1095 
1096 	irq = irq_alloc_descs(-1, 0, nvec, -1);
1097 	if (irq < 0)
1098 		goto out;
1099 
1100 	for (i = 0; i < nvec; i++) {
1101 		info = xen_irq_init(irq + i);
1102 		if (!info) {
1103 			ret = -ENOMEM;
1104 			goto error_irq;
1105 		}
1106 
1107 		irq_set_chip_and_handler_name(irq + i, &xen_pirq_chip, handle_edge_irq, name);
1108 
1109 		ret = xen_irq_info_pirq_setup(info, 0, pirq + i, 0, domid,
1110 					      i == 0 ? 0 : PIRQ_MSI_GROUP);
1111 		if (ret < 0)
1112 			goto error_irq;
1113 	}
1114 
1115 	ret = irq_set_msi_desc(irq, msidesc);
1116 	if (ret < 0)
1117 		goto error_irq;
1118 out:
1119 	mutex_unlock(&irq_mapping_update_lock);
1120 	return irq;
1121 
1122 error_irq:
1123 	while (nvec--) {
1124 		info = info_for_irq(irq + nvec);
1125 		__unbind_from_irq(info, irq + nvec);
1126 	}
1127 	mutex_unlock(&irq_mapping_update_lock);
1128 	return ret;
1129 }
1130 #endif
1131 
xen_destroy_irq(int irq)1132 int xen_destroy_irq(int irq)
1133 {
1134 	struct physdev_unmap_pirq unmap_irq;
1135 	struct irq_info *info = info_for_irq(irq);
1136 	int rc = -ENOENT;
1137 
1138 	mutex_lock(&irq_mapping_update_lock);
1139 
1140 	/*
1141 	 * If trying to remove a vector in a MSI group different
1142 	 * than the first one skip the PIRQ unmap unless this vector
1143 	 * is the first one in the group.
1144 	 */
1145 	if (xen_initial_domain() && !(info->u.pirq.flags & PIRQ_MSI_GROUP)) {
1146 		unmap_irq.pirq = info->u.pirq.pirq;
1147 		unmap_irq.domid = info->u.pirq.domid;
1148 		rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq);
1149 		/* If another domain quits without making the pci_disable_msix
1150 		 * call, the Xen hypervisor takes care of freeing the PIRQs
1151 		 * (free_domain_pirqs).
1152 		 */
1153 		if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF))
1154 			pr_info("domain %d does not have %d anymore\n",
1155 				info->u.pirq.domid, info->u.pirq.pirq);
1156 		else if (rc) {
1157 			pr_warn("unmap irq failed %d\n", rc);
1158 			goto out;
1159 		}
1160 	}
1161 
1162 	xen_free_irq(info);
1163 
1164 out:
1165 	mutex_unlock(&irq_mapping_update_lock);
1166 	return rc;
1167 }
1168 
xen_irq_from_pirq(unsigned pirq)1169 int xen_irq_from_pirq(unsigned pirq)
1170 {
1171 	int irq;
1172 
1173 	struct irq_info *info;
1174 
1175 	mutex_lock(&irq_mapping_update_lock);
1176 
1177 	list_for_each_entry(info, &xen_irq_list_head, list) {
1178 		if (info->type != IRQT_PIRQ)
1179 			continue;
1180 		irq = info->irq;
1181 		if (info->u.pirq.pirq == pirq)
1182 			goto out;
1183 	}
1184 	irq = -1;
1185 out:
1186 	mutex_unlock(&irq_mapping_update_lock);
1187 
1188 	return irq;
1189 }
1190 
1191 
xen_pirq_from_irq(unsigned irq)1192 int xen_pirq_from_irq(unsigned irq)
1193 {
1194 	return pirq_from_irq(irq);
1195 }
1196 EXPORT_SYMBOL_GPL(xen_pirq_from_irq);
1197 
bind_evtchn_to_irq_chip(evtchn_port_t evtchn,struct irq_chip * chip,struct xenbus_device * dev)1198 static int bind_evtchn_to_irq_chip(evtchn_port_t evtchn, struct irq_chip *chip,
1199 				   struct xenbus_device *dev)
1200 {
1201 	int ret = -ENOMEM;
1202 	struct irq_info *info;
1203 
1204 	if (evtchn >= xen_evtchn_max_channels())
1205 		return -ENOMEM;
1206 
1207 	mutex_lock(&irq_mapping_update_lock);
1208 
1209 	info = evtchn_to_info(evtchn);
1210 
1211 	if (!info) {
1212 		info = xen_allocate_irq_dynamic();
1213 		if (!info)
1214 			goto out;
1215 
1216 		irq_set_chip_and_handler_name(info->irq, chip,
1217 					      handle_edge_irq, "event");
1218 
1219 		ret = xen_irq_info_evtchn_setup(info, evtchn, dev);
1220 		if (ret < 0) {
1221 			__unbind_from_irq(info, info->irq);
1222 			goto out;
1223 		}
1224 		/*
1225 		 * New interdomain events are initially bound to vCPU0 This
1226 		 * is required to setup the event channel in the first
1227 		 * place and also important for UP guests because the
1228 		 * affinity setting is not invoked on them so nothing would
1229 		 * bind the channel.
1230 		 */
1231 		bind_evtchn_to_cpu(info, 0, false);
1232 	} else if (!WARN_ON(info->type != IRQT_EVTCHN)) {
1233 		info->refcnt++;
1234 	}
1235 
1236 	ret = info->irq;
1237 
1238 out:
1239 	mutex_unlock(&irq_mapping_update_lock);
1240 
1241 	return ret;
1242 }
1243 
bind_evtchn_to_irq(evtchn_port_t evtchn)1244 int bind_evtchn_to_irq(evtchn_port_t evtchn)
1245 {
1246 	return bind_evtchn_to_irq_chip(evtchn, &xen_dynamic_chip, NULL);
1247 }
1248 EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
1249 
bind_evtchn_to_irq_lateeoi(evtchn_port_t evtchn)1250 int bind_evtchn_to_irq_lateeoi(evtchn_port_t evtchn)
1251 {
1252 	return bind_evtchn_to_irq_chip(evtchn, &xen_lateeoi_chip, NULL);
1253 }
1254 EXPORT_SYMBOL_GPL(bind_evtchn_to_irq_lateeoi);
1255 
bind_ipi_to_irq(unsigned int ipi,unsigned int cpu)1256 static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
1257 {
1258 	struct evtchn_bind_ipi bind_ipi;
1259 	evtchn_port_t evtchn;
1260 	struct irq_info *info;
1261 	int ret;
1262 
1263 	mutex_lock(&irq_mapping_update_lock);
1264 
1265 	ret = per_cpu(ipi_to_irq, cpu)[ipi];
1266 
1267 	if (ret == -1) {
1268 		info = xen_allocate_irq_dynamic();
1269 		if (!info)
1270 			goto out;
1271 
1272 		irq_set_chip_and_handler_name(info->irq, &xen_percpu_chip,
1273 					      handle_percpu_irq, "ipi");
1274 
1275 		bind_ipi.vcpu = xen_vcpu_nr(cpu);
1276 		if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1277 						&bind_ipi) != 0)
1278 			BUG();
1279 		evtchn = bind_ipi.port;
1280 
1281 		ret = xen_irq_info_ipi_setup(info, cpu, evtchn, ipi);
1282 		if (ret < 0) {
1283 			__unbind_from_irq(info, info->irq);
1284 			goto out;
1285 		}
1286 		/*
1287 		 * Force the affinity mask to the target CPU so proc shows
1288 		 * the correct target.
1289 		 */
1290 		bind_evtchn_to_cpu(info, cpu, true);
1291 		ret = info->irq;
1292 	} else {
1293 		info = info_for_irq(ret);
1294 		WARN_ON(info == NULL || info->type != IRQT_IPI);
1295 	}
1296 
1297  out:
1298 	mutex_unlock(&irq_mapping_update_lock);
1299 	return ret;
1300 }
1301 
bind_interdomain_evtchn_to_irq_chip(struct xenbus_device * dev,evtchn_port_t remote_port,struct irq_chip * chip)1302 static int bind_interdomain_evtchn_to_irq_chip(struct xenbus_device *dev,
1303 					       evtchn_port_t remote_port,
1304 					       struct irq_chip *chip)
1305 {
1306 	struct evtchn_bind_interdomain bind_interdomain;
1307 	int err;
1308 
1309 	bind_interdomain.remote_dom  = dev->otherend_id;
1310 	bind_interdomain.remote_port = remote_port;
1311 
1312 	err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
1313 					  &bind_interdomain);
1314 
1315 	return err ? : bind_evtchn_to_irq_chip(bind_interdomain.local_port,
1316 					       chip, dev);
1317 }
1318 
bind_interdomain_evtchn_to_irq_lateeoi(struct xenbus_device * dev,evtchn_port_t remote_port)1319 int bind_interdomain_evtchn_to_irq_lateeoi(struct xenbus_device *dev,
1320 					   evtchn_port_t remote_port)
1321 {
1322 	return bind_interdomain_evtchn_to_irq_chip(dev, remote_port,
1323 						   &xen_lateeoi_chip);
1324 }
1325 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq_lateeoi);
1326 
find_virq(unsigned int virq,unsigned int cpu,evtchn_port_t * evtchn)1327 static int find_virq(unsigned int virq, unsigned int cpu, evtchn_port_t *evtchn)
1328 {
1329 	struct evtchn_status status;
1330 	evtchn_port_t port;
1331 	int rc = -ENOENT;
1332 
1333 	memset(&status, 0, sizeof(status));
1334 	for (port = 0; port < xen_evtchn_max_channels(); port++) {
1335 		status.dom = DOMID_SELF;
1336 		status.port = port;
1337 		rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
1338 		if (rc < 0)
1339 			continue;
1340 		if (status.status != EVTCHNSTAT_virq)
1341 			continue;
1342 		if (status.u.virq == virq && status.vcpu == xen_vcpu_nr(cpu)) {
1343 			*evtchn = port;
1344 			break;
1345 		}
1346 	}
1347 	return rc;
1348 }
1349 
1350 /**
1351  * xen_evtchn_nr_channels - number of usable event channel ports
1352  *
1353  * This may be less than the maximum supported by the current
1354  * hypervisor ABI. Use xen_evtchn_max_channels() for the maximum
1355  * supported.
1356  */
xen_evtchn_nr_channels(void)1357 unsigned xen_evtchn_nr_channels(void)
1358 {
1359         return evtchn_ops->nr_channels();
1360 }
1361 EXPORT_SYMBOL_GPL(xen_evtchn_nr_channels);
1362 
bind_virq_to_irq(unsigned int virq,unsigned int cpu,bool percpu)1363 int bind_virq_to_irq(unsigned int virq, unsigned int cpu, bool percpu)
1364 {
1365 	struct evtchn_bind_virq bind_virq;
1366 	evtchn_port_t evtchn = 0;
1367 	struct irq_info *info;
1368 	int ret;
1369 
1370 	mutex_lock(&irq_mapping_update_lock);
1371 
1372 	ret = per_cpu(virq_to_irq, cpu)[virq];
1373 
1374 	if (ret == -1) {
1375 		info = xen_allocate_irq_dynamic();
1376 		if (!info)
1377 			goto out;
1378 
1379 		if (percpu)
1380 			irq_set_chip_and_handler_name(info->irq, &xen_percpu_chip,
1381 						      handle_percpu_irq, "virq");
1382 		else
1383 			irq_set_chip_and_handler_name(info->irq, &xen_dynamic_chip,
1384 						      handle_edge_irq, "virq");
1385 
1386 		bind_virq.virq = virq;
1387 		bind_virq.vcpu = xen_vcpu_nr(cpu);
1388 		ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1389 						&bind_virq);
1390 		if (ret == 0)
1391 			evtchn = bind_virq.port;
1392 		else {
1393 			if (ret == -EEXIST)
1394 				ret = find_virq(virq, cpu, &evtchn);
1395 			BUG_ON(ret < 0);
1396 		}
1397 
1398 		ret = xen_irq_info_virq_setup(info, cpu, evtchn, virq);
1399 		if (ret < 0) {
1400 			__unbind_from_irq(info, info->irq);
1401 			goto out;
1402 		}
1403 
1404 		/*
1405 		 * Force the affinity mask for percpu interrupts so proc
1406 		 * shows the correct target.
1407 		 */
1408 		bind_evtchn_to_cpu(info, cpu, percpu);
1409 		ret = info->irq;
1410 	} else {
1411 		info = info_for_irq(ret);
1412 		WARN_ON(info == NULL || info->type != IRQT_VIRQ);
1413 	}
1414 
1415 out:
1416 	mutex_unlock(&irq_mapping_update_lock);
1417 
1418 	return ret;
1419 }
1420 
unbind_from_irq(unsigned int irq)1421 static void unbind_from_irq(unsigned int irq)
1422 {
1423 	struct irq_info *info;
1424 
1425 	mutex_lock(&irq_mapping_update_lock);
1426 	info = info_for_irq(irq);
1427 	__unbind_from_irq(info, irq);
1428 	mutex_unlock(&irq_mapping_update_lock);
1429 }
1430 
bind_evtchn_to_irqhandler_chip(evtchn_port_t evtchn,irq_handler_t handler,unsigned long irqflags,const char * devname,void * dev_id,struct irq_chip * chip)1431 static int bind_evtchn_to_irqhandler_chip(evtchn_port_t evtchn,
1432 					  irq_handler_t handler,
1433 					  unsigned long irqflags,
1434 					  const char *devname, void *dev_id,
1435 					  struct irq_chip *chip)
1436 {
1437 	int irq, retval;
1438 
1439 	irq = bind_evtchn_to_irq_chip(evtchn, chip, NULL);
1440 	if (irq < 0)
1441 		return irq;
1442 	retval = request_irq(irq, handler, irqflags, devname, dev_id);
1443 	if (retval != 0) {
1444 		unbind_from_irq(irq);
1445 		return retval;
1446 	}
1447 
1448 	return irq;
1449 }
1450 
bind_evtchn_to_irqhandler(evtchn_port_t evtchn,irq_handler_t handler,unsigned long irqflags,const char * devname,void * dev_id)1451 int bind_evtchn_to_irqhandler(evtchn_port_t evtchn,
1452 			      irq_handler_t handler,
1453 			      unsigned long irqflags,
1454 			      const char *devname, void *dev_id)
1455 {
1456 	return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags,
1457 					      devname, dev_id,
1458 					      &xen_dynamic_chip);
1459 }
1460 EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
1461 
bind_evtchn_to_irqhandler_lateeoi(evtchn_port_t evtchn,irq_handler_t handler,unsigned long irqflags,const char * devname,void * dev_id)1462 int bind_evtchn_to_irqhandler_lateeoi(evtchn_port_t evtchn,
1463 				      irq_handler_t handler,
1464 				      unsigned long irqflags,
1465 				      const char *devname, void *dev_id)
1466 {
1467 	return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags,
1468 					      devname, dev_id,
1469 					      &xen_lateeoi_chip);
1470 }
1471 EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler_lateeoi);
1472 
bind_interdomain_evtchn_to_irqhandler_chip(struct xenbus_device * dev,evtchn_port_t remote_port,irq_handler_t handler,unsigned long irqflags,const char * devname,void * dev_id,struct irq_chip * chip)1473 static int bind_interdomain_evtchn_to_irqhandler_chip(
1474 		struct xenbus_device *dev, evtchn_port_t remote_port,
1475 		irq_handler_t handler, unsigned long irqflags,
1476 		const char *devname, void *dev_id, struct irq_chip *chip)
1477 {
1478 	int irq, retval;
1479 
1480 	irq = bind_interdomain_evtchn_to_irq_chip(dev, remote_port, chip);
1481 	if (irq < 0)
1482 		return irq;
1483 
1484 	retval = request_irq(irq, handler, irqflags, devname, dev_id);
1485 	if (retval != 0) {
1486 		unbind_from_irq(irq);
1487 		return retval;
1488 	}
1489 
1490 	return irq;
1491 }
1492 
bind_interdomain_evtchn_to_irqhandler_lateeoi(struct xenbus_device * dev,evtchn_port_t remote_port,irq_handler_t handler,unsigned long irqflags,const char * devname,void * dev_id)1493 int bind_interdomain_evtchn_to_irqhandler_lateeoi(struct xenbus_device *dev,
1494 						  evtchn_port_t remote_port,
1495 						  irq_handler_t handler,
1496 						  unsigned long irqflags,
1497 						  const char *devname,
1498 						  void *dev_id)
1499 {
1500 	return bind_interdomain_evtchn_to_irqhandler_chip(dev,
1501 				remote_port, handler, irqflags, devname,
1502 				dev_id, &xen_lateeoi_chip);
1503 }
1504 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler_lateeoi);
1505 
bind_virq_to_irqhandler(unsigned int virq,unsigned int cpu,irq_handler_t handler,unsigned long irqflags,const char * devname,void * dev_id)1506 int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
1507 			    irq_handler_t handler,
1508 			    unsigned long irqflags, const char *devname, void *dev_id)
1509 {
1510 	int irq, retval;
1511 
1512 	irq = bind_virq_to_irq(virq, cpu, irqflags & IRQF_PERCPU);
1513 	if (irq < 0)
1514 		return irq;
1515 	retval = request_irq(irq, handler, irqflags, devname, dev_id);
1516 	if (retval != 0) {
1517 		unbind_from_irq(irq);
1518 		return retval;
1519 	}
1520 
1521 	return irq;
1522 }
1523 EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
1524 
bind_ipi_to_irqhandler(enum ipi_vector ipi,unsigned int cpu,irq_handler_t handler,unsigned long irqflags,const char * devname,void * dev_id)1525 int bind_ipi_to_irqhandler(enum ipi_vector ipi,
1526 			   unsigned int cpu,
1527 			   irq_handler_t handler,
1528 			   unsigned long irqflags,
1529 			   const char *devname,
1530 			   void *dev_id)
1531 {
1532 	int irq, retval;
1533 
1534 	irq = bind_ipi_to_irq(ipi, cpu);
1535 	if (irq < 0)
1536 		return irq;
1537 
1538 	irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME;
1539 	retval = request_irq(irq, handler, irqflags, devname, dev_id);
1540 	if (retval != 0) {
1541 		unbind_from_irq(irq);
1542 		return retval;
1543 	}
1544 
1545 	return irq;
1546 }
1547 
unbind_from_irqhandler(unsigned int irq,void * dev_id)1548 void unbind_from_irqhandler(unsigned int irq, void *dev_id)
1549 {
1550 	struct irq_info *info = info_for_irq(irq);
1551 
1552 	if (WARN_ON(!info))
1553 		return;
1554 	free_irq(irq, dev_id);
1555 	unbind_from_irq(irq);
1556 }
1557 EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
1558 
1559 /**
1560  * xen_set_irq_priority() - set an event channel priority.
1561  * @irq:irq bound to an event channel.
1562  * @priority: priority between XEN_IRQ_PRIORITY_MAX and XEN_IRQ_PRIORITY_MIN.
1563  */
xen_set_irq_priority(unsigned irq,unsigned priority)1564 int xen_set_irq_priority(unsigned irq, unsigned priority)
1565 {
1566 	struct evtchn_set_priority set_priority;
1567 
1568 	set_priority.port = evtchn_from_irq(irq);
1569 	set_priority.priority = priority;
1570 
1571 	return HYPERVISOR_event_channel_op(EVTCHNOP_set_priority,
1572 					   &set_priority);
1573 }
1574 EXPORT_SYMBOL_GPL(xen_set_irq_priority);
1575 
evtchn_make_refcounted(evtchn_port_t evtchn,bool is_static)1576 int evtchn_make_refcounted(evtchn_port_t evtchn, bool is_static)
1577 {
1578 	struct irq_info *info = evtchn_to_info(evtchn);
1579 
1580 	if (!info)
1581 		return -ENOENT;
1582 
1583 	WARN_ON(info->refcnt != -1);
1584 
1585 	info->refcnt = 1;
1586 	info->is_static = is_static;
1587 
1588 	return 0;
1589 }
1590 EXPORT_SYMBOL_GPL(evtchn_make_refcounted);
1591 
evtchn_get(evtchn_port_t evtchn)1592 int evtchn_get(evtchn_port_t evtchn)
1593 {
1594 	struct irq_info *info;
1595 	int err = -ENOENT;
1596 
1597 	if (evtchn >= xen_evtchn_max_channels())
1598 		return -EINVAL;
1599 
1600 	mutex_lock(&irq_mapping_update_lock);
1601 
1602 	info = evtchn_to_info(evtchn);
1603 
1604 	if (!info)
1605 		goto done;
1606 
1607 	err = -EINVAL;
1608 	if (info->refcnt <= 0 || info->refcnt == SHRT_MAX)
1609 		goto done;
1610 
1611 	info->refcnt++;
1612 	err = 0;
1613  done:
1614 	mutex_unlock(&irq_mapping_update_lock);
1615 
1616 	return err;
1617 }
1618 EXPORT_SYMBOL_GPL(evtchn_get);
1619 
evtchn_put(evtchn_port_t evtchn)1620 void evtchn_put(evtchn_port_t evtchn)
1621 {
1622 	struct irq_info *info = evtchn_to_info(evtchn);
1623 
1624 	if (WARN_ON(!info))
1625 		return;
1626 	unbind_from_irq(info->irq);
1627 }
1628 EXPORT_SYMBOL_GPL(evtchn_put);
1629 
xen_send_IPI_one(unsigned int cpu,enum ipi_vector vector)1630 void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
1631 {
1632 	evtchn_port_t evtchn;
1633 
1634 #ifdef CONFIG_X86
1635 	if (unlikely(vector == XEN_NMI_VECTOR)) {
1636 		int rc =  HYPERVISOR_vcpu_op(VCPUOP_send_nmi, xen_vcpu_nr(cpu),
1637 					     NULL);
1638 		if (rc < 0)
1639 			printk(KERN_WARNING "Sending nmi to CPU%d failed (rc:%d)\n", cpu, rc);
1640 		return;
1641 	}
1642 #endif
1643 	evtchn = per_cpu(ipi_to_evtchn, cpu)[vector];
1644 	BUG_ON(evtchn == 0);
1645 	notify_remote_via_evtchn(evtchn);
1646 }
1647 
1648 struct evtchn_loop_ctrl {
1649 	ktime_t timeout;
1650 	unsigned count;
1651 	bool defer_eoi;
1652 };
1653 
handle_irq_for_port(evtchn_port_t port,struct evtchn_loop_ctrl * ctrl)1654 void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl)
1655 {
1656 	struct irq_info *info = evtchn_to_info(port);
1657 	struct xenbus_device *dev;
1658 
1659 	if (!info)
1660 		return;
1661 
1662 	/*
1663 	 * Check for timeout every 256 events.
1664 	 * We are setting the timeout value only after the first 256
1665 	 * events in order to not hurt the common case of few loop
1666 	 * iterations. The 256 is basically an arbitrary value.
1667 	 *
1668 	 * In case we are hitting the timeout we need to defer all further
1669 	 * EOIs in order to ensure to leave the event handling loop rather
1670 	 * sooner than later.
1671 	 */
1672 	if (!ctrl->defer_eoi && !(++ctrl->count & 0xff)) {
1673 		ktime_t kt = ktime_get();
1674 
1675 		if (!ctrl->timeout) {
1676 			kt = ktime_add_ms(kt,
1677 					  jiffies_to_msecs(event_loop_timeout));
1678 			ctrl->timeout = kt;
1679 		} else if (kt > ctrl->timeout) {
1680 			ctrl->defer_eoi = true;
1681 		}
1682 	}
1683 
1684 	if (xchg_acquire(&info->is_active, 1))
1685 		return;
1686 
1687 	dev = (info->type == IRQT_EVTCHN) ? info->u.interdomain : NULL;
1688 	if (dev)
1689 		atomic_inc(&dev->events);
1690 
1691 	if (ctrl->defer_eoi) {
1692 		info->eoi_cpu = smp_processor_id();
1693 		info->irq_epoch = __this_cpu_read(irq_epoch);
1694 		info->eoi_time = get_jiffies_64() + event_eoi_delay;
1695 	}
1696 
1697 	generic_handle_irq(info->irq);
1698 }
1699 
xen_evtchn_do_upcall(void)1700 int xen_evtchn_do_upcall(void)
1701 {
1702 	struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
1703 	int ret = vcpu_info->evtchn_upcall_pending ? IRQ_HANDLED : IRQ_NONE;
1704 	int cpu = smp_processor_id();
1705 	struct evtchn_loop_ctrl ctrl = { 0 };
1706 
1707 	/*
1708 	 * When closing an event channel the associated IRQ must not be freed
1709 	 * until all cpus have left the event handling loop. This is ensured
1710 	 * by taking the rcu_read_lock() while handling events, as freeing of
1711 	 * the IRQ is handled via queue_rcu_work() _after_ closing the event
1712 	 * channel.
1713 	 */
1714 	rcu_read_lock();
1715 
1716 	do {
1717 		vcpu_info->evtchn_upcall_pending = 0;
1718 
1719 		xen_evtchn_handle_events(cpu, &ctrl);
1720 
1721 		BUG_ON(!irqs_disabled());
1722 
1723 		virt_rmb(); /* Hypervisor can set upcall pending. */
1724 
1725 	} while (vcpu_info->evtchn_upcall_pending);
1726 
1727 	rcu_read_unlock();
1728 
1729 	/*
1730 	 * Increment irq_epoch only now to defer EOIs only for
1731 	 * xen_irq_lateeoi() invocations occurring from inside the loop
1732 	 * above.
1733 	 */
1734 	__this_cpu_inc(irq_epoch);
1735 
1736 	return ret;
1737 }
1738 EXPORT_SYMBOL_GPL(xen_evtchn_do_upcall);
1739 
1740 /* Rebind a new event channel to an existing irq. */
rebind_evtchn_irq(evtchn_port_t evtchn,int irq)1741 void rebind_evtchn_irq(evtchn_port_t evtchn, int irq)
1742 {
1743 	struct irq_info *info = info_for_irq(irq);
1744 
1745 	if (WARN_ON(!info))
1746 		return;
1747 
1748 	/* Make sure the irq is masked, since the new event channel
1749 	   will also be masked. */
1750 	disable_irq(irq);
1751 
1752 	mutex_lock(&irq_mapping_update_lock);
1753 
1754 	/* After resume the irq<->evtchn mappings are all cleared out */
1755 	BUG_ON(evtchn_to_info(evtchn));
1756 	/* Expect irq to have been bound before,
1757 	   so there should be a proper type */
1758 	BUG_ON(info->type == IRQT_UNBOUND);
1759 
1760 	info->irq = irq;
1761 	(void)xen_irq_info_evtchn_setup(info, evtchn, NULL);
1762 
1763 	mutex_unlock(&irq_mapping_update_lock);
1764 
1765 	bind_evtchn_to_cpu(info, info->cpu, false);
1766 
1767 	/* Unmask the event channel. */
1768 	enable_irq(irq);
1769 }
1770 
1771 /* Rebind an evtchn so that it gets delivered to a specific cpu */
xen_rebind_evtchn_to_cpu(struct irq_info * info,unsigned int tcpu)1772 static int xen_rebind_evtchn_to_cpu(struct irq_info *info, unsigned int tcpu)
1773 {
1774 	struct evtchn_bind_vcpu bind_vcpu;
1775 	evtchn_port_t evtchn = info ? info->evtchn : 0;
1776 
1777 	if (!VALID_EVTCHN(evtchn))
1778 		return -1;
1779 
1780 	if (!xen_support_evtchn_rebind())
1781 		return -1;
1782 
1783 	/* Send future instances of this interrupt to other vcpu. */
1784 	bind_vcpu.port = evtchn;
1785 	bind_vcpu.vcpu = xen_vcpu_nr(tcpu);
1786 
1787 	/*
1788 	 * Mask the event while changing the VCPU binding to prevent
1789 	 * it being delivered on an unexpected VCPU.
1790 	 */
1791 	do_mask(info, EVT_MASK_REASON_TEMPORARY);
1792 
1793 	/*
1794 	 * If this fails, it usually just indicates that we're dealing with a
1795 	 * virq or IPI channel, which don't actually need to be rebound. Ignore
1796 	 * it, but don't do the xenlinux-level rebind in that case.
1797 	 */
1798 	if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
1799 		bind_evtchn_to_cpu(info, tcpu, false);
1800 
1801 	do_unmask(info, EVT_MASK_REASON_TEMPORARY);
1802 
1803 	return 0;
1804 }
1805 
1806 /*
1807  * Find the CPU within @dest mask which has the least number of channels
1808  * assigned. This is not precise as the per cpu counts can be modified
1809  * concurrently.
1810  */
select_target_cpu(const struct cpumask * dest)1811 static unsigned int select_target_cpu(const struct cpumask *dest)
1812 {
1813 	unsigned int cpu, best_cpu = UINT_MAX, minch = UINT_MAX;
1814 
1815 	for_each_cpu_and(cpu, dest, cpu_online_mask) {
1816 		unsigned int curch = atomic_read(&channels_on_cpu[cpu]);
1817 
1818 		if (curch < minch) {
1819 			minch = curch;
1820 			best_cpu = cpu;
1821 		}
1822 	}
1823 
1824 	/*
1825 	 * Catch the unlikely case that dest contains no online CPUs. Can't
1826 	 * recurse.
1827 	 */
1828 	if (best_cpu == UINT_MAX)
1829 		return select_target_cpu(cpu_online_mask);
1830 
1831 	return best_cpu;
1832 }
1833 
set_affinity_irq(struct irq_data * data,const struct cpumask * dest,bool force)1834 static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest,
1835 			    bool force)
1836 {
1837 	unsigned int tcpu = select_target_cpu(dest);
1838 	int ret;
1839 
1840 	ret = xen_rebind_evtchn_to_cpu(info_for_irq(data->irq), tcpu);
1841 	if (!ret)
1842 		irq_data_update_effective_affinity(data, cpumask_of(tcpu));
1843 
1844 	return ret;
1845 }
1846 
enable_dynirq(struct irq_data * data)1847 static void enable_dynirq(struct irq_data *data)
1848 {
1849 	struct irq_info *info = info_for_irq(data->irq);
1850 	evtchn_port_t evtchn = info ? info->evtchn : 0;
1851 
1852 	if (VALID_EVTCHN(evtchn))
1853 		do_unmask(info, EVT_MASK_REASON_EXPLICIT);
1854 }
1855 
disable_dynirq(struct irq_data * data)1856 static void disable_dynirq(struct irq_data *data)
1857 {
1858 	struct irq_info *info = info_for_irq(data->irq);
1859 	evtchn_port_t evtchn = info ? info->evtchn : 0;
1860 
1861 	if (VALID_EVTCHN(evtchn))
1862 		do_mask(info, EVT_MASK_REASON_EXPLICIT);
1863 }
1864 
ack_dynirq(struct irq_data * data)1865 static void ack_dynirq(struct irq_data *data)
1866 {
1867 	struct irq_info *info = info_for_irq(data->irq);
1868 	evtchn_port_t evtchn = info ? info->evtchn : 0;
1869 
1870 	if (VALID_EVTCHN(evtchn))
1871 		event_handler_exit(info);
1872 }
1873 
mask_ack_dynirq(struct irq_data * data)1874 static void mask_ack_dynirq(struct irq_data *data)
1875 {
1876 	disable_dynirq(data);
1877 	ack_dynirq(data);
1878 }
1879 
lateeoi_ack_dynirq(struct irq_data * data)1880 static void lateeoi_ack_dynirq(struct irq_data *data)
1881 {
1882 	struct irq_info *info = info_for_irq(data->irq);
1883 	evtchn_port_t evtchn = info ? info->evtchn : 0;
1884 
1885 	if (VALID_EVTCHN(evtchn)) {
1886 		do_mask(info, EVT_MASK_REASON_EOI_PENDING);
1887 		/*
1888 		 * Don't call event_handler_exit().
1889 		 * Need to keep is_active non-zero in order to ignore re-raised
1890 		 * events after cpu affinity changes while a lateeoi is pending.
1891 		 */
1892 		clear_evtchn(evtchn);
1893 	}
1894 }
1895 
lateeoi_mask_ack_dynirq(struct irq_data * data)1896 static void lateeoi_mask_ack_dynirq(struct irq_data *data)
1897 {
1898 	struct irq_info *info = info_for_irq(data->irq);
1899 	evtchn_port_t evtchn = info ? info->evtchn : 0;
1900 
1901 	if (VALID_EVTCHN(evtchn)) {
1902 		do_mask(info, EVT_MASK_REASON_EXPLICIT);
1903 		event_handler_exit(info);
1904 	}
1905 }
1906 
retrigger_dynirq(struct irq_data * data)1907 static int retrigger_dynirq(struct irq_data *data)
1908 {
1909 	struct irq_info *info = info_for_irq(data->irq);
1910 	evtchn_port_t evtchn = info ? info->evtchn : 0;
1911 
1912 	if (!VALID_EVTCHN(evtchn))
1913 		return 0;
1914 
1915 	do_mask(info, EVT_MASK_REASON_TEMPORARY);
1916 	set_evtchn(evtchn);
1917 	do_unmask(info, EVT_MASK_REASON_TEMPORARY);
1918 
1919 	return 1;
1920 }
1921 
restore_pirqs(void)1922 static void restore_pirqs(void)
1923 {
1924 	int pirq, rc, irq, gsi;
1925 	struct physdev_map_pirq map_irq;
1926 	struct irq_info *info;
1927 
1928 	list_for_each_entry(info, &xen_irq_list_head, list) {
1929 		if (info->type != IRQT_PIRQ)
1930 			continue;
1931 
1932 		pirq = info->u.pirq.pirq;
1933 		gsi = info->u.pirq.gsi;
1934 		irq = info->irq;
1935 
1936 		/* save/restore of PT devices doesn't work, so at this point the
1937 		 * only devices present are GSI based emulated devices */
1938 		if (!gsi)
1939 			continue;
1940 
1941 		map_irq.domid = DOMID_SELF;
1942 		map_irq.type = MAP_PIRQ_TYPE_GSI;
1943 		map_irq.index = gsi;
1944 		map_irq.pirq = pirq;
1945 
1946 		rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq);
1947 		if (rc) {
1948 			pr_warn("xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n",
1949 				gsi, irq, pirq, rc);
1950 			xen_free_irq(info);
1951 			continue;
1952 		}
1953 
1954 		printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq);
1955 
1956 		__startup_pirq(irq);
1957 	}
1958 }
1959 
restore_cpu_virqs(unsigned int cpu)1960 static void restore_cpu_virqs(unsigned int cpu)
1961 {
1962 	struct evtchn_bind_virq bind_virq;
1963 	evtchn_port_t evtchn;
1964 	struct irq_info *info;
1965 	int virq, irq;
1966 
1967 	for (virq = 0; virq < NR_VIRQS; virq++) {
1968 		if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
1969 			continue;
1970 		info = info_for_irq(irq);
1971 
1972 		BUG_ON(virq_from_irq(info) != virq);
1973 
1974 		/* Get a new binding from Xen. */
1975 		bind_virq.virq = virq;
1976 		bind_virq.vcpu = xen_vcpu_nr(cpu);
1977 		if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1978 						&bind_virq) != 0)
1979 			BUG();
1980 		evtchn = bind_virq.port;
1981 
1982 		/* Record the new mapping. */
1983 		xen_irq_info_virq_setup(info, cpu, evtchn, virq);
1984 		/* The affinity mask is still valid */
1985 		bind_evtchn_to_cpu(info, cpu, false);
1986 	}
1987 }
1988 
restore_cpu_ipis(unsigned int cpu)1989 static void restore_cpu_ipis(unsigned int cpu)
1990 {
1991 	struct evtchn_bind_ipi bind_ipi;
1992 	evtchn_port_t evtchn;
1993 	struct irq_info *info;
1994 	int ipi, irq;
1995 
1996 	for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
1997 		if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
1998 			continue;
1999 		info = info_for_irq(irq);
2000 
2001 		BUG_ON(ipi_from_irq(info) != ipi);
2002 
2003 		/* Get a new binding from Xen. */
2004 		bind_ipi.vcpu = xen_vcpu_nr(cpu);
2005 		if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
2006 						&bind_ipi) != 0)
2007 			BUG();
2008 		evtchn = bind_ipi.port;
2009 
2010 		/* Record the new mapping. */
2011 		xen_irq_info_ipi_setup(info, cpu, evtchn, ipi);
2012 		/* The affinity mask is still valid */
2013 		bind_evtchn_to_cpu(info, cpu, false);
2014 	}
2015 }
2016 
2017 /* Clear an irq's pending state, in preparation for polling on it */
xen_clear_irq_pending(int irq)2018 void xen_clear_irq_pending(int irq)
2019 {
2020 	struct irq_info *info = info_for_irq(irq);
2021 	evtchn_port_t evtchn = info ? info->evtchn : 0;
2022 
2023 	if (VALID_EVTCHN(evtchn))
2024 		event_handler_exit(info);
2025 }
2026 EXPORT_SYMBOL(xen_clear_irq_pending);
xen_set_irq_pending(int irq)2027 void xen_set_irq_pending(int irq)
2028 {
2029 	evtchn_port_t evtchn = evtchn_from_irq(irq);
2030 
2031 	if (VALID_EVTCHN(evtchn))
2032 		set_evtchn(evtchn);
2033 }
2034 
xen_test_irq_pending(int irq)2035 bool xen_test_irq_pending(int irq)
2036 {
2037 	evtchn_port_t evtchn = evtchn_from_irq(irq);
2038 	bool ret = false;
2039 
2040 	if (VALID_EVTCHN(evtchn))
2041 		ret = test_evtchn(evtchn);
2042 
2043 	return ret;
2044 }
2045 
2046 /* Poll waiting for an irq to become pending with timeout.  In the usual case,
2047  * the irq will be disabled so it won't deliver an interrupt. */
xen_poll_irq_timeout(int irq,u64 timeout)2048 void xen_poll_irq_timeout(int irq, u64 timeout)
2049 {
2050 	evtchn_port_t evtchn = evtchn_from_irq(irq);
2051 
2052 	if (VALID_EVTCHN(evtchn)) {
2053 		struct sched_poll poll;
2054 
2055 		poll.nr_ports = 1;
2056 		poll.timeout = timeout;
2057 		set_xen_guest_handle(poll.ports, &evtchn);
2058 
2059 		if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
2060 			BUG();
2061 	}
2062 }
2063 EXPORT_SYMBOL(xen_poll_irq_timeout);
2064 /* Poll waiting for an irq to become pending.  In the usual case, the
2065  * irq will be disabled so it won't deliver an interrupt. */
xen_poll_irq(int irq)2066 void xen_poll_irq(int irq)
2067 {
2068 	xen_poll_irq_timeout(irq, 0 /* no timeout */);
2069 }
2070 
2071 /* Check whether the IRQ line is shared with other guests. */
xen_test_irq_shared(int irq)2072 int xen_test_irq_shared(int irq)
2073 {
2074 	struct irq_info *info = info_for_irq(irq);
2075 	struct physdev_irq_status_query irq_status;
2076 
2077 	if (WARN_ON(!info))
2078 		return -ENOENT;
2079 
2080 	irq_status.irq = info->u.pirq.pirq;
2081 
2082 	if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
2083 		return 0;
2084 	return !(irq_status.flags & XENIRQSTAT_shared);
2085 }
2086 EXPORT_SYMBOL_GPL(xen_test_irq_shared);
2087 
xen_irq_resume(void)2088 void xen_irq_resume(void)
2089 {
2090 	unsigned int cpu;
2091 	struct irq_info *info;
2092 
2093 	/* New event-channel space is not 'live' yet. */
2094 	xen_evtchn_resume();
2095 
2096 	/* No IRQ <-> event-channel mappings. */
2097 	list_for_each_entry(info, &xen_irq_list_head, list) {
2098 		/* Zap event-channel binding */
2099 		info->evtchn = 0;
2100 		/* Adjust accounting */
2101 		channels_on_cpu_dec(info);
2102 	}
2103 
2104 	clear_evtchn_to_irq_all();
2105 
2106 	for_each_possible_cpu(cpu) {
2107 		restore_cpu_virqs(cpu);
2108 		restore_cpu_ipis(cpu);
2109 	}
2110 
2111 	restore_pirqs();
2112 }
2113 
2114 static struct irq_chip xen_dynamic_chip __read_mostly = {
2115 	.name			= "xen-dyn",
2116 
2117 	.irq_disable		= disable_dynirq,
2118 	.irq_mask		= disable_dynirq,
2119 	.irq_unmask		= enable_dynirq,
2120 
2121 	.irq_ack		= ack_dynirq,
2122 	.irq_mask_ack		= mask_ack_dynirq,
2123 
2124 	.irq_set_affinity	= set_affinity_irq,
2125 	.irq_retrigger		= retrigger_dynirq,
2126 };
2127 
2128 static struct irq_chip xen_lateeoi_chip __read_mostly = {
2129 	/* The chip name needs to contain "xen-dyn" for irqbalance to work. */
2130 	.name			= "xen-dyn-lateeoi",
2131 
2132 	.irq_disable		= disable_dynirq,
2133 	.irq_mask		= disable_dynirq,
2134 	.irq_unmask		= enable_dynirq,
2135 
2136 	.irq_ack		= lateeoi_ack_dynirq,
2137 	.irq_mask_ack		= lateeoi_mask_ack_dynirq,
2138 
2139 	.irq_set_affinity	= set_affinity_irq,
2140 	.irq_retrigger		= retrigger_dynirq,
2141 };
2142 
2143 static struct irq_chip xen_pirq_chip __read_mostly = {
2144 	.name			= "xen-pirq",
2145 
2146 	.irq_startup		= startup_pirq,
2147 	.irq_shutdown		= shutdown_pirq,
2148 	.irq_enable		= enable_pirq,
2149 	.irq_disable		= disable_pirq,
2150 
2151 	.irq_mask		= disable_dynirq,
2152 	.irq_unmask		= enable_dynirq,
2153 
2154 	.irq_ack		= eoi_pirq,
2155 	.irq_eoi		= eoi_pirq,
2156 	.irq_mask_ack		= mask_ack_pirq,
2157 
2158 	.irq_set_affinity	= set_affinity_irq,
2159 
2160 	.irq_retrigger		= retrigger_dynirq,
2161 };
2162 
2163 static struct irq_chip xen_percpu_chip __read_mostly = {
2164 	.name			= "xen-percpu",
2165 
2166 	.irq_disable		= disable_dynirq,
2167 	.irq_mask		= disable_dynirq,
2168 	.irq_unmask		= enable_dynirq,
2169 
2170 	.irq_ack		= ack_dynirq,
2171 };
2172 
2173 #ifdef CONFIG_X86
2174 #ifdef CONFIG_XEN_PVHVM
2175 /* Vector callbacks are better than PCI interrupts to receive event
2176  * channel notifications because we can receive vector callbacks on any
2177  * vcpu and we don't need PCI support or APIC interactions. */
xen_setup_callback_vector(void)2178 void xen_setup_callback_vector(void)
2179 {
2180 	uint64_t callback_via;
2181 
2182 	if (xen_have_vector_callback) {
2183 		callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR);
2184 		if (xen_set_callback_via(callback_via)) {
2185 			pr_err("Request for Xen HVM callback vector failed\n");
2186 			xen_have_vector_callback = false;
2187 		}
2188 	}
2189 }
2190 
2191 /*
2192  * Setup per-vCPU vector-type callbacks. If this setup is unavailable,
2193  * fallback to the global vector-type callback.
2194  */
xen_init_setup_upcall_vector(void)2195 static __init void xen_init_setup_upcall_vector(void)
2196 {
2197 	if (!xen_have_vector_callback)
2198 		return;
2199 
2200 	if ((cpuid_eax(xen_cpuid_base() + 4) & XEN_HVM_CPUID_UPCALL_VECTOR) &&
2201 	    !xen_set_upcall_vector(0))
2202 		xen_percpu_upcall = true;
2203 	else if (xen_feature(XENFEAT_hvm_callback_vector))
2204 		xen_setup_callback_vector();
2205 	else
2206 		xen_have_vector_callback = false;
2207 }
2208 
xen_set_upcall_vector(unsigned int cpu)2209 int xen_set_upcall_vector(unsigned int cpu)
2210 {
2211 	int rc;
2212 	xen_hvm_evtchn_upcall_vector_t op = {
2213 		.vector = HYPERVISOR_CALLBACK_VECTOR,
2214 		.vcpu = per_cpu(xen_vcpu_id, cpu),
2215 	};
2216 
2217 	rc = HYPERVISOR_hvm_op(HVMOP_set_evtchn_upcall_vector, &op);
2218 	if (rc)
2219 		return rc;
2220 
2221 	/* Trick toolstack to think we are enlightened. */
2222 	if (!cpu)
2223 		rc = xen_set_callback_via(1);
2224 
2225 	return rc;
2226 }
2227 
xen_alloc_callback_vector(void)2228 static __init void xen_alloc_callback_vector(void)
2229 {
2230 	if (!xen_have_vector_callback)
2231 		return;
2232 
2233 	pr_info("Xen HVM callback vector for event delivery is enabled\n");
2234 	alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_xen_hvm_callback);
2235 }
2236 #else
xen_setup_callback_vector(void)2237 void xen_setup_callback_vector(void) {}
xen_init_setup_upcall_vector(void)2238 static inline void xen_init_setup_upcall_vector(void) {}
xen_set_upcall_vector(unsigned int cpu)2239 int xen_set_upcall_vector(unsigned int cpu) {}
xen_alloc_callback_vector(void)2240 static inline void xen_alloc_callback_vector(void) {}
2241 #endif /* CONFIG_XEN_PVHVM */
2242 #endif /* CONFIG_X86 */
2243 
2244 bool xen_fifo_events = true;
2245 module_param_named(fifo_events, xen_fifo_events, bool, 0);
2246 
xen_evtchn_cpu_prepare(unsigned int cpu)2247 static int xen_evtchn_cpu_prepare(unsigned int cpu)
2248 {
2249 	int ret = 0;
2250 
2251 	xen_cpu_init_eoi(cpu);
2252 
2253 	if (evtchn_ops->percpu_init)
2254 		ret = evtchn_ops->percpu_init(cpu);
2255 
2256 	return ret;
2257 }
2258 
xen_evtchn_cpu_dead(unsigned int cpu)2259 static int xen_evtchn_cpu_dead(unsigned int cpu)
2260 {
2261 	int ret = 0;
2262 
2263 	if (evtchn_ops->percpu_deinit)
2264 		ret = evtchn_ops->percpu_deinit(cpu);
2265 
2266 	return ret;
2267 }
2268 
xen_init_IRQ(void)2269 void __init xen_init_IRQ(void)
2270 {
2271 	int ret = -EINVAL;
2272 	evtchn_port_t evtchn;
2273 
2274 	if (xen_fifo_events)
2275 		ret = xen_evtchn_fifo_init();
2276 	if (ret < 0) {
2277 		xen_evtchn_2l_init();
2278 		xen_fifo_events = false;
2279 	}
2280 
2281 	xen_cpu_init_eoi(smp_processor_id());
2282 
2283 	cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE,
2284 				  "xen/evtchn:prepare",
2285 				  xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead);
2286 
2287 	evtchn_to_irq = kcalloc(EVTCHN_ROW(xen_evtchn_max_channels()),
2288 				sizeof(*evtchn_to_irq), GFP_KERNEL);
2289 	BUG_ON(!evtchn_to_irq);
2290 
2291 	/* No event channels are 'live' right now. */
2292 	for (evtchn = 0; evtchn < xen_evtchn_nr_channels(); evtchn++)
2293 		mask_evtchn(evtchn);
2294 
2295 	pirq_needs_eoi = pirq_needs_eoi_flag;
2296 
2297 #ifdef CONFIG_X86
2298 	if (xen_pv_domain()) {
2299 		if (xen_initial_domain())
2300 			pci_xen_initial_domain();
2301 	}
2302 	xen_init_setup_upcall_vector();
2303 	xen_alloc_callback_vector();
2304 
2305 
2306 	if (xen_hvm_domain()) {
2307 		native_init_IRQ();
2308 		/* pci_xen_hvm_init must be called after native_init_IRQ so that
2309 		 * __acpi_register_gsi can point at the right function */
2310 		pci_xen_hvm_init();
2311 	} else {
2312 		int rc;
2313 		struct physdev_pirq_eoi_gmfn eoi_gmfn;
2314 
2315 		pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
2316 		eoi_gmfn.gmfn = virt_to_gfn(pirq_eoi_map);
2317 		rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
2318 		if (rc != 0) {
2319 			free_page((unsigned long) pirq_eoi_map);
2320 			pirq_eoi_map = NULL;
2321 		} else
2322 			pirq_needs_eoi = pirq_check_eoi_map;
2323 	}
2324 #endif
2325 }
2326