1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
4  * Copyright (C) 2005-2006 Thomas Gleixner
5  *
6  * This file contains driver APIs to the irq subsystem.
7  */
8 
9 #define pr_fmt(fmt) "genirq: " fmt
10 
11 #include <linux/irq.h>
12 #include <linux/kthread.h>
13 #include <linux/module.h>
14 #include <linux/random.h>
15 #include <linux/interrupt.h>
16 #include <linux/irqdomain.h>
17 #include <linux/slab.h>
18 #include <linux/sched.h>
19 #include <linux/sched/rt.h>
20 #include <linux/sched/task.h>
21 #include <linux/sched/isolation.h>
22 #include <uapi/linux/sched/types.h>
23 #include <linux/task_work.h>
24 
25 #include "internals.h"
26 
27 #if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)
28 DEFINE_STATIC_KEY_FALSE(force_irqthreads_key);
29 
setup_forced_irqthreads(char * arg)30 static int __init setup_forced_irqthreads(char *arg)
31 {
32 	static_branch_enable(&force_irqthreads_key);
33 	return 0;
34 }
35 early_param("threadirqs", setup_forced_irqthreads);
36 #endif
37 
__synchronize_hardirq(struct irq_desc * desc,bool sync_chip)38 static void __synchronize_hardirq(struct irq_desc *desc, bool sync_chip)
39 {
40 	struct irq_data *irqd = irq_desc_get_irq_data(desc);
41 	bool inprogress;
42 
43 	do {
44 		unsigned long flags;
45 
46 		/*
47 		 * Wait until we're out of the critical section.  This might
48 		 * give the wrong answer due to the lack of memory barriers.
49 		 */
50 		while (irqd_irq_inprogress(&desc->irq_data))
51 			cpu_relax();
52 
53 		/* Ok, that indicated we're done: double-check carefully. */
54 		raw_spin_lock_irqsave(&desc->lock, flags);
55 		inprogress = irqd_irq_inprogress(&desc->irq_data);
56 
57 		/*
58 		 * If requested and supported, check at the chip whether it
59 		 * is in flight at the hardware level, i.e. already pending
60 		 * in a CPU and waiting for service and acknowledge.
61 		 */
62 		if (!inprogress && sync_chip) {
63 			/*
64 			 * Ignore the return code. inprogress is only updated
65 			 * when the chip supports it.
66 			 */
67 			__irq_get_irqchip_state(irqd, IRQCHIP_STATE_ACTIVE,
68 						&inprogress);
69 		}
70 		raw_spin_unlock_irqrestore(&desc->lock, flags);
71 
72 		/* Oops, that failed? */
73 	} while (inprogress);
74 }
75 
76 /**
77  *	synchronize_hardirq - wait for pending hard IRQ handlers (on other CPUs)
78  *	@irq: interrupt number to wait for
79  *
80  *	This function waits for any pending hard IRQ handlers for this
81  *	interrupt to complete before returning. If you use this
82  *	function while holding a resource the IRQ handler may need you
83  *	will deadlock. It does not take associated threaded handlers
84  *	into account.
85  *
86  *	Do not use this for shutdown scenarios where you must be sure
87  *	that all parts (hardirq and threaded handler) have completed.
88  *
89  *	Returns: false if a threaded handler is active.
90  *
91  *	This function may be called - with care - from IRQ context.
92  *
93  *	It does not check whether there is an interrupt in flight at the
94  *	hardware level, but not serviced yet, as this might deadlock when
95  *	called with interrupts disabled and the target CPU of the interrupt
96  *	is the current CPU.
97  */
synchronize_hardirq(unsigned int irq)98 bool synchronize_hardirq(unsigned int irq)
99 {
100 	struct irq_desc *desc = irq_to_desc(irq);
101 
102 	if (desc) {
103 		__synchronize_hardirq(desc, false);
104 		return !atomic_read(&desc->threads_active);
105 	}
106 
107 	return true;
108 }
109 EXPORT_SYMBOL(synchronize_hardirq);
110 
111 /**
112  *	synchronize_irq - wait for pending IRQ handlers (on other CPUs)
113  *	@irq: interrupt number to wait for
114  *
115  *	This function waits for any pending IRQ handlers for this interrupt
116  *	to complete before returning. If you use this function while
117  *	holding a resource the IRQ handler may need you will deadlock.
118  *
119  *	Can only be called from preemptible code as it might sleep when
120  *	an interrupt thread is associated to @irq.
121  *
122  *	It optionally makes sure (when the irq chip supports that method)
123  *	that the interrupt is not pending in any CPU and waiting for
124  *	service.
125  */
synchronize_irq(unsigned int irq)126 void synchronize_irq(unsigned int irq)
127 {
128 	struct irq_desc *desc = irq_to_desc(irq);
129 
130 	if (desc) {
131 		__synchronize_hardirq(desc, true);
132 		/*
133 		 * We made sure that no hardirq handler is
134 		 * running. Now verify that no threaded handlers are
135 		 * active.
136 		 */
137 		wait_event(desc->wait_for_threads,
138 			   !atomic_read(&desc->threads_active));
139 	}
140 }
141 EXPORT_SYMBOL(synchronize_irq);
142 
143 #ifdef CONFIG_SMP
144 cpumask_var_t irq_default_affinity;
145 
__irq_can_set_affinity(struct irq_desc * desc)146 static bool __irq_can_set_affinity(struct irq_desc *desc)
147 {
148 	if (!desc || !irqd_can_balance(&desc->irq_data) ||
149 	    !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)
150 		return false;
151 	return true;
152 }
153 
154 /**
155  *	irq_can_set_affinity - Check if the affinity of a given irq can be set
156  *	@irq:		Interrupt to check
157  *
158  */
irq_can_set_affinity(unsigned int irq)159 int irq_can_set_affinity(unsigned int irq)
160 {
161 	return __irq_can_set_affinity(irq_to_desc(irq));
162 }
163 
164 /**
165  * irq_can_set_affinity_usr - Check if affinity of a irq can be set from user space
166  * @irq:	Interrupt to check
167  *
168  * Like irq_can_set_affinity() above, but additionally checks for the
169  * AFFINITY_MANAGED flag.
170  */
irq_can_set_affinity_usr(unsigned int irq)171 bool irq_can_set_affinity_usr(unsigned int irq)
172 {
173 	struct irq_desc *desc = irq_to_desc(irq);
174 
175 	return __irq_can_set_affinity(desc) &&
176 		!irqd_affinity_is_managed(&desc->irq_data);
177 }
178 
179 /**
180  *	irq_set_thread_affinity - Notify irq threads to adjust affinity
181  *	@desc:		irq descriptor which has affinity changed
182  *
183  *	We just set IRQTF_AFFINITY and delegate the affinity setting
184  *	to the interrupt thread itself. We can not call
185  *	set_cpus_allowed_ptr() here as we hold desc->lock and this
186  *	code can be called from hard interrupt context.
187  */
irq_set_thread_affinity(struct irq_desc * desc)188 void irq_set_thread_affinity(struct irq_desc *desc)
189 {
190 	struct irqaction *action;
191 
192 	for_each_action_of_desc(desc, action)
193 		if (action->thread)
194 			set_bit(IRQTF_AFFINITY, &action->thread_flags);
195 }
196 
197 #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
irq_validate_effective_affinity(struct irq_data * data)198 static void irq_validate_effective_affinity(struct irq_data *data)
199 {
200 	const struct cpumask *m = irq_data_get_effective_affinity_mask(data);
201 	struct irq_chip *chip = irq_data_get_irq_chip(data);
202 
203 	if (!cpumask_empty(m))
204 		return;
205 	pr_warn_once("irq_chip %s did not update eff. affinity mask of irq %u\n",
206 		     chip->name, data->irq);
207 }
208 #else
irq_validate_effective_affinity(struct irq_data * data)209 static inline void irq_validate_effective_affinity(struct irq_data *data) { }
210 #endif
211 
irq_do_set_affinity(struct irq_data * data,const struct cpumask * mask,bool force)212 int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
213 			bool force)
214 {
215 	struct irq_desc *desc = irq_data_to_desc(data);
216 	struct irq_chip *chip = irq_data_get_irq_chip(data);
217 	const struct cpumask  *prog_mask;
218 	int ret;
219 
220 	static DEFINE_RAW_SPINLOCK(tmp_mask_lock);
221 	static struct cpumask tmp_mask;
222 
223 	if (!chip || !chip->irq_set_affinity)
224 		return -EINVAL;
225 
226 	raw_spin_lock(&tmp_mask_lock);
227 	/*
228 	 * If this is a managed interrupt and housekeeping is enabled on
229 	 * it check whether the requested affinity mask intersects with
230 	 * a housekeeping CPU. If so, then remove the isolated CPUs from
231 	 * the mask and just keep the housekeeping CPU(s). This prevents
232 	 * the affinity setter from routing the interrupt to an isolated
233 	 * CPU to avoid that I/O submitted from a housekeeping CPU causes
234 	 * interrupts on an isolated one.
235 	 *
236 	 * If the masks do not intersect or include online CPU(s) then
237 	 * keep the requested mask. The isolated target CPUs are only
238 	 * receiving interrupts when the I/O operation was submitted
239 	 * directly from them.
240 	 *
241 	 * If all housekeeping CPUs in the affinity mask are offline, the
242 	 * interrupt will be migrated by the CPU hotplug code once a
243 	 * housekeeping CPU which belongs to the affinity mask comes
244 	 * online.
245 	 */
246 	if (irqd_affinity_is_managed(data) &&
247 	    housekeeping_enabled(HK_TYPE_MANAGED_IRQ)) {
248 		const struct cpumask *hk_mask;
249 
250 		hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
251 
252 		cpumask_and(&tmp_mask, mask, hk_mask);
253 		if (!cpumask_intersects(&tmp_mask, cpu_online_mask))
254 			prog_mask = mask;
255 		else
256 			prog_mask = &tmp_mask;
257 	} else {
258 		prog_mask = mask;
259 	}
260 
261 	/*
262 	 * Make sure we only provide online CPUs to the irqchip,
263 	 * unless we are being asked to force the affinity (in which
264 	 * case we do as we are told).
265 	 */
266 	cpumask_and(&tmp_mask, prog_mask, cpu_online_mask);
267 	if (!force && !cpumask_empty(&tmp_mask))
268 		ret = chip->irq_set_affinity(data, &tmp_mask, force);
269 	else if (force)
270 		ret = chip->irq_set_affinity(data, mask, force);
271 	else
272 		ret = -EINVAL;
273 
274 	raw_spin_unlock(&tmp_mask_lock);
275 
276 	switch (ret) {
277 	case IRQ_SET_MASK_OK:
278 	case IRQ_SET_MASK_OK_DONE:
279 		cpumask_copy(desc->irq_common_data.affinity, mask);
280 		fallthrough;
281 	case IRQ_SET_MASK_OK_NOCOPY:
282 		irq_validate_effective_affinity(data);
283 		irq_set_thread_affinity(desc);
284 		ret = 0;
285 	}
286 
287 	return ret;
288 }
289 
290 #ifdef CONFIG_GENERIC_PENDING_IRQ
irq_set_affinity_pending(struct irq_data * data,const struct cpumask * dest)291 static inline int irq_set_affinity_pending(struct irq_data *data,
292 					   const struct cpumask *dest)
293 {
294 	struct irq_desc *desc = irq_data_to_desc(data);
295 
296 	irqd_set_move_pending(data);
297 	irq_copy_pending(desc, dest);
298 	return 0;
299 }
300 #else
irq_set_affinity_pending(struct irq_data * data,const struct cpumask * dest)301 static inline int irq_set_affinity_pending(struct irq_data *data,
302 					   const struct cpumask *dest)
303 {
304 	return -EBUSY;
305 }
306 #endif
307 
irq_try_set_affinity(struct irq_data * data,const struct cpumask * dest,bool force)308 static int irq_try_set_affinity(struct irq_data *data,
309 				const struct cpumask *dest, bool force)
310 {
311 	int ret = irq_do_set_affinity(data, dest, force);
312 
313 	/*
314 	 * In case that the underlying vector management is busy and the
315 	 * architecture supports the generic pending mechanism then utilize
316 	 * this to avoid returning an error to user space.
317 	 */
318 	if (ret == -EBUSY && !force)
319 		ret = irq_set_affinity_pending(data, dest);
320 	return ret;
321 }
322 
irq_set_affinity_deactivated(struct irq_data * data,const struct cpumask * mask,bool force)323 static bool irq_set_affinity_deactivated(struct irq_data *data,
324 					 const struct cpumask *mask, bool force)
325 {
326 	struct irq_desc *desc = irq_data_to_desc(data);
327 
328 	/*
329 	 * Handle irq chips which can handle affinity only in activated
330 	 * state correctly
331 	 *
332 	 * If the interrupt is not yet activated, just store the affinity
333 	 * mask and do not call the chip driver at all. On activation the
334 	 * driver has to make sure anyway that the interrupt is in a
335 	 * usable state so startup works.
336 	 */
337 	if (!IS_ENABLED(CONFIG_IRQ_DOMAIN_HIERARCHY) ||
338 	    irqd_is_activated(data) || !irqd_affinity_on_activate(data))
339 		return false;
340 
341 	cpumask_copy(desc->irq_common_data.affinity, mask);
342 	irq_data_update_effective_affinity(data, mask);
343 	irqd_set(data, IRQD_AFFINITY_SET);
344 	return true;
345 }
346 
irq_set_affinity_locked(struct irq_data * data,const struct cpumask * mask,bool force)347 int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,
348 			    bool force)
349 {
350 	struct irq_chip *chip = irq_data_get_irq_chip(data);
351 	struct irq_desc *desc = irq_data_to_desc(data);
352 	int ret = 0;
353 
354 	if (!chip || !chip->irq_set_affinity)
355 		return -EINVAL;
356 
357 	if (irq_set_affinity_deactivated(data, mask, force))
358 		return 0;
359 
360 	if (irq_can_move_pcntxt(data) && !irqd_is_setaffinity_pending(data)) {
361 		ret = irq_try_set_affinity(data, mask, force);
362 	} else {
363 		irqd_set_move_pending(data);
364 		irq_copy_pending(desc, mask);
365 	}
366 
367 	if (desc->affinity_notify) {
368 		kref_get(&desc->affinity_notify->kref);
369 		if (!schedule_work(&desc->affinity_notify->work)) {
370 			/* Work was already scheduled, drop our extra ref */
371 			kref_put(&desc->affinity_notify->kref,
372 				 desc->affinity_notify->release);
373 		}
374 	}
375 	irqd_set(data, IRQD_AFFINITY_SET);
376 
377 	return ret;
378 }
379 
380 /**
381  * irq_update_affinity_desc - Update affinity management for an interrupt
382  * @irq:	The interrupt number to update
383  * @affinity:	Pointer to the affinity descriptor
384  *
385  * This interface can be used to configure the affinity management of
386  * interrupts which have been allocated already.
387  *
388  * There are certain limitations on when it may be used - attempts to use it
389  * for when the kernel is configured for generic IRQ reservation mode (in
390  * config GENERIC_IRQ_RESERVATION_MODE) will fail, as it may conflict with
391  * managed/non-managed interrupt accounting. In addition, attempts to use it on
392  * an interrupt which is already started or which has already been configured
393  * as managed will also fail, as these mean invalid init state or double init.
394  */
irq_update_affinity_desc(unsigned int irq,struct irq_affinity_desc * affinity)395 int irq_update_affinity_desc(unsigned int irq,
396 			     struct irq_affinity_desc *affinity)
397 {
398 	struct irq_desc *desc;
399 	unsigned long flags;
400 	bool activated;
401 	int ret = 0;
402 
403 	/*
404 	 * Supporting this with the reservation scheme used by x86 needs
405 	 * some more thought. Fail it for now.
406 	 */
407 	if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE))
408 		return -EOPNOTSUPP;
409 
410 	desc = irq_get_desc_buslock(irq, &flags, 0);
411 	if (!desc)
412 		return -EINVAL;
413 
414 	/* Requires the interrupt to be shut down */
415 	if (irqd_is_started(&desc->irq_data)) {
416 		ret = -EBUSY;
417 		goto out_unlock;
418 	}
419 
420 	/* Interrupts which are already managed cannot be modified */
421 	if (irqd_affinity_is_managed(&desc->irq_data)) {
422 		ret = -EBUSY;
423 		goto out_unlock;
424 	}
425 
426 	/*
427 	 * Deactivate the interrupt. That's required to undo
428 	 * anything an earlier activation has established.
429 	 */
430 	activated = irqd_is_activated(&desc->irq_data);
431 	if (activated)
432 		irq_domain_deactivate_irq(&desc->irq_data);
433 
434 	if (affinity->is_managed) {
435 		irqd_set(&desc->irq_data, IRQD_AFFINITY_MANAGED);
436 		irqd_set(&desc->irq_data, IRQD_MANAGED_SHUTDOWN);
437 	}
438 
439 	cpumask_copy(desc->irq_common_data.affinity, &affinity->mask);
440 
441 	/* Restore the activation state */
442 	if (activated)
443 		irq_domain_activate_irq(&desc->irq_data, false);
444 
445 out_unlock:
446 	irq_put_desc_busunlock(desc, flags);
447 	return ret;
448 }
449 
__irq_set_affinity(unsigned int irq,const struct cpumask * mask,bool force)450 static int __irq_set_affinity(unsigned int irq, const struct cpumask *mask,
451 			      bool force)
452 {
453 	struct irq_desc *desc = irq_to_desc(irq);
454 	unsigned long flags;
455 	int ret;
456 
457 	if (!desc)
458 		return -EINVAL;
459 
460 	raw_spin_lock_irqsave(&desc->lock, flags);
461 	ret = irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask, force);
462 	raw_spin_unlock_irqrestore(&desc->lock, flags);
463 	return ret;
464 }
465 
466 /**
467  * irq_set_affinity - Set the irq affinity of a given irq
468  * @irq:	Interrupt to set affinity
469  * @cpumask:	cpumask
470  *
471  * Fails if cpumask does not contain an online CPU
472  */
irq_set_affinity(unsigned int irq,const struct cpumask * cpumask)473 int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask)
474 {
475 	return __irq_set_affinity(irq, cpumask, false);
476 }
477 EXPORT_SYMBOL_GPL(irq_set_affinity);
478 
479 /**
480  * irq_force_affinity - Force the irq affinity of a given irq
481  * @irq:	Interrupt to set affinity
482  * @cpumask:	cpumask
483  *
484  * Same as irq_set_affinity, but without checking the mask against
485  * online cpus.
486  *
487  * Solely for low level cpu hotplug code, where we need to make per
488  * cpu interrupts affine before the cpu becomes online.
489  */
irq_force_affinity(unsigned int irq,const struct cpumask * cpumask)490 int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask)
491 {
492 	return __irq_set_affinity(irq, cpumask, true);
493 }
494 EXPORT_SYMBOL_GPL(irq_force_affinity);
495 
__irq_apply_affinity_hint(unsigned int irq,const struct cpumask * m,bool setaffinity)496 int __irq_apply_affinity_hint(unsigned int irq, const struct cpumask *m,
497 			      bool setaffinity)
498 {
499 	unsigned long flags;
500 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
501 
502 	if (!desc)
503 		return -EINVAL;
504 	desc->affinity_hint = m;
505 	irq_put_desc_unlock(desc, flags);
506 	if (m && setaffinity)
507 		__irq_set_affinity(irq, m, false);
508 	return 0;
509 }
510 EXPORT_SYMBOL_GPL(__irq_apply_affinity_hint);
511 
irq_affinity_notify(struct work_struct * work)512 static void irq_affinity_notify(struct work_struct *work)
513 {
514 	struct irq_affinity_notify *notify =
515 		container_of(work, struct irq_affinity_notify, work);
516 	struct irq_desc *desc = irq_to_desc(notify->irq);
517 	cpumask_var_t cpumask;
518 	unsigned long flags;
519 
520 	if (!desc || !alloc_cpumask_var(&cpumask, GFP_KERNEL))
521 		goto out;
522 
523 	raw_spin_lock_irqsave(&desc->lock, flags);
524 	if (irq_move_pending(&desc->irq_data))
525 		irq_get_pending(cpumask, desc);
526 	else
527 		cpumask_copy(cpumask, desc->irq_common_data.affinity);
528 	raw_spin_unlock_irqrestore(&desc->lock, flags);
529 
530 	notify->notify(notify, cpumask);
531 
532 	free_cpumask_var(cpumask);
533 out:
534 	kref_put(&notify->kref, notify->release);
535 }
536 
537 /**
538  *	irq_set_affinity_notifier - control notification of IRQ affinity changes
539  *	@irq:		Interrupt for which to enable/disable notification
540  *	@notify:	Context for notification, or %NULL to disable
541  *			notification.  Function pointers must be initialised;
542  *			the other fields will be initialised by this function.
543  *
544  *	Must be called in process context.  Notification may only be enabled
545  *	after the IRQ is allocated and must be disabled before the IRQ is
546  *	freed using free_irq().
547  */
548 int
irq_set_affinity_notifier(unsigned int irq,struct irq_affinity_notify * notify)549 irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
550 {
551 	struct irq_desc *desc = irq_to_desc(irq);
552 	struct irq_affinity_notify *old_notify;
553 	unsigned long flags;
554 
555 	/* The release function is promised process context */
556 	might_sleep();
557 
558 	if (!desc || desc->istate & IRQS_NMI)
559 		return -EINVAL;
560 
561 	/* Complete initialisation of *notify */
562 	if (notify) {
563 		notify->irq = irq;
564 		kref_init(&notify->kref);
565 		INIT_WORK(&notify->work, irq_affinity_notify);
566 	}
567 
568 	raw_spin_lock_irqsave(&desc->lock, flags);
569 	old_notify = desc->affinity_notify;
570 	desc->affinity_notify = notify;
571 	raw_spin_unlock_irqrestore(&desc->lock, flags);
572 
573 	if (old_notify) {
574 		if (cancel_work_sync(&old_notify->work)) {
575 			/* Pending work had a ref, put that one too */
576 			kref_put(&old_notify->kref, old_notify->release);
577 		}
578 		kref_put(&old_notify->kref, old_notify->release);
579 	}
580 
581 	return 0;
582 }
583 EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
584 
585 #ifndef CONFIG_AUTO_IRQ_AFFINITY
586 /*
587  * Generic version of the affinity autoselector.
588  */
irq_setup_affinity(struct irq_desc * desc)589 int irq_setup_affinity(struct irq_desc *desc)
590 {
591 	struct cpumask *set = irq_default_affinity;
592 	int ret, node = irq_desc_get_node(desc);
593 	static DEFINE_RAW_SPINLOCK(mask_lock);
594 	static struct cpumask mask;
595 
596 	/* Excludes PER_CPU and NO_BALANCE interrupts */
597 	if (!__irq_can_set_affinity(desc))
598 		return 0;
599 
600 	raw_spin_lock(&mask_lock);
601 	/*
602 	 * Preserve the managed affinity setting and a userspace affinity
603 	 * setup, but make sure that one of the targets is online.
604 	 */
605 	if (irqd_affinity_is_managed(&desc->irq_data) ||
606 	    irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
607 		if (cpumask_intersects(desc->irq_common_data.affinity,
608 				       cpu_online_mask))
609 			set = desc->irq_common_data.affinity;
610 		else
611 			irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
612 	}
613 
614 	cpumask_and(&mask, cpu_online_mask, set);
615 	if (cpumask_empty(&mask))
616 		cpumask_copy(&mask, cpu_online_mask);
617 
618 	if (node != NUMA_NO_NODE) {
619 		const struct cpumask *nodemask = cpumask_of_node(node);
620 
621 		/* make sure at least one of the cpus in nodemask is online */
622 		if (cpumask_intersects(&mask, nodemask))
623 			cpumask_and(&mask, &mask, nodemask);
624 	}
625 	ret = irq_do_set_affinity(&desc->irq_data, &mask, false);
626 	raw_spin_unlock(&mask_lock);
627 	return ret;
628 }
629 #else
630 /* Wrapper for ALPHA specific affinity selector magic */
irq_setup_affinity(struct irq_desc * desc)631 int irq_setup_affinity(struct irq_desc *desc)
632 {
633 	return irq_select_affinity(irq_desc_get_irq(desc));
634 }
635 #endif /* CONFIG_AUTO_IRQ_AFFINITY */
636 #endif /* CONFIG_SMP */
637 
638 
639 /**
640  *	irq_set_vcpu_affinity - Set vcpu affinity for the interrupt
641  *	@irq: interrupt number to set affinity
642  *	@vcpu_info: vCPU specific data or pointer to a percpu array of vCPU
643  *	            specific data for percpu_devid interrupts
644  *
645  *	This function uses the vCPU specific data to set the vCPU
646  *	affinity for an irq. The vCPU specific data is passed from
647  *	outside, such as KVM. One example code path is as below:
648  *	KVM -> IOMMU -> irq_set_vcpu_affinity().
649  */
irq_set_vcpu_affinity(unsigned int irq,void * vcpu_info)650 int irq_set_vcpu_affinity(unsigned int irq, void *vcpu_info)
651 {
652 	unsigned long flags;
653 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
654 	struct irq_data *data;
655 	struct irq_chip *chip;
656 	int ret = -ENOSYS;
657 
658 	if (!desc)
659 		return -EINVAL;
660 
661 	data = irq_desc_get_irq_data(desc);
662 	do {
663 		chip = irq_data_get_irq_chip(data);
664 		if (chip && chip->irq_set_vcpu_affinity)
665 			break;
666 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
667 		data = data->parent_data;
668 #else
669 		data = NULL;
670 #endif
671 	} while (data);
672 
673 	if (data)
674 		ret = chip->irq_set_vcpu_affinity(data, vcpu_info);
675 	irq_put_desc_unlock(desc, flags);
676 
677 	return ret;
678 }
679 EXPORT_SYMBOL_GPL(irq_set_vcpu_affinity);
680 
__disable_irq(struct irq_desc * desc)681 void __disable_irq(struct irq_desc *desc)
682 {
683 	if (!desc->depth++)
684 		irq_disable(desc);
685 }
686 
__disable_irq_nosync(unsigned int irq)687 static int __disable_irq_nosync(unsigned int irq)
688 {
689 	unsigned long flags;
690 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
691 
692 	if (!desc)
693 		return -EINVAL;
694 	__disable_irq(desc);
695 	irq_put_desc_busunlock(desc, flags);
696 	return 0;
697 }
698 
699 /**
700  *	disable_irq_nosync - disable an irq without waiting
701  *	@irq: Interrupt to disable
702  *
703  *	Disable the selected interrupt line.  Disables and Enables are
704  *	nested.
705  *	Unlike disable_irq(), this function does not ensure existing
706  *	instances of the IRQ handler have completed before returning.
707  *
708  *	This function may be called from IRQ context.
709  */
disable_irq_nosync(unsigned int irq)710 void disable_irq_nosync(unsigned int irq)
711 {
712 	__disable_irq_nosync(irq);
713 }
714 EXPORT_SYMBOL(disable_irq_nosync);
715 
716 /**
717  *	disable_irq - disable an irq and wait for completion
718  *	@irq: Interrupt to disable
719  *
720  *	Disable the selected interrupt line.  Enables and Disables are
721  *	nested.
722  *	This function waits for any pending IRQ handlers for this interrupt
723  *	to complete before returning. If you use this function while
724  *	holding a resource the IRQ handler may need you will deadlock.
725  *
726  *	This function may be called - with care - from IRQ context.
727  */
disable_irq(unsigned int irq)728 void disable_irq(unsigned int irq)
729 {
730 	if (!__disable_irq_nosync(irq))
731 		synchronize_irq(irq);
732 }
733 EXPORT_SYMBOL(disable_irq);
734 
735 /**
736  *	disable_hardirq - disables an irq and waits for hardirq completion
737  *	@irq: Interrupt to disable
738  *
739  *	Disable the selected interrupt line.  Enables and Disables are
740  *	nested.
741  *	This function waits for any pending hard IRQ handlers for this
742  *	interrupt to complete before returning. If you use this function while
743  *	holding a resource the hard IRQ handler may need you will deadlock.
744  *
745  *	When used to optimistically disable an interrupt from atomic context
746  *	the return value must be checked.
747  *
748  *	Returns: false if a threaded handler is active.
749  *
750  *	This function may be called - with care - from IRQ context.
751  */
disable_hardirq(unsigned int irq)752 bool disable_hardirq(unsigned int irq)
753 {
754 	if (!__disable_irq_nosync(irq))
755 		return synchronize_hardirq(irq);
756 
757 	return false;
758 }
759 EXPORT_SYMBOL_GPL(disable_hardirq);
760 
761 /**
762  *	disable_nmi_nosync - disable an nmi without waiting
763  *	@irq: Interrupt to disable
764  *
765  *	Disable the selected interrupt line. Disables and enables are
766  *	nested.
767  *	The interrupt to disable must have been requested through request_nmi.
768  *	Unlike disable_nmi(), this function does not ensure existing
769  *	instances of the IRQ handler have completed before returning.
770  */
disable_nmi_nosync(unsigned int irq)771 void disable_nmi_nosync(unsigned int irq)
772 {
773 	disable_irq_nosync(irq);
774 }
775 
__enable_irq(struct irq_desc * desc)776 void __enable_irq(struct irq_desc *desc)
777 {
778 	switch (desc->depth) {
779 	case 0:
780  err_out:
781 		WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n",
782 		     irq_desc_get_irq(desc));
783 		break;
784 	case 1: {
785 		if (desc->istate & IRQS_SUSPENDED)
786 			goto err_out;
787 		/* Prevent probing on this irq: */
788 		irq_settings_set_noprobe(desc);
789 		/*
790 		 * Call irq_startup() not irq_enable() here because the
791 		 * interrupt might be marked NOAUTOEN. So irq_startup()
792 		 * needs to be invoked when it gets enabled the first
793 		 * time. If it was already started up, then irq_startup()
794 		 * will invoke irq_enable() under the hood.
795 		 */
796 		irq_startup(desc, IRQ_RESEND, IRQ_START_FORCE);
797 		break;
798 	}
799 	default:
800 		desc->depth--;
801 	}
802 }
803 
804 /**
805  *	enable_irq - enable handling of an irq
806  *	@irq: Interrupt to enable
807  *
808  *	Undoes the effect of one call to disable_irq().  If this
809  *	matches the last disable, processing of interrupts on this
810  *	IRQ line is re-enabled.
811  *
812  *	This function may be called from IRQ context only when
813  *	desc->irq_data.chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
814  */
enable_irq(unsigned int irq)815 void enable_irq(unsigned int irq)
816 {
817 	unsigned long flags;
818 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
819 
820 	if (!desc)
821 		return;
822 	if (WARN(!desc->irq_data.chip,
823 		 KERN_ERR "enable_irq before setup/request_irq: irq %u\n", irq))
824 		goto out;
825 
826 	__enable_irq(desc);
827 out:
828 	irq_put_desc_busunlock(desc, flags);
829 }
830 EXPORT_SYMBOL(enable_irq);
831 
832 /**
833  *	enable_nmi - enable handling of an nmi
834  *	@irq: Interrupt to enable
835  *
836  *	The interrupt to enable must have been requested through request_nmi.
837  *	Undoes the effect of one call to disable_nmi(). If this
838  *	matches the last disable, processing of interrupts on this
839  *	IRQ line is re-enabled.
840  */
enable_nmi(unsigned int irq)841 void enable_nmi(unsigned int irq)
842 {
843 	enable_irq(irq);
844 }
845 
set_irq_wake_real(unsigned int irq,unsigned int on)846 static int set_irq_wake_real(unsigned int irq, unsigned int on)
847 {
848 	struct irq_desc *desc = irq_to_desc(irq);
849 	int ret = -ENXIO;
850 
851 	if (irq_desc_get_chip(desc)->flags &  IRQCHIP_SKIP_SET_WAKE)
852 		return 0;
853 
854 	if (desc->irq_data.chip->irq_set_wake)
855 		ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on);
856 
857 	return ret;
858 }
859 
860 /**
861  *	irq_set_irq_wake - control irq power management wakeup
862  *	@irq:	interrupt to control
863  *	@on:	enable/disable power management wakeup
864  *
865  *	Enable/disable power management wakeup mode, which is
866  *	disabled by default.  Enables and disables must match,
867  *	just as they match for non-wakeup mode support.
868  *
869  *	Wakeup mode lets this IRQ wake the system from sleep
870  *	states like "suspend to RAM".
871  *
872  *	Note: irq enable/disable state is completely orthogonal
873  *	to the enable/disable state of irq wake. An irq can be
874  *	disabled with disable_irq() and still wake the system as
875  *	long as the irq has wake enabled. If this does not hold,
876  *	then the underlying irq chip and the related driver need
877  *	to be investigated.
878  */
irq_set_irq_wake(unsigned int irq,unsigned int on)879 int irq_set_irq_wake(unsigned int irq, unsigned int on)
880 {
881 	unsigned long flags;
882 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
883 	int ret = 0;
884 
885 	if (!desc)
886 		return -EINVAL;
887 
888 	/* Don't use NMIs as wake up interrupts please */
889 	if (desc->istate & IRQS_NMI) {
890 		ret = -EINVAL;
891 		goto out_unlock;
892 	}
893 
894 	/* wakeup-capable irqs can be shared between drivers that
895 	 * don't need to have the same sleep mode behaviors.
896 	 */
897 	if (on) {
898 		if (desc->wake_depth++ == 0) {
899 			ret = set_irq_wake_real(irq, on);
900 			if (ret)
901 				desc->wake_depth = 0;
902 			else
903 				irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE);
904 		}
905 	} else {
906 		if (desc->wake_depth == 0) {
907 			WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
908 		} else if (--desc->wake_depth == 0) {
909 			ret = set_irq_wake_real(irq, on);
910 			if (ret)
911 				desc->wake_depth = 1;
912 			else
913 				irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
914 		}
915 	}
916 
917 out_unlock:
918 	irq_put_desc_busunlock(desc, flags);
919 	return ret;
920 }
921 EXPORT_SYMBOL(irq_set_irq_wake);
922 
923 /*
924  * Internal function that tells the architecture code whether a
925  * particular irq has been exclusively allocated or is available
926  * for driver use.
927  */
can_request_irq(unsigned int irq,unsigned long irqflags)928 int can_request_irq(unsigned int irq, unsigned long irqflags)
929 {
930 	unsigned long flags;
931 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
932 	int canrequest = 0;
933 
934 	if (!desc)
935 		return 0;
936 
937 	if (irq_settings_can_request(desc)) {
938 		if (!desc->action ||
939 		    irqflags & desc->action->flags & IRQF_SHARED)
940 			canrequest = 1;
941 	}
942 	irq_put_desc_unlock(desc, flags);
943 	return canrequest;
944 }
945 
__irq_set_trigger(struct irq_desc * desc,unsigned long flags)946 int __irq_set_trigger(struct irq_desc *desc, unsigned long flags)
947 {
948 	struct irq_chip *chip = desc->irq_data.chip;
949 	int ret, unmask = 0;
950 
951 	if (!chip || !chip->irq_set_type) {
952 		/*
953 		 * IRQF_TRIGGER_* but the PIC does not support multiple
954 		 * flow-types?
955 		 */
956 		pr_debug("No set_type function for IRQ %d (%s)\n",
957 			 irq_desc_get_irq(desc),
958 			 chip ? (chip->name ? : "unknown") : "unknown");
959 		return 0;
960 	}
961 
962 	if (chip->flags & IRQCHIP_SET_TYPE_MASKED) {
963 		if (!irqd_irq_masked(&desc->irq_data))
964 			mask_irq(desc);
965 		if (!irqd_irq_disabled(&desc->irq_data))
966 			unmask = 1;
967 	}
968 
969 	/* Mask all flags except trigger mode */
970 	flags &= IRQ_TYPE_SENSE_MASK;
971 	ret = chip->irq_set_type(&desc->irq_data, flags);
972 
973 	switch (ret) {
974 	case IRQ_SET_MASK_OK:
975 	case IRQ_SET_MASK_OK_DONE:
976 		irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
977 		irqd_set(&desc->irq_data, flags);
978 		fallthrough;
979 
980 	case IRQ_SET_MASK_OK_NOCOPY:
981 		flags = irqd_get_trigger_type(&desc->irq_data);
982 		irq_settings_set_trigger_mask(desc, flags);
983 		irqd_clear(&desc->irq_data, IRQD_LEVEL);
984 		irq_settings_clr_level(desc);
985 		if (flags & IRQ_TYPE_LEVEL_MASK) {
986 			irq_settings_set_level(desc);
987 			irqd_set(&desc->irq_data, IRQD_LEVEL);
988 		}
989 
990 		ret = 0;
991 		break;
992 	default:
993 		pr_err("Setting trigger mode %lu for irq %u failed (%pS)\n",
994 		       flags, irq_desc_get_irq(desc), chip->irq_set_type);
995 	}
996 	if (unmask)
997 		unmask_irq(desc);
998 	return ret;
999 }
1000 
1001 #ifdef CONFIG_HARDIRQS_SW_RESEND
irq_set_parent(int irq,int parent_irq)1002 int irq_set_parent(int irq, int parent_irq)
1003 {
1004 	unsigned long flags;
1005 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
1006 
1007 	if (!desc)
1008 		return -EINVAL;
1009 
1010 	desc->parent_irq = parent_irq;
1011 
1012 	irq_put_desc_unlock(desc, flags);
1013 	return 0;
1014 }
1015 EXPORT_SYMBOL_GPL(irq_set_parent);
1016 #endif
1017 
1018 /*
1019  * Default primary interrupt handler for threaded interrupts. Is
1020  * assigned as primary handler when request_threaded_irq is called
1021  * with handler == NULL. Useful for oneshot interrupts.
1022  */
irq_default_primary_handler(int irq,void * dev_id)1023 static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
1024 {
1025 	return IRQ_WAKE_THREAD;
1026 }
1027 
1028 /*
1029  * Primary handler for nested threaded interrupts. Should never be
1030  * called.
1031  */
irq_nested_primary_handler(int irq,void * dev_id)1032 static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
1033 {
1034 	WARN(1, "Primary handler called for nested irq %d\n", irq);
1035 	return IRQ_NONE;
1036 }
1037 
irq_forced_secondary_handler(int irq,void * dev_id)1038 static irqreturn_t irq_forced_secondary_handler(int irq, void *dev_id)
1039 {
1040 	WARN(1, "Secondary action handler called for irq %d\n", irq);
1041 	return IRQ_NONE;
1042 }
1043 
irq_wait_for_interrupt(struct irqaction * action)1044 static int irq_wait_for_interrupt(struct irqaction *action)
1045 {
1046 	for (;;) {
1047 		set_current_state(TASK_INTERRUPTIBLE);
1048 
1049 		if (kthread_should_stop()) {
1050 			/* may need to run one last time */
1051 			if (test_and_clear_bit(IRQTF_RUNTHREAD,
1052 					       &action->thread_flags)) {
1053 				__set_current_state(TASK_RUNNING);
1054 				return 0;
1055 			}
1056 			__set_current_state(TASK_RUNNING);
1057 			return -1;
1058 		}
1059 
1060 		if (test_and_clear_bit(IRQTF_RUNTHREAD,
1061 				       &action->thread_flags)) {
1062 			__set_current_state(TASK_RUNNING);
1063 			return 0;
1064 		}
1065 		schedule();
1066 	}
1067 }
1068 
1069 /*
1070  * Oneshot interrupts keep the irq line masked until the threaded
1071  * handler finished. unmask if the interrupt has not been disabled and
1072  * is marked MASKED.
1073  */
irq_finalize_oneshot(struct irq_desc * desc,struct irqaction * action)1074 static void irq_finalize_oneshot(struct irq_desc *desc,
1075 				 struct irqaction *action)
1076 {
1077 	if (!(desc->istate & IRQS_ONESHOT) ||
1078 	    action->handler == irq_forced_secondary_handler)
1079 		return;
1080 again:
1081 	chip_bus_lock(desc);
1082 	raw_spin_lock_irq(&desc->lock);
1083 
1084 	/*
1085 	 * Implausible though it may be we need to protect us against
1086 	 * the following scenario:
1087 	 *
1088 	 * The thread is faster done than the hard interrupt handler
1089 	 * on the other CPU. If we unmask the irq line then the
1090 	 * interrupt can come in again and masks the line, leaves due
1091 	 * to IRQS_INPROGRESS and the irq line is masked forever.
1092 	 *
1093 	 * This also serializes the state of shared oneshot handlers
1094 	 * versus "desc->threads_oneshot |= action->thread_mask;" in
1095 	 * irq_wake_thread(). See the comment there which explains the
1096 	 * serialization.
1097 	 */
1098 	if (unlikely(irqd_irq_inprogress(&desc->irq_data))) {
1099 		raw_spin_unlock_irq(&desc->lock);
1100 		chip_bus_sync_unlock(desc);
1101 		cpu_relax();
1102 		goto again;
1103 	}
1104 
1105 	/*
1106 	 * Now check again, whether the thread should run. Otherwise
1107 	 * we would clear the threads_oneshot bit of this thread which
1108 	 * was just set.
1109 	 */
1110 	if (test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
1111 		goto out_unlock;
1112 
1113 	desc->threads_oneshot &= ~action->thread_mask;
1114 
1115 	if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) &&
1116 	    irqd_irq_masked(&desc->irq_data))
1117 		unmask_threaded_irq(desc);
1118 
1119 out_unlock:
1120 	raw_spin_unlock_irq(&desc->lock);
1121 	chip_bus_sync_unlock(desc);
1122 }
1123 
1124 #ifdef CONFIG_SMP
1125 /*
1126  * Check whether we need to change the affinity of the interrupt thread.
1127  */
1128 static void
irq_thread_check_affinity(struct irq_desc * desc,struct irqaction * action)1129 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
1130 {
1131 	cpumask_var_t mask;
1132 	bool valid = true;
1133 
1134 	if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
1135 		return;
1136 
1137 	/*
1138 	 * In case we are out of memory we set IRQTF_AFFINITY again and
1139 	 * try again next time
1140 	 */
1141 	if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
1142 		set_bit(IRQTF_AFFINITY, &action->thread_flags);
1143 		return;
1144 	}
1145 
1146 	raw_spin_lock_irq(&desc->lock);
1147 	/*
1148 	 * This code is triggered unconditionally. Check the affinity
1149 	 * mask pointer. For CPU_MASK_OFFSTACK=n this is optimized out.
1150 	 */
1151 	if (cpumask_available(desc->irq_common_data.affinity)) {
1152 		const struct cpumask *m;
1153 
1154 		m = irq_data_get_effective_affinity_mask(&desc->irq_data);
1155 		cpumask_copy(mask, m);
1156 	} else {
1157 		valid = false;
1158 	}
1159 	raw_spin_unlock_irq(&desc->lock);
1160 
1161 	if (valid)
1162 		set_cpus_allowed_ptr(current, mask);
1163 	free_cpumask_var(mask);
1164 }
1165 #else
1166 static inline void
irq_thread_check_affinity(struct irq_desc * desc,struct irqaction * action)1167 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
1168 #endif
1169 
1170 /*
1171  * Interrupts which are not explicitly requested as threaded
1172  * interrupts rely on the implicit bh/preempt disable of the hard irq
1173  * context. So we need to disable bh here to avoid deadlocks and other
1174  * side effects.
1175  */
1176 static irqreturn_t
irq_forced_thread_fn(struct irq_desc * desc,struct irqaction * action)1177 irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
1178 {
1179 	irqreturn_t ret;
1180 
1181 	local_bh_disable();
1182 	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
1183 		local_irq_disable();
1184 	ret = action->thread_fn(action->irq, action->dev_id);
1185 	if (ret == IRQ_HANDLED)
1186 		atomic_inc(&desc->threads_handled);
1187 
1188 	irq_finalize_oneshot(desc, action);
1189 	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
1190 		local_irq_enable();
1191 	local_bh_enable();
1192 	return ret;
1193 }
1194 
1195 /*
1196  * Interrupts explicitly requested as threaded interrupts want to be
1197  * preemptible - many of them need to sleep and wait for slow busses to
1198  * complete.
1199  */
irq_thread_fn(struct irq_desc * desc,struct irqaction * action)1200 static irqreturn_t irq_thread_fn(struct irq_desc *desc,
1201 		struct irqaction *action)
1202 {
1203 	irqreturn_t ret;
1204 
1205 	ret = action->thread_fn(action->irq, action->dev_id);
1206 	if (ret == IRQ_HANDLED)
1207 		atomic_inc(&desc->threads_handled);
1208 
1209 	irq_finalize_oneshot(desc, action);
1210 	return ret;
1211 }
1212 
wake_threads_waitq(struct irq_desc * desc)1213 static void wake_threads_waitq(struct irq_desc *desc)
1214 {
1215 	if (atomic_dec_and_test(&desc->threads_active))
1216 		wake_up(&desc->wait_for_threads);
1217 }
1218 
irq_thread_dtor(struct callback_head * unused)1219 static void irq_thread_dtor(struct callback_head *unused)
1220 {
1221 	struct task_struct *tsk = current;
1222 	struct irq_desc *desc;
1223 	struct irqaction *action;
1224 
1225 	if (WARN_ON_ONCE(!(current->flags & PF_EXITING)))
1226 		return;
1227 
1228 	action = kthread_data(tsk);
1229 
1230 	pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
1231 	       tsk->comm, tsk->pid, action->irq);
1232 
1233 
1234 	desc = irq_to_desc(action->irq);
1235 	/*
1236 	 * If IRQTF_RUNTHREAD is set, we need to decrement
1237 	 * desc->threads_active and wake possible waiters.
1238 	 */
1239 	if (test_and_clear_bit(IRQTF_RUNTHREAD, &action->thread_flags))
1240 		wake_threads_waitq(desc);
1241 
1242 	/* Prevent a stale desc->threads_oneshot */
1243 	irq_finalize_oneshot(desc, action);
1244 }
1245 
irq_wake_secondary(struct irq_desc * desc,struct irqaction * action)1246 static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action)
1247 {
1248 	struct irqaction *secondary = action->secondary;
1249 
1250 	if (WARN_ON_ONCE(!secondary))
1251 		return;
1252 
1253 	raw_spin_lock_irq(&desc->lock);
1254 	__irq_wake_thread(desc, secondary);
1255 	raw_spin_unlock_irq(&desc->lock);
1256 }
1257 
1258 /*
1259  * Internal function to notify that a interrupt thread is ready.
1260  */
irq_thread_set_ready(struct irq_desc * desc,struct irqaction * action)1261 static void irq_thread_set_ready(struct irq_desc *desc,
1262 				 struct irqaction *action)
1263 {
1264 	set_bit(IRQTF_READY, &action->thread_flags);
1265 	wake_up(&desc->wait_for_threads);
1266 }
1267 
1268 /*
1269  * Internal function to wake up a interrupt thread and wait until it is
1270  * ready.
1271  */
wake_up_and_wait_for_irq_thread_ready(struct irq_desc * desc,struct irqaction * action)1272 static void wake_up_and_wait_for_irq_thread_ready(struct irq_desc *desc,
1273 						  struct irqaction *action)
1274 {
1275 	if (!action || !action->thread)
1276 		return;
1277 
1278 	wake_up_process(action->thread);
1279 	wait_event(desc->wait_for_threads,
1280 		   test_bit(IRQTF_READY, &action->thread_flags));
1281 }
1282 
1283 /*
1284  * Interrupt handler thread
1285  */
irq_thread(void * data)1286 static int irq_thread(void *data)
1287 {
1288 	struct callback_head on_exit_work;
1289 	struct irqaction *action = data;
1290 	struct irq_desc *desc = irq_to_desc(action->irq);
1291 	irqreturn_t (*handler_fn)(struct irq_desc *desc,
1292 			struct irqaction *action);
1293 
1294 	irq_thread_set_ready(desc, action);
1295 
1296 	sched_set_fifo(current);
1297 
1298 	if (force_irqthreads() && test_bit(IRQTF_FORCED_THREAD,
1299 					   &action->thread_flags))
1300 		handler_fn = irq_forced_thread_fn;
1301 	else
1302 		handler_fn = irq_thread_fn;
1303 
1304 	init_task_work(&on_exit_work, irq_thread_dtor);
1305 	task_work_add(current, &on_exit_work, TWA_NONE);
1306 
1307 	irq_thread_check_affinity(desc, action);
1308 
1309 	while (!irq_wait_for_interrupt(action)) {
1310 		irqreturn_t action_ret;
1311 
1312 		irq_thread_check_affinity(desc, action);
1313 
1314 		action_ret = handler_fn(desc, action);
1315 		if (action_ret == IRQ_WAKE_THREAD)
1316 			irq_wake_secondary(desc, action);
1317 
1318 		wake_threads_waitq(desc);
1319 	}
1320 
1321 	/*
1322 	 * This is the regular exit path. __free_irq() is stopping the
1323 	 * thread via kthread_stop() after calling
1324 	 * synchronize_hardirq(). So neither IRQTF_RUNTHREAD nor the
1325 	 * oneshot mask bit can be set.
1326 	 */
1327 	task_work_cancel(current, irq_thread_dtor);
1328 	return 0;
1329 }
1330 
1331 /**
1332  *	irq_wake_thread - wake the irq thread for the action identified by dev_id
1333  *	@irq:		Interrupt line
1334  *	@dev_id:	Device identity for which the thread should be woken
1335  *
1336  */
irq_wake_thread(unsigned int irq,void * dev_id)1337 void irq_wake_thread(unsigned int irq, void *dev_id)
1338 {
1339 	struct irq_desc *desc = irq_to_desc(irq);
1340 	struct irqaction *action;
1341 	unsigned long flags;
1342 
1343 	if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1344 		return;
1345 
1346 	raw_spin_lock_irqsave(&desc->lock, flags);
1347 	for_each_action_of_desc(desc, action) {
1348 		if (action->dev_id == dev_id) {
1349 			if (action->thread)
1350 				__irq_wake_thread(desc, action);
1351 			break;
1352 		}
1353 	}
1354 	raw_spin_unlock_irqrestore(&desc->lock, flags);
1355 }
1356 EXPORT_SYMBOL_GPL(irq_wake_thread);
1357 
irq_setup_forced_threading(struct irqaction * new)1358 static int irq_setup_forced_threading(struct irqaction *new)
1359 {
1360 	if (!force_irqthreads())
1361 		return 0;
1362 	if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
1363 		return 0;
1364 
1365 	/*
1366 	 * No further action required for interrupts which are requested as
1367 	 * threaded interrupts already
1368 	 */
1369 	if (new->handler == irq_default_primary_handler)
1370 		return 0;
1371 
1372 	new->flags |= IRQF_ONESHOT;
1373 
1374 	/*
1375 	 * Handle the case where we have a real primary handler and a
1376 	 * thread handler. We force thread them as well by creating a
1377 	 * secondary action.
1378 	 */
1379 	if (new->handler && new->thread_fn) {
1380 		/* Allocate the secondary action */
1381 		new->secondary = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1382 		if (!new->secondary)
1383 			return -ENOMEM;
1384 		new->secondary->handler = irq_forced_secondary_handler;
1385 		new->secondary->thread_fn = new->thread_fn;
1386 		new->secondary->dev_id = new->dev_id;
1387 		new->secondary->irq = new->irq;
1388 		new->secondary->name = new->name;
1389 	}
1390 	/* Deal with the primary handler */
1391 	set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
1392 	new->thread_fn = new->handler;
1393 	new->handler = irq_default_primary_handler;
1394 	return 0;
1395 }
1396 
irq_request_resources(struct irq_desc * desc)1397 static int irq_request_resources(struct irq_desc *desc)
1398 {
1399 	struct irq_data *d = &desc->irq_data;
1400 	struct irq_chip *c = d->chip;
1401 
1402 	return c->irq_request_resources ? c->irq_request_resources(d) : 0;
1403 }
1404 
irq_release_resources(struct irq_desc * desc)1405 static void irq_release_resources(struct irq_desc *desc)
1406 {
1407 	struct irq_data *d = &desc->irq_data;
1408 	struct irq_chip *c = d->chip;
1409 
1410 	if (c->irq_release_resources)
1411 		c->irq_release_resources(d);
1412 }
1413 
irq_supports_nmi(struct irq_desc * desc)1414 static bool irq_supports_nmi(struct irq_desc *desc)
1415 {
1416 	struct irq_data *d = irq_desc_get_irq_data(desc);
1417 
1418 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1419 	/* Only IRQs directly managed by the root irqchip can be set as NMI */
1420 	if (d->parent_data)
1421 		return false;
1422 #endif
1423 	/* Don't support NMIs for chips behind a slow bus */
1424 	if (d->chip->irq_bus_lock || d->chip->irq_bus_sync_unlock)
1425 		return false;
1426 
1427 	return d->chip->flags & IRQCHIP_SUPPORTS_NMI;
1428 }
1429 
irq_nmi_setup(struct irq_desc * desc)1430 static int irq_nmi_setup(struct irq_desc *desc)
1431 {
1432 	struct irq_data *d = irq_desc_get_irq_data(desc);
1433 	struct irq_chip *c = d->chip;
1434 
1435 	return c->irq_nmi_setup ? c->irq_nmi_setup(d) : -EINVAL;
1436 }
1437 
irq_nmi_teardown(struct irq_desc * desc)1438 static void irq_nmi_teardown(struct irq_desc *desc)
1439 {
1440 	struct irq_data *d = irq_desc_get_irq_data(desc);
1441 	struct irq_chip *c = d->chip;
1442 
1443 	if (c->irq_nmi_teardown)
1444 		c->irq_nmi_teardown(d);
1445 }
1446 
1447 static int
setup_irq_thread(struct irqaction * new,unsigned int irq,bool secondary)1448 setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary)
1449 {
1450 	struct task_struct *t;
1451 
1452 	if (!secondary) {
1453 		t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
1454 				   new->name);
1455 	} else {
1456 		t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq,
1457 				   new->name);
1458 	}
1459 
1460 	if (IS_ERR(t))
1461 		return PTR_ERR(t);
1462 
1463 	/*
1464 	 * We keep the reference to the task struct even if
1465 	 * the thread dies to avoid that the interrupt code
1466 	 * references an already freed task_struct.
1467 	 */
1468 	new->thread = get_task_struct(t);
1469 	/*
1470 	 * Tell the thread to set its affinity. This is
1471 	 * important for shared interrupt handlers as we do
1472 	 * not invoke setup_affinity() for the secondary
1473 	 * handlers as everything is already set up. Even for
1474 	 * interrupts marked with IRQF_NO_BALANCE this is
1475 	 * correct as we want the thread to move to the cpu(s)
1476 	 * on which the requesting code placed the interrupt.
1477 	 */
1478 	set_bit(IRQTF_AFFINITY, &new->thread_flags);
1479 	return 0;
1480 }
1481 
1482 /*
1483  * Internal function to register an irqaction - typically used to
1484  * allocate special interrupts that are part of the architecture.
1485  *
1486  * Locking rules:
1487  *
1488  * desc->request_mutex	Provides serialization against a concurrent free_irq()
1489  *   chip_bus_lock	Provides serialization for slow bus operations
1490  *     desc->lock	Provides serialization against hard interrupts
1491  *
1492  * chip_bus_lock and desc->lock are sufficient for all other management and
1493  * interrupt related functions. desc->request_mutex solely serializes
1494  * request/free_irq().
1495  */
1496 static int
__setup_irq(unsigned int irq,struct irq_desc * desc,struct irqaction * new)1497 __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1498 {
1499 	struct irqaction *old, **old_ptr;
1500 	unsigned long flags, thread_mask = 0;
1501 	int ret, nested, shared = 0;
1502 
1503 	if (!desc)
1504 		return -EINVAL;
1505 
1506 	if (desc->irq_data.chip == &no_irq_chip)
1507 		return -ENOSYS;
1508 	if (!try_module_get(desc->owner))
1509 		return -ENODEV;
1510 
1511 	new->irq = irq;
1512 
1513 	/*
1514 	 * If the trigger type is not specified by the caller,
1515 	 * then use the default for this interrupt.
1516 	 */
1517 	if (!(new->flags & IRQF_TRIGGER_MASK))
1518 		new->flags |= irqd_get_trigger_type(&desc->irq_data);
1519 
1520 	/*
1521 	 * Check whether the interrupt nests into another interrupt
1522 	 * thread.
1523 	 */
1524 	nested = irq_settings_is_nested_thread(desc);
1525 	if (nested) {
1526 		if (!new->thread_fn) {
1527 			ret = -EINVAL;
1528 			goto out_mput;
1529 		}
1530 		/*
1531 		 * Replace the primary handler which was provided from
1532 		 * the driver for non nested interrupt handling by the
1533 		 * dummy function which warns when called.
1534 		 */
1535 		new->handler = irq_nested_primary_handler;
1536 	} else {
1537 		if (irq_settings_can_thread(desc)) {
1538 			ret = irq_setup_forced_threading(new);
1539 			if (ret)
1540 				goto out_mput;
1541 		}
1542 	}
1543 
1544 	/*
1545 	 * Create a handler thread when a thread function is supplied
1546 	 * and the interrupt does not nest into another interrupt
1547 	 * thread.
1548 	 */
1549 	if (new->thread_fn && !nested) {
1550 		ret = setup_irq_thread(new, irq, false);
1551 		if (ret)
1552 			goto out_mput;
1553 		if (new->secondary) {
1554 			ret = setup_irq_thread(new->secondary, irq, true);
1555 			if (ret)
1556 				goto out_thread;
1557 		}
1558 	}
1559 
1560 	/*
1561 	 * Drivers are often written to work w/o knowledge about the
1562 	 * underlying irq chip implementation, so a request for a
1563 	 * threaded irq without a primary hard irq context handler
1564 	 * requires the ONESHOT flag to be set. Some irq chips like
1565 	 * MSI based interrupts are per se one shot safe. Check the
1566 	 * chip flags, so we can avoid the unmask dance at the end of
1567 	 * the threaded handler for those.
1568 	 */
1569 	if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)
1570 		new->flags &= ~IRQF_ONESHOT;
1571 
1572 	/*
1573 	 * Protects against a concurrent __free_irq() call which might wait
1574 	 * for synchronize_hardirq() to complete without holding the optional
1575 	 * chip bus lock and desc->lock. Also protects against handing out
1576 	 * a recycled oneshot thread_mask bit while it's still in use by
1577 	 * its previous owner.
1578 	 */
1579 	mutex_lock(&desc->request_mutex);
1580 
1581 	/*
1582 	 * Acquire bus lock as the irq_request_resources() callback below
1583 	 * might rely on the serialization or the magic power management
1584 	 * functions which are abusing the irq_bus_lock() callback,
1585 	 */
1586 	chip_bus_lock(desc);
1587 
1588 	/* First installed action requests resources. */
1589 	if (!desc->action) {
1590 		ret = irq_request_resources(desc);
1591 		if (ret) {
1592 			pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n",
1593 			       new->name, irq, desc->irq_data.chip->name);
1594 			goto out_bus_unlock;
1595 		}
1596 	}
1597 
1598 	/*
1599 	 * The following block of code has to be executed atomically
1600 	 * protected against a concurrent interrupt and any of the other
1601 	 * management calls which are not serialized via
1602 	 * desc->request_mutex or the optional bus lock.
1603 	 */
1604 	raw_spin_lock_irqsave(&desc->lock, flags);
1605 	old_ptr = &desc->action;
1606 	old = *old_ptr;
1607 	if (old) {
1608 		/*
1609 		 * Can't share interrupts unless both agree to and are
1610 		 * the same type (level, edge, polarity). So both flag
1611 		 * fields must have IRQF_SHARED set and the bits which
1612 		 * set the trigger type must match. Also all must
1613 		 * agree on ONESHOT.
1614 		 * Interrupt lines used for NMIs cannot be shared.
1615 		 */
1616 		unsigned int oldtype;
1617 
1618 		if (desc->istate & IRQS_NMI) {
1619 			pr_err("Invalid attempt to share NMI for %s (irq %d) on irqchip %s.\n",
1620 				new->name, irq, desc->irq_data.chip->name);
1621 			ret = -EINVAL;
1622 			goto out_unlock;
1623 		}
1624 
1625 		/*
1626 		 * If nobody did set the configuration before, inherit
1627 		 * the one provided by the requester.
1628 		 */
1629 		if (irqd_trigger_type_was_set(&desc->irq_data)) {
1630 			oldtype = irqd_get_trigger_type(&desc->irq_data);
1631 		} else {
1632 			oldtype = new->flags & IRQF_TRIGGER_MASK;
1633 			irqd_set_trigger_type(&desc->irq_data, oldtype);
1634 		}
1635 
1636 		if (!((old->flags & new->flags) & IRQF_SHARED) ||
1637 		    (oldtype != (new->flags & IRQF_TRIGGER_MASK)) ||
1638 		    ((old->flags ^ new->flags) & IRQF_ONESHOT))
1639 			goto mismatch;
1640 
1641 		/* All handlers must agree on per-cpuness */
1642 		if ((old->flags & IRQF_PERCPU) !=
1643 		    (new->flags & IRQF_PERCPU))
1644 			goto mismatch;
1645 
1646 		/* add new interrupt at end of irq queue */
1647 		do {
1648 			/*
1649 			 * Or all existing action->thread_mask bits,
1650 			 * so we can find the next zero bit for this
1651 			 * new action.
1652 			 */
1653 			thread_mask |= old->thread_mask;
1654 			old_ptr = &old->next;
1655 			old = *old_ptr;
1656 		} while (old);
1657 		shared = 1;
1658 	}
1659 
1660 	/*
1661 	 * Setup the thread mask for this irqaction for ONESHOT. For
1662 	 * !ONESHOT irqs the thread mask is 0 so we can avoid a
1663 	 * conditional in irq_wake_thread().
1664 	 */
1665 	if (new->flags & IRQF_ONESHOT) {
1666 		/*
1667 		 * Unlikely to have 32 resp 64 irqs sharing one line,
1668 		 * but who knows.
1669 		 */
1670 		if (thread_mask == ~0UL) {
1671 			ret = -EBUSY;
1672 			goto out_unlock;
1673 		}
1674 		/*
1675 		 * The thread_mask for the action is or'ed to
1676 		 * desc->thread_active to indicate that the
1677 		 * IRQF_ONESHOT thread handler has been woken, but not
1678 		 * yet finished. The bit is cleared when a thread
1679 		 * completes. When all threads of a shared interrupt
1680 		 * line have completed desc->threads_active becomes
1681 		 * zero and the interrupt line is unmasked. See
1682 		 * handle.c:irq_wake_thread() for further information.
1683 		 *
1684 		 * If no thread is woken by primary (hard irq context)
1685 		 * interrupt handlers, then desc->threads_active is
1686 		 * also checked for zero to unmask the irq line in the
1687 		 * affected hard irq flow handlers
1688 		 * (handle_[fasteoi|level]_irq).
1689 		 *
1690 		 * The new action gets the first zero bit of
1691 		 * thread_mask assigned. See the loop above which or's
1692 		 * all existing action->thread_mask bits.
1693 		 */
1694 		new->thread_mask = 1UL << ffz(thread_mask);
1695 
1696 	} else if (new->handler == irq_default_primary_handler &&
1697 		   !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) {
1698 		/*
1699 		 * The interrupt was requested with handler = NULL, so
1700 		 * we use the default primary handler for it. But it
1701 		 * does not have the oneshot flag set. In combination
1702 		 * with level interrupts this is deadly, because the
1703 		 * default primary handler just wakes the thread, then
1704 		 * the irq lines is reenabled, but the device still
1705 		 * has the level irq asserted. Rinse and repeat....
1706 		 *
1707 		 * While this works for edge type interrupts, we play
1708 		 * it safe and reject unconditionally because we can't
1709 		 * say for sure which type this interrupt really
1710 		 * has. The type flags are unreliable as the
1711 		 * underlying chip implementation can override them.
1712 		 */
1713 		pr_err("Threaded irq requested with handler=NULL and !ONESHOT for %s (irq %d)\n",
1714 		       new->name, irq);
1715 		ret = -EINVAL;
1716 		goto out_unlock;
1717 	}
1718 
1719 	if (!shared) {
1720 		/* Setup the type (level, edge polarity) if configured: */
1721 		if (new->flags & IRQF_TRIGGER_MASK) {
1722 			ret = __irq_set_trigger(desc,
1723 						new->flags & IRQF_TRIGGER_MASK);
1724 
1725 			if (ret)
1726 				goto out_unlock;
1727 		}
1728 
1729 		/*
1730 		 * Activate the interrupt. That activation must happen
1731 		 * independently of IRQ_NOAUTOEN. request_irq() can fail
1732 		 * and the callers are supposed to handle
1733 		 * that. enable_irq() of an interrupt requested with
1734 		 * IRQ_NOAUTOEN is not supposed to fail. The activation
1735 		 * keeps it in shutdown mode, it merily associates
1736 		 * resources if necessary and if that's not possible it
1737 		 * fails. Interrupts which are in managed shutdown mode
1738 		 * will simply ignore that activation request.
1739 		 */
1740 		ret = irq_activate(desc);
1741 		if (ret)
1742 			goto out_unlock;
1743 
1744 		desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \
1745 				  IRQS_ONESHOT | IRQS_WAITING);
1746 		irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
1747 
1748 		if (new->flags & IRQF_PERCPU) {
1749 			irqd_set(&desc->irq_data, IRQD_PER_CPU);
1750 			irq_settings_set_per_cpu(desc);
1751 			if (new->flags & IRQF_NO_DEBUG)
1752 				irq_settings_set_no_debug(desc);
1753 		}
1754 
1755 		if (noirqdebug)
1756 			irq_settings_set_no_debug(desc);
1757 
1758 		if (new->flags & IRQF_ONESHOT)
1759 			desc->istate |= IRQS_ONESHOT;
1760 
1761 		/* Exclude IRQ from balancing if requested */
1762 		if (new->flags & IRQF_NOBALANCING) {
1763 			irq_settings_set_no_balancing(desc);
1764 			irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1765 		}
1766 
1767 		if (!(new->flags & IRQF_NO_AUTOEN) &&
1768 		    irq_settings_can_autoenable(desc)) {
1769 			irq_startup(desc, IRQ_RESEND, IRQ_START_COND);
1770 		} else {
1771 			/*
1772 			 * Shared interrupts do not go well with disabling
1773 			 * auto enable. The sharing interrupt might request
1774 			 * it while it's still disabled and then wait for
1775 			 * interrupts forever.
1776 			 */
1777 			WARN_ON_ONCE(new->flags & IRQF_SHARED);
1778 			/* Undo nested disables: */
1779 			desc->depth = 1;
1780 		}
1781 
1782 	} else if (new->flags & IRQF_TRIGGER_MASK) {
1783 		unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
1784 		unsigned int omsk = irqd_get_trigger_type(&desc->irq_data);
1785 
1786 		if (nmsk != omsk)
1787 			/* hope the handler works with current  trigger mode */
1788 			pr_warn("irq %d uses trigger mode %u; requested %u\n",
1789 				irq, omsk, nmsk);
1790 	}
1791 
1792 	*old_ptr = new;
1793 
1794 	irq_pm_install_action(desc, new);
1795 
1796 	/* Reset broken irq detection when installing new handler */
1797 	desc->irq_count = 0;
1798 	desc->irqs_unhandled = 0;
1799 
1800 	/*
1801 	 * Check whether we disabled the irq via the spurious handler
1802 	 * before. Reenable it and give it another chance.
1803 	 */
1804 	if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) {
1805 		desc->istate &= ~IRQS_SPURIOUS_DISABLED;
1806 		__enable_irq(desc);
1807 	}
1808 
1809 	raw_spin_unlock_irqrestore(&desc->lock, flags);
1810 	chip_bus_sync_unlock(desc);
1811 	mutex_unlock(&desc->request_mutex);
1812 
1813 	irq_setup_timings(desc, new);
1814 
1815 	wake_up_and_wait_for_irq_thread_ready(desc, new);
1816 	wake_up_and_wait_for_irq_thread_ready(desc, new->secondary);
1817 
1818 	register_irq_proc(irq, desc);
1819 	new->dir = NULL;
1820 	register_handler_proc(irq, new);
1821 	return 0;
1822 
1823 mismatch:
1824 	if (!(new->flags & IRQF_PROBE_SHARED)) {
1825 		pr_err("Flags mismatch irq %d. %08x (%s) vs. %08x (%s)\n",
1826 		       irq, new->flags, new->name, old->flags, old->name);
1827 #ifdef CONFIG_DEBUG_SHIRQ
1828 		dump_stack();
1829 #endif
1830 	}
1831 	ret = -EBUSY;
1832 
1833 out_unlock:
1834 	raw_spin_unlock_irqrestore(&desc->lock, flags);
1835 
1836 	if (!desc->action)
1837 		irq_release_resources(desc);
1838 out_bus_unlock:
1839 	chip_bus_sync_unlock(desc);
1840 	mutex_unlock(&desc->request_mutex);
1841 
1842 out_thread:
1843 	if (new->thread) {
1844 		struct task_struct *t = new->thread;
1845 
1846 		new->thread = NULL;
1847 		kthread_stop(t);
1848 		put_task_struct(t);
1849 	}
1850 	if (new->secondary && new->secondary->thread) {
1851 		struct task_struct *t = new->secondary->thread;
1852 
1853 		new->secondary->thread = NULL;
1854 		kthread_stop(t);
1855 		put_task_struct(t);
1856 	}
1857 out_mput:
1858 	module_put(desc->owner);
1859 	return ret;
1860 }
1861 
1862 /*
1863  * Internal function to unregister an irqaction - used to free
1864  * regular and special interrupts that are part of the architecture.
1865  */
__free_irq(struct irq_desc * desc,void * dev_id)1866 static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id)
1867 {
1868 	unsigned irq = desc->irq_data.irq;
1869 	struct irqaction *action, **action_ptr;
1870 	unsigned long flags;
1871 
1872 	WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1873 
1874 	mutex_lock(&desc->request_mutex);
1875 	chip_bus_lock(desc);
1876 	raw_spin_lock_irqsave(&desc->lock, flags);
1877 
1878 	/*
1879 	 * There can be multiple actions per IRQ descriptor, find the right
1880 	 * one based on the dev_id:
1881 	 */
1882 	action_ptr = &desc->action;
1883 	for (;;) {
1884 		action = *action_ptr;
1885 
1886 		if (!action) {
1887 			WARN(1, "Trying to free already-free IRQ %d\n", irq);
1888 			raw_spin_unlock_irqrestore(&desc->lock, flags);
1889 			chip_bus_sync_unlock(desc);
1890 			mutex_unlock(&desc->request_mutex);
1891 			return NULL;
1892 		}
1893 
1894 		if (action->dev_id == dev_id)
1895 			break;
1896 		action_ptr = &action->next;
1897 	}
1898 
1899 	/* Found it - now remove it from the list of entries: */
1900 	*action_ptr = action->next;
1901 
1902 	irq_pm_remove_action(desc, action);
1903 
1904 	/* If this was the last handler, shut down the IRQ line: */
1905 	if (!desc->action) {
1906 		irq_settings_clr_disable_unlazy(desc);
1907 		/* Only shutdown. Deactivate after synchronize_hardirq() */
1908 		irq_shutdown(desc);
1909 	}
1910 
1911 #ifdef CONFIG_SMP
1912 	/* make sure affinity_hint is cleaned up */
1913 	if (WARN_ON_ONCE(desc->affinity_hint))
1914 		desc->affinity_hint = NULL;
1915 #endif
1916 
1917 	raw_spin_unlock_irqrestore(&desc->lock, flags);
1918 	/*
1919 	 * Drop bus_lock here so the changes which were done in the chip
1920 	 * callbacks above are synced out to the irq chips which hang
1921 	 * behind a slow bus (I2C, SPI) before calling synchronize_hardirq().
1922 	 *
1923 	 * Aside of that the bus_lock can also be taken from the threaded
1924 	 * handler in irq_finalize_oneshot() which results in a deadlock
1925 	 * because kthread_stop() would wait forever for the thread to
1926 	 * complete, which is blocked on the bus lock.
1927 	 *
1928 	 * The still held desc->request_mutex() protects against a
1929 	 * concurrent request_irq() of this irq so the release of resources
1930 	 * and timing data is properly serialized.
1931 	 */
1932 	chip_bus_sync_unlock(desc);
1933 
1934 	unregister_handler_proc(irq, action);
1935 
1936 	/*
1937 	 * Make sure it's not being used on another CPU and if the chip
1938 	 * supports it also make sure that there is no (not yet serviced)
1939 	 * interrupt in flight at the hardware level.
1940 	 */
1941 	__synchronize_hardirq(desc, true);
1942 
1943 #ifdef CONFIG_DEBUG_SHIRQ
1944 	/*
1945 	 * It's a shared IRQ -- the driver ought to be prepared for an IRQ
1946 	 * event to happen even now it's being freed, so let's make sure that
1947 	 * is so by doing an extra call to the handler ....
1948 	 *
1949 	 * ( We do this after actually deregistering it, to make sure that a
1950 	 *   'real' IRQ doesn't run in parallel with our fake. )
1951 	 */
1952 	if (action->flags & IRQF_SHARED) {
1953 		local_irq_save(flags);
1954 		action->handler(irq, dev_id);
1955 		local_irq_restore(flags);
1956 	}
1957 #endif
1958 
1959 	/*
1960 	 * The action has already been removed above, but the thread writes
1961 	 * its oneshot mask bit when it completes. Though request_mutex is
1962 	 * held across this which prevents __setup_irq() from handing out
1963 	 * the same bit to a newly requested action.
1964 	 */
1965 	if (action->thread) {
1966 		kthread_stop(action->thread);
1967 		put_task_struct(action->thread);
1968 		if (action->secondary && action->secondary->thread) {
1969 			kthread_stop(action->secondary->thread);
1970 			put_task_struct(action->secondary->thread);
1971 		}
1972 	}
1973 
1974 	/* Last action releases resources */
1975 	if (!desc->action) {
1976 		/*
1977 		 * Reacquire bus lock as irq_release_resources() might
1978 		 * require it to deallocate resources over the slow bus.
1979 		 */
1980 		chip_bus_lock(desc);
1981 		/*
1982 		 * There is no interrupt on the fly anymore. Deactivate it
1983 		 * completely.
1984 		 */
1985 		raw_spin_lock_irqsave(&desc->lock, flags);
1986 		irq_domain_deactivate_irq(&desc->irq_data);
1987 		raw_spin_unlock_irqrestore(&desc->lock, flags);
1988 
1989 		irq_release_resources(desc);
1990 		chip_bus_sync_unlock(desc);
1991 		irq_remove_timings(desc);
1992 	}
1993 
1994 	mutex_unlock(&desc->request_mutex);
1995 
1996 	irq_chip_pm_put(&desc->irq_data);
1997 	module_put(desc->owner);
1998 	kfree(action->secondary);
1999 	return action;
2000 }
2001 
2002 /**
2003  *	free_irq - free an interrupt allocated with request_irq
2004  *	@irq: Interrupt line to free
2005  *	@dev_id: Device identity to free
2006  *
2007  *	Remove an interrupt handler. The handler is removed and if the
2008  *	interrupt line is no longer in use by any driver it is disabled.
2009  *	On a shared IRQ the caller must ensure the interrupt is disabled
2010  *	on the card it drives before calling this function. The function
2011  *	does not return until any executing interrupts for this IRQ
2012  *	have completed.
2013  *
2014  *	This function must not be called from interrupt context.
2015  *
2016  *	Returns the devname argument passed to request_irq.
2017  */
free_irq(unsigned int irq,void * dev_id)2018 const void *free_irq(unsigned int irq, void *dev_id)
2019 {
2020 	struct irq_desc *desc = irq_to_desc(irq);
2021 	struct irqaction *action;
2022 	const char *devname;
2023 
2024 	if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
2025 		return NULL;
2026 
2027 #ifdef CONFIG_SMP
2028 	if (WARN_ON(desc->affinity_notify))
2029 		desc->affinity_notify = NULL;
2030 #endif
2031 
2032 	action = __free_irq(desc, dev_id);
2033 
2034 	if (!action)
2035 		return NULL;
2036 
2037 	devname = action->name;
2038 	kfree(action);
2039 	return devname;
2040 }
2041 EXPORT_SYMBOL(free_irq);
2042 
2043 /* This function must be called with desc->lock held */
__cleanup_nmi(unsigned int irq,struct irq_desc * desc)2044 static const void *__cleanup_nmi(unsigned int irq, struct irq_desc *desc)
2045 {
2046 	const char *devname = NULL;
2047 
2048 	desc->istate &= ~IRQS_NMI;
2049 
2050 	if (!WARN_ON(desc->action == NULL)) {
2051 		irq_pm_remove_action(desc, desc->action);
2052 		devname = desc->action->name;
2053 		unregister_handler_proc(irq, desc->action);
2054 
2055 		kfree(desc->action);
2056 		desc->action = NULL;
2057 	}
2058 
2059 	irq_settings_clr_disable_unlazy(desc);
2060 	irq_shutdown_and_deactivate(desc);
2061 
2062 	irq_release_resources(desc);
2063 
2064 	irq_chip_pm_put(&desc->irq_data);
2065 	module_put(desc->owner);
2066 
2067 	return devname;
2068 }
2069 
free_nmi(unsigned int irq,void * dev_id)2070 const void *free_nmi(unsigned int irq, void *dev_id)
2071 {
2072 	struct irq_desc *desc = irq_to_desc(irq);
2073 	unsigned long flags;
2074 	const void *devname;
2075 
2076 	if (!desc || WARN_ON(!(desc->istate & IRQS_NMI)))
2077 		return NULL;
2078 
2079 	if (WARN_ON(irq_settings_is_per_cpu_devid(desc)))
2080 		return NULL;
2081 
2082 	/* NMI still enabled */
2083 	if (WARN_ON(desc->depth == 0))
2084 		disable_nmi_nosync(irq);
2085 
2086 	raw_spin_lock_irqsave(&desc->lock, flags);
2087 
2088 	irq_nmi_teardown(desc);
2089 	devname = __cleanup_nmi(irq, desc);
2090 
2091 	raw_spin_unlock_irqrestore(&desc->lock, flags);
2092 
2093 	return devname;
2094 }
2095 
2096 /**
2097  *	request_threaded_irq - allocate an interrupt line
2098  *	@irq: Interrupt line to allocate
2099  *	@handler: Function to be called when the IRQ occurs.
2100  *		  Primary handler for threaded interrupts.
2101  *		  If handler is NULL and thread_fn != NULL
2102  *		  the default primary handler is installed.
2103  *	@thread_fn: Function called from the irq handler thread
2104  *		    If NULL, no irq thread is created
2105  *	@irqflags: Interrupt type flags
2106  *	@devname: An ascii name for the claiming device
2107  *	@dev_id: A cookie passed back to the handler function
2108  *
2109  *	This call allocates interrupt resources and enables the
2110  *	interrupt line and IRQ handling. From the point this
2111  *	call is made your handler function may be invoked. Since
2112  *	your handler function must clear any interrupt the board
2113  *	raises, you must take care both to initialise your hardware
2114  *	and to set up the interrupt handler in the right order.
2115  *
2116  *	If you want to set up a threaded irq handler for your device
2117  *	then you need to supply @handler and @thread_fn. @handler is
2118  *	still called in hard interrupt context and has to check
2119  *	whether the interrupt originates from the device. If yes it
2120  *	needs to disable the interrupt on the device and return
2121  *	IRQ_WAKE_THREAD which will wake up the handler thread and run
2122  *	@thread_fn. This split handler design is necessary to support
2123  *	shared interrupts.
2124  *
2125  *	Dev_id must be globally unique. Normally the address of the
2126  *	device data structure is used as the cookie. Since the handler
2127  *	receives this value it makes sense to use it.
2128  *
2129  *	If your interrupt is shared you must pass a non NULL dev_id
2130  *	as this is required when freeing the interrupt.
2131  *
2132  *	Flags:
2133  *
2134  *	IRQF_SHARED		Interrupt is shared
2135  *	IRQF_TRIGGER_*		Specify active edge(s) or level
2136  *	IRQF_ONESHOT		Run thread_fn with interrupt line masked
2137  */
request_threaded_irq(unsigned int irq,irq_handler_t handler,irq_handler_t thread_fn,unsigned long irqflags,const char * devname,void * dev_id)2138 int request_threaded_irq(unsigned int irq, irq_handler_t handler,
2139 			 irq_handler_t thread_fn, unsigned long irqflags,
2140 			 const char *devname, void *dev_id)
2141 {
2142 	struct irqaction *action;
2143 	struct irq_desc *desc;
2144 	int retval;
2145 
2146 	if (irq == IRQ_NOTCONNECTED)
2147 		return -ENOTCONN;
2148 
2149 	/*
2150 	 * Sanity-check: shared interrupts must pass in a real dev-ID,
2151 	 * otherwise we'll have trouble later trying to figure out
2152 	 * which interrupt is which (messes up the interrupt freeing
2153 	 * logic etc).
2154 	 *
2155 	 * Also shared interrupts do not go well with disabling auto enable.
2156 	 * The sharing interrupt might request it while it's still disabled
2157 	 * and then wait for interrupts forever.
2158 	 *
2159 	 * Also IRQF_COND_SUSPEND only makes sense for shared interrupts and
2160 	 * it cannot be set along with IRQF_NO_SUSPEND.
2161 	 */
2162 	if (((irqflags & IRQF_SHARED) && !dev_id) ||
2163 	    ((irqflags & IRQF_SHARED) && (irqflags & IRQF_NO_AUTOEN)) ||
2164 	    (!(irqflags & IRQF_SHARED) && (irqflags & IRQF_COND_SUSPEND)) ||
2165 	    ((irqflags & IRQF_NO_SUSPEND) && (irqflags & IRQF_COND_SUSPEND)))
2166 		return -EINVAL;
2167 
2168 	desc = irq_to_desc(irq);
2169 	if (!desc)
2170 		return -EINVAL;
2171 
2172 	if (!irq_settings_can_request(desc) ||
2173 	    WARN_ON(irq_settings_is_per_cpu_devid(desc)))
2174 		return -EINVAL;
2175 
2176 	if (!handler) {
2177 		if (!thread_fn)
2178 			return -EINVAL;
2179 		handler = irq_default_primary_handler;
2180 	}
2181 
2182 	action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2183 	if (!action)
2184 		return -ENOMEM;
2185 
2186 	action->handler = handler;
2187 	action->thread_fn = thread_fn;
2188 	action->flags = irqflags;
2189 	action->name = devname;
2190 	action->dev_id = dev_id;
2191 
2192 	retval = irq_chip_pm_get(&desc->irq_data);
2193 	if (retval < 0) {
2194 		kfree(action);
2195 		return retval;
2196 	}
2197 
2198 	retval = __setup_irq(irq, desc, action);
2199 
2200 	if (retval) {
2201 		irq_chip_pm_put(&desc->irq_data);
2202 		kfree(action->secondary);
2203 		kfree(action);
2204 	}
2205 
2206 #ifdef CONFIG_DEBUG_SHIRQ_FIXME
2207 	if (!retval && (irqflags & IRQF_SHARED)) {
2208 		/*
2209 		 * It's a shared IRQ -- the driver ought to be prepared for it
2210 		 * to happen immediately, so let's make sure....
2211 		 * We disable the irq to make sure that a 'real' IRQ doesn't
2212 		 * run in parallel with our fake.
2213 		 */
2214 		unsigned long flags;
2215 
2216 		disable_irq(irq);
2217 		local_irq_save(flags);
2218 
2219 		handler(irq, dev_id);
2220 
2221 		local_irq_restore(flags);
2222 		enable_irq(irq);
2223 	}
2224 #endif
2225 	return retval;
2226 }
2227 EXPORT_SYMBOL(request_threaded_irq);
2228 
2229 /**
2230  *	request_any_context_irq - allocate an interrupt line
2231  *	@irq: Interrupt line to allocate
2232  *	@handler: Function to be called when the IRQ occurs.
2233  *		  Threaded handler for threaded interrupts.
2234  *	@flags: Interrupt type flags
2235  *	@name: An ascii name for the claiming device
2236  *	@dev_id: A cookie passed back to the handler function
2237  *
2238  *	This call allocates interrupt resources and enables the
2239  *	interrupt line and IRQ handling. It selects either a
2240  *	hardirq or threaded handling method depending on the
2241  *	context.
2242  *
2243  *	On failure, it returns a negative value. On success,
2244  *	it returns either IRQC_IS_HARDIRQ or IRQC_IS_NESTED.
2245  */
request_any_context_irq(unsigned int irq,irq_handler_t handler,unsigned long flags,const char * name,void * dev_id)2246 int request_any_context_irq(unsigned int irq, irq_handler_t handler,
2247 			    unsigned long flags, const char *name, void *dev_id)
2248 {
2249 	struct irq_desc *desc;
2250 	int ret;
2251 
2252 	if (irq == IRQ_NOTCONNECTED)
2253 		return -ENOTCONN;
2254 
2255 	desc = irq_to_desc(irq);
2256 	if (!desc)
2257 		return -EINVAL;
2258 
2259 	if (irq_settings_is_nested_thread(desc)) {
2260 		ret = request_threaded_irq(irq, NULL, handler,
2261 					   flags, name, dev_id);
2262 		return !ret ? IRQC_IS_NESTED : ret;
2263 	}
2264 
2265 	ret = request_irq(irq, handler, flags, name, dev_id);
2266 	return !ret ? IRQC_IS_HARDIRQ : ret;
2267 }
2268 EXPORT_SYMBOL_GPL(request_any_context_irq);
2269 
2270 /**
2271  *	request_nmi - allocate an interrupt line for NMI delivery
2272  *	@irq: Interrupt line to allocate
2273  *	@handler: Function to be called when the IRQ occurs.
2274  *		  Threaded handler for threaded interrupts.
2275  *	@irqflags: Interrupt type flags
2276  *	@name: An ascii name for the claiming device
2277  *	@dev_id: A cookie passed back to the handler function
2278  *
2279  *	This call allocates interrupt resources and enables the
2280  *	interrupt line and IRQ handling. It sets up the IRQ line
2281  *	to be handled as an NMI.
2282  *
2283  *	An interrupt line delivering NMIs cannot be shared and IRQ handling
2284  *	cannot be threaded.
2285  *
2286  *	Interrupt lines requested for NMI delivering must produce per cpu
2287  *	interrupts and have auto enabling setting disabled.
2288  *
2289  *	Dev_id must be globally unique. Normally the address of the
2290  *	device data structure is used as the cookie. Since the handler
2291  *	receives this value it makes sense to use it.
2292  *
2293  *	If the interrupt line cannot be used to deliver NMIs, function
2294  *	will fail and return a negative value.
2295  */
request_nmi(unsigned int irq,irq_handler_t handler,unsigned long irqflags,const char * name,void * dev_id)2296 int request_nmi(unsigned int irq, irq_handler_t handler,
2297 		unsigned long irqflags, const char *name, void *dev_id)
2298 {
2299 	struct irqaction *action;
2300 	struct irq_desc *desc;
2301 	unsigned long flags;
2302 	int retval;
2303 
2304 	if (irq == IRQ_NOTCONNECTED)
2305 		return -ENOTCONN;
2306 
2307 	/* NMI cannot be shared, used for Polling */
2308 	if (irqflags & (IRQF_SHARED | IRQF_COND_SUSPEND | IRQF_IRQPOLL))
2309 		return -EINVAL;
2310 
2311 	if (!(irqflags & IRQF_PERCPU))
2312 		return -EINVAL;
2313 
2314 	if (!handler)
2315 		return -EINVAL;
2316 
2317 	desc = irq_to_desc(irq);
2318 
2319 	if (!desc || (irq_settings_can_autoenable(desc) &&
2320 	    !(irqflags & IRQF_NO_AUTOEN)) ||
2321 	    !irq_settings_can_request(desc) ||
2322 	    WARN_ON(irq_settings_is_per_cpu_devid(desc)) ||
2323 	    !irq_supports_nmi(desc))
2324 		return -EINVAL;
2325 
2326 	action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2327 	if (!action)
2328 		return -ENOMEM;
2329 
2330 	action->handler = handler;
2331 	action->flags = irqflags | IRQF_NO_THREAD | IRQF_NOBALANCING;
2332 	action->name = name;
2333 	action->dev_id = dev_id;
2334 
2335 	retval = irq_chip_pm_get(&desc->irq_data);
2336 	if (retval < 0)
2337 		goto err_out;
2338 
2339 	retval = __setup_irq(irq, desc, action);
2340 	if (retval)
2341 		goto err_irq_setup;
2342 
2343 	raw_spin_lock_irqsave(&desc->lock, flags);
2344 
2345 	/* Setup NMI state */
2346 	desc->istate |= IRQS_NMI;
2347 	retval = irq_nmi_setup(desc);
2348 	if (retval) {
2349 		__cleanup_nmi(irq, desc);
2350 		raw_spin_unlock_irqrestore(&desc->lock, flags);
2351 		return -EINVAL;
2352 	}
2353 
2354 	raw_spin_unlock_irqrestore(&desc->lock, flags);
2355 
2356 	return 0;
2357 
2358 err_irq_setup:
2359 	irq_chip_pm_put(&desc->irq_data);
2360 err_out:
2361 	kfree(action);
2362 
2363 	return retval;
2364 }
2365 
enable_percpu_irq(unsigned int irq,unsigned int type)2366 void enable_percpu_irq(unsigned int irq, unsigned int type)
2367 {
2368 	unsigned int cpu = smp_processor_id();
2369 	unsigned long flags;
2370 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
2371 
2372 	if (!desc)
2373 		return;
2374 
2375 	/*
2376 	 * If the trigger type is not specified by the caller, then
2377 	 * use the default for this interrupt.
2378 	 */
2379 	type &= IRQ_TYPE_SENSE_MASK;
2380 	if (type == IRQ_TYPE_NONE)
2381 		type = irqd_get_trigger_type(&desc->irq_data);
2382 
2383 	if (type != IRQ_TYPE_NONE) {
2384 		int ret;
2385 
2386 		ret = __irq_set_trigger(desc, type);
2387 
2388 		if (ret) {
2389 			WARN(1, "failed to set type for IRQ%d\n", irq);
2390 			goto out;
2391 		}
2392 	}
2393 
2394 	irq_percpu_enable(desc, cpu);
2395 out:
2396 	irq_put_desc_unlock(desc, flags);
2397 }
2398 EXPORT_SYMBOL_GPL(enable_percpu_irq);
2399 
enable_percpu_nmi(unsigned int irq,unsigned int type)2400 void enable_percpu_nmi(unsigned int irq, unsigned int type)
2401 {
2402 	enable_percpu_irq(irq, type);
2403 }
2404 
2405 /**
2406  * irq_percpu_is_enabled - Check whether the per cpu irq is enabled
2407  * @irq:	Linux irq number to check for
2408  *
2409  * Must be called from a non migratable context. Returns the enable
2410  * state of a per cpu interrupt on the current cpu.
2411  */
irq_percpu_is_enabled(unsigned int irq)2412 bool irq_percpu_is_enabled(unsigned int irq)
2413 {
2414 	unsigned int cpu = smp_processor_id();
2415 	struct irq_desc *desc;
2416 	unsigned long flags;
2417 	bool is_enabled;
2418 
2419 	desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
2420 	if (!desc)
2421 		return false;
2422 
2423 	is_enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
2424 	irq_put_desc_unlock(desc, flags);
2425 
2426 	return is_enabled;
2427 }
2428 EXPORT_SYMBOL_GPL(irq_percpu_is_enabled);
2429 
disable_percpu_irq(unsigned int irq)2430 void disable_percpu_irq(unsigned int irq)
2431 {
2432 	unsigned int cpu = smp_processor_id();
2433 	unsigned long flags;
2434 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
2435 
2436 	if (!desc)
2437 		return;
2438 
2439 	irq_percpu_disable(desc, cpu);
2440 	irq_put_desc_unlock(desc, flags);
2441 }
2442 EXPORT_SYMBOL_GPL(disable_percpu_irq);
2443 
disable_percpu_nmi(unsigned int irq)2444 void disable_percpu_nmi(unsigned int irq)
2445 {
2446 	disable_percpu_irq(irq);
2447 }
2448 
2449 /*
2450  * Internal function to unregister a percpu irqaction.
2451  */
__free_percpu_irq(unsigned int irq,void __percpu * dev_id)2452 static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_id)
2453 {
2454 	struct irq_desc *desc = irq_to_desc(irq);
2455 	struct irqaction *action;
2456 	unsigned long flags;
2457 
2458 	WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
2459 
2460 	if (!desc)
2461 		return NULL;
2462 
2463 	raw_spin_lock_irqsave(&desc->lock, flags);
2464 
2465 	action = desc->action;
2466 	if (!action || action->percpu_dev_id != dev_id) {
2467 		WARN(1, "Trying to free already-free IRQ %d\n", irq);
2468 		goto bad;
2469 	}
2470 
2471 	if (!cpumask_empty(desc->percpu_enabled)) {
2472 		WARN(1, "percpu IRQ %d still enabled on CPU%d!\n",
2473 		     irq, cpumask_first(desc->percpu_enabled));
2474 		goto bad;
2475 	}
2476 
2477 	/* Found it - now remove it from the list of entries: */
2478 	desc->action = NULL;
2479 
2480 	desc->istate &= ~IRQS_NMI;
2481 
2482 	raw_spin_unlock_irqrestore(&desc->lock, flags);
2483 
2484 	unregister_handler_proc(irq, action);
2485 
2486 	irq_chip_pm_put(&desc->irq_data);
2487 	module_put(desc->owner);
2488 	return action;
2489 
2490 bad:
2491 	raw_spin_unlock_irqrestore(&desc->lock, flags);
2492 	return NULL;
2493 }
2494 
2495 /**
2496  *	remove_percpu_irq - free a per-cpu interrupt
2497  *	@irq: Interrupt line to free
2498  *	@act: irqaction for the interrupt
2499  *
2500  * Used to remove interrupts statically setup by the early boot process.
2501  */
remove_percpu_irq(unsigned int irq,struct irqaction * act)2502 void remove_percpu_irq(unsigned int irq, struct irqaction *act)
2503 {
2504 	struct irq_desc *desc = irq_to_desc(irq);
2505 
2506 	if (desc && irq_settings_is_per_cpu_devid(desc))
2507 	    __free_percpu_irq(irq, act->percpu_dev_id);
2508 }
2509 
2510 /**
2511  *	free_percpu_irq - free an interrupt allocated with request_percpu_irq
2512  *	@irq: Interrupt line to free
2513  *	@dev_id: Device identity to free
2514  *
2515  *	Remove a percpu interrupt handler. The handler is removed, but
2516  *	the interrupt line is not disabled. This must be done on each
2517  *	CPU before calling this function. The function does not return
2518  *	until any executing interrupts for this IRQ have completed.
2519  *
2520  *	This function must not be called from interrupt context.
2521  */
free_percpu_irq(unsigned int irq,void __percpu * dev_id)2522 void free_percpu_irq(unsigned int irq, void __percpu *dev_id)
2523 {
2524 	struct irq_desc *desc = irq_to_desc(irq);
2525 
2526 	if (!desc || !irq_settings_is_per_cpu_devid(desc))
2527 		return;
2528 
2529 	chip_bus_lock(desc);
2530 	kfree(__free_percpu_irq(irq, dev_id));
2531 	chip_bus_sync_unlock(desc);
2532 }
2533 EXPORT_SYMBOL_GPL(free_percpu_irq);
2534 
free_percpu_nmi(unsigned int irq,void __percpu * dev_id)2535 void free_percpu_nmi(unsigned int irq, void __percpu *dev_id)
2536 {
2537 	struct irq_desc *desc = irq_to_desc(irq);
2538 
2539 	if (!desc || !irq_settings_is_per_cpu_devid(desc))
2540 		return;
2541 
2542 	if (WARN_ON(!(desc->istate & IRQS_NMI)))
2543 		return;
2544 
2545 	kfree(__free_percpu_irq(irq, dev_id));
2546 }
2547 
2548 /**
2549  *	setup_percpu_irq - setup a per-cpu interrupt
2550  *	@irq: Interrupt line to setup
2551  *	@act: irqaction for the interrupt
2552  *
2553  * Used to statically setup per-cpu interrupts in the early boot process.
2554  */
setup_percpu_irq(unsigned int irq,struct irqaction * act)2555 int setup_percpu_irq(unsigned int irq, struct irqaction *act)
2556 {
2557 	struct irq_desc *desc = irq_to_desc(irq);
2558 	int retval;
2559 
2560 	if (!desc || !irq_settings_is_per_cpu_devid(desc))
2561 		return -EINVAL;
2562 
2563 	retval = irq_chip_pm_get(&desc->irq_data);
2564 	if (retval < 0)
2565 		return retval;
2566 
2567 	retval = __setup_irq(irq, desc, act);
2568 
2569 	if (retval)
2570 		irq_chip_pm_put(&desc->irq_data);
2571 
2572 	return retval;
2573 }
2574 
2575 /**
2576  *	__request_percpu_irq - allocate a percpu interrupt line
2577  *	@irq: Interrupt line to allocate
2578  *	@handler: Function to be called when the IRQ occurs.
2579  *	@flags: Interrupt type flags (IRQF_TIMER only)
2580  *	@devname: An ascii name for the claiming device
2581  *	@dev_id: A percpu cookie passed back to the handler function
2582  *
2583  *	This call allocates interrupt resources and enables the
2584  *	interrupt on the local CPU. If the interrupt is supposed to be
2585  *	enabled on other CPUs, it has to be done on each CPU using
2586  *	enable_percpu_irq().
2587  *
2588  *	Dev_id must be globally unique. It is a per-cpu variable, and
2589  *	the handler gets called with the interrupted CPU's instance of
2590  *	that variable.
2591  */
__request_percpu_irq(unsigned int irq,irq_handler_t handler,unsigned long flags,const char * devname,void __percpu * dev_id)2592 int __request_percpu_irq(unsigned int irq, irq_handler_t handler,
2593 			 unsigned long flags, const char *devname,
2594 			 void __percpu *dev_id)
2595 {
2596 	struct irqaction *action;
2597 	struct irq_desc *desc;
2598 	int retval;
2599 
2600 	if (!dev_id)
2601 		return -EINVAL;
2602 
2603 	desc = irq_to_desc(irq);
2604 	if (!desc || !irq_settings_can_request(desc) ||
2605 	    !irq_settings_is_per_cpu_devid(desc))
2606 		return -EINVAL;
2607 
2608 	if (flags && flags != IRQF_TIMER)
2609 		return -EINVAL;
2610 
2611 	action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2612 	if (!action)
2613 		return -ENOMEM;
2614 
2615 	action->handler = handler;
2616 	action->flags = flags | IRQF_PERCPU | IRQF_NO_SUSPEND;
2617 	action->name = devname;
2618 	action->percpu_dev_id = dev_id;
2619 
2620 	retval = irq_chip_pm_get(&desc->irq_data);
2621 	if (retval < 0) {
2622 		kfree(action);
2623 		return retval;
2624 	}
2625 
2626 	retval = __setup_irq(irq, desc, action);
2627 
2628 	if (retval) {
2629 		irq_chip_pm_put(&desc->irq_data);
2630 		kfree(action);
2631 	}
2632 
2633 	return retval;
2634 }
2635 EXPORT_SYMBOL_GPL(__request_percpu_irq);
2636 
2637 /**
2638  *	request_percpu_nmi - allocate a percpu interrupt line for NMI delivery
2639  *	@irq: Interrupt line to allocate
2640  *	@handler: Function to be called when the IRQ occurs.
2641  *	@name: An ascii name for the claiming device
2642  *	@dev_id: A percpu cookie passed back to the handler function
2643  *
2644  *	This call allocates interrupt resources for a per CPU NMI. Per CPU NMIs
2645  *	have to be setup on each CPU by calling prepare_percpu_nmi() before
2646  *	being enabled on the same CPU by using enable_percpu_nmi().
2647  *
2648  *	Dev_id must be globally unique. It is a per-cpu variable, and
2649  *	the handler gets called with the interrupted CPU's instance of
2650  *	that variable.
2651  *
2652  *	Interrupt lines requested for NMI delivering should have auto enabling
2653  *	setting disabled.
2654  *
2655  *	If the interrupt line cannot be used to deliver NMIs, function
2656  *	will fail returning a negative value.
2657  */
request_percpu_nmi(unsigned int irq,irq_handler_t handler,const char * name,void __percpu * dev_id)2658 int request_percpu_nmi(unsigned int irq, irq_handler_t handler,
2659 		       const char *name, void __percpu *dev_id)
2660 {
2661 	struct irqaction *action;
2662 	struct irq_desc *desc;
2663 	unsigned long flags;
2664 	int retval;
2665 
2666 	if (!handler)
2667 		return -EINVAL;
2668 
2669 	desc = irq_to_desc(irq);
2670 
2671 	if (!desc || !irq_settings_can_request(desc) ||
2672 	    !irq_settings_is_per_cpu_devid(desc) ||
2673 	    irq_settings_can_autoenable(desc) ||
2674 	    !irq_supports_nmi(desc))
2675 		return -EINVAL;
2676 
2677 	/* The line cannot already be NMI */
2678 	if (desc->istate & IRQS_NMI)
2679 		return -EINVAL;
2680 
2681 	action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
2682 	if (!action)
2683 		return -ENOMEM;
2684 
2685 	action->handler = handler;
2686 	action->flags = IRQF_PERCPU | IRQF_NO_SUSPEND | IRQF_NO_THREAD
2687 		| IRQF_NOBALANCING;
2688 	action->name = name;
2689 	action->percpu_dev_id = dev_id;
2690 
2691 	retval = irq_chip_pm_get(&desc->irq_data);
2692 	if (retval < 0)
2693 		goto err_out;
2694 
2695 	retval = __setup_irq(irq, desc, action);
2696 	if (retval)
2697 		goto err_irq_setup;
2698 
2699 	raw_spin_lock_irqsave(&desc->lock, flags);
2700 	desc->istate |= IRQS_NMI;
2701 	raw_spin_unlock_irqrestore(&desc->lock, flags);
2702 
2703 	return 0;
2704 
2705 err_irq_setup:
2706 	irq_chip_pm_put(&desc->irq_data);
2707 err_out:
2708 	kfree(action);
2709 
2710 	return retval;
2711 }
2712 
2713 /**
2714  *	prepare_percpu_nmi - performs CPU local setup for NMI delivery
2715  *	@irq: Interrupt line to prepare for NMI delivery
2716  *
2717  *	This call prepares an interrupt line to deliver NMI on the current CPU,
2718  *	before that interrupt line gets enabled with enable_percpu_nmi().
2719  *
2720  *	As a CPU local operation, this should be called from non-preemptible
2721  *	context.
2722  *
2723  *	If the interrupt line cannot be used to deliver NMIs, function
2724  *	will fail returning a negative value.
2725  */
prepare_percpu_nmi(unsigned int irq)2726 int prepare_percpu_nmi(unsigned int irq)
2727 {
2728 	unsigned long flags;
2729 	struct irq_desc *desc;
2730 	int ret = 0;
2731 
2732 	WARN_ON(preemptible());
2733 
2734 	desc = irq_get_desc_lock(irq, &flags,
2735 				 IRQ_GET_DESC_CHECK_PERCPU);
2736 	if (!desc)
2737 		return -EINVAL;
2738 
2739 	if (WARN(!(desc->istate & IRQS_NMI),
2740 		 KERN_ERR "prepare_percpu_nmi called for a non-NMI interrupt: irq %u\n",
2741 		 irq)) {
2742 		ret = -EINVAL;
2743 		goto out;
2744 	}
2745 
2746 	ret = irq_nmi_setup(desc);
2747 	if (ret) {
2748 		pr_err("Failed to setup NMI delivery: irq %u\n", irq);
2749 		goto out;
2750 	}
2751 
2752 out:
2753 	irq_put_desc_unlock(desc, flags);
2754 	return ret;
2755 }
2756 
2757 /**
2758  *	teardown_percpu_nmi - undoes NMI setup of IRQ line
2759  *	@irq: Interrupt line from which CPU local NMI configuration should be
2760  *	      removed
2761  *
2762  *	This call undoes the setup done by prepare_percpu_nmi().
2763  *
2764  *	IRQ line should not be enabled for the current CPU.
2765  *
2766  *	As a CPU local operation, this should be called from non-preemptible
2767  *	context.
2768  */
teardown_percpu_nmi(unsigned int irq)2769 void teardown_percpu_nmi(unsigned int irq)
2770 {
2771 	unsigned long flags;
2772 	struct irq_desc *desc;
2773 
2774 	WARN_ON(preemptible());
2775 
2776 	desc = irq_get_desc_lock(irq, &flags,
2777 				 IRQ_GET_DESC_CHECK_PERCPU);
2778 	if (!desc)
2779 		return;
2780 
2781 	if (WARN_ON(!(desc->istate & IRQS_NMI)))
2782 		goto out;
2783 
2784 	irq_nmi_teardown(desc);
2785 out:
2786 	irq_put_desc_unlock(desc, flags);
2787 }
2788 
__irq_get_irqchip_state(struct irq_data * data,enum irqchip_irq_state which,bool * state)2789 int __irq_get_irqchip_state(struct irq_data *data, enum irqchip_irq_state which,
2790 			    bool *state)
2791 {
2792 	struct irq_chip *chip;
2793 	int err = -EINVAL;
2794 
2795 	do {
2796 		chip = irq_data_get_irq_chip(data);
2797 		if (WARN_ON_ONCE(!chip))
2798 			return -ENODEV;
2799 		if (chip->irq_get_irqchip_state)
2800 			break;
2801 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
2802 		data = data->parent_data;
2803 #else
2804 		data = NULL;
2805 #endif
2806 	} while (data);
2807 
2808 	if (data)
2809 		err = chip->irq_get_irqchip_state(data, which, state);
2810 	return err;
2811 }
2812 
2813 /**
2814  *	irq_get_irqchip_state - returns the irqchip state of a interrupt.
2815  *	@irq: Interrupt line that is forwarded to a VM
2816  *	@which: One of IRQCHIP_STATE_* the caller wants to know about
2817  *	@state: a pointer to a boolean where the state is to be stored
2818  *
2819  *	This call snapshots the internal irqchip state of an
2820  *	interrupt, returning into @state the bit corresponding to
2821  *	stage @which
2822  *
2823  *	This function should be called with preemption disabled if the
2824  *	interrupt controller has per-cpu registers.
2825  */
irq_get_irqchip_state(unsigned int irq,enum irqchip_irq_state which,bool * state)2826 int irq_get_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
2827 			  bool *state)
2828 {
2829 	struct irq_desc *desc;
2830 	struct irq_data *data;
2831 	unsigned long flags;
2832 	int err = -EINVAL;
2833 
2834 	desc = irq_get_desc_buslock(irq, &flags, 0);
2835 	if (!desc)
2836 		return err;
2837 
2838 	data = irq_desc_get_irq_data(desc);
2839 
2840 	err = __irq_get_irqchip_state(data, which, state);
2841 
2842 	irq_put_desc_busunlock(desc, flags);
2843 	return err;
2844 }
2845 EXPORT_SYMBOL_GPL(irq_get_irqchip_state);
2846 
2847 /**
2848  *	irq_set_irqchip_state - set the state of a forwarded interrupt.
2849  *	@irq: Interrupt line that is forwarded to a VM
2850  *	@which: State to be restored (one of IRQCHIP_STATE_*)
2851  *	@val: Value corresponding to @which
2852  *
2853  *	This call sets the internal irqchip state of an interrupt,
2854  *	depending on the value of @which.
2855  *
2856  *	This function should be called with migration disabled if the
2857  *	interrupt controller has per-cpu registers.
2858  */
irq_set_irqchip_state(unsigned int irq,enum irqchip_irq_state which,bool val)2859 int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
2860 			  bool val)
2861 {
2862 	struct irq_desc *desc;
2863 	struct irq_data *data;
2864 	struct irq_chip *chip;
2865 	unsigned long flags;
2866 	int err = -EINVAL;
2867 
2868 	desc = irq_get_desc_buslock(irq, &flags, 0);
2869 	if (!desc)
2870 		return err;
2871 
2872 	data = irq_desc_get_irq_data(desc);
2873 
2874 	do {
2875 		chip = irq_data_get_irq_chip(data);
2876 		if (WARN_ON_ONCE(!chip)) {
2877 			err = -ENODEV;
2878 			goto out_unlock;
2879 		}
2880 		if (chip->irq_set_irqchip_state)
2881 			break;
2882 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
2883 		data = data->parent_data;
2884 #else
2885 		data = NULL;
2886 #endif
2887 	} while (data);
2888 
2889 	if (data)
2890 		err = chip->irq_set_irqchip_state(data, which, val);
2891 
2892 out_unlock:
2893 	irq_put_desc_busunlock(desc, flags);
2894 	return err;
2895 }
2896 EXPORT_SYMBOL_GPL(irq_set_irqchip_state);
2897 
2898 /**
2899  * irq_has_action - Check whether an interrupt is requested
2900  * @irq:	The linux irq number
2901  *
2902  * Returns: A snapshot of the current state
2903  */
irq_has_action(unsigned int irq)2904 bool irq_has_action(unsigned int irq)
2905 {
2906 	bool res;
2907 
2908 	rcu_read_lock();
2909 	res = irq_desc_has_action(irq_to_desc(irq));
2910 	rcu_read_unlock();
2911 	return res;
2912 }
2913 EXPORT_SYMBOL_GPL(irq_has_action);
2914 
2915 /**
2916  * irq_check_status_bit - Check whether bits in the irq descriptor status are set
2917  * @irq:	The linux irq number
2918  * @bitmask:	The bitmask to evaluate
2919  *
2920  * Returns: True if one of the bits in @bitmask is set
2921  */
irq_check_status_bit(unsigned int irq,unsigned int bitmask)2922 bool irq_check_status_bit(unsigned int irq, unsigned int bitmask)
2923 {
2924 	struct irq_desc *desc;
2925 	bool res = false;
2926 
2927 	rcu_read_lock();
2928 	desc = irq_to_desc(irq);
2929 	if (desc)
2930 		res = !!(desc->status_use_accessors & bitmask);
2931 	rcu_read_unlock();
2932 	return res;
2933 }
2934 EXPORT_SYMBOL_GPL(irq_check_status_bit);
2935