57 static bool irq_work_claim(struct irq_work *work)  in irq_work_claim()  argument
61 	oflags = atomic_fetch_or(IRQ_WORK_CLAIMED | CSD_TYPE_IRQ_WORK, &work->node.a_flags);  in irq_work_claim()
79 static __always_inline void irq_work_raise(struct irq_work *work)  in irq_work_raise()  argument
82 		trace_ipi_send_cpu(smp_processor_id(), _RET_IP_, work->func);  in irq_work_raise()
88 static void __irq_work_queue_local(struct irq_work *work)  in __irq_work_queue_local()  argument
95 	work_flags = atomic_read(&work->node.a_flags);  in __irq_work_queue_local()
107 	if (!llist_add(&work->node.llist, list))  in __irq_work_queue_local()
112 		irq_work_raise(work);  in __irq_work_queue_local()
116 bool irq_work_queue(struct irq_work *work)  in irq_work_queue()  argument
119 	if (!irq_work_claim(work))  in irq_work_queue()
124 	__irq_work_queue_local(work);  in irq_work_queue()
137 bool irq_work_queue_on(struct irq_work *work, int cpu)  in irq_work_queue_on()  argument
140 	return irq_work_queue(work);  in irq_work_queue_on()
147 	if (!irq_work_claim(work))  in irq_work_queue_on()
150 	kasan_record_aux_stack_noalloc(work);  in irq_work_queue_on()
163 		    !(atomic_read(&work->node.a_flags) & IRQ_WORK_HARD_IRQ)) {  in irq_work_queue_on()
165 			if (!llist_add(&work->node.llist, &per_cpu(lazy_list, cpu)))  in irq_work_queue_on()
168 			work = &per_cpu(irq_work_wakeup, cpu);  in irq_work_queue_on()
169 			if (!irq_work_claim(work))  in irq_work_queue_on()
173 		__smp_call_single_queue(cpu, &work->node.llist);  in irq_work_queue_on()
175 		__irq_work_queue_local(work);  in irq_work_queue_on()
203 	struct irq_work *work = arg;  in irq_work_single()  local
211 	flags = atomic_read(&work->node.a_flags);  in irq_work_single()
213 	atomic_set(&work->node.a_flags, flags);  in irq_work_single()
221 	work->func(work);  in irq_work_single()
228 	(void)atomic_cmpxchg(&work->node.a_flags, flags, flags & ~IRQ_WORK_BUSY);  in irq_work_single()
230 	if ((IS_ENABLED(CONFIG_PREEMPT_RT) && !irq_work_is_hard(work)) ||  in irq_work_single()
232 		rcuwait_wake_up(&work->irqwait);  in irq_work_single()
237 	struct irq_work *work, *tmp;  in irq_work_run_list()  local
251 	llist_for_each_entry_safe(work, tmp, llnode, node.llist)  in irq_work_run_list()
252 		irq_work_single(work);  in irq_work_run_list()
286 void irq_work_sync(struct irq_work *work)  in irq_work_sync()  argument
291 	if ((IS_ENABLED(CONFIG_PREEMPT_RT) && !irq_work_is_hard(work)) ||  in irq_work_sync()
293 		rcuwait_wait_event(&work->irqwait, !irq_work_is_busy(work),  in irq_work_sync()
298 	while (irq_work_is_busy(work))  in irq_work_sync()