19 static bool irq_enable(struct intel_breadcrumbs *b)  in irq_enable()  argument
21 	return intel_engine_irq_enable(b->irq_engine);  in irq_enable()
24 static void irq_disable(struct intel_breadcrumbs *b)  in irq_disable()  argument
26 	intel_engine_irq_disable(b->irq_engine);  in irq_disable()
29 static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)  in __intel_breadcrumbs_arm_irq()  argument
35 	if (GEM_WARN_ON(!intel_gt_pm_get_if_awake(b->irq_engine->gt)))  in __intel_breadcrumbs_arm_irq()
44 	WRITE_ONCE(b->irq_armed, true);  in __intel_breadcrumbs_arm_irq()
47 	if (!b->irq_enabled++ && b->irq_enable(b))  in __intel_breadcrumbs_arm_irq()
48 		irq_work_queue(&b->irq_work);  in __intel_breadcrumbs_arm_irq()
51 static void intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)  in intel_breadcrumbs_arm_irq()  argument
53 	if (!b->irq_engine)  in intel_breadcrumbs_arm_irq()
56 	spin_lock(&b->irq_lock);  in intel_breadcrumbs_arm_irq()
57 	if (!b->irq_armed)  in intel_breadcrumbs_arm_irq()
58 		__intel_breadcrumbs_arm_irq(b);  in intel_breadcrumbs_arm_irq()
59 	spin_unlock(&b->irq_lock);  in intel_breadcrumbs_arm_irq()
62 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)  in __intel_breadcrumbs_disarm_irq()  argument
64 	GEM_BUG_ON(!b->irq_enabled);  in __intel_breadcrumbs_disarm_irq()
65 	if (!--b->irq_enabled)  in __intel_breadcrumbs_disarm_irq()
66 		b->irq_disable(b);  in __intel_breadcrumbs_disarm_irq()
68 	WRITE_ONCE(b->irq_armed, false);  in __intel_breadcrumbs_disarm_irq()
69 	intel_gt_pm_put_async(b->irq_engine->gt);  in __intel_breadcrumbs_disarm_irq()
72 static void intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)  in intel_breadcrumbs_disarm_irq()  argument
74 	spin_lock(&b->irq_lock);  in intel_breadcrumbs_disarm_irq()
75 	if (b->irq_armed)  in intel_breadcrumbs_disarm_irq()
76 		__intel_breadcrumbs_disarm_irq(b);  in intel_breadcrumbs_disarm_irq()
77 	spin_unlock(&b->irq_lock);  in intel_breadcrumbs_disarm_irq()
80 static void add_signaling_context(struct intel_breadcrumbs *b,  in add_signaling_context()  argument
85 	spin_lock(&b->signalers_lock);  in add_signaling_context()
86 	list_add_rcu(&ce->signal_link, &b->signalers);  in add_signaling_context()
87 	spin_unlock(&b->signalers_lock);  in add_signaling_context()
90 static bool remove_signaling_context(struct intel_breadcrumbs *b,  in remove_signaling_context()  argument
98 	spin_lock(&b->signalers_lock);  in remove_signaling_context()
100 	spin_unlock(&b->signalers_lock);  in remove_signaling_context()
152 static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl)  in add_retire()  argument
154 	if (b->irq_engine)  in add_retire()
155 		intel_engine_add_retire(b->irq_engine, tl);  in add_retire()
167 	struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work);  in signal_irq_work()  local
173 	if (unlikely(!llist_empty(&b->signaled_requests)))  in signal_irq_work()
174 		signal = llist_del_all(&b->signaled_requests);  in signal_irq_work()
201 	if (!signal && READ_ONCE(b->irq_armed) && list_empty(&b->signalers))  in signal_irq_work()
202 		intel_breadcrumbs_disarm_irq(b);  in signal_irq_work()
205 	atomic_inc(&b->signaler_active);  in signal_irq_work()
206 	list_for_each_entry_rcu(ce, &b->signalers, signal_link) {  in signal_irq_work()
226 			release = remove_signaling_context(b, ce);  in signal_irq_work()
230 					add_retire(b, ce->timeline);  in signal_irq_work()
241 	atomic_dec(&b->signaler_active);  in signal_irq_work()
261 	if (!READ_ONCE(b->irq_armed) && !list_empty(&b->signalers))  in signal_irq_work()
262 		intel_breadcrumbs_arm_irq(b);  in signal_irq_work()
268 	struct intel_breadcrumbs *b;  in intel_breadcrumbs_create()  local
270 	b = kzalloc(sizeof(*b), GFP_KERNEL);  in intel_breadcrumbs_create()
271 	if (!b)  in intel_breadcrumbs_create()
274 	kref_init(&b->ref);  in intel_breadcrumbs_create()
276 	spin_lock_init(&b->signalers_lock);  in intel_breadcrumbs_create()
277 	INIT_LIST_HEAD(&b->signalers);  in intel_breadcrumbs_create()
278 	init_llist_head(&b->signaled_requests);  in intel_breadcrumbs_create()
280 	spin_lock_init(&b->irq_lock);  in intel_breadcrumbs_create()
281 	init_irq_work(&b->irq_work, signal_irq_work);  in intel_breadcrumbs_create()
283 	b->irq_engine = irq_engine;  in intel_breadcrumbs_create()
284 	b->irq_enable = irq_enable;  in intel_breadcrumbs_create()
285 	b->irq_disable = irq_disable;  in intel_breadcrumbs_create()
287 	return b;  in intel_breadcrumbs_create()
290 void intel_breadcrumbs_reset(struct intel_breadcrumbs *b)  in intel_breadcrumbs_reset()  argument
294 	if (!b->irq_engine)  in intel_breadcrumbs_reset()
297 	spin_lock_irqsave(&b->irq_lock, flags);  in intel_breadcrumbs_reset()
299 	if (b->irq_enabled)  in intel_breadcrumbs_reset()
300 		b->irq_enable(b);  in intel_breadcrumbs_reset()
302 		b->irq_disable(b);  in intel_breadcrumbs_reset()
304 	spin_unlock_irqrestore(&b->irq_lock, flags);  in intel_breadcrumbs_reset()
307 void __intel_breadcrumbs_park(struct intel_breadcrumbs *b)  in __intel_breadcrumbs_park()  argument
309 	if (!READ_ONCE(b->irq_armed))  in __intel_breadcrumbs_park()
313 	irq_work_sync(&b->irq_work);  in __intel_breadcrumbs_park()
314 	while (READ_ONCE(b->irq_armed) && !atomic_read(&b->active)) {  in __intel_breadcrumbs_park()
316 		signal_irq_work(&b->irq_work);  in __intel_breadcrumbs_park()
324 	struct intel_breadcrumbs *b = container_of(kref, typeof(*b), ref);  in intel_breadcrumbs_free()  local
326 	irq_work_sync(&b->irq_work);  in intel_breadcrumbs_free()
327 	GEM_BUG_ON(!list_empty(&b->signalers));  in intel_breadcrumbs_free()
328 	GEM_BUG_ON(b->irq_armed);  in intel_breadcrumbs_free()
330 	kfree(b);  in intel_breadcrumbs_free()
334 			       struct intel_breadcrumbs *b)  in irq_signal_request()  argument
340 	if (llist_add(&rq->signal_node, &b->signaled_requests))  in irq_signal_request()
341 		irq_work_queue(&b->irq_work);  in irq_signal_request()
346 	struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs;  in insert_breadcrumb()  local
359 		irq_signal_request(rq, b);  in insert_breadcrumb()
365 		add_signaling_context(b, ce);  in insert_breadcrumb()
402 	if (!b->irq_armed || __i915_request_is_complete(rq))  in insert_breadcrumb()
403 		irq_work_queue(&b->irq_work);  in insert_breadcrumb()
433 	struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs;  in i915_request_cancel_breadcrumb()  local
444 	release = remove_signaling_context(b, ce);  in i915_request_cancel_breadcrumb()
450 		irq_signal_request(rq, b);  in i915_request_cancel_breadcrumb()
456 				      struct intel_breadcrumbs *b)  in intel_context_remove_breadcrumbs()  argument
474 		irq_signal_request(rq, b);  in intel_context_remove_breadcrumbs()
477 	release = remove_signaling_context(b, ce);  in intel_context_remove_breadcrumbs()
484 	while (atomic_read(&b->signaler_active))  in intel_context_remove_breadcrumbs()
488 static void print_signals(struct intel_breadcrumbs *b, struct drm_printer *p)  in print_signals()  argument
496 	list_for_each_entry_rcu(ce, &b->signalers, signal_link) {  in print_signals()
511 	struct intel_breadcrumbs *b;  in intel_engine_print_breadcrumbs()  local
513 	b = engine->breadcrumbs;  in intel_engine_print_breadcrumbs()
514 	if (!b)  in intel_engine_print_breadcrumbs()
517 	drm_printf(p, "IRQ: %s\n", str_enabled_disabled(b->irq_armed));  in intel_engine_print_breadcrumbs()
518 	if (!list_empty(&b->signalers))  in intel_engine_print_breadcrumbs()
519 		print_signals(b, p);  in intel_engine_print_breadcrumbs()