61 static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)  in __intel_breadcrumbs_arm_irq()  argument
67 	if (GEM_WARN_ON(!intel_gt_pm_get_if_awake(b->irq_engine->gt)))  in __intel_breadcrumbs_arm_irq()
76 	WRITE_ONCE(b->irq_armed, true);  in __intel_breadcrumbs_arm_irq()
79 	if (!b->irq_enabled++ && irq_enable(b->irq_engine))  in __intel_breadcrumbs_arm_irq()
80 		irq_work_queue(&b->irq_work);  in __intel_breadcrumbs_arm_irq()
83 static void intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)  in intel_breadcrumbs_arm_irq()  argument
85 	if (!b->irq_engine)  in intel_breadcrumbs_arm_irq()
88 	spin_lock(&b->irq_lock);  in intel_breadcrumbs_arm_irq()
89 	if (!b->irq_armed)  in intel_breadcrumbs_arm_irq()
90 		__intel_breadcrumbs_arm_irq(b);  in intel_breadcrumbs_arm_irq()
91 	spin_unlock(&b->irq_lock);  in intel_breadcrumbs_arm_irq()
94 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)  in __intel_breadcrumbs_disarm_irq()  argument
96 	GEM_BUG_ON(!b->irq_enabled);  in __intel_breadcrumbs_disarm_irq()
97 	if (!--b->irq_enabled)  in __intel_breadcrumbs_disarm_irq()
98 		irq_disable(b->irq_engine);  in __intel_breadcrumbs_disarm_irq()
100 	WRITE_ONCE(b->irq_armed, false);  in __intel_breadcrumbs_disarm_irq()
101 	intel_gt_pm_put_async(b->irq_engine->gt);  in __intel_breadcrumbs_disarm_irq()
104 static void intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)  in intel_breadcrumbs_disarm_irq()  argument
106 	spin_lock(&b->irq_lock);  in intel_breadcrumbs_disarm_irq()
107 	if (b->irq_armed)  in intel_breadcrumbs_disarm_irq()
108 		__intel_breadcrumbs_disarm_irq(b);  in intel_breadcrumbs_disarm_irq()
109 	spin_unlock(&b->irq_lock);  in intel_breadcrumbs_disarm_irq()
112 static void add_signaling_context(struct intel_breadcrumbs *b,  in add_signaling_context()  argument
117 	spin_lock(&b->signalers_lock);  in add_signaling_context()
118 	list_add_rcu(&ce->signal_link, &b->signalers);  in add_signaling_context()
119 	spin_unlock(&b->signalers_lock);  in add_signaling_context()
122 static bool remove_signaling_context(struct intel_breadcrumbs *b,  in remove_signaling_context()  argument
130 	spin_lock(&b->signalers_lock);  in remove_signaling_context()
132 	spin_unlock(&b->signalers_lock);  in remove_signaling_context()
189 static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl)  in add_retire()  argument
191 	if (b->irq_engine)  in add_retire()
192 		intel_engine_add_retire(b->irq_engine, tl);  in add_retire()
216 	struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work);  in signal_irq_work()  local
222 	if (unlikely(!llist_empty(&b->signaled_requests)))  in signal_irq_work()
223 		signal = llist_del_all(&b->signaled_requests);  in signal_irq_work()
250 	if (!signal && READ_ONCE(b->irq_armed) && list_empty(&b->signalers))  in signal_irq_work()
251 		intel_breadcrumbs_disarm_irq(b);  in signal_irq_work()
254 	list_for_each_entry_rcu(ce, &b->signalers, signal_link) {  in signal_irq_work()
274 			release = remove_signaling_context(b, ce);  in signal_irq_work()
282 				add_retire(b, ce->timeline);  in signal_irq_work()
303 	if (!READ_ONCE(b->irq_armed) && !list_empty(&b->signalers))  in signal_irq_work()
304 		intel_breadcrumbs_arm_irq(b);  in signal_irq_work()
310 	struct intel_breadcrumbs *b;  in intel_breadcrumbs_create()  local
312 	b = kzalloc(sizeof(*b), GFP_KERNEL);  in intel_breadcrumbs_create()
313 	if (!b)  in intel_breadcrumbs_create()
316 	b->irq_engine = irq_engine;  in intel_breadcrumbs_create()
318 	spin_lock_init(&b->signalers_lock);  in intel_breadcrumbs_create()
319 	INIT_LIST_HEAD(&b->signalers);  in intel_breadcrumbs_create()
320 	init_llist_head(&b->signaled_requests);  in intel_breadcrumbs_create()
322 	spin_lock_init(&b->irq_lock);  in intel_breadcrumbs_create()
323 	init_irq_work(&b->irq_work, signal_irq_work);  in intel_breadcrumbs_create()
325 	return b;  in intel_breadcrumbs_create()
328 void intel_breadcrumbs_reset(struct intel_breadcrumbs *b)  in intel_breadcrumbs_reset()  argument
332 	if (!b->irq_engine)  in intel_breadcrumbs_reset()
335 	spin_lock_irqsave(&b->irq_lock, flags);  in intel_breadcrumbs_reset()
337 	if (b->irq_enabled)  in intel_breadcrumbs_reset()
338 		irq_enable(b->irq_engine);  in intel_breadcrumbs_reset()
340 		irq_disable(b->irq_engine);  in intel_breadcrumbs_reset()
342 	spin_unlock_irqrestore(&b->irq_lock, flags);  in intel_breadcrumbs_reset()
345 void intel_breadcrumbs_park(struct intel_breadcrumbs *b)  in intel_breadcrumbs_park()  argument
348 	irq_work_sync(&b->irq_work);  in intel_breadcrumbs_park()
349 	while (unlikely(READ_ONCE(b->irq_armed))) {  in intel_breadcrumbs_park()
351 		signal_irq_work(&b->irq_work);  in intel_breadcrumbs_park()
355 	GEM_BUG_ON(!list_empty(&b->signalers));  in intel_breadcrumbs_park()
358 void intel_breadcrumbs_free(struct intel_breadcrumbs *b)  in intel_breadcrumbs_free()  argument
360 	irq_work_sync(&b->irq_work);  in intel_breadcrumbs_free()
361 	GEM_BUG_ON(!list_empty(&b->signalers));  in intel_breadcrumbs_free()
362 	GEM_BUG_ON(b->irq_armed);  in intel_breadcrumbs_free()
363 	kfree(b);  in intel_breadcrumbs_free()
368 	struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs;  in insert_breadcrumb()  local
384 		    llist_add(&rq->signal_node, &b->signaled_requests))  in insert_breadcrumb()
385 			irq_work_queue(&b->irq_work);  in insert_breadcrumb()
391 		add_signaling_context(b, ce);  in insert_breadcrumb()
426 	irq_work_queue(&b->irq_work);  in insert_breadcrumb()
472 static void print_signals(struct intel_breadcrumbs *b, struct drm_printer *p)  in print_signals()  argument
480 	list_for_each_entry_rcu(ce, &b->signalers, signal_link) {  in print_signals()
495 	struct intel_breadcrumbs *b;  in intel_engine_print_breadcrumbs()  local
497 	b = engine->breadcrumbs;  in intel_engine_print_breadcrumbs()
498 	if (!b)  in intel_engine_print_breadcrumbs()
501 	drm_printf(p, "IRQ: %s\n", enableddisabled(b->irq_armed));  in intel_engine_print_breadcrumbs()
502 	if (!list_empty(&b->signalers))  in intel_engine_print_breadcrumbs()
503 		print_signals(b, p);  in intel_engine_print_breadcrumbs()