36 static unsigned int __intel_breadcrumbs_wakeup(struct intel_breadcrumbs *b)  in __intel_breadcrumbs_wakeup()  argument
41 	lockdep_assert_held(&b->irq_lock);  in __intel_breadcrumbs_wakeup()
43 	wait = b->irq_wait;  in __intel_breadcrumbs_wakeup()
67 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_wakeup()  local
71 	spin_lock_irqsave(&b->irq_lock, flags);  in intel_engine_wakeup()
72 	result = __intel_breadcrumbs_wakeup(b);  in intel_engine_wakeup()
73 	spin_unlock_irqrestore(&b->irq_lock, flags);  in intel_engine_wakeup()
100 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_breadcrumbs_hangcheck()  local
103 	if (!b->irq_armed)  in intel_breadcrumbs_hangcheck()
106 	irq_count = READ_ONCE(b->irq_count);  in intel_breadcrumbs_hangcheck()
107 	if (b->hangcheck_interrupts != irq_count) {  in intel_breadcrumbs_hangcheck()
108 		b->hangcheck_interrupts = irq_count;  in intel_breadcrumbs_hangcheck()
109 		mod_timer(&b->hangcheck, wait_timeout());  in intel_breadcrumbs_hangcheck()
127 		mod_timer(&b->fake_irq, jiffies + 1);  in intel_breadcrumbs_hangcheck()
129 		mod_timer(&b->hangcheck, wait_timeout());  in intel_breadcrumbs_hangcheck()
137 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_breadcrumbs_fake_irq()  local
147 	spin_lock_irq(&b->irq_lock);  in intel_breadcrumbs_fake_irq()
148 	if (b->irq_armed && !__intel_breadcrumbs_wakeup(b))  in intel_breadcrumbs_fake_irq()
150 	spin_unlock_irq(&b->irq_lock);  in intel_breadcrumbs_fake_irq()
151 	if (!b->irq_armed)  in intel_breadcrumbs_fake_irq()
156 		mod_timer(&b->hangcheck, wait_timeout());  in intel_breadcrumbs_fake_irq()
160 	mod_timer(&b->fake_irq, jiffies + 1);  in intel_breadcrumbs_fake_irq()
199 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in __intel_engine_disarm_breadcrumbs()  local
201 	lockdep_assert_held(&b->irq_lock);  in __intel_engine_disarm_breadcrumbs()
202 	GEM_BUG_ON(b->irq_wait);  in __intel_engine_disarm_breadcrumbs()
203 	GEM_BUG_ON(!b->irq_armed);  in __intel_engine_disarm_breadcrumbs()
205 	GEM_BUG_ON(!b->irq_enabled);  in __intel_engine_disarm_breadcrumbs()
206 	if (!--b->irq_enabled)  in __intel_engine_disarm_breadcrumbs()
209 	b->irq_armed = false;  in __intel_engine_disarm_breadcrumbs()
214 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_pin_breadcrumbs_irq()  local
216 	spin_lock_irq(&b->irq_lock);  in intel_engine_pin_breadcrumbs_irq()
217 	if (!b->irq_enabled++)  in intel_engine_pin_breadcrumbs_irq()
219 	GEM_BUG_ON(!b->irq_enabled); /* no overflow! */  in intel_engine_pin_breadcrumbs_irq()
220 	spin_unlock_irq(&b->irq_lock);  in intel_engine_pin_breadcrumbs_irq()
225 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_unpin_breadcrumbs_irq()  local
227 	spin_lock_irq(&b->irq_lock);  in intel_engine_unpin_breadcrumbs_irq()
228 	GEM_BUG_ON(!b->irq_enabled); /* no underflow! */  in intel_engine_unpin_breadcrumbs_irq()
229 	if (!--b->irq_enabled)  in intel_engine_unpin_breadcrumbs_irq()
231 	spin_unlock_irq(&b->irq_lock);  in intel_engine_unpin_breadcrumbs_irq()
236 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_disarm_breadcrumbs()  local
239 	if (!b->irq_armed)  in intel_engine_disarm_breadcrumbs()
250 	spin_lock_irq(&b->rb_lock);  in intel_engine_disarm_breadcrumbs()
252 	spin_lock(&b->irq_lock);  in intel_engine_disarm_breadcrumbs()
253 	b->irq_wait = NULL;  in intel_engine_disarm_breadcrumbs()
254 	if (b->irq_armed)  in intel_engine_disarm_breadcrumbs()
256 	spin_unlock(&b->irq_lock);  in intel_engine_disarm_breadcrumbs()
258 	rbtree_postorder_for_each_entry_safe(wait, n, &b->waiters, node) {  in intel_engine_disarm_breadcrumbs()
264 	b->waiters = RB_ROOT;  in intel_engine_disarm_breadcrumbs()
266 	spin_unlock_irq(&b->rb_lock);  in intel_engine_disarm_breadcrumbs()
269 static bool use_fake_irq(const struct intel_breadcrumbs *b)  in use_fake_irq()  argument
272 		container_of(b, struct intel_engine_cs, breadcrumbs);  in use_fake_irq()
284 	return READ_ONCE(b->irq_count) == b->hangcheck_interrupts;  in use_fake_irq()
287 static void enable_fake_irq(struct intel_breadcrumbs *b)  in enable_fake_irq()  argument
290 	if (!b->irq_enabled || use_fake_irq(b))  in enable_fake_irq()
291 		mod_timer(&b->fake_irq, jiffies + 1);  in enable_fake_irq()
293 		mod_timer(&b->hangcheck, wait_timeout());  in enable_fake_irq()
296 static bool __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)  in __intel_breadcrumbs_enable_irq()  argument
299 		container_of(b, struct intel_engine_cs, breadcrumbs);  in __intel_breadcrumbs_enable_irq()
303 	lockdep_assert_held(&b->irq_lock);  in __intel_breadcrumbs_enable_irq()
304 	if (b->irq_armed)  in __intel_breadcrumbs_enable_irq()
312 	b->irq_armed = true;  in __intel_breadcrumbs_enable_irq()
314 	if (I915_SELFTEST_ONLY(b->mock)) {  in __intel_breadcrumbs_enable_irq()
334 	if (!b->irq_enabled++ &&  in __intel_breadcrumbs_enable_irq()
340 	enable_fake_irq(b);  in __intel_breadcrumbs_enable_irq()
349 static inline void __intel_breadcrumbs_finish(struct intel_breadcrumbs *b,  in __intel_breadcrumbs_finish()  argument
352 	lockdep_assert_held(&b->rb_lock);  in __intel_breadcrumbs_finish()
353 	GEM_BUG_ON(b->irq_wait == wait);  in __intel_breadcrumbs_finish()
364 	rb_erase(&wait->node, &b->waiters);  in __intel_breadcrumbs_finish()
374 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in __intel_breadcrumbs_next()  local
376 	spin_lock(&b->irq_lock);  in __intel_breadcrumbs_next()
377 	GEM_BUG_ON(!b->irq_armed);  in __intel_breadcrumbs_next()
378 	GEM_BUG_ON(!b->irq_wait);  in __intel_breadcrumbs_next()
379 	b->irq_wait = to_wait(next);  in __intel_breadcrumbs_next()
380 	spin_unlock(&b->irq_lock);  in __intel_breadcrumbs_next()
393 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in __intel_engine_add_wait()  local
428 	p = &b->waiters.rb_node;  in __intel_engine_add_wait()
455 	rb_insert_color(&wait->node, &b->waiters);  in __intel_engine_add_wait()
458 		spin_lock(&b->irq_lock);  in __intel_engine_add_wait()
459 		b->irq_wait = wait;  in __intel_engine_add_wait()
468 		armed = __intel_breadcrumbs_enable_irq(b);  in __intel_engine_add_wait()
469 		spin_unlock(&b->irq_lock);  in __intel_engine_add_wait()
487 			__intel_breadcrumbs_finish(b, crumb);  in __intel_engine_add_wait()
491 	GEM_BUG_ON(!b->irq_wait);  in __intel_engine_add_wait()
492 	GEM_BUG_ON(!b->irq_armed);  in __intel_engine_add_wait()
493 	GEM_BUG_ON(rb_first(&b->waiters) != &b->irq_wait->node);  in __intel_engine_add_wait()
501 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_add_wait()  local
504 	spin_lock_irq(&b->rb_lock);  in intel_engine_add_wait()
506 	spin_unlock_irq(&b->rb_lock);  in intel_engine_add_wait()
520 static inline int wakeup_priority(struct intel_breadcrumbs *b,  in wakeup_priority()  argument
523 	if (tsk == b->signaler)  in wakeup_priority()
532 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in __intel_engine_remove_wait()  local
534 	lockdep_assert_held(&b->rb_lock);  in __intel_engine_remove_wait()
539 	if (b->irq_wait == wait) {  in __intel_engine_remove_wait()
540 		const int priority = wakeup_priority(b, wait->tsk);  in __intel_engine_remove_wait()
568 				__intel_breadcrumbs_finish(b, to_wait(next));  in __intel_engine_remove_wait()
577 		GEM_BUG_ON(rb_first(&b->waiters) == &wait->node);  in __intel_engine_remove_wait()
581 	rb_erase(&wait->node, &b->waiters);  in __intel_engine_remove_wait()
585 	GEM_BUG_ON(b->irq_wait == wait);  in __intel_engine_remove_wait()
586 	GEM_BUG_ON(rb_first(&b->waiters) !=  in __intel_engine_remove_wait()
587 		   (b->irq_wait ? &b->irq_wait->node : NULL));  in __intel_engine_remove_wait()
593 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_remove_wait()  local
600 		GEM_BUG_ON(READ_ONCE(b->irq_wait) == wait);  in intel_engine_remove_wait()
604 	spin_lock_irq(&b->rb_lock);  in intel_engine_remove_wait()
606 	spin_unlock_irq(&b->rb_lock);  in intel_engine_remove_wait()
619 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_breadcrumbs_signaler()  local
631 		if (list_empty(&b->signals))  in intel_breadcrumbs_signaler()
645 		spin_lock_irq(&b->rb_lock);  in intel_breadcrumbs_signaler()
646 		list_for_each_entry_safe(rq, n, &b->signals, signaling.link) {  in intel_breadcrumbs_signaler()
669 		spin_unlock_irq(&b->rb_lock);  in intel_breadcrumbs_signaler()
694 			    !list_empty(&b->signals) &&  in intel_breadcrumbs_signaler()
717 static void insert_signal(struct intel_breadcrumbs *b,  in insert_signal()  argument
723 	lockdep_assert_held(&b->rb_lock);  in insert_signal()
736 	list_for_each_entry_reverse(iter, &b->signals, signaling.link)  in insert_signal()
746 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_enable_signaling()  local
767 	wait->tsk = b->signaler;  in intel_engine_enable_signaling()
780 	spin_lock(&b->rb_lock);  in intel_engine_enable_signaling()
781 	insert_signal(b, request, seqno);  in intel_engine_enable_signaling()
783 	spin_unlock(&b->rb_lock);  in intel_engine_enable_signaling()
786 		wake_up_process(b->signaler);  in intel_engine_enable_signaling()
796 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_cancel_signaling()  local
804 	spin_lock(&b->rb_lock);  in intel_engine_cancel_signaling()
808 	spin_unlock(&b->rb_lock);  in intel_engine_cancel_signaling()
813 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_init_breadcrumbs()  local
816 	spin_lock_init(&b->rb_lock);  in intel_engine_init_breadcrumbs()
817 	spin_lock_init(&b->irq_lock);  in intel_engine_init_breadcrumbs()
819 	timer_setup(&b->fake_irq, intel_breadcrumbs_fake_irq, 0);  in intel_engine_init_breadcrumbs()
820 	timer_setup(&b->hangcheck, intel_breadcrumbs_hangcheck, 0);  in intel_engine_init_breadcrumbs()
822 	INIT_LIST_HEAD(&b->signals);  in intel_engine_init_breadcrumbs()
835 	b->signaler = tsk;  in intel_engine_init_breadcrumbs()
842 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in cancel_fake_irq()  local
844 	del_timer_sync(&b->fake_irq); /* may queue b->hangcheck */  in cancel_fake_irq()
845 	del_timer_sync(&b->hangcheck);  in cancel_fake_irq()
851 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_reset_breadcrumbs()  local
854 	spin_lock_irqsave(&b->irq_lock, flags);  in intel_engine_reset_breadcrumbs()
863 	if (b->irq_enabled)  in intel_engine_reset_breadcrumbs()
878 	spin_unlock_irqrestore(&b->irq_lock, flags);  in intel_engine_reset_breadcrumbs()
883 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_fini_breadcrumbs()  local
886 	WARN_ON(READ_ONCE(b->irq_wait));  in intel_engine_fini_breadcrumbs()
887 	WARN_ON(!RB_EMPTY_ROOT(&b->waiters));  in intel_engine_fini_breadcrumbs()
888 	WARN_ON(!list_empty(&b->signals));  in intel_engine_fini_breadcrumbs()
890 	if (!IS_ERR_OR_NULL(b->signaler))  in intel_engine_fini_breadcrumbs()
891 		kthread_stop(b->signaler);  in intel_engine_fini_breadcrumbs()