137 static inline bool __request_completed(const struct i915_request *rq)  in __request_completed()  argument
139 	return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);  in __request_completed()
143 check_signal_order(struct intel_context *ce, struct i915_request *rq)  in check_signal_order()  argument
145 	if (rq->context != ce)  in check_signal_order()
148 	if (!list_is_last(&rq->signal_link, &ce->signals) &&  in check_signal_order()
149 	    i915_seqno_passed(rq->fence.seqno,  in check_signal_order()
150 			      list_next_entry(rq, signal_link)->fence.seqno))  in check_signal_order()
153 	if (!list_is_first(&rq->signal_link, &ce->signals) &&  in check_signal_order()
154 	    i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,  in check_signal_order()
155 			      rq->fence.seqno))  in check_signal_order()
195 static bool __signal_request(struct i915_request *rq)  in __signal_request()  argument
197 	GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags));  in __signal_request()
199 	if (!__dma_fence_signal(&rq->fence)) {  in __signal_request()
200 		i915_request_put(rq);  in __signal_request()
255 		struct i915_request *rq;  in signal_irq_work()  local
257 		list_for_each_entry_rcu(rq, &ce->signals, signal_link) {  in signal_irq_work()
260 			if (!__request_completed(rq))  in signal_irq_work()
264 						&rq->fence.flags))  in signal_irq_work()
273 			list_del_rcu(&rq->signal_link);  in signal_irq_work()
277 			if (__signal_request(rq))  in signal_irq_work()
279 				signal = slist_add(&rq->signal_node, signal);  in signal_irq_work()
290 		struct i915_request *rq =  in signal_irq_work()  local
291 			llist_entry(signal, typeof(*rq), signal_node);  in signal_irq_work()
294 		spin_lock(&rq->lock);  in signal_irq_work()
295 		list_replace(&rq->fence.cb_list, &cb_list);  in signal_irq_work()
296 		__dma_fence_signal__timestamp(&rq->fence, timestamp);  in signal_irq_work()
297 		__dma_fence_signal__notify(&rq->fence, &cb_list);  in signal_irq_work()
298 		spin_unlock(&rq->lock);  in signal_irq_work()
300 		i915_request_put(rq);  in signal_irq_work()
366 static void insert_breadcrumb(struct i915_request *rq)  in insert_breadcrumb()  argument
368 	struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs;  in insert_breadcrumb()
369 	struct intel_context *ce = rq->context;  in insert_breadcrumb()
372 	if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags))  in insert_breadcrumb()
375 	i915_request_get(rq);  in insert_breadcrumb()
382 	if (__request_completed(rq)) {  in insert_breadcrumb()
383 		if (__signal_request(rq) &&  in insert_breadcrumb()
384 		    llist_add(&rq->signal_node, &b->signaled_requests))  in insert_breadcrumb()
412 			if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))  in insert_breadcrumb()
416 	list_add_rcu(&rq->signal_link, pos);  in insert_breadcrumb()
417 	GEM_BUG_ON(!check_signal_order(ce, rq));  in insert_breadcrumb()
418 	GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags));  in insert_breadcrumb()
419 	set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);  in insert_breadcrumb()
429 bool i915_request_enable_breadcrumb(struct i915_request *rq)  in i915_request_enable_breadcrumb()  argument
431 	struct intel_context *ce = rq->context;  in i915_request_enable_breadcrumb()
434 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))  in i915_request_enable_breadcrumb()
443 	if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags))  in i915_request_enable_breadcrumb()
447 	if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags))  in i915_request_enable_breadcrumb()
448 		insert_breadcrumb(rq);  in i915_request_enable_breadcrumb()
454 void i915_request_cancel_breadcrumb(struct i915_request *rq)  in i915_request_cancel_breadcrumb()  argument
456 	struct intel_context *ce = rq->context;  in i915_request_cancel_breadcrumb()
459 	if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags))  in i915_request_cancel_breadcrumb()
463 	list_del_rcu(&rq->signal_link);  in i915_request_cancel_breadcrumb()
464 	release = remove_signaling_context(rq->engine->breadcrumbs, ce);  in i915_request_cancel_breadcrumb()
469 	i915_request_put(rq);  in i915_request_cancel_breadcrumb()
475 	struct i915_request *rq;  in print_signals()  local
481 		list_for_each_entry_rcu(rq, &ce->signals, signal_link)  in print_signals()
483 				   rq->fence.context, rq->fence.seqno,  in print_signals()
484 				   i915_request_completed(rq) ? "!" :  in print_signals()
485 				   i915_request_started(rq) ? "*" :  in print_signals()
487 				   jiffies_to_msecs(jiffies - rq->emitted_jiffies));  in print_signals()