98 i915_request_remove_from_client(struct i915_request *request)  in i915_request_remove_from_client()  argument
102 	file_priv = request->file_priv;  in i915_request_remove_from_client()
107 	if (request->file_priv) {  in i915_request_remove_from_client()
108 		list_del(&request->client_link);  in i915_request_remove_from_client()
109 		request->file_priv = NULL;  in i915_request_remove_from_client()
295 			  struct i915_request *request)  in i915_gem_retire_noop()  argument
300 static void advance_ring(struct i915_request *request)  in advance_ring()  argument
302 	struct intel_ring *ring = request->ring;  in advance_ring()
314 	GEM_BUG_ON(!list_is_first(&request->ring_link, &ring->request_list));  in advance_ring()
315 	if (list_is_last(&request->ring_link, &ring->request_list)) {  in advance_ring()
325 		tail = READ_ONCE(request->tail);  in advance_ring()
328 		tail = request->postfix;  in advance_ring()
330 	list_del_init(&request->ring_link);  in advance_ring()
335 static void free_capture_list(struct i915_request *request)  in free_capture_list()  argument
339 	capture = request->capture_list;  in free_capture_list()
409 static void i915_request_retire(struct i915_request *request)  in i915_request_retire()  argument
414 		  request->engine->name,  in i915_request_retire()
415 		  request->fence.context, request->fence.seqno,  in i915_request_retire()
416 		  request->global_seqno,  in i915_request_retire()
417 		  intel_engine_get_seqno(request->engine));  in i915_request_retire()
419 	lockdep_assert_held(&request->i915->drm.struct_mutex);  in i915_request_retire()
420 	GEM_BUG_ON(!i915_sw_fence_signaled(&request->submit));  in i915_request_retire()
421 	GEM_BUG_ON(!i915_request_completed(request));  in i915_request_retire()
423 	trace_i915_request_retire(request);  in i915_request_retire()
425 	advance_ring(request);  in i915_request_retire()
426 	free_capture_list(request);  in i915_request_retire()
438 	list_for_each_entry_safe(active, next, &request->active_list, link) {  in i915_request_retire()
452 		RCU_INIT_POINTER(active->request, NULL);  in i915_request_retire()
454 		active->retire(active, request);  in i915_request_retire()
457 	i915_request_remove_from_client(request);  in i915_request_retire()
460 	atomic_dec_if_positive(&request->gem_context->ban_score);  in i915_request_retire()
461 	intel_context_unpin(request->hw_context);  in i915_request_retire()
463 	__retire_engine_upto(request->engine, request);  in i915_request_retire()
465 	unreserve_gt(request->i915);  in i915_request_retire()
467 	i915_sched_node_fini(request->i915, &request->sched);  in i915_request_retire()
468 	i915_request_put(request);  in i915_request_retire()
501 static void move_to_timeline(struct i915_request *request,  in move_to_timeline()  argument
504 	GEM_BUG_ON(request->timeline == &request->engine->timeline);  in move_to_timeline()
505 	lockdep_assert_held(&request->engine->timeline.lock);  in move_to_timeline()
507 	spin_lock(&request->timeline->lock);  in move_to_timeline()
508 	list_move_tail(&request->link, &timeline->requests);  in move_to_timeline()
509 	spin_unlock(&request->timeline->lock);  in move_to_timeline()
512 void __i915_request_submit(struct i915_request *request)  in __i915_request_submit()  argument
514 	struct intel_engine_cs *engine = request->engine;  in __i915_request_submit()
519 		  request->fence.context, request->fence.seqno,  in __i915_request_submit()
526 	GEM_BUG_ON(request->global_seqno);  in __i915_request_submit()
533 	spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);  in __i915_request_submit()
534 	request->global_seqno = seqno;  in __i915_request_submit()
535 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))  in __i915_request_submit()
536 		intel_engine_enable_signaling(request, false);  in __i915_request_submit()
537 	spin_unlock(&request->lock);  in __i915_request_submit()
539 	engine->emit_breadcrumb(request,  in __i915_request_submit()
540 				request->ring->vaddr + request->postfix);  in __i915_request_submit()
543 	move_to_timeline(request, &engine->timeline);  in __i915_request_submit()
545 	trace_i915_request_execute(request);  in __i915_request_submit()
547 	wake_up_all(&request->execute);  in __i915_request_submit()
550 void i915_request_submit(struct i915_request *request)  in i915_request_submit()  argument
552 	struct intel_engine_cs *engine = request->engine;  in i915_request_submit()
558 	__i915_request_submit(request);  in i915_request_submit()
563 void __i915_request_unsubmit(struct i915_request *request)  in __i915_request_unsubmit()  argument
565 	struct intel_engine_cs *engine = request->engine;  in __i915_request_unsubmit()
569 		  request->fence.context, request->fence.seqno,  in __i915_request_unsubmit()
570 		  request->global_seqno,  in __i915_request_unsubmit()
580 	GEM_BUG_ON(!request->global_seqno);  in __i915_request_unsubmit()
581 	GEM_BUG_ON(request->global_seqno != engine->timeline.seqno);  in __i915_request_unsubmit()
583 				     request->global_seqno));  in __i915_request_unsubmit()
587 	spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);  in __i915_request_unsubmit()
588 	request->global_seqno = 0;  in __i915_request_unsubmit()
589 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))  in __i915_request_unsubmit()
590 		intel_engine_cancel_signaling(request);  in __i915_request_unsubmit()
591 	spin_unlock(&request->lock);  in __i915_request_unsubmit()
594 	move_to_timeline(request, request->timeline);  in __i915_request_unsubmit()
605 void i915_request_unsubmit(struct i915_request *request)  in i915_request_unsubmit()  argument
607 	struct intel_engine_cs *engine = request->engine;  in i915_request_unsubmit()
613 	__i915_request_unsubmit(request);  in i915_request_unsubmit()
621 	struct i915_request *request =  in submit_notify()  local
622 		container_of(fence, typeof(*request), submit);  in submit_notify()
626 		trace_i915_request_submit(request);  in submit_notify()
636 		request->engine->submit_request(request);  in submit_notify()
641 		i915_request_put(request);  in submit_notify()
1044 void i915_request_add(struct i915_request *request)  in i915_request_add()  argument
1046 	struct intel_engine_cs *engine = request->engine;  in i915_request_add()
1047 	struct i915_timeline *timeline = request->timeline;  in i915_request_add()
1048 	struct intel_ring *ring = request->ring;  in i915_request_add()
1053 		  engine->name, request->fence.context, request->fence.seqno);  in i915_request_add()
1055 	lockdep_assert_held(&request->i915->drm.struct_mutex);  in i915_request_add()
1056 	trace_i915_request_add(request);  in i915_request_add()
1063 	GEM_BUG_ON(timeline->seqno != request->fence.seqno);  in i915_request_add()
1070 	request->reserved_space = 0;  in i915_request_add()
1071 	engine->emit_flush(request, EMIT_FLUSH);  in i915_request_add()
1079 	cs = intel_ring_begin(request, engine->emit_breadcrumb_sz);  in i915_request_add()
1081 	request->postfix = intel_ring_offset(request, cs);  in i915_request_add()
1091 				   &request->i915->drm.struct_mutex);  in i915_request_add()
1093 		i915_sw_fence_await_sw_fence(&request->submit, &prev->submit,  in i915_request_add()
1094 					     &request->submitq);  in i915_request_add()
1096 			__i915_sched_node_add_dependency(&request->sched,  in i915_request_add()
1098 							 &request->dep,  in i915_request_add()
1103 	list_add_tail(&request->link, &timeline->requests);  in i915_request_add()
1106 	GEM_BUG_ON(timeline->seqno != request->fence.seqno);  in i915_request_add()
1107 	i915_gem_active_set(&timeline->last_request, request);  in i915_request_add()
1109 	list_add_tail(&request->ring_link, &ring->request_list);  in i915_request_add()
1110 	if (list_is_first(&request->ring_link, &ring->request_list)) {  in i915_request_add()
1112 		list_add(&ring->active_link, &request->i915->gt.active_rings);  in i915_request_add()
1114 	request->emitted_jiffies = jiffies;  in i915_request_add()
1130 		engine->schedule(request, &request->gem_context->sched);  in i915_request_add()
1132 	i915_sw_fence_commit(&request->submit);  in i915_request_add()
1249 static bool __i915_wait_request_check_and_reset(struct i915_request *request)  in __i915_wait_request_check_and_reset()  argument
1251 	struct i915_gpu_error *error = &request->i915->gpu_error;  in __i915_wait_request_check_and_reset()
1257 	i915_reset(request->i915, error->stalled_mask, error->reason);  in __i915_wait_request_check_and_reset()
1422 	struct i915_request *request, *next;  in ring_retire_requests()  local
1424 	list_for_each_entry_safe(request, next,  in ring_retire_requests()
1426 		if (!i915_request_completed(request))  in ring_retire_requests()
1429 		i915_request_retire(request);  in ring_retire_requests()