59 static void update_shadow_pdps(struct intel_vgpu_workload *workload)  in update_shadow_pdps()  argument
62 	struct intel_context *ctx = workload->req->context;  in update_shadow_pdps()
64 	if (WARN_ON(!workload->shadow_mm))  in update_shadow_pdps()
67 	if (WARN_ON(!atomic_read(&workload->shadow_mm->pincount)))  in update_shadow_pdps()
72 			(void *)workload->shadow_mm->ppgtt_mm.shadow_pdps);  in update_shadow_pdps()
80 static void sr_oa_regs(struct intel_vgpu_workload *workload,  in sr_oa_regs()  argument
83 	struct drm_i915_private *dev_priv = workload->vgpu->gvt->gt->i915;  in sr_oa_regs()
97 	if (workload->engine->id != RCS0)  in sr_oa_regs()
101 		workload->oactxctrl = reg_state[ctx_oactxctrl + 1];  in sr_oa_regs()
103 		for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) {  in sr_oa_regs()
106 			workload->flex_mmio[i] = reg_state[state_offset + 1];  in sr_oa_regs()
111 		reg_state[ctx_oactxctrl + 1] = workload->oactxctrl;  in sr_oa_regs()
113 		for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) {  in sr_oa_regs()
118 			reg_state[state_offset + 1] = workload->flex_mmio[i];  in sr_oa_regs()
123 static int populate_shadow_context(struct intel_vgpu_workload *workload)  in populate_shadow_context()  argument
125 	struct intel_vgpu *vgpu = workload->vgpu;  in populate_shadow_context()
127 	struct intel_context *ctx = workload->req->context;  in populate_shadow_context()
137 	int ring_id = workload->engine->id;  in populate_shadow_context()
146 	sr_oa_regs(workload, (u32 *)shadow_ring_context, true);  in populate_shadow_context()
148 	intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \  in populate_shadow_context()
151 		intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \  in populate_shadow_context()
160 	if (workload->engine->id == RCS0) {  in populate_shadow_context()
169 			workload->ring_context_gpa +  in populate_shadow_context()
175 	sr_oa_regs(workload, (u32 *)shadow_ring_context, false);  in populate_shadow_context()
178 			workload->engine->name, workload->ctx_desc.lrca,  in populate_shadow_context()
179 			workload->ctx_desc.context_id,  in populate_shadow_context()
180 			workload->ring_context_gpa);  in populate_shadow_context()
190 				workload->ctx_desc.lrca) &&  in populate_shadow_context()
192 				workload->ring_context_gpa))  in populate_shadow_context()
195 	s->last_ctx[ring_id].lrca = workload->ctx_desc.lrca;  in populate_shadow_context()
196 	s->last_ctx[ring_id].ring_context_gpa = workload->ring_context_gpa;  in populate_shadow_context()
202 	context_page_num = workload->engine->context_size;  in populate_shadow_context()
205 	if (IS_BROADWELL(gvt->gt->i915) && workload->engine->id == RCS0)  in populate_shadow_context()
214 				(u32)((workload->ctx_desc.lrca + i) <<  in populate_shadow_context()
276 	struct intel_vgpu_workload *workload;  in shadow_context_status_change()  local
293 	workload = scheduler->current_workload[ring_id];  in shadow_context_status_change()
294 	if (unlikely(!workload))  in shadow_context_status_change()
300 		if (workload->vgpu != scheduler->engine_owner[ring_id]) {  in shadow_context_status_change()
303 					      workload->vgpu, rq->engine);  in shadow_context_status_change()
304 			scheduler->engine_owner[ring_id] = workload->vgpu;  in shadow_context_status_change()
307 				      ring_id, workload->vgpu->id);  in shadow_context_status_change()
309 		atomic_set(&workload->shadow_ctx_active, 1);  in shadow_context_status_change()
312 		save_ring_hw_state(workload->vgpu, rq->engine);  in shadow_context_status_change()
313 		atomic_set(&workload->shadow_ctx_active, 0);  in shadow_context_status_change()
316 		save_ring_hw_state(workload->vgpu, rq->engine);  in shadow_context_status_change()
322 	wake_up(&workload->shadow_ctx_status_wq);  in shadow_context_status_change()
328 				 struct intel_vgpu_workload *workload)  in shadow_context_descriptor_update()  argument
337 	desc |= (u64)workload->ctx_desc.addressing_mode <<  in shadow_context_descriptor_update()
343 static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload)  in copy_workload_to_ring_buffer()  argument
345 	struct intel_vgpu *vgpu = workload->vgpu;  in copy_workload_to_ring_buffer()
346 	struct i915_request *req = workload->req;  in copy_workload_to_ring_buffer()
373 	cs = intel_ring_begin(workload->req, workload->rb_len / sizeof(u32));  in copy_workload_to_ring_buffer()
376 			workload->rb_len);  in copy_workload_to_ring_buffer()
380 	shadow_ring_buffer_va = workload->shadow_ring_buffer_va;  in copy_workload_to_ring_buffer()
383 	workload->shadow_ring_buffer_va = cs;  in copy_workload_to_ring_buffer()
386 			workload->rb_len);  in copy_workload_to_ring_buffer()
388 	cs += workload->rb_len / sizeof(u32);  in copy_workload_to_ring_buffer()
389 	intel_ring_advance(workload->req, cs);  in copy_workload_to_ring_buffer()
414 static void set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload,  in set_context_ppgtt_from_shadow()  argument
417 	struct intel_vgpu_mm *mm = workload->shadow_mm;  in set_context_ppgtt_from_shadow()
439 intel_gvt_workload_req_alloc(struct intel_vgpu_workload *workload)  in intel_gvt_workload_req_alloc()  argument
441 	struct intel_vgpu *vgpu = workload->vgpu;  in intel_gvt_workload_req_alloc()
445 	if (workload->req)  in intel_gvt_workload_req_alloc()
448 	rq = i915_request_create(s->shadow[workload->engine->id]);  in intel_gvt_workload_req_alloc()
454 	workload->req = i915_request_get(rq);  in intel_gvt_workload_req_alloc()
466 int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)  in intel_gvt_scan_and_shadow_workload()  argument
468 	struct intel_vgpu *vgpu = workload->vgpu;  in intel_gvt_scan_and_shadow_workload()
474 	if (workload->shadow)  in intel_gvt_scan_and_shadow_workload()
477 	if (!test_and_set_bit(workload->engine->id, s->shadow_ctx_desc_updated))  in intel_gvt_scan_and_shadow_workload()
478 		shadow_context_descriptor_update(s->shadow[workload->engine->id],  in intel_gvt_scan_and_shadow_workload()
479 						 workload);  in intel_gvt_scan_and_shadow_workload()
481 	ret = intel_gvt_scan_and_shadow_ringbuffer(workload);  in intel_gvt_scan_and_shadow_workload()
485 	if (workload->engine->id == RCS0 &&  in intel_gvt_scan_and_shadow_workload()
486 	    workload->wa_ctx.indirect_ctx.size) {  in intel_gvt_scan_and_shadow_workload()
487 		ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx);  in intel_gvt_scan_and_shadow_workload()
492 	workload->shadow = true;  in intel_gvt_scan_and_shadow_workload()
496 	release_shadow_wa_ctx(&workload->wa_ctx);  in intel_gvt_scan_and_shadow_workload()
500 static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload);
502 static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)  in prepare_shadow_batch_buffer()  argument
504 	struct intel_gvt *gvt = workload->vgpu->gvt;  in prepare_shadow_batch_buffer()
509 	list_for_each_entry(bb, &workload->shadow_bb, list) {  in prepare_shadow_batch_buffer()
518 			bb->bb_start_cmd_va = workload->shadow_ring_buffer_va  in prepare_shadow_batch_buffer()
544 						      workload->req,  in prepare_shadow_batch_buffer()
555 	release_shadow_batch_buffer(workload);  in prepare_shadow_batch_buffer()
561 	struct intel_vgpu_workload *workload =  in update_wa_ctx_2_shadow_ctx()  local
563 	struct i915_request *rq = workload->req;  in update_wa_ctx_2_shadow_ctx()
604 static void update_vreg_in_ctx(struct intel_vgpu_workload *workload)  in update_vreg_in_ctx()  argument
606 	vgpu_vreg_t(workload->vgpu, RING_START(workload->engine->mmio_base)) =  in update_vreg_in_ctx()
607 		workload->rb_start;  in update_vreg_in_ctx()
610 static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload)  in release_shadow_batch_buffer()  argument
614 	if (list_empty(&workload->shadow_bb))  in release_shadow_batch_buffer()
617 	bb = list_first_entry(&workload->shadow_bb,  in release_shadow_batch_buffer()
620 	list_for_each_entry_safe(bb, pos, &workload->shadow_bb, list) {  in release_shadow_batch_buffer()
636 intel_vgpu_shadow_mm_pin(struct intel_vgpu_workload *workload)  in intel_vgpu_shadow_mm_pin()  argument
638 	struct intel_vgpu *vgpu = workload->vgpu;  in intel_vgpu_shadow_mm_pin()
642 	ret = intel_vgpu_pin_mm(workload->shadow_mm);  in intel_vgpu_shadow_mm_pin()
648 	if (workload->shadow_mm->type != INTEL_GVT_MM_PPGTT ||  in intel_vgpu_shadow_mm_pin()
649 	    !workload->shadow_mm->ppgtt_mm.shadowed) {  in intel_vgpu_shadow_mm_pin()
654 	if (!list_empty(&workload->lri_shadow_mm)) {  in intel_vgpu_shadow_mm_pin()
655 		list_for_each_entry(m, &workload->lri_shadow_mm,  in intel_vgpu_shadow_mm_pin()
660 								 &workload->lri_shadow_mm,  in intel_vgpu_shadow_mm_pin()
670 		intel_vgpu_unpin_mm(workload->shadow_mm);  in intel_vgpu_shadow_mm_pin()
676 intel_vgpu_shadow_mm_unpin(struct intel_vgpu_workload *workload)  in intel_vgpu_shadow_mm_unpin()  argument
680 	if (!list_empty(&workload->lri_shadow_mm)) {  in intel_vgpu_shadow_mm_unpin()
681 		list_for_each_entry(m, &workload->lri_shadow_mm,  in intel_vgpu_shadow_mm_unpin()
685 	intel_vgpu_unpin_mm(workload->shadow_mm);  in intel_vgpu_shadow_mm_unpin()
688 static int prepare_workload(struct intel_vgpu_workload *workload)  in prepare_workload()  argument
690 	struct intel_vgpu *vgpu = workload->vgpu;  in prepare_workload()
694 	ret = intel_vgpu_shadow_mm_pin(workload);  in prepare_workload()
700 	update_shadow_pdps(workload);  in prepare_workload()
702 	set_context_ppgtt_from_shadow(workload, s->shadow[workload->engine->id]);  in prepare_workload()
704 	ret = intel_vgpu_sync_oos_pages(workload->vgpu);  in prepare_workload()
710 	ret = intel_vgpu_flush_post_shadow(workload->vgpu);  in prepare_workload()
716 	ret = copy_workload_to_ring_buffer(workload);  in prepare_workload()
722 	ret = prepare_shadow_batch_buffer(workload);  in prepare_workload()
728 	ret = prepare_shadow_wa_ctx(&workload->wa_ctx);  in prepare_workload()
734 	if (workload->prepare) {  in prepare_workload()
735 		ret = workload->prepare(workload);  in prepare_workload()
742 	release_shadow_wa_ctx(&workload->wa_ctx);  in prepare_workload()
744 	release_shadow_batch_buffer(workload);  in prepare_workload()
746 	intel_vgpu_shadow_mm_unpin(workload);  in prepare_workload()
750 static int dispatch_workload(struct intel_vgpu_workload *workload)  in dispatch_workload()  argument
752 	struct intel_vgpu *vgpu = workload->vgpu;  in dispatch_workload()
757 		      workload->engine->name, workload);  in dispatch_workload()
761 	ret = intel_gvt_workload_req_alloc(workload);  in dispatch_workload()
765 	ret = intel_gvt_scan_and_shadow_workload(workload);  in dispatch_workload()
769 	ret = populate_shadow_context(workload);  in dispatch_workload()
771 		release_shadow_wa_ctx(&workload->wa_ctx);  in dispatch_workload()
775 	ret = prepare_workload(workload);  in dispatch_workload()
781 		rq = fetch_and_zero(&workload->req);  in dispatch_workload()
785 	if (!IS_ERR_OR_NULL(workload->req)) {  in dispatch_workload()
787 			      workload->engine->name, workload->req);  in dispatch_workload()
788 		i915_request_add(workload->req);  in dispatch_workload()
789 		workload->dispatched = true;  in dispatch_workload()
793 		workload->status = ret;  in dispatch_workload()
802 	struct intel_vgpu_workload *workload = NULL;  in pick_next_workload()  local
829 		workload = scheduler->current_workload[engine->id];  in pick_next_workload()
831 			      engine->name, workload);  in pick_next_workload()
846 	workload = scheduler->current_workload[engine->id];  in pick_next_workload()
848 	gvt_dbg_sched("ring %s pick new workload %p\n", engine->name, workload);  in pick_next_workload()
850 	atomic_inc(&workload->vgpu->submission.running_workload_num);  in pick_next_workload()
853 	return workload;  in pick_next_workload()
887 static void update_guest_context(struct intel_vgpu_workload *workload)  in update_guest_context()  argument
889 	struct i915_request *rq = workload->req;  in update_guest_context()
890 	struct intel_vgpu *vgpu = workload->vgpu;  in update_guest_context()
892 	struct intel_context *ctx = workload->req->context;  in update_guest_context()
904 		      workload->ctx_desc.lrca);  in update_guest_context()
908 	head = workload->rb_head;  in update_guest_context()
909 	tail = workload->rb_tail;  in update_guest_context()
910 	wrap_count = workload->guest_rb_head >> RB_HEAD_WRAP_CNT_OFF;  in update_guest_context()
940 				(u32)((workload->ctx_desc.lrca + i) <<  in update_guest_context()
967 	intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa +  in update_guest_context()
968 		RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4);  in update_guest_context()
972 	if (!list_empty(&workload->lri_shadow_mm)) {  in update_guest_context()
973 		struct intel_vgpu_mm *m = list_last_entry(&workload->lri_shadow_mm,  in update_guest_context()
977 		update_guest_pdps(vgpu, workload->ring_context_gpa,  in update_guest_context()
982 	intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \  in update_guest_context()
991 			workload->ring_context_gpa +  in update_guest_context()
1021 	struct intel_vgpu_workload *workload =  in complete_current_workload()  local
1023 	struct intel_vgpu *vgpu = workload->vgpu;  in complete_current_workload()
1025 	struct i915_request *rq = workload->req;  in complete_current_workload()
1036 		wait_event(workload->shadow_ctx_status_wq,  in complete_current_workload()
1037 			   !atomic_read(&workload->shadow_ctx_active));  in complete_current_workload()
1044 		if (likely(workload->status == -EINPROGRESS)) {  in complete_current_workload()
1045 			if (workload->req->fence.error == -EIO)  in complete_current_workload()
1046 				workload->status = -EIO;  in complete_current_workload()
1048 				workload->status = 0;  in complete_current_workload()
1051 		if (!workload->status &&  in complete_current_workload()
1053 			update_guest_context(workload);  in complete_current_workload()
1055 			for_each_set_bit(event, workload->pending_events,  in complete_current_workload()
1060 		i915_request_put(fetch_and_zero(&workload->req));  in complete_current_workload()
1064 			ring_id, workload, workload->status);  in complete_current_workload()
1068 	list_del_init(&workload->list);  in complete_current_workload()
1070 	if (workload->status || vgpu->resetting_eng & BIT(ring_id)) {  in complete_current_workload()
1087 	workload->complete(workload);  in complete_current_workload()
1089 	intel_vgpu_shadow_mm_unpin(workload);  in complete_current_workload()
1090 	intel_vgpu_destroy_workload(workload);  in complete_current_workload()
1108 	struct intel_vgpu_workload *workload = NULL;  in workload_thread()  local
1120 			workload = pick_next_workload(gvt, engine);  in workload_thread()
1121 			if (workload)  in workload_thread()
1128 		if (!workload)  in workload_thread()
1132 			      engine->name, workload,  in workload_thread()
1133 			      workload->vgpu->id);  in workload_thread()
1138 			      engine->name, workload);  in workload_thread()
1149 		update_vreg_in_ctx(workload);  in workload_thread()
1151 		ret = dispatch_workload(workload);  in workload_thread()
1154 			vgpu = workload->vgpu;  in workload_thread()
1160 			      engine->name, workload);  in workload_thread()
1161 		i915_request_wait(workload->req, 0, MAX_SCHEDULE_TIMEOUT);  in workload_thread()
1165 			      workload, workload->status);  in workload_thread()
1472 void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload)  in intel_vgpu_destroy_workload()  argument
1474 	struct intel_vgpu_submission *s = &workload->vgpu->submission;  in intel_vgpu_destroy_workload()
1476 	intel_context_unpin(s->shadow[workload->engine->id]);  in intel_vgpu_destroy_workload()
1477 	release_shadow_batch_buffer(workload);  in intel_vgpu_destroy_workload()
1478 	release_shadow_wa_ctx(&workload->wa_ctx);  in intel_vgpu_destroy_workload()
1480 	if (!list_empty(&workload->lri_shadow_mm)) {  in intel_vgpu_destroy_workload()
1482 		list_for_each_entry_safe(m, mm, &workload->lri_shadow_mm,  in intel_vgpu_destroy_workload()
1489 	GEM_BUG_ON(!list_empty(&workload->lri_shadow_mm));  in intel_vgpu_destroy_workload()
1490 	if (workload->shadow_mm)  in intel_vgpu_destroy_workload()
1491 		intel_vgpu_mm_put(workload->shadow_mm);  in intel_vgpu_destroy_workload()
1493 	kmem_cache_free(s->workloads, workload);  in intel_vgpu_destroy_workload()
1500 	struct intel_vgpu_workload *workload;  in alloc_workload()  local
1502 	workload = kmem_cache_zalloc(s->workloads, GFP_KERNEL);  in alloc_workload()
1503 	if (!workload)  in alloc_workload()
1506 	INIT_LIST_HEAD(&workload->list);  in alloc_workload()
1507 	INIT_LIST_HEAD(&workload->shadow_bb);  in alloc_workload()
1508 	INIT_LIST_HEAD(&workload->lri_shadow_mm);  in alloc_workload()
1510 	init_waitqueue_head(&workload->shadow_ctx_status_wq);  in alloc_workload()
1511 	atomic_set(&workload->shadow_ctx_active, 0);  in alloc_workload()
1513 	workload->status = -EINPROGRESS;  in alloc_workload()
1514 	workload->vgpu = vgpu;  in alloc_workload()
1516 	return workload;  in alloc_workload()
1535 static int prepare_mm(struct intel_vgpu_workload *workload)  in prepare_mm()  argument
1537 	struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc;  in prepare_mm()
1539 	struct intel_vgpu *vgpu = workload->vgpu;  in prepare_mm()
1555 	read_guest_pdps(workload->vgpu, workload->ring_context_gpa, (void *)pdps);  in prepare_mm()
1557 	mm = intel_vgpu_get_ppgtt_mm(workload->vgpu, root_entry_type, pdps);  in prepare_mm()
1561 	workload->shadow_mm = mm;  in prepare_mm()
1589 	struct intel_vgpu_workload *workload = NULL;  in intel_vgpu_create_workload()  local
1645 	workload = alloc_workload(vgpu);  in intel_vgpu_create_workload()
1646 	if (IS_ERR(workload))  in intel_vgpu_create_workload()
1647 		return workload;  in intel_vgpu_create_workload()
1649 	workload->engine = engine;  in intel_vgpu_create_workload()
1650 	workload->ctx_desc = *desc;  in intel_vgpu_create_workload()
1651 	workload->ring_context_gpa = ring_context_gpa;  in intel_vgpu_create_workload()
1652 	workload->rb_head = head;  in intel_vgpu_create_workload()
1653 	workload->guest_rb_head = guest_head;  in intel_vgpu_create_workload()
1654 	workload->rb_tail = tail;  in intel_vgpu_create_workload()
1655 	workload->rb_start = start;  in intel_vgpu_create_workload()
1656 	workload->rb_ctl = ctl;  in intel_vgpu_create_workload()
1664 		workload->wa_ctx.indirect_ctx.guest_gma =  in intel_vgpu_create_workload()
1666 		workload->wa_ctx.indirect_ctx.size =  in intel_vgpu_create_workload()
1670 		if (workload->wa_ctx.indirect_ctx.size != 0) {  in intel_vgpu_create_workload()
1672 				workload->wa_ctx.indirect_ctx.guest_gma,  in intel_vgpu_create_workload()
1673 				workload->wa_ctx.indirect_ctx.size)) {  in intel_vgpu_create_workload()
1675 				    workload->wa_ctx.indirect_ctx.guest_gma);  in intel_vgpu_create_workload()
1676 				kmem_cache_free(s->workloads, workload);  in intel_vgpu_create_workload()
1681 		workload->wa_ctx.per_ctx.guest_gma =  in intel_vgpu_create_workload()
1683 		workload->wa_ctx.per_ctx.valid = per_ctx & 1;  in intel_vgpu_create_workload()
1684 		if (workload->wa_ctx.per_ctx.valid) {  in intel_vgpu_create_workload()
1686 				workload->wa_ctx.per_ctx.guest_gma,  in intel_vgpu_create_workload()
1689 					workload->wa_ctx.per_ctx.guest_gma);  in intel_vgpu_create_workload()
1690 				kmem_cache_free(s->workloads, workload);  in intel_vgpu_create_workload()
1697 		   workload, engine->name, head, tail, start, ctl);  in intel_vgpu_create_workload()
1699 	ret = prepare_mm(workload);  in intel_vgpu_create_workload()
1701 		kmem_cache_free(s->workloads, workload);  in intel_vgpu_create_workload()
1712 			ret = intel_gvt_scan_and_shadow_workload(workload);  in intel_vgpu_create_workload()
1718 		intel_vgpu_destroy_workload(workload);  in intel_vgpu_create_workload()
1724 		intel_vgpu_destroy_workload(workload);  in intel_vgpu_create_workload()
1728 	return workload;  in intel_vgpu_create_workload()
1735 void intel_vgpu_queue_workload(struct intel_vgpu_workload *workload)  in intel_vgpu_queue_workload()  argument
1737 	list_add_tail(&workload->list,  in intel_vgpu_queue_workload()
1738 		      workload_q_head(workload->vgpu, workload->engine));  in intel_vgpu_queue_workload()
1739 	intel_gvt_kick_schedule(workload->vgpu->gvt);  in intel_vgpu_queue_workload()
1740 	wake_up(&workload->vgpu->gvt->scheduler.waitq[workload->engine->id]);  in intel_vgpu_queue_workload()