49 			const struct intel_engine_cs *engine,  in set_offsets()  argument
60 	const u32 base = engine->mmio_base;  in set_offsets()
78 		if (GRAPHICS_VER(engine->i915) >= 11)  in set_offsets()
101 		if (GRAPHICS_VER(engine->i915) >= 11)  in set_offsets()
682 static const u8 *reg_offsets(const struct intel_engine_cs *engine)  in reg_offsets()  argument
690 	GEM_BUG_ON(GRAPHICS_VER(engine->i915) >= 12 &&  in reg_offsets()
691 		   !intel_engine_has_relative_mmio(engine));  in reg_offsets()
693 	if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE) {  in reg_offsets()
694 		if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 70))  in reg_offsets()
696 		else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))  in reg_offsets()
698 		else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))  in reg_offsets()
700 		else if (GRAPHICS_VER(engine->i915) >= 12)  in reg_offsets()
702 		else if (GRAPHICS_VER(engine->i915) >= 11)  in reg_offsets()
704 		else if (GRAPHICS_VER(engine->i915) >= 9)  in reg_offsets()
709 		if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))  in reg_offsets()
711 		else if (GRAPHICS_VER(engine->i915) >= 12)  in reg_offsets()
713 		else if (GRAPHICS_VER(engine->i915) >= 9)  in reg_offsets()
720 static int lrc_ring_mi_mode(const struct intel_engine_cs *engine)  in lrc_ring_mi_mode()  argument
722 	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))  in lrc_ring_mi_mode()
724 	else if (GRAPHICS_VER(engine->i915) >= 12)  in lrc_ring_mi_mode()
726 	else if (GRAPHICS_VER(engine->i915) >= 9)  in lrc_ring_mi_mode()
728 	else if (engine->class == RENDER_CLASS)  in lrc_ring_mi_mode()
734 static int lrc_ring_bb_offset(const struct intel_engine_cs *engine)  in lrc_ring_bb_offset()  argument
736 	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))  in lrc_ring_bb_offset()
738 	else if (GRAPHICS_VER(engine->i915) >= 12)  in lrc_ring_bb_offset()
740 	else if (GRAPHICS_VER(engine->i915) >= 9)  in lrc_ring_bb_offset()
742 	else if (GRAPHICS_VER(engine->i915) >= 8 &&  in lrc_ring_bb_offset()
743 		 engine->class == RENDER_CLASS)  in lrc_ring_bb_offset()
749 static int lrc_ring_gpr0(const struct intel_engine_cs *engine)  in lrc_ring_gpr0()  argument
751 	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))  in lrc_ring_gpr0()
753 	else if (GRAPHICS_VER(engine->i915) >= 12)  in lrc_ring_gpr0()
755 	else if (GRAPHICS_VER(engine->i915) >= 9)  in lrc_ring_gpr0()
757 	else if (engine->class == RENDER_CLASS)  in lrc_ring_gpr0()
763 static int lrc_ring_wa_bb_per_ctx(const struct intel_engine_cs *engine)  in lrc_ring_wa_bb_per_ctx()  argument
765 	if (GRAPHICS_VER(engine->i915) >= 12)  in lrc_ring_wa_bb_per_ctx()
767 	else if (GRAPHICS_VER(engine->i915) >= 9 || engine->class == RENDER_CLASS)  in lrc_ring_wa_bb_per_ctx()
773 static int lrc_ring_indirect_ptr(const struct intel_engine_cs *engine)  in lrc_ring_indirect_ptr()  argument
777 	x = lrc_ring_wa_bb_per_ctx(engine);  in lrc_ring_indirect_ptr()
784 static int lrc_ring_indirect_offset(const struct intel_engine_cs *engine)  in lrc_ring_indirect_offset()  argument
788 	x = lrc_ring_indirect_ptr(engine);  in lrc_ring_indirect_offset()
795 static int lrc_ring_cmd_buf_cctl(const struct intel_engine_cs *engine)  in lrc_ring_cmd_buf_cctl()  argument
798 	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))  in lrc_ring_cmd_buf_cctl()
804 	else if (engine->class != RENDER_CLASS)  in lrc_ring_cmd_buf_cctl()
806 	else if (GRAPHICS_VER(engine->i915) >= 12)  in lrc_ring_cmd_buf_cctl()
808 	else if (GRAPHICS_VER(engine->i915) >= 11)  in lrc_ring_cmd_buf_cctl()
815 lrc_ring_indirect_offset_default(const struct intel_engine_cs *engine)  in lrc_ring_indirect_offset_default()  argument
817 	if (GRAPHICS_VER(engine->i915) >= 12)  in lrc_ring_indirect_offset_default()
819 	else if (GRAPHICS_VER(engine->i915) >= 11)  in lrc_ring_indirect_offset_default()
821 	else if (GRAPHICS_VER(engine->i915) >= 9)  in lrc_ring_indirect_offset_default()
823 	else if (GRAPHICS_VER(engine->i915) >= 8)  in lrc_ring_indirect_offset_default()
826 	GEM_BUG_ON(GRAPHICS_VER(engine->i915) < 8);  in lrc_ring_indirect_offset_default()
833 		       const struct intel_engine_cs *engine,  in lrc_setup_indirect_ctx()  argument
839 	GEM_BUG_ON(lrc_ring_indirect_ptr(engine) == -1);  in lrc_setup_indirect_ctx()
840 	regs[lrc_ring_indirect_ptr(engine) + 1] =  in lrc_setup_indirect_ctx()
843 	GEM_BUG_ON(lrc_ring_indirect_offset(engine) == -1);  in lrc_setup_indirect_ctx()
844 	regs[lrc_ring_indirect_offset(engine) + 1] =  in lrc_setup_indirect_ctx()
845 		lrc_ring_indirect_offset_default(engine) << 6;  in lrc_setup_indirect_ctx()
850 			     const struct intel_engine_cs *engine,  in init_common_regs()  argument
860 	if (GRAPHICS_VER(engine->i915) < 11)  in init_common_regs()
867 	loc = lrc_ring_bb_offset(engine);  in init_common_regs()
873 			    const struct intel_engine_cs *engine)  in init_wa_bb_regs()  argument
875 	const struct i915_ctx_workarounds * const wa_ctx = &engine->wa_ctx;  in init_wa_bb_regs()
880 		GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1);  in init_wa_bb_regs()
881 		regs[lrc_ring_wa_bb_per_ctx(engine) + 1] =  in init_wa_bb_regs()
886 		lrc_setup_indirect_ctx(regs, engine,  in init_wa_bb_regs()
917 static void __reset_stop_ring(u32 *regs, const struct intel_engine_cs *engine)  in __reset_stop_ring()  argument
921 	x = lrc_ring_mi_mode(engine);  in __reset_stop_ring()
930 			    const struct intel_engine_cs *engine,  in __lrc_init_regs()  argument
947 	set_offsets(regs, reg_offsets(engine), engine, inhibit);  in __lrc_init_regs()
949 	init_common_regs(regs, ce, engine, inhibit);  in __lrc_init_regs()
952 	init_wa_bb_regs(regs, engine);  in __lrc_init_regs()
954 	__reset_stop_ring(regs, engine);  in __lrc_init_regs()
958 		   const struct intel_engine_cs *engine,  in lrc_init_regs()  argument
961 	__lrc_init_regs(ce->lrc_reg_state, ce, engine, inhibit);  in lrc_init_regs()
965 		    const struct intel_engine_cs *engine)  in lrc_reset_regs()  argument
967 	__reset_stop_ring(ce->lrc_reg_state, engine);  in lrc_reset_regs()
971 set_redzone(void *vaddr, const struct intel_engine_cs *engine)  in set_redzone()  argument
976 	vaddr += engine->context_size;  in set_redzone()
982 check_redzone(const void *vaddr, const struct intel_engine_cs *engine)  in check_redzone()  argument
987 	vaddr += engine->context_size;  in check_redzone()
990 		drm_err_once(&engine->i915->drm,  in check_redzone()
992 			     engine->name);  in check_redzone()
1014 		    struct intel_engine_cs *engine,  in lrc_init_state()  argument
1019 	set_redzone(state, engine);  in lrc_init_state()
1021 	if (engine->default_state) {  in lrc_init_state()
1022 		shmem_read(engine->default_state, 0,  in lrc_init_state()
1023 			   state, engine->context_size);  in lrc_init_state()
1039 	__lrc_init_regs(state + LRC_STATE_OFFSET, ce, engine, inhibit);  in lrc_init_state()
1072 __lrc_alloc_state(struct intel_context *ce, struct intel_engine_cs *engine)  in __lrc_alloc_state()  argument
1078 	context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE);  in __lrc_alloc_state()
1083 	if (GRAPHICS_VER(engine->i915) >= 12) {  in __lrc_alloc_state()
1088 	if (intel_context_is_parent(ce) && intel_engine_uses_guc(engine)) {  in __lrc_alloc_state()
1093 	obj = i915_gem_object_create_lmem(engine->i915, context_size,  in __lrc_alloc_state()
1096 		obj = i915_gem_object_create_shmem(engine->i915, context_size);  in __lrc_alloc_state()
1105 		if (intel_gt_needs_wa_22016122933(engine->gt))  in __lrc_alloc_state()
1109 	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);  in __lrc_alloc_state()
1119 pinned_timeline(struct intel_context *ce, struct intel_engine_cs *engine)  in pinned_timeline()  argument
1123 	return intel_timeline_create_from_engine(engine, page_unmask_bits(tl));  in pinned_timeline()
1126 int lrc_alloc(struct intel_context *ce, struct intel_engine_cs *engine)  in lrc_alloc()  argument
1134 	vma = __lrc_alloc_state(ce, engine);  in lrc_alloc()
1138 	ring = intel_engine_create_ring(engine, ce->ring_size);  in lrc_alloc()
1152 			tl = pinned_timeline(ce, engine);  in lrc_alloc()
1154 			tl = intel_timeline_create(engine->gt);  in lrc_alloc()
1182 	lrc_init_regs(ce, ce->engine, true);  in lrc_reset()
1183 	ce->lrc.lrca = lrc_update_regs(ce, ce->engine, ce->ring->tail);  in lrc_reset()
1188 	    struct intel_engine_cs *engine,  in lrc_pre_pin()  argument
1196 					 intel_gt_coherent_map_type(ce->engine->gt,  in lrc_pre_pin()
1206 	struct intel_engine_cs *engine,  in lrc_pin()  argument
1212 		lrc_init_state(ce, engine, vaddr);  in lrc_pin()
1214 	ce->lrc.lrca = lrc_update_regs(ce, engine, ce->ring->tail);  in lrc_pin()
1225 		      ce->engine);  in lrc_unpin()
1284 	GEM_BUG_ON(lrc_ring_gpr0(ce->engine) == -1);  in gen12_emit_restore_scratch()
1291 		(lrc_ring_gpr0(ce->engine) + 1) * sizeof(u32);  in gen12_emit_restore_scratch()
1300 	GEM_BUG_ON(lrc_ring_cmd_buf_cctl(ce->engine) == -1);  in gen12_emit_cmd_buf_wa()
1307 		(lrc_ring_cmd_buf_cctl(ce->engine) + 1) * sizeof(u32);  in gen12_emit_cmd_buf_wa()
1328 	*cs++ = i915_mmio_reg_offset(GEN12_STATE_ACK_DEBUG(ce->engine->mmio_base));  in dg2_emit_rcs_hang_wabb()
1332 	*cs++ = i915_mmio_reg_offset(RING_NOPID(ce->engine->mmio_base));  in dg2_emit_rcs_hang_wabb()
1336 	*cs++ = i915_mmio_reg_offset(RING_NOPID(ce->engine->mmio_base));  in dg2_emit_rcs_hang_wabb()
1367 	if (IS_DG2_GRAPHICS_STEP(ce->engine->i915, G10, STEP_A0, STEP_B0) ||  in gen12_emit_indirect_ctx_rcs()
1368 	    IS_DG2_GRAPHICS_STEP(ce->engine->i915, G11, STEP_A0, STEP_B0))  in gen12_emit_indirect_ctx_rcs()
1372 	if (IS_DG2_GRAPHICS_STEP(ce->engine->i915, G10, STEP_B0, STEP_C0) ||  in gen12_emit_indirect_ctx_rcs()
1373 	    IS_DG2_G11(ce->engine->i915))  in gen12_emit_indirect_ctx_rcs()
1376 	cs = gen12_emit_aux_table_inv(ce->engine, cs);  in gen12_emit_indirect_ctx_rcs()
1379 	if (IS_MTL_GRAPHICS_STEP(ce->engine->i915, M, STEP_A0, STEP_B0) ||  in gen12_emit_indirect_ctx_rcs()
1380 	    IS_MTL_GRAPHICS_STEP(ce->engine->i915, P, STEP_A0, STEP_B0) ||  in gen12_emit_indirect_ctx_rcs()
1381 	    IS_DG2(ce->engine->i915))  in gen12_emit_indirect_ctx_rcs()
1394 	if (IS_DG2_GRAPHICS_STEP(ce->engine->i915, G10, STEP_B0, STEP_C0) ||  in gen12_emit_indirect_ctx_xcs()
1395 	    IS_DG2_G11(ce->engine->i915))  in gen12_emit_indirect_ctx_xcs()
1396 		if (ce->engine->class == COMPUTE_CLASS)  in gen12_emit_indirect_ctx_xcs()
1401 	return gen12_emit_aux_table_inv(ce->engine, cs);  in gen12_emit_indirect_ctx_xcs()
1406 		      const struct intel_engine_cs *engine,  in setup_indirect_ctx_bb()  argument
1420 	lrc_setup_indirect_ctx(ce->lrc_reg_state, engine,  in setup_indirect_ctx_bb()
1476 		    const struct intel_engine_cs *engine,  in lrc_update_regs()  argument
1491 	if (engine->class == RENDER_CLASS) {  in lrc_update_regs()
1493 			intel_sseu_make_rpcs(engine->gt, &ce->sseu);  in lrc_update_regs()
1495 		i915_oa_init_reg_state(ce, engine);  in lrc_update_regs()
1502 		if (ce->engine->class == RENDER_CLASS)  in lrc_update_regs()
1506 		GEM_BUG_ON(engine->wa_ctx.indirect_ctx.size);  in lrc_update_regs()
1507 		setup_indirect_ctx_bb(ce, engine, fn);  in lrc_update_regs()
1514 			struct intel_engine_cs *engine)  in lrc_update_offsets()  argument
1516 	set_offsets(ce->lrc_reg_state, reg_offsets(engine), engine, false);  in lrc_update_offsets()
1520 		    const struct intel_engine_cs *engine,  in lrc_check_regs()  argument
1530 		       engine->name,  in lrc_check_regs()
1540 		       engine->name,  in lrc_check_regs()
1547 	x = lrc_ring_mi_mode(engine);  in lrc_check_regs()
1550 		       engine->name, regs[x + 1]);  in lrc_check_regs()
1576 gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, u32 *batch)  in gen8_emit_flush_coherentl3_wa()  argument
1581 	*batch++ = intel_gt_scratch_offset(engine->gt,  in gen8_emit_flush_coherentl3_wa()
1596 	*batch++ = intel_gt_scratch_offset(engine->gt,  in gen8_emit_flush_coherentl3_wa()
1618 static u32 *gen8_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)  in gen8_init_indirectctx_bb()  argument
1624 	if (IS_BROADWELL(engine->i915))  in gen8_init_indirectctx_bb()
1625 		batch = gen8_emit_flush_coherentl3_wa(engine, batch);  in gen8_init_indirectctx_bb()
1670 static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)  in gen9_init_indirectctx_bb()  argument
1698 	batch = gen8_emit_flush_coherentl3_wa(engine, batch);  in gen9_init_indirectctx_bb()
1711 	if (HAS_POOLED_EU(engine->i915)) {  in gen9_init_indirectctx_bb()
1744 static int lrc_create_wa_ctx(struct intel_engine_cs *engine)  in lrc_create_wa_ctx()  argument
1750 	obj = i915_gem_object_create_shmem(engine->i915, CTX_WA_BB_SIZE);  in lrc_create_wa_ctx()
1754 	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);  in lrc_create_wa_ctx()
1760 	engine->wa_ctx.vma = vma;  in lrc_create_wa_ctx()
1768 void lrc_fini_wa_ctx(struct intel_engine_cs *engine)  in lrc_fini_wa_ctx()  argument
1770 	i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0);  in lrc_fini_wa_ctx()
1773 typedef u32 *(*wa_bb_func_t)(struct intel_engine_cs *engine, u32 *batch);
1775 void lrc_init_wa_ctx(struct intel_engine_cs *engine)  in lrc_init_wa_ctx()  argument
1777 	struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;  in lrc_init_wa_ctx()
1787 	if (GRAPHICS_VER(engine->i915) >= 11 ||  in lrc_init_wa_ctx()
1788 	    !(engine->flags & I915_ENGINE_HAS_RCS_REG_STATE))  in lrc_init_wa_ctx()
1791 	if (GRAPHICS_VER(engine->i915) == 9) {  in lrc_init_wa_ctx()
1794 	} else if (GRAPHICS_VER(engine->i915) == 8) {  in lrc_init_wa_ctx()
1799 	err = lrc_create_wa_ctx(engine);  in lrc_init_wa_ctx()
1806 		drm_err(&engine->i915->drm,  in lrc_init_wa_ctx()
1812 	if (!engine->wa_ctx.vma)  in lrc_init_wa_ctx()
1843 			batch_ptr = wa_bb_fn[i](engine, batch_ptr);  in lrc_init_wa_ctx()
1853 		err = i915_inject_probe_error(engine->i915, -ENODEV);  in lrc_init_wa_ctx()
1867 		i915_vma_put(engine->wa_ctx.vma);  in lrc_init_wa_ctx()