25 			const struct intel_engine_cs *engine,  in set_offsets()  argument
36 	const u32 base = engine->mmio_base;  in set_offsets()
54 		if (GRAPHICS_VER(engine->i915) >= 11)  in set_offsets()
77 		if (GRAPHICS_VER(engine->i915) >= 11)  in set_offsets()
615 static const u8 *reg_offsets(const struct intel_engine_cs *engine)  in reg_offsets()  argument
623 	GEM_BUG_ON(GRAPHICS_VER(engine->i915) >= 12 &&  in reg_offsets()
624 		   !intel_engine_has_relative_mmio(engine));  in reg_offsets()
626 	if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE) {  in reg_offsets()
627 		if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))  in reg_offsets()
629 		else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))  in reg_offsets()
631 		else if (GRAPHICS_VER(engine->i915) >= 12)  in reg_offsets()
633 		else if (GRAPHICS_VER(engine->i915) >= 11)  in reg_offsets()
635 		else if (GRAPHICS_VER(engine->i915) >= 9)  in reg_offsets()
640 		if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))  in reg_offsets()
642 		else if (GRAPHICS_VER(engine->i915) >= 12)  in reg_offsets()
644 		else if (GRAPHICS_VER(engine->i915) >= 9)  in reg_offsets()
651 static int lrc_ring_mi_mode(const struct intel_engine_cs *engine)  in lrc_ring_mi_mode()  argument
653 	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))  in lrc_ring_mi_mode()
655 	else if (GRAPHICS_VER(engine->i915) >= 12)  in lrc_ring_mi_mode()
657 	else if (GRAPHICS_VER(engine->i915) >= 9)  in lrc_ring_mi_mode()
659 	else if (engine->class == RENDER_CLASS)  in lrc_ring_mi_mode()
665 static int lrc_ring_bb_offset(const struct intel_engine_cs *engine)  in lrc_ring_bb_offset()  argument
667 	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))  in lrc_ring_bb_offset()
669 	else if (GRAPHICS_VER(engine->i915) >= 12)  in lrc_ring_bb_offset()
671 	else if (GRAPHICS_VER(engine->i915) >= 9)  in lrc_ring_bb_offset()
673 	else if (GRAPHICS_VER(engine->i915) >= 8 &&  in lrc_ring_bb_offset()
674 		 engine->class == RENDER_CLASS)  in lrc_ring_bb_offset()
680 static int lrc_ring_gpr0(const struct intel_engine_cs *engine)  in lrc_ring_gpr0()  argument
682 	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))  in lrc_ring_gpr0()
684 	else if (GRAPHICS_VER(engine->i915) >= 12)  in lrc_ring_gpr0()
686 	else if (GRAPHICS_VER(engine->i915) >= 9)  in lrc_ring_gpr0()
688 	else if (engine->class == RENDER_CLASS)  in lrc_ring_gpr0()
694 static int lrc_ring_wa_bb_per_ctx(const struct intel_engine_cs *engine)  in lrc_ring_wa_bb_per_ctx()  argument
696 	if (GRAPHICS_VER(engine->i915) >= 12)  in lrc_ring_wa_bb_per_ctx()
698 	else if (GRAPHICS_VER(engine->i915) >= 9 || engine->class == RENDER_CLASS)  in lrc_ring_wa_bb_per_ctx()
704 static int lrc_ring_indirect_ptr(const struct intel_engine_cs *engine)  in lrc_ring_indirect_ptr()  argument
708 	x = lrc_ring_wa_bb_per_ctx(engine);  in lrc_ring_indirect_ptr()
715 static int lrc_ring_indirect_offset(const struct intel_engine_cs *engine)  in lrc_ring_indirect_offset()  argument
719 	x = lrc_ring_indirect_ptr(engine);  in lrc_ring_indirect_offset()
726 static int lrc_ring_cmd_buf_cctl(const struct intel_engine_cs *engine)  in lrc_ring_cmd_buf_cctl()  argument
729 	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))  in lrc_ring_cmd_buf_cctl()
735 	else if (engine->class != RENDER_CLASS)  in lrc_ring_cmd_buf_cctl()
737 	else if (GRAPHICS_VER(engine->i915) >= 12)  in lrc_ring_cmd_buf_cctl()
739 	else if (GRAPHICS_VER(engine->i915) >= 11)  in lrc_ring_cmd_buf_cctl()
746 lrc_ring_indirect_offset_default(const struct intel_engine_cs *engine)  in lrc_ring_indirect_offset_default()  argument
748 	switch (GRAPHICS_VER(engine->i915)) {  in lrc_ring_indirect_offset_default()
750 		MISSING_CASE(GRAPHICS_VER(engine->i915));  in lrc_ring_indirect_offset_default()
765 		       const struct intel_engine_cs *engine,  in lrc_setup_indirect_ctx()  argument
771 	GEM_BUG_ON(lrc_ring_indirect_ptr(engine) == -1);  in lrc_setup_indirect_ctx()
772 	regs[lrc_ring_indirect_ptr(engine) + 1] =  in lrc_setup_indirect_ctx()
775 	GEM_BUG_ON(lrc_ring_indirect_offset(engine) == -1);  in lrc_setup_indirect_ctx()
776 	regs[lrc_ring_indirect_offset(engine) + 1] =  in lrc_setup_indirect_ctx()
777 		lrc_ring_indirect_offset_default(engine) << 6;  in lrc_setup_indirect_ctx()
782 			     const struct intel_engine_cs *engine,  in init_common_regs()  argument
792 	if (GRAPHICS_VER(engine->i915) < 11)  in init_common_regs()
799 	loc = lrc_ring_bb_offset(engine);  in init_common_regs()
805 			    const struct intel_engine_cs *engine)  in init_wa_bb_regs()  argument
807 	const struct i915_ctx_workarounds * const wa_ctx = &engine->wa_ctx;  in init_wa_bb_regs()
812 		GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1);  in init_wa_bb_regs()
813 		regs[lrc_ring_wa_bb_per_ctx(engine) + 1] =  in init_wa_bb_regs()
818 		lrc_setup_indirect_ctx(regs, engine,  in init_wa_bb_regs()
849 static void __reset_stop_ring(u32 *regs, const struct intel_engine_cs *engine)  in __reset_stop_ring()  argument
853 	x = lrc_ring_mi_mode(engine);  in __reset_stop_ring()
862 			    const struct intel_engine_cs *engine,  in __lrc_init_regs()  argument
879 	set_offsets(regs, reg_offsets(engine), engine, inhibit);  in __lrc_init_regs()
881 	init_common_regs(regs, ce, engine, inhibit);  in __lrc_init_regs()
884 	init_wa_bb_regs(regs, engine);  in __lrc_init_regs()
886 	__reset_stop_ring(regs, engine);  in __lrc_init_regs()
890 		   const struct intel_engine_cs *engine,  in lrc_init_regs()  argument
893 	__lrc_init_regs(ce->lrc_reg_state, ce, engine, inhibit);  in lrc_init_regs()
897 		    const struct intel_engine_cs *engine)  in lrc_reset_regs()  argument
899 	__reset_stop_ring(ce->lrc_reg_state, engine);  in lrc_reset_regs()
903 set_redzone(void *vaddr, const struct intel_engine_cs *engine)  in set_redzone()  argument
908 	vaddr += engine->context_size;  in set_redzone()
914 check_redzone(const void *vaddr, const struct intel_engine_cs *engine)  in check_redzone()  argument
919 	vaddr += engine->context_size;  in check_redzone()
922 		drm_err_once(&engine->i915->drm,  in check_redzone()
924 			     engine->name);  in check_redzone()
946 		    struct intel_engine_cs *engine,  in lrc_init_state()  argument
951 	set_redzone(state, engine);  in lrc_init_state()
953 	if (engine->default_state) {  in lrc_init_state()
954 		shmem_read(engine->default_state, 0,  in lrc_init_state()
955 			   state, engine->context_size);  in lrc_init_state()
971 	__lrc_init_regs(state + LRC_STATE_OFFSET, ce, engine, inhibit);  in lrc_init_state()
1004 __lrc_alloc_state(struct intel_context *ce, struct intel_engine_cs *engine)  in __lrc_alloc_state()  argument
1010 	context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE);  in __lrc_alloc_state()
1015 	if (GRAPHICS_VER(engine->i915) == 12) {  in __lrc_alloc_state()
1020 	if (intel_context_is_parent(ce) && intel_engine_uses_guc(engine)) {  in __lrc_alloc_state()
1025 	obj = i915_gem_object_create_lmem(engine->i915, context_size,  in __lrc_alloc_state()
1028 		obj = i915_gem_object_create_shmem(engine->i915, context_size);  in __lrc_alloc_state()
1032 	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);  in __lrc_alloc_state()
1042 pinned_timeline(struct intel_context *ce, struct intel_engine_cs *engine)  in pinned_timeline()  argument
1046 	return intel_timeline_create_from_engine(engine, page_unmask_bits(tl));  in pinned_timeline()
1049 int lrc_alloc(struct intel_context *ce, struct intel_engine_cs *engine)  in lrc_alloc()  argument
1057 	vma = __lrc_alloc_state(ce, engine);  in lrc_alloc()
1061 	ring = intel_engine_create_ring(engine, ce->ring_size);  in lrc_alloc()
1075 			tl = pinned_timeline(ce, engine);  in lrc_alloc()
1077 			tl = intel_timeline_create(engine->gt);  in lrc_alloc()
1105 	lrc_init_regs(ce, ce->engine, true);  in lrc_reset()
1106 	ce->lrc.lrca = lrc_update_regs(ce, ce->engine, ce->ring->tail);  in lrc_reset()
1111 	    struct intel_engine_cs *engine,  in lrc_pre_pin()  argument
1119 					 i915_coherent_map_type(ce->engine->i915,  in lrc_pre_pin()
1129 	struct intel_engine_cs *engine,  in lrc_pin()  argument
1135 		lrc_init_state(ce, engine, vaddr);  in lrc_pin()
1137 	ce->lrc.lrca = lrc_update_regs(ce, engine, ce->ring->tail);  in lrc_pin()
1148 		      ce->engine);  in lrc_unpin()
1207 	GEM_BUG_ON(lrc_ring_gpr0(ce->engine) == -1);  in gen12_emit_restore_scratch()
1214 		(lrc_ring_gpr0(ce->engine) + 1) * sizeof(u32);  in gen12_emit_restore_scratch()
1223 	GEM_BUG_ON(lrc_ring_cmd_buf_cctl(ce->engine) == -1);  in gen12_emit_cmd_buf_wa()
1230 		(lrc_ring_cmd_buf_cctl(ce->engine) + 1) * sizeof(u32);  in gen12_emit_cmd_buf_wa()
1255 	*cs++ = i915_mmio_reg_offset(RING_NOPID(ce->engine->mmio_base));  in dg2_emit_rcs_hang_wabb()
1259 	*cs++ = i915_mmio_reg_offset(RING_NOPID(ce->engine->mmio_base));  in dg2_emit_rcs_hang_wabb()
1290 	if (IS_DG2_GRAPHICS_STEP(ce->engine->i915, G10, STEP_A0, STEP_B0) ||  in gen12_emit_indirect_ctx_rcs()
1291 	    IS_DG2_GRAPHICS_STEP(ce->engine->i915, G11, STEP_A0, STEP_B0))  in gen12_emit_indirect_ctx_rcs()
1295 	if (IS_DG2_GRAPHICS_STEP(ce->engine->i915, G10, STEP_B0, STEP_C0) ||  in gen12_emit_indirect_ctx_rcs()
1296 	    IS_DG2_G11(ce->engine->i915))  in gen12_emit_indirect_ctx_rcs()
1300 	if (!HAS_FLAT_CCS(ce->engine->i915))  in gen12_emit_indirect_ctx_rcs()
1301 		cs = gen12_emit_aux_table_inv(ce->engine->gt,  in gen12_emit_indirect_ctx_rcs()
1305 	if (IS_DG2(ce->engine->i915))  in gen12_emit_indirect_ctx_rcs()
1318 	if (IS_DG2_GRAPHICS_STEP(ce->engine->i915, G10, STEP_B0, STEP_C0) ||  in gen12_emit_indirect_ctx_xcs()
1319 	    IS_DG2_G11(ce->engine->i915))  in gen12_emit_indirect_ctx_xcs()
1320 		if (ce->engine->class == COMPUTE_CLASS)  in gen12_emit_indirect_ctx_xcs()
1326 	if (!HAS_FLAT_CCS(ce->engine->i915)) {  in gen12_emit_indirect_ctx_xcs()
1327 		if (ce->engine->class == VIDEO_DECODE_CLASS)  in gen12_emit_indirect_ctx_xcs()
1328 			cs = gen12_emit_aux_table_inv(ce->engine->gt,  in gen12_emit_indirect_ctx_xcs()
1330 		else if (ce->engine->class == VIDEO_ENHANCEMENT_CLASS)  in gen12_emit_indirect_ctx_xcs()
1331 			cs = gen12_emit_aux_table_inv(ce->engine->gt,  in gen12_emit_indirect_ctx_xcs()
1340 		      const struct intel_engine_cs *engine,  in setup_indirect_ctx_bb()  argument
1354 	lrc_setup_indirect_ctx(ce->lrc_reg_state, engine,  in setup_indirect_ctx_bb()
1410 		    const struct intel_engine_cs *engine,  in lrc_update_regs()  argument
1425 	if (engine->class == RENDER_CLASS) {  in lrc_update_regs()
1427 			intel_sseu_make_rpcs(engine->gt, &ce->sseu);  in lrc_update_regs()
1429 		i915_oa_init_reg_state(ce, engine);  in lrc_update_regs()
1436 		if (ce->engine->class == RENDER_CLASS)  in lrc_update_regs()
1440 		GEM_BUG_ON(engine->wa_ctx.indirect_ctx.size);  in lrc_update_regs()
1441 		setup_indirect_ctx_bb(ce, engine, fn);  in lrc_update_regs()
1448 			struct intel_engine_cs *engine)  in lrc_update_offsets()  argument
1450 	set_offsets(ce->lrc_reg_state, reg_offsets(engine), engine, false);  in lrc_update_offsets()
1454 		    const struct intel_engine_cs *engine,  in lrc_check_regs()  argument
1464 		       engine->name,  in lrc_check_regs()
1474 		       engine->name,  in lrc_check_regs()
1481 	x = lrc_ring_mi_mode(engine);  in lrc_check_regs()
1484 		       engine->name, regs[x + 1]);  in lrc_check_regs()
1510 gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, u32 *batch)  in gen8_emit_flush_coherentl3_wa()  argument
1515 	*batch++ = intel_gt_scratch_offset(engine->gt,  in gen8_emit_flush_coherentl3_wa()
1530 	*batch++ = intel_gt_scratch_offset(engine->gt,  in gen8_emit_flush_coherentl3_wa()
1552 static u32 *gen8_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)  in gen8_init_indirectctx_bb()  argument
1558 	if (IS_BROADWELL(engine->i915))  in gen8_init_indirectctx_bb()
1559 		batch = gen8_emit_flush_coherentl3_wa(engine, batch);  in gen8_init_indirectctx_bb()
1604 static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)  in gen9_init_indirectctx_bb()  argument
1632 	batch = gen8_emit_flush_coherentl3_wa(engine, batch);  in gen9_init_indirectctx_bb()
1645 	if (HAS_POOLED_EU(engine->i915)) {  in gen9_init_indirectctx_bb()
1678 static int lrc_create_wa_ctx(struct intel_engine_cs *engine)  in lrc_create_wa_ctx()  argument
1684 	obj = i915_gem_object_create_shmem(engine->i915, CTX_WA_BB_SIZE);  in lrc_create_wa_ctx()
1688 	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);  in lrc_create_wa_ctx()
1694 	engine->wa_ctx.vma = vma;  in lrc_create_wa_ctx()
1702 void lrc_fini_wa_ctx(struct intel_engine_cs *engine)  in lrc_fini_wa_ctx()  argument
1704 	i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0);  in lrc_fini_wa_ctx()
1707 typedef u32 *(*wa_bb_func_t)(struct intel_engine_cs *engine, u32 *batch);
1709 void lrc_init_wa_ctx(struct intel_engine_cs *engine)  in lrc_init_wa_ctx()  argument
1711 	struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;  in lrc_init_wa_ctx()
1721 	if (!(engine->flags & I915_ENGINE_HAS_RCS_REG_STATE))  in lrc_init_wa_ctx()
1724 	switch (GRAPHICS_VER(engine->i915)) {  in lrc_init_wa_ctx()
1737 		MISSING_CASE(GRAPHICS_VER(engine->i915));  in lrc_init_wa_ctx()
1741 	err = lrc_create_wa_ctx(engine);  in lrc_init_wa_ctx()
1748 		drm_err(&engine->i915->drm,  in lrc_init_wa_ctx()
1754 	if (!engine->wa_ctx.vma)  in lrc_init_wa_ctx()
1785 			batch_ptr = wa_bb_fn[i](engine, batch_ptr);  in lrc_init_wa_ctx()
1795 		err = i915_inject_probe_error(engine->i915, -ENODEV);  in lrc_init_wa_ctx()
1809 		i915_vma_put(engine->wa_ctx.vma);  in lrc_init_wa_ctx()