Lines Matching refs:wal
58 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name) in wa_init_start() argument
60 wal->name = name; in wa_init_start()
61 wal->engine_name = engine_name; in wa_init_start()
66 static void wa_init_finish(struct i915_wa_list *wal) in wa_init_finish() argument
69 if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) { in wa_init_finish()
70 struct i915_wa *list = kmemdup(wal->list, in wa_init_finish()
71 wal->count * sizeof(*list), in wa_init_finish()
75 kfree(wal->list); in wa_init_finish()
76 wal->list = list; in wa_init_finish()
80 if (!wal->count) in wa_init_finish()
84 wal->wa_count, wal->name, wal->engine_name); in wa_init_finish()
87 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa) in _wa_add() argument
90 unsigned int start = 0, end = wal->count; in _wa_add()
96 if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */ in _wa_add()
99 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa), in _wa_add()
106 if (wal->list) { in _wa_add()
107 memcpy(list, wal->list, sizeof(*wa) * wal->count); in _wa_add()
108 kfree(wal->list); in _wa_add()
111 wal->list = list; in _wa_add()
117 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) { in _wa_add()
119 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) { in _wa_add()
122 wa_ = &wal->list[mid]; in _wa_add()
132 wal->wa_count++; in _wa_add()
140 wal->wa_count++; in _wa_add()
141 wa_ = &wal->list[wal->count++]; in _wa_add()
144 while (wa_-- > wal->list) { in _wa_add()
155 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg, in wa_add() argument
166 _wa_add(wal, &wa); in wa_add()
170 wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set) in wa_write_clr_set() argument
172 wa_add(wal, reg, clear, set, clear, false); in wa_write_clr_set()
176 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set) in wa_write() argument
178 wa_write_clr_set(wal, reg, ~0, set); in wa_write()
182 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set) in wa_write_or() argument
184 wa_write_clr_set(wal, reg, set, set); in wa_write_or()
188 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr) in wa_write_clr() argument
190 wa_write_clr_set(wal, reg, clr, 0); in wa_write_clr()
205 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val) in wa_masked_en() argument
207 wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val, true); in wa_masked_en()
211 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val) in wa_masked_dis() argument
213 wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val, true); in wa_masked_dis()
217 wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg, in wa_masked_field_set() argument
220 wa_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask, true); in wa_masked_field_set()
224 struct i915_wa_list *wal) in gen6_ctx_workarounds_init() argument
226 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING); in gen6_ctx_workarounds_init()
230 struct i915_wa_list *wal) in gen7_ctx_workarounds_init() argument
232 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING); in gen7_ctx_workarounds_init()
236 struct i915_wa_list *wal) in gen8_ctx_workarounds_init() argument
238 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING); in gen8_ctx_workarounds_init()
241 wa_masked_en(wal, RING_MI_MODE(RENDER_RING_BASE), ASYNC_FLIP_PERF_DISABLE); in gen8_ctx_workarounds_init()
244 wa_masked_en(wal, GEN8_ROW_CHICKEN, in gen8_ctx_workarounds_init()
253 wa_masked_en(wal, HDC_CHICKEN0, in gen8_ctx_workarounds_init()
265 wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE); in gen8_ctx_workarounds_init()
268 wa_masked_en(wal, CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE); in gen8_ctx_workarounds_init()
278 wa_masked_field_set(wal, GEN7_GT_MODE, in gen8_ctx_workarounds_init()
284 struct i915_wa_list *wal) in bdw_ctx_workarounds_init() argument
288 gen8_ctx_workarounds_init(engine, wal); in bdw_ctx_workarounds_init()
291 wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); in bdw_ctx_workarounds_init()
298 wa_masked_en(wal, GEN7_ROW_CHICKEN2, in bdw_ctx_workarounds_init()
301 wa_masked_en(wal, HALF_SLICE_CHICKEN3, in bdw_ctx_workarounds_init()
304 wa_masked_en(wal, HDC_CHICKEN0, in bdw_ctx_workarounds_init()
312 struct i915_wa_list *wal) in chv_ctx_workarounds_init() argument
314 gen8_ctx_workarounds_init(engine, wal); in chv_ctx_workarounds_init()
317 wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); in chv_ctx_workarounds_init()
320 wa_masked_en(wal, HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X); in chv_ctx_workarounds_init()
324 struct i915_wa_list *wal) in gen9_ctx_workarounds_init() argument
334 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, in gen9_ctx_workarounds_init()
336 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7, in gen9_ctx_workarounds_init()
342 wa_masked_en(wal, GEN8_ROW_CHICKEN, in gen9_ctx_workarounds_init()
348 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7, in gen9_ctx_workarounds_init()
354 wa_masked_en(wal, CACHE_MODE_1, in gen9_ctx_workarounds_init()
359 wa_masked_dis(wal, GEN9_HALF_SLICE_CHICKEN5, in gen9_ctx_workarounds_init()
363 wa_masked_en(wal, HDC_CHICKEN0, in gen9_ctx_workarounds_init()
381 wa_masked_en(wal, HDC_CHICKEN0, in gen9_ctx_workarounds_init()
389 wa_masked_en(wal, HALF_SLICE_CHICKEN3, in gen9_ctx_workarounds_init()
393 wa_masked_en(wal, HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE); in gen9_ctx_workarounds_init()
407 wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL); in gen9_ctx_workarounds_init()
410 wa_masked_field_set(wal, GEN8_CS_CHICKEN1, in gen9_ctx_workarounds_init()
416 wa_masked_en(wal, GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ); in gen9_ctx_workarounds_init()
420 struct i915_wa_list *wal) in skl_tune_iz_hashing() argument
450 wa_masked_field_set(wal, GEN7_GT_MODE, in skl_tune_iz_hashing()
460 struct i915_wa_list *wal) in skl_ctx_workarounds_init() argument
462 gen9_ctx_workarounds_init(engine, wal); in skl_ctx_workarounds_init()
463 skl_tune_iz_hashing(engine, wal); in skl_ctx_workarounds_init()
467 struct i915_wa_list *wal) in bxt_ctx_workarounds_init() argument
469 gen9_ctx_workarounds_init(engine, wal); in bxt_ctx_workarounds_init()
472 wa_masked_en(wal, GEN8_ROW_CHICKEN, in bxt_ctx_workarounds_init()
476 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, in bxt_ctx_workarounds_init()
481 struct i915_wa_list *wal) in kbl_ctx_workarounds_init() argument
485 gen9_ctx_workarounds_init(engine, wal); in kbl_ctx_workarounds_init()
489 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, in kbl_ctx_workarounds_init()
493 wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1, in kbl_ctx_workarounds_init()
498 struct i915_wa_list *wal) in glk_ctx_workarounds_init() argument
500 gen9_ctx_workarounds_init(engine, wal); in glk_ctx_workarounds_init()
503 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, in glk_ctx_workarounds_init()
508 struct i915_wa_list *wal) in cfl_ctx_workarounds_init() argument
510 gen9_ctx_workarounds_init(engine, wal); in cfl_ctx_workarounds_init()
513 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, in cfl_ctx_workarounds_init()
517 wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1, in cfl_ctx_workarounds_init()
522 struct i915_wa_list *wal) in icl_ctx_workarounds_init() argument
525 wa_write(wal, in icl_ctx_workarounds_init()
537 wa_masked_en(wal, ICL_HDC_MODE, HDC_FORCE_NON_COHERENT); in icl_ctx_workarounds_init()
540 wa_add(wal, GEN10_CACHE_MODE_SS, 0, in icl_ctx_workarounds_init()
546 wa_masked_field_set(wal, GEN8_CS_CHICKEN1, in icl_ctx_workarounds_init()
551 wa_masked_en(wal, GEN10_SAMPLER_MODE, in icl_ctx_workarounds_init()
555 wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID); in icl_ctx_workarounds_init()
556 wa_write_clr_set(wal, IVB_FBC_RT_BASE_UPPER, in icl_ctx_workarounds_init()
561 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU); in icl_ctx_workarounds_init()
569 struct i915_wa_list *wal) in dg2_ctx_gt_tuning_init() argument
571 wa_masked_en(wal, CHICKEN_RASTER_2, TBIMR_FAST_CLIP); in dg2_ctx_gt_tuning_init()
572 wa_write_clr_set(wal, GEN11_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK, in dg2_ctx_gt_tuning_init()
574 wa_add(wal, in dg2_ctx_gt_tuning_init()
586 struct i915_wa_list *wal) in gen12_ctx_gt_tuning_init() argument
601 wa_add(wal, in gen12_ctx_gt_tuning_init()
609 struct i915_wa_list *wal) in gen12_ctx_workarounds_init() argument
611 gen12_ctx_gt_tuning_init(engine, wal); in gen12_ctx_workarounds_init()
625 wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3, in gen12_ctx_workarounds_init()
629 wa_masked_field_set(wal, GEN8_CS_CHICKEN1, in gen12_ctx_workarounds_init()
639 wa_add(wal, in gen12_ctx_workarounds_init()
647 struct i915_wa_list *wal) in dg1_ctx_workarounds_init() argument
649 gen12_ctx_workarounds_init(engine, wal); in dg1_ctx_workarounds_init()
652 wa_masked_dis(wal, GEN11_COMMON_SLICE_CHICKEN3, in dg1_ctx_workarounds_init()
656 wa_masked_en(wal, HIZ_CHICKEN, in dg1_ctx_workarounds_init()
661 struct i915_wa_list *wal) in dg2_ctx_workarounds_init() argument
663 dg2_ctx_gt_tuning_init(engine, wal); in dg2_ctx_workarounds_init()
667 wa_masked_dis(wal, VFLSKPD, DIS_MULT_MISS_RD_SQUASH); in dg2_ctx_workarounds_init()
668 wa_masked_en(wal, VFLSKPD, DIS_OVER_FETCH_CACHE); in dg2_ctx_workarounds_init()
673 wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3, in dg2_ctx_workarounds_init()
681 wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3, in dg2_ctx_workarounds_init()
686 wa_masked_en(wal, SLICE_COMMON_ECO_CHICKEN1, in dg2_ctx_workarounds_init()
692 wa_masked_field_set(wal, VF_PREEMPTION, PREEMPTION_VERTEX_COUNT, 0x4000); in dg2_ctx_workarounds_init()
695 wa_masked_en(wal, CHICKEN_RASTER_1, DIS_SF_ROUND_NEAREST_EVEN); in dg2_ctx_workarounds_init()
699 struct i915_wa_list *wal) in fakewa_disable_nestedbb_mode() argument
726 wa_masked_dis(wal, RING_MI_MODE(engine->mmio_base), TGL_NESTED_BB_EN); in fakewa_disable_nestedbb_mode()
730 struct i915_wa_list *wal) in gen12_ctx_gt_mocs_init() argument
741 wa_write_clr_set(wal, in gen12_ctx_gt_mocs_init()
756 struct i915_wa_list *wal) in gen12_ctx_gt_fake_wa_init() argument
759 fakewa_disable_nestedbb_mode(engine, wal); in gen12_ctx_gt_fake_wa_init()
761 gen12_ctx_gt_mocs_init(engine, wal); in gen12_ctx_gt_fake_wa_init()
766 struct i915_wa_list *wal, in __intel_engine_init_ctx_wa() argument
771 wa_init_start(wal, name, engine->name); in __intel_engine_init_ctx_wa()
779 gen12_ctx_gt_fake_wa_init(engine, wal); in __intel_engine_init_ctx_wa()
787 dg2_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
791 dg1_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
793 gen12_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
795 icl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
797 cfl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
799 glk_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
801 kbl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
803 bxt_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
805 skl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
807 chv_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
809 bdw_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
811 gen7_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
813 gen6_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
820 wa_init_finish(wal); in __intel_engine_init_ctx_wa()
830 struct i915_wa_list *wal = &rq->engine->ctx_wa_list; in intel_engine_emit_ctx_wa() local
836 if (wal->count == 0) in intel_engine_emit_ctx_wa()
843 cs = intel_ring_begin(rq, (wal->count * 2 + 2)); in intel_engine_emit_ctx_wa()
847 *cs++ = MI_LOAD_REGISTER_IMM(wal->count); in intel_engine_emit_ctx_wa()
848 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in intel_engine_emit_ctx_wa()
865 struct i915_wa_list *wal) in gen4_gt_workarounds_init() argument
868 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE); in gen4_gt_workarounds_init()
872 g4x_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in g4x_gt_workarounds_init() argument
874 gen4_gt_workarounds_init(gt, wal); in g4x_gt_workarounds_init()
877 wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE); in g4x_gt_workarounds_init()
881 ilk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in ilk_gt_workarounds_init() argument
883 g4x_gt_workarounds_init(gt, wal); in ilk_gt_workarounds_init()
885 wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED); in ilk_gt_workarounds_init()
889 snb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in snb_gt_workarounds_init() argument
894 ivb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in ivb_gt_workarounds_init() argument
897 wa_masked_dis(wal, in ivb_gt_workarounds_init()
902 wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL); in ivb_gt_workarounds_init()
903 wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE); in ivb_gt_workarounds_init()
906 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE); in ivb_gt_workarounds_init()
910 vlv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in vlv_gt_workarounds_init() argument
913 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE); in vlv_gt_workarounds_init()
919 wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE); in vlv_gt_workarounds_init()
923 hsw_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in hsw_gt_workarounds_init() argument
926 wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); in hsw_gt_workarounds_init()
928 wa_add(wal, in hsw_gt_workarounds_init()
934 wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME); in hsw_gt_workarounds_init()
938 gen9_wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal) in gen9_wa_init_mcr() argument
972 wa_write_clr_set(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr); in gen9_wa_init_mcr()
976 gen9_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in gen9_gt_workarounds_init() argument
981 gen9_wa_init_mcr(i915, wal); in gen9_gt_workarounds_init()
985 wa_write_or(wal, in gen9_gt_workarounds_init()
995 wa_write_or(wal, in gen9_gt_workarounds_init()
1001 wa_write_or(wal, in gen9_gt_workarounds_init()
1007 skl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in skl_gt_workarounds_init() argument
1009 gen9_gt_workarounds_init(gt, wal); in skl_gt_workarounds_init()
1012 wa_write_or(wal, in skl_gt_workarounds_init()
1018 wa_write_or(wal, in skl_gt_workarounds_init()
1024 kbl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in kbl_gt_workarounds_init() argument
1026 gen9_gt_workarounds_init(gt, wal); in kbl_gt_workarounds_init()
1030 wa_write_or(wal, in kbl_gt_workarounds_init()
1035 wa_write_or(wal, in kbl_gt_workarounds_init()
1040 wa_write_or(wal, in kbl_gt_workarounds_init()
1046 glk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in glk_gt_workarounds_init() argument
1048 gen9_gt_workarounds_init(gt, wal); in glk_gt_workarounds_init()
1052 cfl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in cfl_gt_workarounds_init() argument
1054 gen9_gt_workarounds_init(gt, wal); in cfl_gt_workarounds_init()
1057 wa_write_or(wal, in cfl_gt_workarounds_init()
1062 wa_write_or(wal, in cfl_gt_workarounds_init()
1067 static void __set_mcr_steering(struct i915_wa_list *wal, in __set_mcr_steering() argument
1076 wa_write_clr_set(wal, steering_reg, mcr_mask, mcr); in __set_mcr_steering()
1079 static void __add_mcr_wa(struct intel_gt *gt, struct i915_wa_list *wal, in __add_mcr_wa() argument
1084 __set_mcr_steering(wal, GEN8_MCR_SELECTOR, slice, subslice); in __add_mcr_wa()
1094 icl_wa_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) in icl_wa_init_mcr() argument
1121 __add_mcr_wa(gt, wal, 0, subslice); in icl_wa_init_mcr()
1125 xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) in xehp_init_mcr() argument
1188 __add_mcr_wa(gt, wal, slice, subslice); in xehp_init_mcr()
1199 __set_mcr_steering(wal, MCFG_MCR_SELECTOR, 0, 2); in xehp_init_mcr()
1200 __set_mcr_steering(wal, SF_MCR_SELECTOR, 0, 2); in xehp_init_mcr()
1204 pvc_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) in pvc_init_mcr() argument
1214 __add_mcr_wa(gt, wal, dss / GEN_DSS_PER_CSLICE, dss % GEN_DSS_PER_CSLICE); in pvc_init_mcr()
1218 icl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in icl_gt_workarounds_init() argument
1222 icl_wa_init_mcr(gt, wal); in icl_gt_workarounds_init()
1225 wa_write_clr_set(wal, in icl_gt_workarounds_init()
1233 wa_write_or(wal, in icl_gt_workarounds_init()
1241 wa_write_or(wal, in icl_gt_workarounds_init()
1248 wa_write_or(wal, in icl_gt_workarounds_init()
1253 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, in icl_gt_workarounds_init()
1257 wa_write_or(wal, in icl_gt_workarounds_init()
1264 wa_write_or(wal, in icl_gt_workarounds_init()
1272 wa_write_clr(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE); in icl_gt_workarounds_init()
1282 wa_14011060649(struct intel_gt *gt, struct i915_wa_list *wal) in wa_14011060649() argument
1292 wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base), in wa_14011060649()
1298 gen12_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in gen12_gt_workarounds_init() argument
1300 icl_wa_init_mcr(gt, wal); in gen12_gt_workarounds_init()
1303 wa_14011060649(gt, wal); in gen12_gt_workarounds_init()
1306 wa_write_or(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE); in gen12_gt_workarounds_init()
1310 tgl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in tgl_gt_workarounds_init() argument
1314 gen12_gt_workarounds_init(gt, wal); in tgl_gt_workarounds_init()
1318 wa_write_or(wal, in tgl_gt_workarounds_init()
1324 wa_write_or(wal, in tgl_gt_workarounds_init()
1330 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, in tgl_gt_workarounds_init()
1335 dg1_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in dg1_gt_workarounds_init() argument
1339 gen12_gt_workarounds_init(gt, wal); in dg1_gt_workarounds_init()
1343 wa_write_or(wal, in dg1_gt_workarounds_init()
1349 wa_write_or(wal, in dg1_gt_workarounds_init()
1356 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, in dg1_gt_workarounds_init()
1361 xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in xehpsdv_gt_workarounds_init() argument
1365 xehp_init_mcr(gt, wal); in xehpsdv_gt_workarounds_init()
1368 wa_write_or(wal, SCCGCTL94DC, CG3DDISURB); in xehpsdv_gt_workarounds_init()
1372 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, in xehpsdv_gt_workarounds_init()
1377 wa_write_or(wal, UNSLCGCTL9440, GAMTLBOACS_CLKGATE_DIS | in xehpsdv_gt_workarounds_init()
1389 wa_write_or(wal, UNSLCGCTL9444, GAMTLBGFXA0_CLKGATE_DIS | in xehpsdv_gt_workarounds_init()
1408 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, VFUNIT_CLKGATE_DIS); in xehpsdv_gt_workarounds_init()
1411 wa_14011060649(gt, wal); in xehpsdv_gt_workarounds_init()
1415 dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in dg2_gt_workarounds_init() argument
1420 xehp_init_mcr(gt, wal); in dg2_gt_workarounds_init()
1423 wa_14011060649(gt, wal); in dg2_gt_workarounds_init()
1438 wa_write_or(wal, VDBOX_CGCTL3F18(engine->mmio_base), in dg2_gt_workarounds_init()
1444 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, in dg2_gt_workarounds_init()
1448 wa_write_or(wal, SUBSLICE_UNIT_LEVEL_CLKGATE, in dg2_gt_workarounds_init()
1454 wa_write_or(wal, UNSLCGCTL9430, MSQDUNIT_CLKGATE_DIS); in dg2_gt_workarounds_init()
1457 wa_write_or(wal, UNSLCGCTL9444, LTCDD_CLKGATE_DIS); in dg2_gt_workarounds_init()
1460 wa_write_or(wal, SLICE_UNIT_LEVEL_CLKGATE, NODEDSS_CLKGATE_DIS); in dg2_gt_workarounds_init()
1463 wa_write_or(wal, UNSLCGCTL9440, GAMTLBOACS_CLKGATE_DIS | in dg2_gt_workarounds_init()
1475 wa_write_or(wal, UNSLCGCTL9444, GAMTLBGFXA0_CLKGATE_DIS | in dg2_gt_workarounds_init()
1492 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, in dg2_gt_workarounds_init()
1496 wa_write_or(wal, SSMCGCTL9530, RTFUNIT_CLKGATE_DIS); in dg2_gt_workarounds_init()
1500 wa_write_clr(wal, SARB_CHICKEN1, COMP_CKN_IN); in dg2_gt_workarounds_init()
1507 wa_write_or(wal, GEN12_SQCM, EN_32B_ACCESS); in dg2_gt_workarounds_init()
1510 wa_write_clr(wal, GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE); in dg2_gt_workarounds_init()
1514 pvc_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in pvc_gt_workarounds_init() argument
1516 pvc_init_mcr(gt, wal); in pvc_gt_workarounds_init()
1519 wa_write_clr(wal, GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE); in pvc_gt_workarounds_init()
1523 gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal) in gt_init_workarounds() argument
1528 pvc_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1530 dg2_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1532 xehpsdv_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1534 dg1_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1536 tgl_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1538 gen12_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1540 icl_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1542 cfl_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1544 glk_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1546 kbl_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1548 gen9_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1550 skl_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1552 hsw_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1554 vlv_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1556 ivb_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1558 snb_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1560 ilk_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1562 g4x_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1564 gen4_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1573 struct i915_wa_list *wal = >->wa_list; in intel_gt_init_workarounds() local
1575 wa_init_start(wal, "GT", "global"); in intel_gt_init_workarounds()
1576 gt_init_workarounds(gt, wal); in intel_gt_init_workarounds()
1577 wa_init_finish(wal); in intel_gt_init_workarounds()
1581 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal) in wal_get_fw_for_rmw() argument
1587 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) in wal_get_fw_for_rmw()
1611 wa_list_apply(struct intel_gt *gt, const struct i915_wa_list *wal) in wa_list_apply() argument
1619 if (!wal->count) in wa_list_apply()
1622 fw = wal_get_fw_for_rmw(uncore, wal); in wa_list_apply()
1627 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in wa_list_apply()
1638 wal->name, "application"); in wa_list_apply()
1651 const struct i915_wa_list *wal, in wa_list_verify() argument
1661 fw = wal_get_fw_for_rmw(uncore, wal); in wa_list_verify()
1666 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) in wa_list_verify()
1669 wal->name, from); in wa_list_verify()
1698 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags) in whitelist_reg_ext() argument
1704 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS)) in whitelist_reg_ext()
1711 _wa_add(wal, &wa); in whitelist_reg_ext()
1715 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg) in whitelist_reg() argument
1717 whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW); in whitelist_reg()
2026 const struct i915_wa_list *wal = &engine->whitelist; in intel_engine_apply_whitelist() local
2032 if (!wal->count) in intel_engine_apply_whitelist()
2035 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) in intel_engine_apply_whitelist()
2055 engine_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in engine_fake_wa_init() argument
2084 wa_masked_field_set(wal, in engine_fake_wa_init()
2098 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in rcs_engine_wa_init() argument
2104 wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS | in rcs_engine_wa_init()
2110 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_ENABLE_LARGE_GRF_MODE); in rcs_engine_wa_init()
2113 wa_write_or(wal, LSC_CHICKEN_BIT_0_UDW, DIS_CHAIN_2XSIMD8); in rcs_engine_wa_init()
2119 wa_masked_en(wal, GEN10_SAMPLER_MODE, in rcs_engine_wa_init()
2126 wa_masked_en(wal, GEN9_ROW_CHICKEN4, in rcs_engine_wa_init()
2136 wa_write_clr_set(wal, LSC_CHICKEN_BIT_0_UDW, in rcs_engine_wa_init()
2141 wa_write_or(wal, LSC_CHICKEN_BIT_0, in rcs_engine_wa_init()
2148 wa_masked_dis(wal, GEN12_CS_DEBUG_MODE1_CCCSUNIT_BE_COMMON, in rcs_engine_wa_init()
2155 wa_write_or(wal, LSC_CHICKEN_BIT_0_UDW, in rcs_engine_wa_init()
2159 wa_masked_en(wal, GEN7_ROW_CHICKEN2, in rcs_engine_wa_init()
2166 wa_masked_dis(wal, GEN12_HDC_CHICKEN0, in rcs_engine_wa_init()
2175 wa_masked_en(wal, GEN8_ROW_CHICKEN, in rcs_engine_wa_init()
2184 wa_add(wal, LSC_CHICKEN_BIT_0_UDW, 0, in rcs_engine_wa_init()
2191 wa_masked_en(wal, in rcs_engine_wa_init()
2196 wa_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE); in rcs_engine_wa_init()
2202 wa_write_or(wal, RT_CTRL, DIS_NULL_QUERY); in rcs_engine_wa_init()
2207 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7, in rcs_engine_wa_init()
2212 wa_write_or(wal, GEN12_GAMSTLB_CTRL, CONTROL_BLOCK_CLKGATE_DIS | in rcs_engine_wa_init()
2219 wa_write_or(wal, GEN12_MERT_MOD_CTRL, FORCE_MISS_FTLB); in rcs_engine_wa_init()
2225 wa_add(wal, GEN10_CACHE_MODE_SS, 0, in rcs_engine_wa_init()
2238 wa_write_or(wal, in rcs_engine_wa_init()
2248 wa_write_or(wal, in rcs_engine_wa_init()
2256 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ); in rcs_engine_wa_init()
2264 wa_write_or(wal, GEN7_FF_THREAD_MODE, in rcs_engine_wa_init()
2276 wa_masked_en(wal, in rcs_engine_wa_init()
2285 wa_masked_en(wal, GEN7_ROW_CHICKEN2, in rcs_engine_wa_init()
2292 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH); in rcs_engine_wa_init()
2307 wa_masked_en(wal, in rcs_engine_wa_init()
2316 wa_masked_en(wal, in rcs_engine_wa_init()
2323 wa_masked_en(wal, in rcs_engine_wa_init()
2331 wa_write_or(wal, in rcs_engine_wa_init()
2339 wa_write_clr_set(wal, in rcs_engine_wa_init()
2343 wa_write_clr_set(wal, in rcs_engine_wa_init()
2352 wa_write_or(wal, in rcs_engine_wa_init()
2357 wa_write_or(wal, in rcs_engine_wa_init()
2362 wa_write_clr_set(wal, in rcs_engine_wa_init()
2368 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS, in rcs_engine_wa_init()
2375 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, in rcs_engine_wa_init()
2382 wa_write_or(wal, in rcs_engine_wa_init()
2387 wa_masked_en(wal, in rcs_engine_wa_init()
2394 wa_masked_en(wal, in rcs_engine_wa_init()
2404 wa_write_or(wal, in rcs_engine_wa_init()
2411 wa_masked_en(wal, in rcs_engine_wa_init()
2418 wa_masked_en(wal, in rcs_engine_wa_init()
2423 wa_write_or(wal, in rcs_engine_wa_init()
2429 wa_write_clr_set(wal, in rcs_engine_wa_init()
2436 wa_write_or(wal, in rcs_engine_wa_init()
2441 wa_write_clr_set(wal, GEN9_SCRATCH_LNCF1, in rcs_engine_wa_init()
2443 wa_write_clr_set(wal, GEN8_L3SQCREG4, in rcs_engine_wa_init()
2445 wa_write_clr_set(wal, GEN9_SCRATCH1, in rcs_engine_wa_init()
2451 wa_masked_en(wal, in rcs_engine_wa_init()
2454 wa_masked_dis(wal, in rcs_engine_wa_init()
2462 wa_masked_en(wal, in rcs_engine_wa_init()
2472 wa_write_clr_set(wal, in rcs_engine_wa_init()
2481 wa_masked_en(wal, in rcs_engine_wa_init()
2489 wa_masked_en(wal, in rcs_engine_wa_init()
2495 wa_masked_dis(wal, in rcs_engine_wa_init()
2506 wa_write_clr_set(wal, in rcs_engine_wa_init()
2515 wa_masked_en(wal, in rcs_engine_wa_init()
2522 wa_masked_en(wal, in rcs_engine_wa_init()
2527 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE); in rcs_engine_wa_init()
2534 wa_masked_en(wal, in rcs_engine_wa_init()
2546 wa_masked_field_set(wal, in rcs_engine_wa_init()
2560 wa_masked_en(wal, in rcs_engine_wa_init()
2570 wa_masked_en(wal, in rcs_engine_wa_init()
2575 wa_masked_en(wal, in rcs_engine_wa_init()
2579 wa_masked_en(wal, in rcs_engine_wa_init()
2599 wa_masked_field_set(wal, in rcs_engine_wa_init()
2605 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE); in rcs_engine_wa_init()
2613 wa_masked_dis(wal, in rcs_engine_wa_init()
2620 wa_add(wal, RING_MI_MODE(RENDER_RING_BASE), in rcs_engine_wa_init()
2636 wa_add(wal, ECOSKPD(RENDER_RING_BASE), in rcs_engine_wa_init()
2643 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in xcs_engine_wa_init() argument
2649 wa_write(wal, in xcs_engine_wa_init()
2656 ccs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in ccs_engine_wa_init() argument
2660 wa_masked_en(wal, GEN10_CACHE_MODE_SS, DISABLE_ECC); in ccs_engine_wa_init()
2678 struct i915_wa_list *wal) in add_render_compute_tuning_settings() argument
2681 wa_write(wal, XEHPC_L3SCRUB, in add_render_compute_tuning_settings()
2686 wa_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS); in add_render_compute_tuning_settings()
2687 wa_write_clr_set(wal, RT_CTRL, STACKID_CTRL, STACKID_CTRL_512); in add_render_compute_tuning_settings()
2698 wa_add(wal, GEN10_CACHE_MODE_SS, 0, in add_render_compute_tuning_settings()
2710 wa_masked_field_set(wal, GEN9_ROW_CHICKEN4, THREAD_EX_ARB_MODE, in add_render_compute_tuning_settings()
2724 general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in general_render_compute_wa_init() argument
2728 add_render_compute_tuning_settings(i915, wal); in general_render_compute_wa_init()
2732 wa_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_OVRLSCCC); in general_render_compute_wa_init()
2737 wa_masked_en(wal, in general_render_compute_wa_init()
2742 wa_masked_en(wal, in general_render_compute_wa_init()
2747 wa_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE); in general_render_compute_wa_init()
2750 wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1, in general_render_compute_wa_init()
2755 wa_masked_dis(wal, MLTICTXCTL, TDONRENDER); in general_render_compute_wa_init()
2756 wa_write_or(wal, L3SQCREG1_CCS0, FLUSHALLNONCOH); in general_render_compute_wa_init()
2760 wa_write_or(wal, GEN12_MERT_MOD_CTRL, FORCE_MISS_FTLB); in general_render_compute_wa_init()
2763 wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS | in general_render_compute_wa_init()
2769 wa_masked_en(wal, GEN9_ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE); in general_render_compute_wa_init()
2772 wa_write_or(wal, LSC_CHICKEN_BIT_0, DISABLE_D8_D16_COASLESCE); in general_render_compute_wa_init()
2775 wa_masked_en(wal, FF_SLICE_CS_CHICKEN2, GEN12_PERF_FIX_BALANCING_CFE_DISABLE); in general_render_compute_wa_init()
2778 wa_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB); in general_render_compute_wa_init()
2779 wa_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB); in general_render_compute_wa_init()
2780 wa_write_or(wal, VDBX_MOD_CTRL, FORCE_MISS_FTLB); in general_render_compute_wa_init()
2781 wa_write_or(wal, VEBX_MOD_CTRL, FORCE_MISS_FTLB); in general_render_compute_wa_init()
2786 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal) in engine_init_workarounds() argument
2791 engine_fake_wa_init(engine, wal); in engine_init_workarounds()
2799 general_render_compute_wa_init(engine, wal); in engine_init_workarounds()
2802 ccs_engine_wa_init(engine, wal); in engine_init_workarounds()
2804 rcs_engine_wa_init(engine, wal); in engine_init_workarounds()
2806 xcs_engine_wa_init(engine, wal); in engine_init_workarounds()
2811 struct i915_wa_list *wal = &engine->wa_list; in intel_engine_init_workarounds() local
2816 wa_init_start(wal, "engine", engine->name); in intel_engine_init_workarounds()
2817 engine_init_workarounds(engine, wal); in intel_engine_init_workarounds()
2818 wa_init_finish(wal); in intel_engine_init_workarounds()
2890 const struct i915_wa_list *wal, in wa_list_srm() argument
2902 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in wa_list_srm()
2911 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in wa_list_srm()
2928 const struct i915_wa_list * const wal, in engine_wa_list_verify() argument
2939 if (!wal->count) in engine_wa_list_verify()
2943 wal->count * sizeof(u32)); in engine_wa_list_verify()
2971 err = wa_list_srm(rq, wal, vma); in engine_wa_list_verify()
2993 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in engine_wa_list_verify()
2997 if (!wa_verify(wa, results[i], wal->name, from)) in engine_wa_list_verify()