Lines Matching +full:cs +full:- +full:x
1 // SPDX-License-Identifier: MIT
23 #define NOP(x) (BIT(7) | (x)) in set_offsets() argument
26 #define REG(x) (((x) >> 2) | BUILD_BUG_ON_ZERO(x >= 0x200)) in set_offsets() argument
27 #define REG16(x) \ in set_offsets() argument
28 (((x) >> 9) | BIT(7) | BUILD_BUG_ON_ZERO(x >= 0x10000)), \ in set_offsets()
29 (((x) >> 2) & 0x7f) in set_offsets()
32 const u32 base = engine->mmio_base; in set_offsets()
50 if (GRAPHICS_VER(engine->i915) >= 11) in set_offsets()
67 } while (--count); in set_offsets()
73 if (GRAPHICS_VER(engine->i915) >= 11) in set_offsets()
542 GEM_BUG_ON(GRAPHICS_VER(engine->i915) >= 12 && in reg_offsets()
545 if (engine->class == RENDER_CLASS) { in reg_offsets()
546 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in reg_offsets()
548 else if (GRAPHICS_VER(engine->i915) >= 12) in reg_offsets()
550 else if (GRAPHICS_VER(engine->i915) >= 11) in reg_offsets()
552 else if (GRAPHICS_VER(engine->i915) >= 9) in reg_offsets()
557 if (GRAPHICS_VER(engine->i915) >= 12) in reg_offsets()
559 else if (GRAPHICS_VER(engine->i915) >= 9) in reg_offsets()
568 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in lrc_ring_mi_mode()
570 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_mi_mode()
572 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_mi_mode()
574 else if (engine->class == RENDER_CLASS) in lrc_ring_mi_mode()
577 return -1; in lrc_ring_mi_mode()
582 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in lrc_ring_gpr0()
584 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_gpr0()
586 else if (GRAPHICS_VER(engine->i915) >= 9) in lrc_ring_gpr0()
588 else if (engine->class == RENDER_CLASS) in lrc_ring_gpr0()
591 return -1; in lrc_ring_gpr0()
596 if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_wa_bb_per_ctx()
598 else if (GRAPHICS_VER(engine->i915) >= 9 || engine->class == RENDER_CLASS) in lrc_ring_wa_bb_per_ctx()
601 return -1; in lrc_ring_wa_bb_per_ctx()
606 int x; in lrc_ring_indirect_ptr() local
608 x = lrc_ring_wa_bb_per_ctx(engine); in lrc_ring_indirect_ptr()
609 if (x < 0) in lrc_ring_indirect_ptr()
610 return x; in lrc_ring_indirect_ptr()
612 return x + 2; in lrc_ring_indirect_ptr()
617 int x; in lrc_ring_indirect_offset() local
619 x = lrc_ring_indirect_ptr(engine); in lrc_ring_indirect_offset()
620 if (x < 0) in lrc_ring_indirect_offset()
621 return x; in lrc_ring_indirect_offset()
623 return x + 2; in lrc_ring_indirect_offset()
629 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) in lrc_ring_cmd_buf_cctl()
635 else if (engine->class != RENDER_CLASS) in lrc_ring_cmd_buf_cctl()
636 return -1; in lrc_ring_cmd_buf_cctl()
637 else if (GRAPHICS_VER(engine->i915) >= 12) in lrc_ring_cmd_buf_cctl()
639 else if (GRAPHICS_VER(engine->i915) >= 11) in lrc_ring_cmd_buf_cctl()
642 return -1; in lrc_ring_cmd_buf_cctl()
648 switch (GRAPHICS_VER(engine->i915)) { in lrc_ring_indirect_offset_default()
650 MISSING_CASE(GRAPHICS_VER(engine->i915)); in lrc_ring_indirect_offset_default()
671 GEM_BUG_ON(lrc_ring_indirect_ptr(engine) == -1); in lrc_setup_indirect_ctx()
675 GEM_BUG_ON(lrc_ring_indirect_offset(engine) == -1); in lrc_setup_indirect_ctx()
691 if (GRAPHICS_VER(engine->i915) < 11) in init_common_regs()
696 regs[CTX_TIMESTAMP] = ce->runtime.last; in init_common_regs()
702 const struct i915_ctx_workarounds * const wa_ctx = &engine->wa_ctx; in init_wa_bb_regs()
704 if (wa_ctx->per_ctx.size) { in init_wa_bb_regs()
705 const u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma); in init_wa_bb_regs()
707 GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1); in init_wa_bb_regs()
709 (ggtt_offset + wa_ctx->per_ctx.offset) | 0x01; in init_wa_bb_regs()
712 if (wa_ctx->indirect_ctx.size) { in init_wa_bb_regs()
714 i915_ggtt_offset(wa_ctx->vma) + in init_wa_bb_regs()
715 wa_ctx->indirect_ctx.offset, in init_wa_bb_regs()
716 wa_ctx->indirect_ctx.size); in init_wa_bb_regs()
722 if (i915_vm_is_4lvl(&ppgtt->vm)) { in init_ppgtt_regs()
739 return i915_vm_to_ggtt(vm)->alias; in vm_alias()
746 int x; in __reset_stop_ring() local
748 x = lrc_ring_mi_mode(engine); in __reset_stop_ring()
749 if (x != -1) { in __reset_stop_ring()
750 regs[x + 1] &= ~STOP_RING; in __reset_stop_ring()
751 regs[x + 1] |= STOP_RING << 16; in __reset_stop_ring()
777 init_ppgtt_regs(regs, vm_alias(ce->vm)); in __lrc_init_regs()
788 __lrc_init_regs(ce->lrc_reg_state, ce, engine, inhibit); in lrc_init_regs()
794 __reset_stop_ring(ce->lrc_reg_state, engine); in lrc_reset_regs()
803 vaddr += engine->context_size; in set_redzone()
814 vaddr += engine->context_size; in check_redzone()
817 drm_err_once(&engine->i915->drm, in check_redzone()
819 engine->name); in check_redzone()
830 if (engine->default_state) { in lrc_init_state()
831 shmem_read(engine->default_state, 0, in lrc_init_state()
832 state, engine->context_size); in lrc_init_state()
833 __set_bit(CONTEXT_VALID_BIT, &ce->flags); in lrc_init_state()
837 /* Clear the ppHWSP (inc. per-context counters) */ in lrc_init_state()
854 context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE); in __lrc_alloc_state()
859 if (GRAPHICS_VER(engine->i915) == 12) { in __lrc_alloc_state()
860 ce->wa_bb_page = context_size / PAGE_SIZE; in __lrc_alloc_state()
864 obj = i915_gem_object_create_lmem(engine->i915, context_size, 0); in __lrc_alloc_state()
866 obj = i915_gem_object_create_shmem(engine->i915, context_size); in __lrc_alloc_state()
870 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); in __lrc_alloc_state()
882 struct intel_timeline *tl = fetch_and_zero(&ce->timeline); in pinned_timeline()
893 GEM_BUG_ON(ce->state); in lrc_alloc()
899 ring = intel_engine_create_ring(engine, ce->ring_size); in lrc_alloc()
905 if (!page_mask_bits(ce->timeline)) { in lrc_alloc()
912 if (unlikely(ce->timeline)) in lrc_alloc()
915 tl = intel_timeline_create(engine->gt); in lrc_alloc()
921 ce->timeline = tl; in lrc_alloc()
924 ce->ring = ring; in lrc_alloc()
925 ce->state = vma; in lrc_alloc()
940 intel_ring_reset(ce->ring, ce->ring->emit); in lrc_reset()
943 lrc_init_regs(ce, ce->engine, true); in lrc_reset()
944 ce->lrc.lrca = lrc_update_regs(ce, ce->engine, ce->ring->tail); in lrc_reset()
953 GEM_BUG_ON(!ce->state); in lrc_pre_pin()
954 GEM_BUG_ON(!i915_vma_is_pinned(ce->state)); in lrc_pre_pin()
956 *vaddr = i915_gem_object_pin_map(ce->state->obj, in lrc_pre_pin()
957 i915_coherent_map_type(ce->engine->i915, in lrc_pre_pin()
958 ce->state->obj, in lrc_pre_pin()
970 ce->lrc_reg_state = vaddr + LRC_STATE_OFFSET; in lrc_pin()
972 if (!__test_and_set_bit(CONTEXT_INIT_BIT, &ce->flags)) in lrc_pin()
975 ce->lrc.lrca = lrc_update_regs(ce, engine, ce->ring->tail); in lrc_pin()
981 check_redzone((void *)ce->lrc_reg_state - LRC_STATE_OFFSET, in lrc_unpin()
982 ce->engine); in lrc_unpin()
987 i915_gem_object_unpin_map(ce->state->obj); in lrc_post_unpin()
992 if (!ce->state) in lrc_fini()
995 intel_ring_put(fetch_and_zero(&ce->ring)); in lrc_fini()
996 i915_vma_put(fetch_and_zero(&ce->state)); in lrc_fini()
1003 GEM_BUG_ON(!i915_active_is_idle(&ce->active)); in lrc_destroy()
1013 gen12_emit_timestamp_wa(const struct intel_context *ce, u32 *cs) in gen12_emit_timestamp_wa() argument
1015 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_timestamp_wa()
1018 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1019 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_timestamp_wa()
1021 *cs++ = 0; in gen12_emit_timestamp_wa()
1023 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_timestamp_wa()
1026 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1027 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); in gen12_emit_timestamp_wa()
1029 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_timestamp_wa()
1032 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_timestamp_wa()
1033 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); in gen12_emit_timestamp_wa()
1035 return cs; in gen12_emit_timestamp_wa()
1039 gen12_emit_restore_scratch(const struct intel_context *ce, u32 *cs) in gen12_emit_restore_scratch() argument
1041 GEM_BUG_ON(lrc_ring_gpr0(ce->engine) == -1); in gen12_emit_restore_scratch()
1043 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_restore_scratch()
1046 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_restore_scratch()
1047 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_restore_scratch()
1048 (lrc_ring_gpr0(ce->engine) + 1) * sizeof(u32); in gen12_emit_restore_scratch()
1049 *cs++ = 0; in gen12_emit_restore_scratch()
1051 return cs; in gen12_emit_restore_scratch()
1055 gen12_emit_cmd_buf_wa(const struct intel_context *ce, u32 *cs) in gen12_emit_cmd_buf_wa() argument
1057 GEM_BUG_ON(lrc_ring_cmd_buf_cctl(ce->engine) == -1); in gen12_emit_cmd_buf_wa()
1059 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | in gen12_emit_cmd_buf_wa()
1062 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_cmd_buf_wa()
1063 *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + in gen12_emit_cmd_buf_wa()
1064 (lrc_ring_cmd_buf_cctl(ce->engine) + 1) * sizeof(u32); in gen12_emit_cmd_buf_wa()
1065 *cs++ = 0; in gen12_emit_cmd_buf_wa()
1067 *cs++ = MI_LOAD_REGISTER_REG | in gen12_emit_cmd_buf_wa()
1070 *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); in gen12_emit_cmd_buf_wa()
1071 *cs++ = i915_mmio_reg_offset(RING_CMD_BUF_CCTL(0)); in gen12_emit_cmd_buf_wa()
1073 return cs; in gen12_emit_cmd_buf_wa()
1077 gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs) in gen12_emit_indirect_ctx_rcs() argument
1079 cs = gen12_emit_timestamp_wa(ce, cs); in gen12_emit_indirect_ctx_rcs()
1080 cs = gen12_emit_cmd_buf_wa(ce, cs); in gen12_emit_indirect_ctx_rcs()
1081 cs = gen12_emit_restore_scratch(ce, cs); in gen12_emit_indirect_ctx_rcs()
1083 return cs; in gen12_emit_indirect_ctx_rcs()
1087 gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs) in gen12_emit_indirect_ctx_xcs() argument
1089 cs = gen12_emit_timestamp_wa(ce, cs); in gen12_emit_indirect_ctx_xcs()
1090 cs = gen12_emit_restore_scratch(ce, cs); in gen12_emit_indirect_ctx_xcs()
1092 return cs; in gen12_emit_indirect_ctx_xcs()
1097 return PAGE_SIZE * ce->wa_bb_page; in context_wa_bb_offset()
1104 GEM_BUG_ON(!ce->wa_bb_page); in context_indirect_bb()
1106 ptr = ce->lrc_reg_state; in context_indirect_bb()
1107 ptr -= LRC_STATE_OFFSET; /* back to start of context image */ in context_indirect_bb()
1119 u32 *cs; in setup_indirect_ctx_bb() local
1121 cs = emit(ce, start); in setup_indirect_ctx_bb()
1122 GEM_BUG_ON(cs - start > I915_GTT_PAGE_SIZE / sizeof(*cs)); in setup_indirect_ctx_bb()
1123 while ((unsigned long)cs % CACHELINE_BYTES) in setup_indirect_ctx_bb()
1124 *cs++ = MI_NOOP; in setup_indirect_ctx_bb()
1126 lrc_setup_indirect_ctx(ce->lrc_reg_state, engine, in setup_indirect_ctx_bb()
1127 i915_ggtt_offset(ce->state) + in setup_indirect_ctx_bb()
1129 (cs - start) * sizeof(*cs)); in setup_indirect_ctx_bb()
1140 * bits 0-11: flags, GEN8_CTX_* (cached in ctx->desc_template)
1141 * bits 12-31: LRCA, GTT address of (the HWSP of) this context
1142 * bits 32-52: ctx ID, a globally unique tag (highest bit used by GuC)
1143 * bits 53-54: mbz, reserved for use by hardware
1144 * bits 55-63: group ID, currently unused and set to 0
1148 * bits 32-36: reserved
1149 * bits 37-47: SW context ID
1152 * bits 55-60: SW counter
1153 * bits 61-63: engine class
1157 * bits 32-37: virtual function number
1159 * bits 39-54: SW context ID
1160 * bits 55-57: reserved
1161 * bits 58-63: SW counter
1171 if (i915_vm_is_4lvl(ce->vm)) in lrc_descriptor()
1176 if (GRAPHICS_VER(ce->vm->i915) == 8) in lrc_descriptor()
1179 return i915_ggtt_offset(ce->state) | desc; in lrc_descriptor()
1186 struct intel_ring *ring = ce->ring; in lrc_update_regs()
1187 u32 *regs = ce->lrc_reg_state; in lrc_update_regs()
1190 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail)); in lrc_update_regs()
1192 regs[CTX_RING_START] = i915_ggtt_offset(ring->vma); in lrc_update_regs()
1194 regs[CTX_RING_TAIL] = ring->tail; in lrc_update_regs()
1195 regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; in lrc_update_regs()
1198 if (engine->class == RENDER_CLASS) { in lrc_update_regs()
1200 intel_sseu_make_rpcs(engine->gt, &ce->sseu); in lrc_update_regs()
1205 if (ce->wa_bb_page) { in lrc_update_regs()
1206 u32 *(*fn)(const struct intel_context *ce, u32 *cs); in lrc_update_regs()
1209 if (ce->engine->class == RENDER_CLASS) in lrc_update_regs()
1213 GEM_BUG_ON(engine->wa_ctx.indirect_ctx.size); in lrc_update_regs()
1223 set_offsets(ce->lrc_reg_state, reg_offsets(engine), engine, false); in lrc_update_offsets()
1230 const struct intel_ring *ring = ce->ring; in lrc_check_regs()
1231 u32 *regs = ce->lrc_reg_state; in lrc_check_regs()
1233 int x; in lrc_check_regs() local
1235 if (regs[CTX_RING_START] != i915_ggtt_offset(ring->vma)) { in lrc_check_regs()
1236 pr_err("%s: context submitted with incorrect RING_START [%08x], expected %08x\n", in lrc_check_regs()
1237 engine->name, in lrc_check_regs()
1239 i915_ggtt_offset(ring->vma)); in lrc_check_regs()
1240 regs[CTX_RING_START] = i915_ggtt_offset(ring->vma); in lrc_check_regs()
1245 (RING_CTL_SIZE(ring->size) | RING_VALID)) { in lrc_check_regs()
1246 pr_err("%s: context submitted with incorrect RING_CTL [%08x], expected %08x\n", in lrc_check_regs()
1247 engine->name, in lrc_check_regs()
1249 (u32)(RING_CTL_SIZE(ring->size) | RING_VALID)); in lrc_check_regs()
1250 regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; in lrc_check_regs()
1254 x = lrc_ring_mi_mode(engine); in lrc_check_regs()
1255 if (x != -1 && regs[x + 1] & (regs[x + 1] >> 16) & STOP_RING) { in lrc_check_regs()
1256 pr_err("%s: context submitted with STOP_RING [%08x] in RING_MI_MODE\n", in lrc_check_regs()
1257 engine->name, regs[x + 1]); in lrc_check_regs()
1258 regs[x + 1] &= ~STOP_RING; in lrc_check_regs()
1259 regs[x + 1] |= STOP_RING << 16; in lrc_check_regs()
1275 * it for a short period and this batch in non-premptible. We can ofcourse
1288 *batch++ = intel_gt_scratch_offset(engine->gt, in gen8_emit_flush_coherentl3_wa()
1303 *batch++ = intel_gt_scratch_offset(engine->gt, in gen8_emit_flush_coherentl3_wa()
1331 if (IS_BROADWELL(engine->i915)) in gen8_init_indirectctx_bb()
1369 *batch++ = i915_mmio_reg_offset(lri->reg); in emit_lri()
1370 *batch++ = lri->value; in emit_lri()
1371 } while (lri++, --count); in emit_lri()
1418 if (HAS_POOLED_EU(engine->i915)) { in gen9_init_indirectctx_bb()
1457 obj = i915_gem_object_create_shmem(engine->i915, CTX_WA_BB_SIZE); in lrc_create_wa_ctx()
1461 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); in lrc_create_wa_ctx()
1467 engine->wa_ctx.vma = vma; in lrc_create_wa_ctx()
1477 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0); in lrc_fini_wa_ctx()
1484 struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx; in lrc_init_wa_ctx()
1486 &wa_ctx->indirect_ctx, &wa_ctx->per_ctx in lrc_init_wa_ctx()
1494 if (engine->class != RENDER_CLASS) in lrc_init_wa_ctx()
1497 switch (GRAPHICS_VER(engine->i915)) { in lrc_init_wa_ctx()
1510 MISSING_CASE(GRAPHICS_VER(engine->i915)); in lrc_init_wa_ctx()
1521 drm_err(&engine->i915->drm, in lrc_init_wa_ctx()
1527 if (!engine->wa_ctx.vma) in lrc_init_wa_ctx()
1532 err = i915_gem_object_lock(wa_ctx->vma->obj, &ww); in lrc_init_wa_ctx()
1534 err = i915_ggtt_pin(wa_ctx->vma, &ww, 0, PIN_HIGH); in lrc_init_wa_ctx()
1538 batch = i915_gem_object_pin_map(wa_ctx->vma->obj, I915_MAP_WB); in lrc_init_wa_ctx()
1551 wa_bb[i]->offset = batch_ptr - batch; in lrc_init_wa_ctx()
1552 if (GEM_DEBUG_WARN_ON(!IS_ALIGNED(wa_bb[i]->offset, in lrc_init_wa_ctx()
1554 err = -EINVAL; in lrc_init_wa_ctx()
1559 wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset); in lrc_init_wa_ctx()
1561 GEM_BUG_ON(batch_ptr - batch > CTX_WA_BB_SIZE); in lrc_init_wa_ctx()
1563 __i915_gem_object_flush_map(wa_ctx->vma->obj, 0, batch_ptr - batch); in lrc_init_wa_ctx()
1564 __i915_gem_object_release_map(wa_ctx->vma->obj); in lrc_init_wa_ctx()
1568 err = i915_inject_probe_error(engine->i915, -ENODEV); in lrc_init_wa_ctx()
1572 i915_vma_unpin(wa_ctx->vma); in lrc_init_wa_ctx()
1574 if (err == -EDEADLK) { in lrc_init_wa_ctx()
1582 i915_vma_put(engine->wa_ctx.vma); in lrc_init_wa_ctx()
1592 ce->runtime.num_underflow++; in st_update_runtime_underflow()
1593 ce->runtime.max_underflow = max_t(u32, ce->runtime.max_underflow, -dt); in st_update_runtime_underflow()
1605 old = ce->runtime.last; in lrc_update_runtime()
1606 ce->runtime.last = lrc_get_runtime(ce); in lrc_update_runtime()
1607 dt = ce->runtime.last - old; in lrc_update_runtime()
1611 old, ce->runtime.last, dt); in lrc_update_runtime()
1616 ewma_runtime_add(&ce->runtime.avg, dt); in lrc_update_runtime()
1617 ce->runtime.total += dt; in lrc_update_runtime()