Lines Matching +full:0 +full:xf0ffffff
55 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
56 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1 0x1388
58 #define regCGTT_WD_CLK_CTRL 0x5086
60 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1 0x4e7e
83 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010),
84 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010),
85 SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200),
86 SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988),
87 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007),
88 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008),
89 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100),
90 SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000),
91 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a)
129 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) | in gfx11_kiq_set_resources()
130 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */ in gfx11_kiq_set_resources()
133 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ in gfx11_kiq_set_resources()
134 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ in gfx11_kiq_set_resources()
135 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ in gfx11_kiq_set_resources()
136 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ in gfx11_kiq_set_resources()
144 uint32_t me = 0, eng_sel = 0; in gfx11_kiq_map_queues()
149 eng_sel = 0; in gfx11_kiq_map_queues()
152 me = 0; in gfx11_kiq_map_queues()
164 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ in gfx11_kiq_map_queues()
165 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ in gfx11_kiq_map_queues()
166 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ in gfx11_kiq_map_queues()
167 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ in gfx11_kiq_map_queues()
171 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */ in gfx11_kiq_map_queues()
172 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */ in gfx11_kiq_map_queues()
188 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; in gfx11_kiq_unmap_queues()
196 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ in gfx11_kiq_unmap_queues()
198 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | in gfx11_kiq_unmap_queues()
209 amdgpu_ring_write(kiq_ring, 0); in gfx11_kiq_unmap_queues()
210 amdgpu_ring_write(kiq_ring, 0); in gfx11_kiq_unmap_queues()
211 amdgpu_ring_write(kiq_ring, 0); in gfx11_kiq_unmap_queues()
220 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; in gfx11_kiq_query_status()
224 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | in gfx11_kiq_query_status()
225 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | in gfx11_kiq_query_status()
227 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ in gfx11_kiq_query_status()
263 switch (adev->ip_versions[GC_HWIP][0]) { in gfx_v11_0_init_golden_registers()
264 case IP_VERSION(11, 0, 1): in gfx_v11_0_init_golden_registers()
279 WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0)); in gfx_v11_0_write_data_to_reg()
281 amdgpu_ring_write(ring, 0); in gfx_v11_0_write_data_to_reg()
292 /* memory (1) or register (0) */ in gfx_v11_0_wait_reg_mem()
299 BUG_ON(addr0 & 0x3); /* Dword align */ in gfx_v11_0_wait_reg_mem()
310 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0); in gfx_v11_0_ring_test_ring()
311 uint32_t tmp = 0; in gfx_v11_0_ring_test_ring()
315 WREG32(scratch, 0xCAFEDEAD); in gfx_v11_0_ring_test_ring()
324 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF); in gfx_v11_0_ring_test_ring()
329 amdgpu_ring_write(ring, 0xDEADBEEF); in gfx_v11_0_ring_test_ring()
333 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v11_0_ring_test_ring()
335 if (tmp == 0xDEADBEEF) in gfx_v11_0_ring_test_ring()
361 return 0; in gfx_v11_0_ring_test_ib()
363 memset(&ib, 0, sizeof(ib)); in gfx_v11_0_ring_test_ib()
377 *cpu_ptr = cpu_to_le32(0xCAFEDEAD); in gfx_v11_0_ring_test_ib()
384 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); in gfx_v11_0_ring_test_ib()
394 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); in gfx_v11_0_ring_test_ib()
398 ib.ptr[4] = 0xDEADBEEF; in gfx_v11_0_ring_test_ib()
406 if (r == 0) { in gfx_v11_0_ring_test_ib()
409 } else if (r < 0) { in gfx_v11_0_ring_test_ib()
413 if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF) in gfx_v11_0_ring_test_ib()
414 r = 0; in gfx_v11_0_ring_test_ib()
464 adev->gfx.pfp_fw->data, 2, 0); in gfx_v11_0_init_microcode()
523 /* only one MEC for gfx 11.0.0. */ in gfx_v11_0_init_microcode()
547 int err = 0; in gfx_v11_0_init_toc_microcode()
568 return 0; in gfx_v11_0_init_toc_microcode()
578 u32 count = 0; in gfx_v11_0_get_csb_size()
592 return 0; in gfx_v11_0_get_csb_size()
609 u32 count = 0, i; in gfx_v11_0_get_csb_buffer()
619 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); in gfx_v11_0_get_csb_buffer()
623 buffer[count++] = cpu_to_le32(0x80000000); in gfx_v11_0_get_csb_buffer()
624 buffer[count++] = cpu_to_le32(0x80000000); in gfx_v11_0_get_csb_buffer()
633 for (i = 0; i < ext->reg_count; i++) in gfx_v11_0_get_csb_buffer()
642 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START; in gfx_v11_0_get_csb_buffer()
647 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); in gfx_v11_0_get_csb_buffer()
650 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); in gfx_v11_0_get_csb_buffer()
651 buffer[count++] = cpu_to_le32(0); in gfx_v11_0_get_csb_buffer()
672 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0); in gfx_v11_0_init_rlcg_reg_access_ctrl()
673 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1); in gfx_v11_0_init_rlcg_reg_access_ctrl()
674 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2); in gfx_v11_0_init_rlcg_reg_access_ctrl()
675 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3); in gfx_v11_0_init_rlcg_reg_access_ctrl()
676 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL); in gfx_v11_0_init_rlcg_reg_access_ctrl()
677 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX); in gfx_v11_0_init_rlcg_reg_access_ctrl()
678 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0); in gfx_v11_0_init_rlcg_reg_access_ctrl()
698 /* init spm vmid with 0xf */ in gfx_v11_0_rlc_init()
700 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); in gfx_v11_0_rlc_init()
702 return 0; in gfx_v11_0_rlc_init()
751 memset(hpd, 0, mec_hpd_size); in gfx_v11_0_mec_init()
757 return 0; in gfx_v11_0_mec_init()
762 WREG32_SOC15(GC, 0, regSQ_IND_INDEX, in wave_read_ind()
765 return RREG32_SOC15(GC, 0, regSQ_IND_DATA); in wave_read_ind()
772 WREG32_SOC15(GC, 0, regSQ_IND_INDEX, in wave_read_regs()
778 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA); in wave_read_regs()
786 WARN_ON(simd != 0); in gfx_v11_0_read_wave_data()
811 WARN_ON(simd != 0); in gfx_v11_0_read_wave_sgprs()
814 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size, in gfx_v11_0_read_wave_sgprs()
848 switch (adev->ip_versions[GC_HWIP][0]) { in gfx_v11_0_gpu_early_init()
849 case IP_VERSION(11, 0, 0): in gfx_v11_0_gpu_early_init()
850 case IP_VERSION(11, 0, 2): in gfx_v11_0_gpu_early_init()
851 case IP_VERSION(11, 0, 3): in gfx_v11_0_gpu_early_init()
853 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v11_0_gpu_early_init()
854 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v11_0_gpu_early_init()
855 adev->gfx.config.sc_hiz_tile_fifo_size = 0; in gfx_v11_0_gpu_early_init()
856 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v11_0_gpu_early_init()
858 case IP_VERSION(11, 0, 1): in gfx_v11_0_gpu_early_init()
860 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v11_0_gpu_early_init()
861 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v11_0_gpu_early_init()
862 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; in gfx_v11_0_gpu_early_init()
863 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300; in gfx_v11_0_gpu_early_init()
870 return 0; in gfx_v11_0_gpu_early_init()
900 return 0; in gfx_v11_0_gfx_ring_init()
936 return 0; in gfx_v11_0_compute_ring_init()
961 uint32_t total_size = 0; in gfx_v11_0_calc_toc_total_size()
995 return 0; in gfx_v11_0_rlc_autoload_buffer_init()
1014 if (fw_size == 0) in gfx_v11_0_rlc_backdoor_autoload_copy_ucode()
1023 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size); in gfx_v11_0_rlc_backdoor_autoload_copy_ucode()
1036 *(uint64_t *)fw_autoload_mask |= 0x1; in gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode()
1038 DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask); in gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode()
1185 adev->sdma.instance[0].fw->data; in gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode()
1186 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data + in gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode()
1193 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data + in gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode()
1209 for (pipe = 0; pipe < 2; pipe++) { in gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode()
1210 if (pipe==0) { in gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode()
1243 memset(autoload_fw_id, 0, sizeof(uint32_t) * 2); in gfx_v11_0_rlc_backdoor_autoload_enable()
1255 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr)); in gfx_v11_0_rlc_backdoor_autoload_enable()
1256 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr)); in gfx_v11_0_rlc_backdoor_autoload_enable()
1258 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size); in gfx_v11_0_rlc_backdoor_autoload_enable()
1272 return 0; in gfx_v11_0_rlc_backdoor_autoload_enable()
1277 int i, j, k, r, ring_id = 0; in gfx_v11_0_sw_init()
1283 switch (adev->ip_versions[GC_HWIP][0]) { in gfx_v11_0_sw_init()
1284 case IP_VERSION(11, 0, 0): in gfx_v11_0_sw_init()
1285 case IP_VERSION(11, 0, 1): in gfx_v11_0_sw_init()
1286 case IP_VERSION(11, 0, 2): in gfx_v11_0_sw_init()
1287 case IP_VERSION(11, 0, 3): in gfx_v11_0_sw_init()
1353 for (i = 0; i < adev->gfx.me.num_me; i++) { in gfx_v11_0_sw_init()
1354 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) { in gfx_v11_0_sw_init()
1355 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) { in gfx_v11_0_sw_init()
1368 ring_id = 0; in gfx_v11_0_sw_init()
1370 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { in gfx_v11_0_sw_init()
1371 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { in gfx_v11_0_sw_init()
1372 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { in gfx_v11_0_sw_init()
1418 return 0; in gfx_v11_0_sw_init()
1455 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v11_0_sw_fini()
1457 for (i = 0; i < adev->gfx.num_compute_rings; i++) in gfx_v11_0_sw_fini()
1477 return 0; in gfx_v11_0_sw_fini()
1485 if (instance == 0xffffffff) in gfx_v11_0_select_se_sh()
1486 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, in gfx_v11_0_select_se_sh()
1489 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, in gfx_v11_0_select_se_sh()
1492 if (se_num == 0xffffffff) in gfx_v11_0_select_se_sh()
1498 if (sh_num == 0xffffffff) in gfx_v11_0_select_se_sh()
1504 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data); in gfx_v11_0_select_se_sh()
1511 data = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE); in gfx_v11_0_get_rb_active_bitmap()
1512 data |= RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE); in gfx_v11_0_get_rb_active_bitmap()
1527 u32 active_rbs = 0; in gfx_v11_0_setup_rb()
1532 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v11_0_setup_rb()
1533 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v11_0_setup_rb()
1534 gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff); in gfx_v11_0_setup_rb()
1540 gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); in gfx_v11_0_setup_rb()
1547 #define DEFAULT_SH_MEM_BASES (0x6000)
1548 #define LDS_APP_BASE 0x1
1549 #define SCRATCH_APP_BASE 0x2
1559 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) in gfx_v11_0_init_compute_vmid()
1560 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) in gfx_v11_0_init_compute_vmid()
1561 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) in gfx_v11_0_init_compute_vmid()
1568 soc21_grbm_select(adev, 0, 0, 0, i); in gfx_v11_0_init_compute_vmid()
1570 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); in gfx_v11_0_init_compute_vmid()
1571 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases); in gfx_v11_0_init_compute_vmid()
1574 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL); in gfx_v11_0_init_compute_vmid()
1577 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_init_compute_vmid()
1583 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0); in gfx_v11_0_init_compute_vmid()
1584 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0); in gfx_v11_0_init_compute_vmid()
1585 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0); in gfx_v11_0_init_compute_vmid()
1586 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0); in gfx_v11_0_init_compute_vmid()
1601 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0); in gfx_v11_0_init_gds_vmid()
1602 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0); in gfx_v11_0_init_gds_vmid()
1603 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0); in gfx_v11_0_init_gds_vmid()
1604 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0); in gfx_v11_0_init_gds_vmid()
1616 uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) | in gfx_v11_0_get_tcc_info()
1617 RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE); in gfx_v11_0_get_tcc_info()
1629 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); in gfx_v11_0_constants_init()
1634 adev->gfx.config.pa_sc_tile_steering_override = 0; in gfx_v11_0_constants_init()
1639 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) { in gfx_v11_0_constants_init()
1640 soc21_grbm_select(adev, 0, 0, 0, i); in gfx_v11_0_constants_init()
1642 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); in gfx_v11_0_constants_init()
1643 if (i != 0) { in gfx_v11_0_constants_init()
1644 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, in gfx_v11_0_constants_init()
1648 WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp); in gfx_v11_0_constants_init()
1651 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_constants_init()
1667 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0); in gfx_v11_0_enable_gui_idle_interrupt()
1670 enable ? 1 : 0); in gfx_v11_0_enable_gui_idle_interrupt()
1672 enable ? 1 : 0); in gfx_v11_0_enable_gui_idle_interrupt()
1674 enable ? 1 : 0); in gfx_v11_0_enable_gui_idle_interrupt()
1676 enable ? 1 : 0); in gfx_v11_0_enable_gui_idle_interrupt()
1678 WREG32_SOC15(GC, 0, regCP_INT_CNTL_RING0, tmp); in gfx_v11_0_enable_gui_idle_interrupt()
1685 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI, in gfx_v11_0_init_csb()
1687 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO, in gfx_v11_0_init_csb()
1688 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); in gfx_v11_0_init_csb()
1689 WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size); in gfx_v11_0_init_csb()
1691 return 0; in gfx_v11_0_init_csb()
1696 u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL); in gfx_v11_0_rlc_stop()
1698 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0); in gfx_v11_0_rlc_stop()
1699 WREG32_SOC15(GC, 0, regRLC_CNTL, tmp); in gfx_v11_0_rlc_stop()
1704 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); in gfx_v11_0_rlc_reset()
1706 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); in gfx_v11_0_rlc_reset()
1715 rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL); in gfx_v11_0_rlc_smu_handshake_cntl()
1718 /* RLC_PG_CNTL[23] = 0 (default) in gfx_v11_0_rlc_smu_handshake_cntl()
1729 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl); in gfx_v11_0_rlc_smu_handshake_cntl()
1739 WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); in gfx_v11_0_rlc_start()
1748 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL)); in gfx_v11_0_rlc_enable_srm()
1751 WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp); in gfx_v11_0_rlc_enable_srm()
1765 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, in gfx_v11_0_load_rlcg_microcode()
1768 for (i = 0; i < fw_size; i++) in gfx_v11_0_load_rlcg_microcode()
1769 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA, in gfx_v11_0_load_rlcg_microcode()
1772 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); in gfx_v11_0_load_rlcg_microcode()
1788 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0); in gfx_v11_0_load_rlc_iram_dram_microcode()
1790 for (i = 0; i < fw_size; i++) { in gfx_v11_0_load_rlc_iram_dram_microcode()
1793 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA, in gfx_v11_0_load_rlc_iram_dram_microcode()
1797 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); in gfx_v11_0_load_rlc_iram_dram_microcode()
1803 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0); in gfx_v11_0_load_rlc_iram_dram_microcode()
1804 for (i = 0; i < fw_size; i++) { in gfx_v11_0_load_rlc_iram_dram_microcode()
1807 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA, in gfx_v11_0_load_rlc_iram_dram_microcode()
1811 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); in gfx_v11_0_load_rlc_iram_dram_microcode()
1813 tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL); in gfx_v11_0_load_rlc_iram_dram_microcode()
1815 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0); in gfx_v11_0_load_rlc_iram_dram_microcode()
1816 WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp); in gfx_v11_0_load_rlc_iram_dram_microcode()
1832 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0); in gfx_v11_0_load_rlcp_rlcv_microcode()
1834 for (i = 0; i < fw_size; i++) { in gfx_v11_0_load_rlcp_rlcv_microcode()
1837 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA, in gfx_v11_0_load_rlcp_rlcv_microcode()
1841 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version); in gfx_v11_0_load_rlcp_rlcv_microcode()
1843 tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE); in gfx_v11_0_load_rlcp_rlcv_microcode()
1845 WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp); in gfx_v11_0_load_rlcp_rlcv_microcode()
1851 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0); in gfx_v11_0_load_rlcp_rlcv_microcode()
1853 for (i = 0; i < fw_size; i++) { in gfx_v11_0_load_rlcp_rlcv_microcode()
1856 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA, in gfx_v11_0_load_rlcp_rlcv_microcode()
1860 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version); in gfx_v11_0_load_rlcp_rlcv_microcode()
1862 tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL); in gfx_v11_0_load_rlcp_rlcv_microcode()
1864 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp); in gfx_v11_0_load_rlcp_rlcv_microcode()
1891 return 0; in gfx_v11_0_rlc_load_microcode()
1909 return 0; in gfx_v11_0_rlc_resume()
1915 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0); in gfx_v11_0_rlc_resume()
1918 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0); in gfx_v11_0_rlc_resume()
1931 return 0; in gfx_v11_0_rlc_resume()
1941 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); in gfx_v11_0_config_me_cache()
1943 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); in gfx_v11_0_config_me_cache()
1946 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_config_me_cache()
1947 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); in gfx_v11_0_config_me_cache()
1962 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); in gfx_v11_0_config_me_cache()
1963 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); in gfx_v11_0_config_me_cache()
1964 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_config_me_cache()
1965 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); in gfx_v11_0_config_me_cache()
1967 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); in gfx_v11_0_config_me_cache()
1970 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, in gfx_v11_0_config_me_cache()
1971 lower_32_bits(addr) & 0xFFFFF000); in gfx_v11_0_config_me_cache()
1972 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, in gfx_v11_0_config_me_cache()
1975 return 0; in gfx_v11_0_config_me_cache()
1985 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); in gfx_v11_0_config_pfp_cache()
1987 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); in gfx_v11_0_config_pfp_cache()
1990 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_config_pfp_cache()
1991 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); in gfx_v11_0_config_pfp_cache()
2006 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); in gfx_v11_0_config_pfp_cache()
2007 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); in gfx_v11_0_config_pfp_cache()
2008 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_config_pfp_cache()
2009 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); in gfx_v11_0_config_pfp_cache()
2011 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); in gfx_v11_0_config_pfp_cache()
2014 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, in gfx_v11_0_config_pfp_cache()
2015 lower_32_bits(addr) & 0xFFFFF000); in gfx_v11_0_config_pfp_cache()
2016 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, in gfx_v11_0_config_pfp_cache()
2019 return 0; in gfx_v11_0_config_pfp_cache()
2029 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); in gfx_v11_0_config_mec_cache()
2032 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); in gfx_v11_0_config_mec_cache()
2035 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_config_mec_cache()
2036 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); in gfx_v11_0_config_mec_cache()
2051 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); in gfx_v11_0_config_mec_cache()
2052 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_config_mec_cache()
2053 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); in gfx_v11_0_config_mec_cache()
2055 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); in gfx_v11_0_config_mec_cache()
2058 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, in gfx_v11_0_config_mec_cache()
2059 lower_32_bits(addr) & 0xFFFFF000); in gfx_v11_0_config_mec_cache()
2060 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, in gfx_v11_0_config_mec_cache()
2063 return 0; in gfx_v11_0_config_mec_cache()
2076 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, in gfx_v11_0_config_pfp_cache_rs64()
2078 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, in gfx_v11_0_config_pfp_cache_rs64()
2081 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); in gfx_v11_0_config_pfp_cache_rs64()
2082 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); in gfx_v11_0_config_pfp_cache_rs64()
2083 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_config_pfp_cache_rs64()
2084 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); in gfx_v11_0_config_pfp_cache_rs64()
2085 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); in gfx_v11_0_config_pfp_cache_rs64()
2092 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_config_pfp_cache_rs64()
2093 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); in gfx_v11_0_config_pfp_cache_rs64()
2106 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); in gfx_v11_0_config_pfp_cache_rs64()
2108 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); in gfx_v11_0_config_pfp_cache_rs64()
2110 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_config_pfp_cache_rs64()
2111 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); in gfx_v11_0_config_pfp_cache_rs64()
2124 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { in gfx_v11_0_config_pfp_cache_rs64()
2125 soc21_grbm_select(adev, 0, pipe_id, 0, 0); in gfx_v11_0_config_pfp_cache_rs64()
2126 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, in gfx_v11_0_config_pfp_cache_rs64()
2129 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, in gfx_v11_0_config_pfp_cache_rs64()
2136 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); in gfx_v11_0_config_pfp_cache_rs64()
2137 if (pipe_id == 0) in gfx_v11_0_config_pfp_cache_rs64()
2143 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); in gfx_v11_0_config_pfp_cache_rs64()
2146 if (pipe_id == 0) in gfx_v11_0_config_pfp_cache_rs64()
2148 PFP_PIPE0_RESET, 0); in gfx_v11_0_config_pfp_cache_rs64()
2151 PFP_PIPE1_RESET, 0); in gfx_v11_0_config_pfp_cache_rs64()
2152 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); in gfx_v11_0_config_pfp_cache_rs64()
2154 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO, in gfx_v11_0_config_pfp_cache_rs64()
2156 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI, in gfx_v11_0_config_pfp_cache_rs64()
2159 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_config_pfp_cache_rs64()
2162 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); in gfx_v11_0_config_pfp_cache_rs64()
2163 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); in gfx_v11_0_config_pfp_cache_rs64()
2164 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_config_pfp_cache_rs64()
2165 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); in gfx_v11_0_config_pfp_cache_rs64()
2168 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); in gfx_v11_0_config_pfp_cache_rs64()
2170 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); in gfx_v11_0_config_pfp_cache_rs64()
2172 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_config_pfp_cache_rs64()
2173 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); in gfx_v11_0_config_pfp_cache_rs64()
2185 return 0; in gfx_v11_0_config_pfp_cache_rs64()
2198 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, in gfx_v11_0_config_me_cache_rs64()
2200 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, in gfx_v11_0_config_me_cache_rs64()
2203 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); in gfx_v11_0_config_me_cache_rs64()
2204 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); in gfx_v11_0_config_me_cache_rs64()
2205 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_config_me_cache_rs64()
2206 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); in gfx_v11_0_config_me_cache_rs64()
2207 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); in gfx_v11_0_config_me_cache_rs64()
2214 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_config_me_cache_rs64()
2215 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); in gfx_v11_0_config_me_cache_rs64()
2228 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); in gfx_v11_0_config_me_cache_rs64()
2230 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); in gfx_v11_0_config_me_cache_rs64()
2233 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_config_me_cache_rs64()
2234 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); in gfx_v11_0_config_me_cache_rs64()
2247 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { in gfx_v11_0_config_me_cache_rs64()
2248 soc21_grbm_select(adev, 0, pipe_id, 0, 0); in gfx_v11_0_config_me_cache_rs64()
2249 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, in gfx_v11_0_config_me_cache_rs64()
2252 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, in gfx_v11_0_config_me_cache_rs64()
2259 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); in gfx_v11_0_config_me_cache_rs64()
2260 if (pipe_id == 0) in gfx_v11_0_config_me_cache_rs64()
2266 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); in gfx_v11_0_config_me_cache_rs64()
2269 if (pipe_id == 0) in gfx_v11_0_config_me_cache_rs64()
2271 ME_PIPE0_RESET, 0); in gfx_v11_0_config_me_cache_rs64()
2274 ME_PIPE1_RESET, 0); in gfx_v11_0_config_me_cache_rs64()
2275 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); in gfx_v11_0_config_me_cache_rs64()
2277 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO, in gfx_v11_0_config_me_cache_rs64()
2279 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI, in gfx_v11_0_config_me_cache_rs64()
2282 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_config_me_cache_rs64()
2285 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); in gfx_v11_0_config_me_cache_rs64()
2286 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); in gfx_v11_0_config_me_cache_rs64()
2287 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_config_me_cache_rs64()
2288 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); in gfx_v11_0_config_me_cache_rs64()
2291 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); in gfx_v11_0_config_me_cache_rs64()
2293 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); in gfx_v11_0_config_me_cache_rs64()
2295 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_config_me_cache_rs64()
2296 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); in gfx_v11_0_config_me_cache_rs64()
2308 return 0; in gfx_v11_0_config_me_cache_rs64()
2321 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); in gfx_v11_0_config_mec_cache_rs64()
2322 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); in gfx_v11_0_config_mec_cache_rs64()
2323 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); in gfx_v11_0_config_mec_cache_rs64()
2324 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_config_mec_cache_rs64()
2325 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); in gfx_v11_0_config_mec_cache_rs64()
2327 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL); in gfx_v11_0_config_mec_cache_rs64()
2328 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0); in gfx_v11_0_config_mec_cache_rs64()
2329 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_config_mec_cache_rs64()
2330 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp); in gfx_v11_0_config_mec_cache_rs64()
2333 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { in gfx_v11_0_config_mec_cache_rs64()
2334 soc21_grbm_select(adev, 1, i, 0, 0); in gfx_v11_0_config_mec_cache_rs64()
2336 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2); in gfx_v11_0_config_mec_cache_rs64()
2337 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI, in gfx_v11_0_config_mec_cache_rs64()
2340 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, in gfx_v11_0_config_mec_cache_rs64()
2343 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, in gfx_v11_0_config_mec_cache_rs64()
2346 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr); in gfx_v11_0_config_mec_cache_rs64()
2347 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, in gfx_v11_0_config_mec_cache_rs64()
2351 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_config_mec_cache_rs64()
2354 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); in gfx_v11_0_config_mec_cache_rs64()
2356 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp); in gfx_v11_0_config_mec_cache_rs64()
2359 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_config_mec_cache_rs64()
2360 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); in gfx_v11_0_config_mec_cache_rs64()
2373 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); in gfx_v11_0_config_mec_cache_rs64()
2375 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); in gfx_v11_0_config_mec_cache_rs64()
2378 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_config_mec_cache_rs64()
2379 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); in gfx_v11_0_config_mec_cache_rs64()
2391 return 0; in gfx_v11_0_config_mec_cache_rs64()
2409 for (pipe_id = 0; pipe_id < 2; pipe_id++) { in gfx_v11_0_config_gfx_rs64()
2410 soc21_grbm_select(adev, 0, pipe_id, 0, 0); in gfx_v11_0_config_gfx_rs64()
2411 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, in gfx_v11_0_config_gfx_rs64()
2414 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, in gfx_v11_0_config_gfx_rs64()
2417 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_config_gfx_rs64()
2420 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); in gfx_v11_0_config_gfx_rs64()
2423 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); in gfx_v11_0_config_gfx_rs64()
2426 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0); in gfx_v11_0_config_gfx_rs64()
2427 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0); in gfx_v11_0_config_gfx_rs64()
2428 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); in gfx_v11_0_config_gfx_rs64()
2431 for (pipe_id = 0; pipe_id < 2; pipe_id++) { in gfx_v11_0_config_gfx_rs64()
2432 soc21_grbm_select(adev, 0, pipe_id, 0, 0); in gfx_v11_0_config_gfx_rs64()
2433 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, in gfx_v11_0_config_gfx_rs64()
2436 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, in gfx_v11_0_config_gfx_rs64()
2439 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_config_gfx_rs64()
2442 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); in gfx_v11_0_config_gfx_rs64()
2445 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); in gfx_v11_0_config_gfx_rs64()
2448 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0); in gfx_v11_0_config_gfx_rs64()
2449 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0); in gfx_v11_0_config_gfx_rs64()
2450 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); in gfx_v11_0_config_gfx_rs64()
2453 for (pipe_id = 0; pipe_id < 4; pipe_id++) { in gfx_v11_0_config_gfx_rs64()
2454 soc21_grbm_select(adev, 1, pipe_id, 0, 0); in gfx_v11_0_config_gfx_rs64()
2455 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, in gfx_v11_0_config_gfx_rs64()
2458 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, in gfx_v11_0_config_gfx_rs64()
2461 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_config_gfx_rs64()
2464 tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); in gfx_v11_0_config_gfx_rs64()
2469 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp); in gfx_v11_0_config_gfx_rs64()
2472 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0); in gfx_v11_0_config_gfx_rs64()
2473 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0); in gfx_v11_0_config_gfx_rs64()
2474 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0); in gfx_v11_0_config_gfx_rs64()
2475 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0); in gfx_v11_0_config_gfx_rs64()
2476 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp); in gfx_v11_0_config_gfx_rs64()
2486 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v11_0_wait_for_rlc_autoload_complete()
2487 cp_status = RREG32_SOC15(GC, 0, regCP_STAT); in gfx_v11_0_wait_for_rlc_autoload_complete()
2489 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(11, 0, 1)) in gfx_v11_0_wait_for_rlc_autoload_complete()
2490 bootload_status = RREG32_SOC15(GC, 0, in gfx_v11_0_wait_for_rlc_autoload_complete()
2493 bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS); in gfx_v11_0_wait_for_rlc_autoload_complete()
2495 if ((cp_status == 0) && in gfx_v11_0_wait_for_rlc_autoload_complete()
2550 return 0; in gfx_v11_0_wait_for_rlc_autoload_complete()
2556 u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); in gfx_v11_0_cp_gfx_enable()
2558 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); in gfx_v11_0_cp_gfx_enable()
2559 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); in gfx_v11_0_cp_gfx_enable()
2560 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); in gfx_v11_0_cp_gfx_enable()
2562 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v11_0_cp_gfx_enable()
2563 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0) in gfx_v11_0_cp_gfx_enable()
2571 return 0; in gfx_v11_0_cp_gfx_enable()
2608 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0); in gfx_v11_0_cp_gfx_load_pfp_microcode()
2610 for (i = 0; i < pfp_hdr->jt_size; i++) in gfx_v11_0_cp_gfx_load_pfp_microcode()
2611 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA, in gfx_v11_0_cp_gfx_load_pfp_microcode()
2614 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); in gfx_v11_0_cp_gfx_load_pfp_microcode()
2616 return 0; in gfx_v11_0_cp_gfx_load_pfp_microcode()
2676 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2678 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2681 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2682 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2683 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2684 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2685 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2692 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2693 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2706 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2708 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2710 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2711 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2724 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2725 soc21_grbm_select(adev, 0, pipe_id, 0, 0); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2726 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2729 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2736 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2737 if (pipe_id == 0) in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2743 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2746 if (pipe_id == 0) in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2748 PFP_PIPE0_RESET, 0); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2751 PFP_PIPE1_RESET, 0); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2752 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2754 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO, in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2756 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI, in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2759 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2762 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2763 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2764 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2765 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2768 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2770 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2772 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2773 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2785 return 0; in gfx_v11_0_cp_gfx_load_pfp_microcode_rs64()
2822 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0); in gfx_v11_0_cp_gfx_load_me_microcode()
2824 for (i = 0; i < me_hdr->jt_size; i++) in gfx_v11_0_cp_gfx_load_me_microcode()
2825 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA, in gfx_v11_0_cp_gfx_load_me_microcode()
2828 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version); in gfx_v11_0_cp_gfx_load_me_microcode()
2830 return 0; in gfx_v11_0_cp_gfx_load_me_microcode()
2890 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2892 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2895 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2896 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2897 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2898 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2899 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2906 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2907 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2920 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2922 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2925 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2926 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2939 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2940 soc21_grbm_select(adev, 0, pipe_id, 0, 0); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2941 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2944 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2951 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2952 if (pipe_id == 0) in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2958 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2961 if (pipe_id == 0) in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2963 ME_PIPE0_RESET, 0); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2966 ME_PIPE1_RESET, 0); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2967 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2969 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO, in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2971 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI, in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2974 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2977 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2978 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2979 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2980 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2983 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2985 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2987 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
2988 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
3000 return 0; in gfx_v11_0_cp_gfx_load_me_microcode_rs64()
3030 return 0; in gfx_v11_0_cp_gfx_load_microcode()
3042 WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT, in gfx_v11_0_cp_gfx_start()
3044 WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1); in gfx_v11_0_cp_gfx_start()
3049 ring = &adev->gfx.gfx_ring[0]; in gfx_v11_0_cp_gfx_start()
3056 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); in gfx_v11_0_cp_gfx_start()
3060 amdgpu_ring_write(ring, 0x80000000); in gfx_v11_0_cp_gfx_start()
3061 amdgpu_ring_write(ring, 0x80000000); in gfx_v11_0_cp_gfx_start()
3071 for (i = 0; i < ext->reg_count; i++) in gfx_v11_0_cp_gfx_start()
3078 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START; in gfx_v11_0_cp_gfx_start()
3083 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); in gfx_v11_0_cp_gfx_start()
3086 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); in gfx_v11_0_cp_gfx_start()
3087 amdgpu_ring_write(ring, 0); in gfx_v11_0_cp_gfx_start()
3091 /* submit cs packet to copy state 0 to next available state */ in gfx_v11_0_cp_gfx_start()
3101 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); in gfx_v11_0_cp_gfx_start()
3102 amdgpu_ring_write(ring, 0); in gfx_v11_0_cp_gfx_start()
3106 return 0; in gfx_v11_0_cp_gfx_start()
3114 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL); in gfx_v11_0_cp_gfx_switch_pipe()
3117 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp); in gfx_v11_0_cp_gfx_switch_pipe()
3125 tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL); in gfx_v11_0_cp_gfx_set_doorbell()
3133 DOORBELL_EN, 0); in gfx_v11_0_cp_gfx_set_doorbell()
3135 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp); in gfx_v11_0_cp_gfx_set_doorbell()
3137 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, in gfx_v11_0_cp_gfx_set_doorbell()
3139 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp); in gfx_v11_0_cp_gfx_set_doorbell()
3141 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER, in gfx_v11_0_cp_gfx_set_doorbell()
3154 WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0); in gfx_v11_0_cp_gfx_resume()
3156 /* set the RB to use vmid 0 */ in gfx_v11_0_cp_gfx_resume()
3157 WREG32_SOC15(GC, 0, regCP_RB_VMID, 0); in gfx_v11_0_cp_gfx_resume()
3159 /* Init gfx ring 0 for pipe 0 */ in gfx_v11_0_cp_gfx_resume()
3164 ring = &adev->gfx.gfx_ring[0]; in gfx_v11_0_cp_gfx_resume()
3166 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); in gfx_v11_0_cp_gfx_resume()
3168 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); in gfx_v11_0_cp_gfx_resume()
3171 ring->wptr = 0; in gfx_v11_0_cp_gfx_resume()
3172 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr)); in gfx_v11_0_cp_gfx_resume()
3173 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); in gfx_v11_0_cp_gfx_resume()
3177 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); in gfx_v11_0_cp_gfx_resume()
3178 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & in gfx_v11_0_cp_gfx_resume()
3182 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, in gfx_v11_0_cp_gfx_resume()
3184 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, in gfx_v11_0_cp_gfx_resume()
3188 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); in gfx_v11_0_cp_gfx_resume()
3191 WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr); in gfx_v11_0_cp_gfx_resume()
3192 WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr)); in gfx_v11_0_cp_gfx_resume()
3194 WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1); in gfx_v11_0_cp_gfx_resume()
3206 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz); in gfx_v11_0_cp_gfx_resume()
3208 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp); in gfx_v11_0_cp_gfx_resume()
3210 ring->wptr = 0; in gfx_v11_0_cp_gfx_resume()
3211 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr)); in gfx_v11_0_cp_gfx_resume()
3212 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr)); in gfx_v11_0_cp_gfx_resume()
3215 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr)); in gfx_v11_0_cp_gfx_resume()
3216 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & in gfx_v11_0_cp_gfx_resume()
3219 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, in gfx_v11_0_cp_gfx_resume()
3221 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, in gfx_v11_0_cp_gfx_resume()
3225 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp); in gfx_v11_0_cp_gfx_resume()
3228 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr); in gfx_v11_0_cp_gfx_resume()
3229 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr)); in gfx_v11_0_cp_gfx_resume()
3230 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1); in gfx_v11_0_cp_gfx_resume()
3235 /* Switch to pipe 0 */ in gfx_v11_0_cp_gfx_resume()
3243 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v11_0_cp_gfx_resume()
3248 return 0; in gfx_v11_0_cp_gfx_resume()
3256 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); in gfx_v11_0_cp_compute_enable()
3258 enable ? 0 : 1); in gfx_v11_0_cp_compute_enable()
3260 enable ? 0 : 1); in gfx_v11_0_cp_compute_enable()
3262 enable ? 0 : 1); in gfx_v11_0_cp_compute_enable()
3264 enable ? 0 : 1); in gfx_v11_0_cp_compute_enable()
3266 enable ? 0 : 1); in gfx_v11_0_cp_compute_enable()
3268 enable ? 1 : 0); in gfx_v11_0_cp_compute_enable()
3270 enable ? 1 : 0); in gfx_v11_0_cp_compute_enable()
3272 enable ? 1 : 0); in gfx_v11_0_cp_compute_enable()
3274 enable ? 1 : 0); in gfx_v11_0_cp_compute_enable()
3276 enable ? 0 : 1); in gfx_v11_0_cp_compute_enable()
3277 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data); in gfx_v11_0_cp_compute_enable()
3279 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL); in gfx_v11_0_cp_compute_enable()
3282 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0); in gfx_v11_0_cp_compute_enable()
3285 MEC_ME2_HALT, 0); in gfx_v11_0_cp_compute_enable()
3290 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data); in gfx_v11_0_cp_compute_enable()
3338 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0); in gfx_v11_0_cp_compute_load_microcode()
3340 for (i = 0; i < mec_hdr->jt_size; i++) in gfx_v11_0_cp_compute_load_microcode()
3341 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA, in gfx_v11_0_cp_compute_load_microcode()
3344 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version); in gfx_v11_0_cp_compute_load_microcode()
3346 return 0; in gfx_v11_0_cp_compute_load_microcode()
3404 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); in gfx_v11_0_cp_compute_load_microcode_rs64()
3405 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); in gfx_v11_0_cp_compute_load_microcode_rs64()
3406 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); in gfx_v11_0_cp_compute_load_microcode_rs64()
3407 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_cp_compute_load_microcode_rs64()
3408 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); in gfx_v11_0_cp_compute_load_microcode_rs64()
3410 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL); in gfx_v11_0_cp_compute_load_microcode_rs64()
3411 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0); in gfx_v11_0_cp_compute_load_microcode_rs64()
3412 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0); in gfx_v11_0_cp_compute_load_microcode_rs64()
3413 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp); in gfx_v11_0_cp_compute_load_microcode_rs64()
3416 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { in gfx_v11_0_cp_compute_load_microcode_rs64()
3417 soc21_grbm_select(adev, 1, i, 0, 0); in gfx_v11_0_cp_compute_load_microcode_rs64()
3419 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr); in gfx_v11_0_cp_compute_load_microcode_rs64()
3420 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI, in gfx_v11_0_cp_compute_load_microcode_rs64()
3423 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, in gfx_v11_0_cp_compute_load_microcode_rs64()
3426 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, in gfx_v11_0_cp_compute_load_microcode_rs64()
3429 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr); in gfx_v11_0_cp_compute_load_microcode_rs64()
3430 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, in gfx_v11_0_cp_compute_load_microcode_rs64()
3434 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_cp_compute_load_microcode_rs64()
3437 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); in gfx_v11_0_cp_compute_load_microcode_rs64()
3439 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp); in gfx_v11_0_cp_compute_load_microcode_rs64()
3442 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_cp_compute_load_microcode_rs64()
3443 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); in gfx_v11_0_cp_compute_load_microcode_rs64()
3456 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); in gfx_v11_0_cp_compute_load_microcode_rs64()
3458 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); in gfx_v11_0_cp_compute_load_microcode_rs64()
3461 for (i = 0; i < usec_timeout; i++) { in gfx_v11_0_cp_compute_load_microcode_rs64()
3462 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); in gfx_v11_0_cp_compute_load_microcode_rs64()
3474 return 0; in gfx_v11_0_cp_compute_load_microcode_rs64()
3483 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS); in gfx_v11_0_kiq_setting()
3484 tmp &= 0xffffff00; in gfx_v11_0_kiq_setting()
3486 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp); in gfx_v11_0_kiq_setting()
3487 tmp |= 0x80; in gfx_v11_0_kiq_setting()
3488 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp); in gfx_v11_0_kiq_setting()
3494 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, in gfx_v11_0_cp_set_doorbell_range()
3496 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER, in gfx_v11_0_cp_set_doorbell_range()
3500 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER, in gfx_v11_0_cp_set_doorbell_range()
3502 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER, in gfx_v11_0_cp_set_doorbell_range()
3515 mqd->cp_gfx_hqd_wptr = 0; in gfx_v11_0_gfx_mqd_init()
3516 mqd->cp_gfx_hqd_wptr_hi = 0; in gfx_v11_0_gfx_mqd_init()
3519 mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc; in gfx_v11_0_gfx_mqd_init()
3523 tmp = RREG32_SOC15(GC, 0, regCP_GFX_MQD_CONTROL); in gfx_v11_0_gfx_mqd_init()
3524 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0); in gfx_v11_0_gfx_mqd_init()
3526 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0); in gfx_v11_0_gfx_mqd_init()
3529 /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */ in gfx_v11_0_gfx_mqd_init()
3530 tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_VMID); in gfx_v11_0_gfx_mqd_init()
3531 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0); in gfx_v11_0_gfx_mqd_init()
3532 mqd->cp_gfx_hqd_vmid = 0; in gfx_v11_0_gfx_mqd_init()
3535 * 0x0 = low priority, 0x1 = high priority */ in gfx_v11_0_gfx_mqd_init()
3536 tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY); in gfx_v11_0_gfx_mqd_init()
3537 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, 0); in gfx_v11_0_gfx_mqd_init()
3541 tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUANTUM); in gfx_v11_0_gfx_mqd_init()
3552 mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc; in gfx_v11_0_gfx_mqd_init()
3554 upper_32_bits(wb_gpu_addr) & 0xffff; in gfx_v11_0_gfx_mqd_init()
3558 mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; in gfx_v11_0_gfx_mqd_init()
3559 mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; in gfx_v11_0_gfx_mqd_init()
3563 tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_CNTL); in gfx_v11_0_gfx_mqd_init()
3572 tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL); in gfx_v11_0_gfx_mqd_init()
3580 DOORBELL_EN, 0); in gfx_v11_0_gfx_mqd_init()
3584 mqd->cp_gfx_hqd_rptr = RREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR); in gfx_v11_0_gfx_mqd_init()
3589 return 0; in gfx_v11_0_gfx_mqd_init()
3598 /* set mmCP_GFX_HQD_WPTR/_HI to 0 */ in gfx_v11_0_gfx_queue_init_register()
3599 WREG32_SOC15(GC, 0, regCP_GFX_HQD_WPTR, mqd->cp_gfx_hqd_wptr); in gfx_v11_0_gfx_queue_init_register()
3600 WREG32_SOC15(GC, 0, regCP_GFX_HQD_WPTR_HI, mqd->cp_gfx_hqd_wptr_hi); in gfx_v11_0_gfx_queue_init_register()
3603 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr); in gfx_v11_0_gfx_queue_init_register()
3604 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi); in gfx_v11_0_gfx_queue_init_register()
3607 WREG32_SOC15(GC, 0, regCP_GFX_MQD_CONTROL, mqd->cp_gfx_mqd_control); in gfx_v11_0_gfx_queue_init_register()
3609 /* set GFX_HQD_VMID to 0 */ in gfx_v11_0_gfx_queue_init_register()
3610 WREG32_SOC15(GC, 0, regCP_GFX_HQD_VMID, mqd->cp_gfx_hqd_vmid); in gfx_v11_0_gfx_queue_init_register()
3612 WREG32_SOC15(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY, in gfx_v11_0_gfx_queue_init_register()
3614 WREG32_SOC15(GC, 0, regCP_GFX_HQD_QUANTUM, mqd->cp_gfx_hqd_quantum); in gfx_v11_0_gfx_queue_init_register()
3617 WREG32_SOC15(GC, 0, regCP_GFX_HQD_BASE, mqd->cp_gfx_hqd_base); in gfx_v11_0_gfx_queue_init_register()
3618 WREG32_SOC15(GC, 0, regCP_GFX_HQD_BASE_HI, mqd->cp_gfx_hqd_base_hi); in gfx_v11_0_gfx_queue_init_register()
3621 WREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR_ADDR, mqd->cp_gfx_hqd_rptr_addr); in gfx_v11_0_gfx_queue_init_register()
3622 WREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR_ADDR_HI, mqd->cp_gfx_hqd_rptr_addr_hi); in gfx_v11_0_gfx_queue_init_register()
3625 WREG32_SOC15(GC, 0, regCP_GFX_HQD_CNTL, mqd->cp_gfx_hqd_cntl); in gfx_v11_0_gfx_queue_init_register()
3628 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, mqd->cp_rb_wptr_poll_addr_lo); in gfx_v11_0_gfx_queue_init_register()
3629 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, mqd->cp_rb_wptr_poll_addr_hi); in gfx_v11_0_gfx_queue_init_register()
3632 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, mqd->cp_rb_doorbell_control); in gfx_v11_0_gfx_queue_init_register()
3635 WREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE, mqd->cp_gfx_hqd_active); in gfx_v11_0_gfx_queue_init_register()
3637 return 0; in gfx_v11_0_gfx_queue_init_register()
3645 int mqd_idx = ring - &adev->gfx.gfx_ring[0]; in gfx_v11_0_gfx_init_queue()
3648 memset((void *)mqd, 0, sizeof(*mqd)); in gfx_v11_0_gfx_init_queue()
3650 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); in gfx_v11_0_gfx_init_queue()
3655 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_gfx_init_queue()
3664 ring->wptr = 0; in gfx_v11_0_gfx_init_queue()
3665 *ring->wptr_cpu_addr = 0; in gfx_v11_0_gfx_init_queue()
3669 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); in gfx_v11_0_gfx_init_queue()
3671 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_gfx_init_queue()
3678 return 0; in gfx_v11_0_gfx_init_queue()
3698 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v11_0_kiq_enable_kgq()
3710 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v11_0_cp_async_gfx_ring_resume()
3714 if (unlikely(r != 0)) in gfx_v11_0_cp_async_gfx_ring_resume()
3736 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v11_0_cp_async_gfx_ring_resume()
3751 mqd->header = 0xC0310800; in gfx_v11_0_compute_mqd_init()
3752 mqd->compute_pipelinestat_enable = 0x00000001; in gfx_v11_0_compute_mqd_init()
3753 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; in gfx_v11_0_compute_mqd_init()
3754 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; in gfx_v11_0_compute_mqd_init()
3755 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; in gfx_v11_0_compute_mqd_init()
3756 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; in gfx_v11_0_compute_mqd_init()
3757 mqd->compute_misc_reserved = 0x00000007; in gfx_v11_0_compute_mqd_init()
3764 tmp = RREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL); in gfx_v11_0_compute_mqd_init()
3771 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL); in gfx_v11_0_compute_mqd_init()
3779 DOORBELL_SOURCE, 0); in gfx_v11_0_compute_mqd_init()
3781 DOORBELL_HIT, 0); in gfx_v11_0_compute_mqd_init()
3784 DOORBELL_EN, 0); in gfx_v11_0_compute_mqd_init()
3790 mqd->cp_hqd_dequeue_request = 0; in gfx_v11_0_compute_mqd_init()
3791 mqd->cp_hqd_pq_rptr = 0; in gfx_v11_0_compute_mqd_init()
3792 mqd->cp_hqd_pq_wptr_lo = 0; in gfx_v11_0_compute_mqd_init()
3793 mqd->cp_hqd_pq_wptr_hi = 0; in gfx_v11_0_compute_mqd_init()
3796 mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc; in gfx_v11_0_compute_mqd_init()
3799 /* set MQD vmid to 0 */ in gfx_v11_0_compute_mqd_init()
3800 tmp = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL); in gfx_v11_0_compute_mqd_init()
3801 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); in gfx_v11_0_compute_mqd_init()
3810 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL); in gfx_v11_0_compute_mqd_init()
3815 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); in gfx_v11_0_compute_mqd_init()
3816 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0); in gfx_v11_0_compute_mqd_init()
3823 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; in gfx_v11_0_compute_mqd_init()
3825 upper_32_bits(wb_gpu_addr) & 0xffff; in gfx_v11_0_compute_mqd_init()
3829 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; in gfx_v11_0_compute_mqd_init()
3830 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; in gfx_v11_0_compute_mqd_init()
3832 tmp = 0; in gfx_v11_0_compute_mqd_init()
3835 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL); in gfx_v11_0_compute_mqd_init()
3842 DOORBELL_SOURCE, 0); in gfx_v11_0_compute_mqd_init()
3844 DOORBELL_HIT, 0); in gfx_v11_0_compute_mqd_init()
3850 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR); in gfx_v11_0_compute_mqd_init()
3853 mqd->cp_hqd_vmid = 0; in gfx_v11_0_compute_mqd_init()
3855 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE); in gfx_v11_0_compute_mqd_init()
3856 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55); in gfx_v11_0_compute_mqd_init()
3860 tmp = RREG32_SOC15(GC, 0, regCP_HQD_IB_CONTROL); in gfx_v11_0_compute_mqd_init()
3870 return 0; in gfx_v11_0_compute_mqd_init()
3881 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0); in gfx_v11_0_kiq_init_register()
3884 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); in gfx_v11_0_kiq_init_register()
3887 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR, in gfx_v11_0_kiq_init_register()
3889 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI, in gfx_v11_0_kiq_init_register()
3893 WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL, in gfx_v11_0_kiq_init_register()
3897 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, in gfx_v11_0_kiq_init_register()
3901 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) { in gfx_v11_0_kiq_init_register()
3902 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1); in gfx_v11_0_kiq_init_register()
3903 for (j = 0; j < adev->usec_timeout; j++) { in gfx_v11_0_kiq_init_register()
3904 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1)) in gfx_v11_0_kiq_init_register()
3908 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, in gfx_v11_0_kiq_init_register()
3910 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, in gfx_v11_0_kiq_init_register()
3912 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, in gfx_v11_0_kiq_init_register()
3914 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, in gfx_v11_0_kiq_init_register()
3919 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, in gfx_v11_0_kiq_init_register()
3921 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, in gfx_v11_0_kiq_init_register()
3924 /* set MQD vmid to 0 */ in gfx_v11_0_kiq_init_register()
3925 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, in gfx_v11_0_kiq_init_register()
3929 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, in gfx_v11_0_kiq_init_register()
3931 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, in gfx_v11_0_kiq_init_register()
3935 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, in gfx_v11_0_kiq_init_register()
3939 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR, in gfx_v11_0_kiq_init_register()
3941 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI, in gfx_v11_0_kiq_init_register()
3945 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR, in gfx_v11_0_kiq_init_register()
3947 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI, in gfx_v11_0_kiq_init_register()
3952 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER, in gfx_v11_0_kiq_init_register()
3954 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER, in gfx_v11_0_kiq_init_register()
3958 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, in gfx_v11_0_kiq_init_register()
3962 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, in gfx_v11_0_kiq_init_register()
3964 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, in gfx_v11_0_kiq_init_register()
3968 WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid); in gfx_v11_0_kiq_init_register()
3970 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, in gfx_v11_0_kiq_init_register()
3974 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, in gfx_v11_0_kiq_init_register()
3978 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); in gfx_v11_0_kiq_init_register()
3980 return 0; in gfx_v11_0_kiq_init_register()
3997 ring->wptr = 0; in gfx_v11_0_kiq_init_queue()
4001 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); in gfx_v11_0_kiq_init_queue()
4003 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_kiq_init_queue()
4006 memset((void *)mqd, 0, sizeof(*mqd)); in gfx_v11_0_kiq_init_queue()
4008 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); in gfx_v11_0_kiq_init_queue()
4011 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_kiq_init_queue()
4018 return 0; in gfx_v11_0_kiq_init_queue()
4025 int mqd_idx = ring - &adev->gfx.compute_ring[0]; in gfx_v11_0_kcq_init_queue()
4028 memset((void *)mqd, 0, sizeof(*mqd)); in gfx_v11_0_kcq_init_queue()
4030 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); in gfx_v11_0_kcq_init_queue()
4032 soc21_grbm_select(adev, 0, 0, 0, 0); in gfx_v11_0_kcq_init_queue()
4043 ring->wptr = 0; in gfx_v11_0_kcq_init_queue()
4044 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); in gfx_v11_0_kcq_init_queue()
4050 return 0; in gfx_v11_0_kcq_init_queue()
4061 if (unlikely(r != 0)) in gfx_v11_0_kiq_resume()
4065 if (unlikely(r != 0)) { in gfx_v11_0_kiq_resume()
4075 return 0; in gfx_v11_0_kiq_resume()
4081 int r = 0, i; in gfx_v11_0_kcq_resume()
4086 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v11_0_kcq_resume()
4090 if (unlikely(r != 0)) in gfx_v11_0_kcq_resume()
4158 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v11_0_cp_resume()
4165 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v11_0_cp_resume()
4172 return 0; in gfx_v11_0_cp_resume()
4196 amdgpu_gmc_flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB_0, 0); in gfx_v11_0_gfxhub_enable()
4198 return 0; in gfx_v11_0_gfxhub_enable()
4207 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL); in gfx_v11_0_select_cp_fw_arch()
4209 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp); in gfx_v11_0_select_cp_fw_arch()
4211 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL); in gfx_v11_0_select_cp_fw_arch()
4213 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp); in gfx_v11_0_select_cp_fw_arch()
4224 gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG); in get_gb_addr_config()
4225 if (gb_addr_config == 0) in get_gb_addr_config()
4253 return 0; in get_gb_addr_config()
4260 data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG); in gfx_v11_0_disable_gpa_mode()
4262 WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data); in gfx_v11_0_disable_gpa_mode()
4264 data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG); in gfx_v11_0_disable_gpa_mode()
4266 WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data); in gfx_v11_0_disable_gpa_mode()
4286 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) { in gfx_v11_0_hw_init()
4369 int i, r = 0; in gfx_v11_0_kiq_disable_kgq()
4378 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v11_0_kiq_disable_kgq()
4380 PREEMPT_QUEUES, 0, 0); in gfx_v11_0_kiq_disable_kgq()
4395 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); in gfx_v11_0_hw_fini()
4396 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); in gfx_v11_0_hw_fini()
4415 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS); in gfx_v11_0_hw_fini()
4416 tmp &= 0xffffff00; in gfx_v11_0_hw_fini()
4417 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp); in gfx_v11_0_hw_fini()
4419 return 0; in gfx_v11_0_hw_fini()
4428 return 0; in gfx_v11_0_hw_fini()
4445 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS), in gfx_v11_0_is_idle()
4458 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v11_0_wait_for_idle()
4460 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) & in gfx_v11_0_wait_for_idle()
4464 return 0; in gfx_v11_0_wait_for_idle()
4472 u32 grbm_soft_reset = 0; in gfx_v11_0_soft_reset()
4477 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL); in gfx_v11_0_soft_reset()
4478 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0); in gfx_v11_0_soft_reset()
4479 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0); in gfx_v11_0_soft_reset()
4480 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0); in gfx_v11_0_soft_reset()
4481 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0); in gfx_v11_0_soft_reset()
4482 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp); in gfx_v11_0_soft_reset()
4486 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { in gfx_v11_0_soft_reset()
4487 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { in gfx_v11_0_soft_reset()
4488 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { in gfx_v11_0_soft_reset()
4489 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL); in gfx_v11_0_soft_reset()
4493 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp); in gfx_v11_0_soft_reset()
4495 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2); in gfx_v11_0_soft_reset()
4496 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1); in gfx_v11_0_soft_reset()
4500 for (i = 0; i < adev->gfx.me.num_me; ++i) { in gfx_v11_0_soft_reset()
4501 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) { in gfx_v11_0_soft_reset()
4502 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) { in gfx_v11_0_soft_reset()
4503 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL); in gfx_v11_0_soft_reset()
4507 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp); in gfx_v11_0_soft_reset()
4509 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1); in gfx_v11_0_soft_reset()
4514 WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe); in gfx_v11_0_soft_reset()
4517 // to get sufficient time for GFX_HQD_ACTIVE reach 0 in gfx_v11_0_soft_reset()
4518 RREG32_SOC15(GC, 0, regCP_VMID_RESET); in gfx_v11_0_soft_reset()
4519 RREG32_SOC15(GC, 0, regCP_VMID_RESET); in gfx_v11_0_soft_reset()
4520 RREG32_SOC15(GC, 0, regCP_VMID_RESET); in gfx_v11_0_soft_reset()
4522 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v11_0_soft_reset()
4523 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) && in gfx_v11_0_soft_reset()
4524 !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE)) in gfx_v11_0_soft_reset()
4534 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET); in gfx_v11_0_soft_reset()
4545 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset); in gfx_v11_0_soft_reset()
4547 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET); in gfx_v11_0_soft_reset()
4549 SOFT_RESET_CP, 0); in gfx_v11_0_soft_reset()
4551 SOFT_RESET_GFX, 0); in gfx_v11_0_soft_reset()
4553 SOFT_RESET_CPF, 0); in gfx_v11_0_soft_reset()
4555 SOFT_RESET_CPC, 0); in gfx_v11_0_soft_reset()
4557 SOFT_RESET_CPG, 0); in gfx_v11_0_soft_reset()
4558 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset); in gfx_v11_0_soft_reset()
4560 tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL); in gfx_v11_0_soft_reset()
4561 tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1); in gfx_v11_0_soft_reset()
4562 WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp); in gfx_v11_0_soft_reset()
4564 WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0); in gfx_v11_0_soft_reset()
4565 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0); in gfx_v11_0_soft_reset()
4567 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v11_0_soft_reset()
4568 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET)) in gfx_v11_0_soft_reset()
4573 printk("Failed to wait CP_VMID_RESET to 0\n"); in gfx_v11_0_soft_reset()
4577 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL); in gfx_v11_0_soft_reset()
4582 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp); in gfx_v11_0_soft_reset()
4596 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v11_0_check_soft_reset()
4603 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v11_0_check_soft_reset()
4619 clock = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER) | in gfx_v11_0_get_gpu_clock_counter()
4620 ((uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER) << 32ULL); in gfx_v11_0_get_gpu_clock_counter()
4635 gfx_v11_0_write_data_to_reg(ring, 0, false, in gfx_v11_0_ring_emit_gds_switch()
4636 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid, in gfx_v11_0_ring_emit_gds_switch()
4640 gfx_v11_0_write_data_to_reg(ring, 0, false, in gfx_v11_0_ring_emit_gds_switch()
4641 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid, in gfx_v11_0_ring_emit_gds_switch()
4645 gfx_v11_0_write_data_to_reg(ring, 0, false, in gfx_v11_0_ring_emit_gds_switch()
4646 SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid, in gfx_v11_0_ring_emit_gds_switch()
4650 gfx_v11_0_write_data_to_reg(ring, 0, false, in gfx_v11_0_ring_emit_gds_switch()
4651 SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid, in gfx_v11_0_ring_emit_gds_switch()
4673 return 0; in gfx_v11_0_early_init()
4681 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); in gfx_v11_0_late_init()
4685 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); in gfx_v11_0_late_init()
4689 return 0; in gfx_v11_0_late_init()
4697 rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL); in gfx_v11_0_is_rlc_enabled()
4709 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data); in gfx_v11_0_set_safe_mode()
4712 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v11_0_set_safe_mode()
4713 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE), in gfx_v11_0_set_safe_mode()
4722 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK); in gfx_v11_0_unset_safe_mode()
4733 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); in gfx_v11_0_update_perf_clk()
4741 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); in gfx_v11_0_update_perf_clk()
4752 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); in gfx_v11_0_update_sram_fgcg()
4760 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); in gfx_v11_0_update_sram_fgcg()
4771 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); in gfx_v11_0_update_repeater_fgcg()
4779 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); in gfx_v11_0_update_repeater_fgcg()
4794 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); in gfx_v11_0_update_medium_grain_clock_gating()
4801 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); in gfx_v11_0_update_medium_grain_clock_gating()
4805 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); in gfx_v11_0_update_medium_grain_clock_gating()
4812 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); in gfx_v11_0_update_medium_grain_clock_gating()
4830 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); in gfx_v11_0_update_coarse_grain_clock_gating()
4843 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); in gfx_v11_0_update_coarse_grain_clock_gating()
4845 /* enable cgcg FSM(0x0000363F) */ in gfx_v11_0_update_coarse_grain_clock_gating()
4846 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); in gfx_v11_0_update_coarse_grain_clock_gating()
4850 data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | in gfx_v11_0_update_coarse_grain_clock_gating()
4856 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | in gfx_v11_0_update_coarse_grain_clock_gating()
4861 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data); in gfx_v11_0_update_coarse_grain_clock_gating()
4864 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); in gfx_v11_0_update_coarse_grain_clock_gating()
4868 data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | in gfx_v11_0_update_coarse_grain_clock_gating()
4874 data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | in gfx_v11_0_update_coarse_grain_clock_gating()
4879 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data); in gfx_v11_0_update_coarse_grain_clock_gating()
4881 /* set IDLE_POLL_COUNT(0x00900100) */ in gfx_v11_0_update_coarse_grain_clock_gating()
4882 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL); in gfx_v11_0_update_coarse_grain_clock_gating()
4885 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | in gfx_v11_0_update_coarse_grain_clock_gating()
4886 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); in gfx_v11_0_update_coarse_grain_clock_gating()
4889 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data); in gfx_v11_0_update_coarse_grain_clock_gating()
4891 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL); in gfx_v11_0_update_coarse_grain_clock_gating()
4896 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data); in gfx_v11_0_update_coarse_grain_clock_gating()
4898 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL); in gfx_v11_0_update_coarse_grain_clock_gating()
4900 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data); in gfx_v11_0_update_coarse_grain_clock_gating()
4904 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL); in gfx_v11_0_update_coarse_grain_clock_gating()
4906 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data); in gfx_v11_0_update_coarse_grain_clock_gating()
4910 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); in gfx_v11_0_update_coarse_grain_clock_gating()
4919 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data); in gfx_v11_0_update_coarse_grain_clock_gating()
4922 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); in gfx_v11_0_update_coarse_grain_clock_gating()
4930 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data); in gfx_v11_0_update_coarse_grain_clock_gating()
4932 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL); in gfx_v11_0_update_coarse_grain_clock_gating()
4934 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data); in gfx_v11_0_update_coarse_grain_clock_gating()
4938 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL); in gfx_v11_0_update_coarse_grain_clock_gating()
4940 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data); in gfx_v11_0_update_coarse_grain_clock_gating()
4970 return 0; in gfx_v11_0_update_gfx_clock_gating()
4979 reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL); in gfx_v11_0_update_spm_vmid()
4989 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data); in gfx_v11_0_update_spm_vmid()
4991 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data); in gfx_v11_0_update_spm_vmid()
5012 u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL); in gfx_v11_cntl_power_gating()
5019 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data); in gfx_v11_cntl_power_gating()
5023 switch (adev->ip_versions[GC_HWIP][0]) { in gfx_v11_cntl_power_gating()
5024 case IP_VERSION(11, 0, 1): in gfx_v11_cntl_power_gating()
5025 WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1); in gfx_v11_cntl_power_gating()
5049 return 0; in gfx_v11_0_set_powergating_state()
5051 switch (adev->ip_versions[GC_HWIP][0]) { in gfx_v11_0_set_powergating_state()
5052 case IP_VERSION(11, 0, 0): in gfx_v11_0_set_powergating_state()
5053 case IP_VERSION(11, 0, 2): in gfx_v11_0_set_powergating_state()
5054 case IP_VERSION(11, 0, 3): in gfx_v11_0_set_powergating_state()
5057 case IP_VERSION(11, 0, 1): in gfx_v11_0_set_powergating_state()
5065 return 0; in gfx_v11_0_set_powergating_state()
5074 return 0; in gfx_v11_0_set_clockgating_state()
5076 switch (adev->ip_versions[GC_HWIP][0]) { in gfx_v11_0_set_clockgating_state()
5077 case IP_VERSION(11, 0, 0): in gfx_v11_0_set_clockgating_state()
5078 case IP_VERSION(11, 0, 1): in gfx_v11_0_set_clockgating_state()
5079 case IP_VERSION(11, 0, 2): in gfx_v11_0_set_clockgating_state()
5080 case IP_VERSION(11, 0, 3): in gfx_v11_0_set_clockgating_state()
5088 return 0; in gfx_v11_0_set_clockgating_state()
5097 data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); in gfx_v11_0_get_clockgating_state()
5114 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); in gfx_v11_0_get_clockgating_state()
5123 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); in gfx_v11_0_get_clockgating_state()
5147 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR); in gfx_v11_0_ring_get_wptr_gfx()
5148 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32; in gfx_v11_0_ring_get_wptr_gfx()
5191 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, in gfx_v11_0_ring_set_wptr_gfx()
5193 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, in gfx_v11_0_ring_set_wptr_gfx()
5276 reg_mem_engine = 0; in gfx_v11_0_ring_emit_hdp_flush()
5282 gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, in gfx_v11_0_ring_emit_hdp_flush()
5285 ref_and_mask, ref_and_mask, 0x20); in gfx_v11_0_ring_emit_hdp_flush()
5294 u32 header, control = 0; in gfx_v11_0_ring_emit_ib_gfx()
5315 control |= 0x400000; in gfx_v11_0_ring_emit_ib_gfx()
5318 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ in gfx_v11_0_ring_emit_ib_gfx()
5321 (2 << 0) | in gfx_v11_0_ring_emit_ib_gfx()
5338 control |= 0x40000000; in gfx_v11_0_ring_emit_ib_compute()
5348 * GDS to 0 for this ring (me/pipe). in gfx_v11_0_ring_emit_ib_compute()
5357 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ in gfx_v11_0_ring_emit_ib_compute()
5360 (2 << 0) | in gfx_v11_0_ring_emit_ib_compute()
5387 PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0))); in gfx_v11_0_ring_emit_fence()
5394 BUG_ON(addr & 0x7); in gfx_v11_0_ring_emit_fence()
5396 BUG_ON(addr & 0x3); in gfx_v11_0_ring_emit_fence()
5402 (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0); in gfx_v11_0_ring_emit_fence()
5411 gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr), in gfx_v11_0_ring_emit_pipeline_sync()
5412 upper_32_bits(addr), seq, 0xffffffff, 4); in gfx_v11_0_ring_emit_pipeline_sync()
5419 amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); in gfx_v11_0_ring_invalidate_tlbs()
5431 gfx_v11_0_ring_invalidate_tlbs(ring, 0, 0, false, 0); in gfx_v11_0_ring_emit_vm_flush()
5438 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); in gfx_v11_0_ring_emit_vm_flush()
5439 amdgpu_ring_write(ring, 0x0); in gfx_v11_0_ring_emit_vm_flush()
5453 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | in gfx_v11_0_ring_emit_fence_kiq()
5462 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | in gfx_v11_0_ring_emit_fence_kiq()
5463 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); in gfx_v11_0_ring_emit_fence_kiq()
5464 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS)); in gfx_v11_0_ring_emit_fence_kiq()
5465 amdgpu_ring_write(ring, 0); in gfx_v11_0_ring_emit_fence_kiq()
5466 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ in gfx_v11_0_ring_emit_fence_kiq()
5473 uint32_t dw2 = 0; in gfx_v11_0_ring_emit_cntxcntl()
5475 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ in gfx_v11_0_ring_emit_cntxcntl()
5478 dw2 |= 0x8001; in gfx_v11_0_ring_emit_cntxcntl()
5480 dw2 |= 0x01000000; in gfx_v11_0_ring_emit_cntxcntl()
5482 dw2 |= 0x10002; in gfx_v11_0_ring_emit_cntxcntl()
5487 amdgpu_ring_write(ring, 0); in gfx_v11_0_ring_emit_cntxcntl()
5497 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ in gfx_v11_0_ring_emit_init_cond_exec()
5499 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ in gfx_v11_0_ring_emit_init_cond_exec()
5508 BUG_ON(ring->ring[offset] != 0x55aa55aa); in gfx_v11_0_ring_emit_patch_cond_exec()
5519 int i, r = 0; in gfx_v11_0_ring_preempt_ib()
5547 for (i = 0; i < adev->usec_timeout; i++) { in gfx_v11_0_ring_preempt_ib()
5567 struct v10_de_ib_state de_payload = {0}; in gfx_v11_0_ring_emit_de_meta()
5574 gfx[0].gfx_meta_data) + in gfx_v11_0_ring_emit_de_meta()
5582 gfx[0].gds_backup) + in gfx_v11_0_ring_emit_de_meta()
5603 WRITE_DATA_CACHE_POLICY(0)); in gfx_v11_0_ring_emit_de_meta()
5618 uint32_t v = secure ? FRAME_TMZ : 0; in gfx_v11_0_ring_emit_frame_cntl()
5620 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); in gfx_v11_0_ring_emit_frame_cntl()
5621 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); in gfx_v11_0_ring_emit_frame_cntl()
5630 amdgpu_ring_write(ring, 0 | /* src: register*/ in gfx_v11_0_ring_emit_rreg()
5634 amdgpu_ring_write(ring, 0); in gfx_v11_0_ring_emit_rreg()
5644 uint32_t cmd = 0; in gfx_v11_0_ring_emit_wreg()
5660 amdgpu_ring_write(ring, 0); in gfx_v11_0_ring_emit_wreg()
5667 gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); in gfx_v11_0_ring_emit_reg_wait()
5676 gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, in gfx_v11_0_ring_emit_reg_write_reg_wait()
5677 ref, mask, 0x20); in gfx_v11_0_ring_emit_reg_write_reg_wait()
5684 uint32_t value = 0; in gfx_v11_0_ring_soft_recovery()
5686 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); in gfx_v11_0_ring_soft_recovery()
5687 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); in gfx_v11_0_ring_soft_recovery()
5690 WREG32_SOC15(GC, 0, regSQ_CMD, value); in gfx_v11_0_ring_soft_recovery()
5702 case 0: in gfx_v11_0_set_gfx_eop_interrupt_state()
5703 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0); in gfx_v11_0_set_gfx_eop_interrupt_state()
5706 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1); in gfx_v11_0_set_gfx_eop_interrupt_state()
5721 TIME_STAMP_INT_ENABLE, 0); in gfx_v11_0_set_gfx_eop_interrupt_state()
5723 GENERIC0_INT_ENABLE, 0); in gfx_v11_0_set_gfx_eop_interrupt_state()
5753 case 0: in gfx_v11_0_set_compute_eop_interrupt_state()
5754 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); in gfx_v11_0_set_compute_eop_interrupt_state()
5757 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL); in gfx_v11_0_set_compute_eop_interrupt_state()
5760 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL); in gfx_v11_0_set_compute_eop_interrupt_state()
5763 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL); in gfx_v11_0_set_compute_eop_interrupt_state()
5778 TIME_STAMP_INT_ENABLE, 0); in gfx_v11_0_set_compute_eop_interrupt_state()
5780 GENERIC0_INT_ENABLE, 0); in gfx_v11_0_set_compute_eop_interrupt_state()
5803 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state); in gfx_v11_0_set_eop_interrupt_state()
5806 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state); in gfx_v11_0_set_eop_interrupt_state()
5809 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state); in gfx_v11_0_set_eop_interrupt_state()
5823 return 0; in gfx_v11_0_set_eop_interrupt_state()
5833 uint32_t mes_queue_id = entry->src_data[0]; in gfx_v11_0_eop_irq()
5850 me_id = (entry->ring_id & 0x0c) >> 2; in gfx_v11_0_eop_irq()
5851 pipe_id = (entry->ring_id & 0x03) >> 0; in gfx_v11_0_eop_irq()
5852 queue_id = (entry->ring_id & 0x70) >> 4; in gfx_v11_0_eop_irq()
5855 case 0: in gfx_v11_0_eop_irq()
5856 if (pipe_id == 0) in gfx_v11_0_eop_irq()
5857 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); in gfx_v11_0_eop_irq()
5863 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v11_0_eop_irq()
5878 return 0; in gfx_v11_0_eop_irq()
5889 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0, in gfx_v11_0_set_priv_reg_fault_state()
5891 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); in gfx_v11_0_set_priv_reg_fault_state()
5897 return 0; in gfx_v11_0_set_priv_reg_fault_state()
5908 WREG32_FIELD15_PREREG(GC, 0, CP_INT_CNTL_RING0, in gfx_v11_0_set_priv_inst_fault_state()
5910 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); in gfx_v11_0_set_priv_inst_fault_state()
5916 return 0; in gfx_v11_0_set_priv_inst_fault_state()
5926 me_id = (entry->ring_id & 0x0c) >> 2; in gfx_v11_0_handle_priv_fault()
5927 pipe_id = (entry->ring_id & 0x03) >> 0; in gfx_v11_0_handle_priv_fault()
5928 queue_id = (entry->ring_id & 0x70) >> 4; in gfx_v11_0_handle_priv_fault()
5931 case 0: in gfx_v11_0_handle_priv_fault()
5932 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v11_0_handle_priv_fault()
5941 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v11_0_handle_priv_fault()
5960 return 0; in gfx_v11_0_priv_reg_irq()
5969 return 0; in gfx_v11_0_priv_inst_irq()
5972 #if 0
5981 target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
5987 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
5989 GENERIC2_INT_ENABLE, 0);
5990 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
5994 GENERIC2_INT_ENABLE, 0);
5997 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6000 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6012 return 0;
6030 amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */ in gfx_v11_0_emit_mem_sync()
6031 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ in gfx_v11_0_emit_mem_sync()
6032 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ in gfx_v11_0_emit_mem_sync()
6033 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ in gfx_v11_0_emit_mem_sync()
6034 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ in gfx_v11_0_emit_mem_sync()
6035 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ in gfx_v11_0_emit_mem_sync()
6060 .align_mask = 0xff,
6061 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6108 .align_mask = 0xff,
6109 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6144 .align_mask = 0xff,
6145 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6179 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v11_0_set_ring_funcs()
6182 for (i = 0; i < adev->gfx.num_compute_rings; i++) in gfx_v11_0_set_ring_funcs()
6234 adev->gds.gds_size = 0x1000; in gfx_v11_0_set_gds_init()
6265 WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data); in gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh()
6271 data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG); in gfx_v11_0_get_wgp_active_bitmap_per_sh()
6272 data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG); in gfx_v11_0_get_wgp_active_bitmap_per_sh()
6289 cu_active_bitmap = 0; in gfx_v11_0_get_cu_active_bitmap_per_sh()
6291 for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) { in gfx_v11_0_get_cu_active_bitmap_per_sh()
6304 int i, j, k, counter, active_cu_number = 0; in gfx_v11_0_get_cu_info()
6314 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v11_0_get_cu_info()
6315 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v11_0_get_cu_info()
6317 counter = 0; in gfx_v11_0_get_cu_info()
6318 gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff); in gfx_v11_0_get_cu_info()
6330 * SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]} in gfx_v11_0_get_cu_info()
6331 * SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]} in gfx_v11_0_get_cu_info()
6332 * SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]} in gfx_v11_0_get_cu_info()
6333 * SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]} in gfx_v11_0_get_cu_info()
6334 * SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]} in gfx_v11_0_get_cu_info()
6341 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) { in gfx_v11_0_get_cu_info()
6350 gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); in gfx_v11_0_get_cu_info()
6356 return 0; in gfx_v11_0_get_cu_info()
6363 .minor = 0,
6364 .rev = 0,