Lines Matching full:gfx
40 #include "ivsrcid/gfx/irqsrcs_gfx_10_1.h"
3571 if (adev->enable_mes && !adev->gfx.kiq.ring.sched.ready) { in gfx10_kiq_unmap_queues()
3639 adev->gfx.kiq.pmf = &gfx_v10_0_kiq_pm4_funcs; in gfx_v10_0_set_kiq_pm4_funcs()
3894 release_firmware(adev->gfx.pfp_fw); in gfx_v10_0_free_microcode()
3895 adev->gfx.pfp_fw = NULL; in gfx_v10_0_free_microcode()
3896 release_firmware(adev->gfx.me_fw); in gfx_v10_0_free_microcode()
3897 adev->gfx.me_fw = NULL; in gfx_v10_0_free_microcode()
3898 release_firmware(adev->gfx.ce_fw); in gfx_v10_0_free_microcode()
3899 adev->gfx.ce_fw = NULL; in gfx_v10_0_free_microcode()
3900 release_firmware(adev->gfx.rlc_fw); in gfx_v10_0_free_microcode()
3901 adev->gfx.rlc_fw = NULL; in gfx_v10_0_free_microcode()
3902 release_firmware(adev->gfx.mec_fw); in gfx_v10_0_free_microcode()
3903 adev->gfx.mec_fw = NULL; in gfx_v10_0_free_microcode()
3904 release_firmware(adev->gfx.mec2_fw); in gfx_v10_0_free_microcode()
3905 adev->gfx.mec2_fw = NULL; in gfx_v10_0_free_microcode()
3907 kfree(adev->gfx.rlc.register_list_format); in gfx_v10_0_free_microcode()
3912 adev->gfx.cp_fw_write_wait = false; in gfx_v10_0_check_fw_write_wait()
3920 if ((adev->gfx.me_fw_version >= 0x00000046) && in gfx_v10_0_check_fw_write_wait()
3921 (adev->gfx.me_feature_version >= 27) && in gfx_v10_0_check_fw_write_wait()
3922 (adev->gfx.pfp_fw_version >= 0x00000068) && in gfx_v10_0_check_fw_write_wait()
3923 (adev->gfx.pfp_feature_version >= 27) && in gfx_v10_0_check_fw_write_wait()
3924 (adev->gfx.mec_fw_version >= 0x0000005b) && in gfx_v10_0_check_fw_write_wait()
3925 (adev->gfx.mec_feature_version >= 27)) in gfx_v10_0_check_fw_write_wait()
3926 adev->gfx.cp_fw_write_wait = true; in gfx_v10_0_check_fw_write_wait()
3936 adev->gfx.cp_fw_write_wait = true; in gfx_v10_0_check_fw_write_wait()
3942 if (!adev->gfx.cp_fw_write_wait) in gfx_v10_0_check_fw_write_wait()
4033 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); in gfx_v10_0_init_microcode()
4036 err = amdgpu_ucode_validate(adev->gfx.pfp_fw); in gfx_v10_0_init_microcode()
4042 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); in gfx_v10_0_init_microcode()
4045 err = amdgpu_ucode_validate(adev->gfx.me_fw); in gfx_v10_0_init_microcode()
4051 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); in gfx_v10_0_init_microcode()
4054 err = amdgpu_ucode_validate(adev->gfx.ce_fw); in gfx_v10_0_init_microcode()
4061 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); in gfx_v10_0_init_microcode()
4067 err = amdgpu_ucode_validate(adev->gfx.rlc_fw); in gfx_v10_0_init_microcode()
4072 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; in gfx_v10_0_init_microcode()
4081 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); in gfx_v10_0_init_microcode()
4084 err = amdgpu_ucode_validate(adev->gfx.mec_fw); in gfx_v10_0_init_microcode()
4091 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); in gfx_v10_0_init_microcode()
4093 err = amdgpu_ucode_validate(adev->gfx.mec2_fw); in gfx_v10_0_init_microcode()
4100 adev->gfx.mec2_fw = NULL; in gfx_v10_0_init_microcode()
4109 release_firmware(adev->gfx.pfp_fw); in gfx_v10_0_init_microcode()
4110 adev->gfx.pfp_fw = NULL; in gfx_v10_0_init_microcode()
4111 release_firmware(adev->gfx.me_fw); in gfx_v10_0_init_microcode()
4112 adev->gfx.me_fw = NULL; in gfx_v10_0_init_microcode()
4113 release_firmware(adev->gfx.ce_fw); in gfx_v10_0_init_microcode()
4114 adev->gfx.ce_fw = NULL; in gfx_v10_0_init_microcode()
4115 release_firmware(adev->gfx.rlc_fw); in gfx_v10_0_init_microcode()
4116 adev->gfx.rlc_fw = NULL; in gfx_v10_0_init_microcode()
4117 release_firmware(adev->gfx.mec_fw); in gfx_v10_0_init_microcode()
4118 adev->gfx.mec_fw = NULL; in gfx_v10_0_init_microcode()
4119 release_firmware(adev->gfx.mec2_fw); in gfx_v10_0_init_microcode()
4120 adev->gfx.mec2_fw = NULL; in gfx_v10_0_init_microcode()
4166 if (adev->gfx.rlc.cs_data == NULL) in gfx_v10_0_get_csb_buffer()
4178 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { in gfx_v10_0_get_csb_buffer()
4197 buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override); in gfx_v10_0_get_csb_buffer()
4209 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, in gfx_v10_0_rlc_fini()
4210 &adev->gfx.rlc.clear_state_gpu_addr, in gfx_v10_0_rlc_fini()
4211 (void **)&adev->gfx.rlc.cs_ptr); in gfx_v10_0_rlc_fini()
4214 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, in gfx_v10_0_rlc_fini()
4215 &adev->gfx.rlc.cp_table_gpu_addr, in gfx_v10_0_rlc_fini()
4216 (void **)&adev->gfx.rlc.cp_table_ptr); in gfx_v10_0_rlc_fini()
4223 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl; in gfx_v10_0_init_rlcg_reg_access_ctrl()
4240 adev->gfx.rlc.rlcg_reg_access_supported = true; in gfx_v10_0_init_rlcg_reg_access_ctrl()
4248 adev->gfx.rlc.cs_data = gfx10_cs_data; in gfx_v10_0_rlc_init()
4250 cs_data = adev->gfx.rlc.cs_data; in gfx_v10_0_rlc_init()
4260 if (adev->gfx.rlc.funcs->update_spm_vmid) in gfx_v10_0_rlc_init()
4261 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); in gfx_v10_0_rlc_init()
4269 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); in gfx_v10_0_mec_fini()
4270 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); in gfx_v10_0_mec_fini()
4277 bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES); in gfx_v10_0_me_init()
4283 DRM_ERROR("Failed to load gfx firmware!\n"); in gfx_v10_0_me_init()
4299 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); in gfx_v10_0_mec_init()
4303 mec_hpd_size = adev->gfx.num_compute_rings * GFX10_MEC_HPD_SIZE; in gfx_v10_0_mec_init()
4308 &adev->gfx.mec.hpd_eop_obj, in gfx_v10_0_mec_init()
4309 &adev->gfx.mec.hpd_eop_gpu_addr, in gfx_v10_0_mec_init()
4319 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); in gfx_v10_0_mec_init()
4320 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); in gfx_v10_0_mec_init()
4324 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; in gfx_v10_0_mec_init()
4326 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + in gfx_v10_0_mec_init()
4332 &adev->gfx.mec.mec_fw_obj, in gfx_v10_0_mec_init()
4333 &adev->gfx.mec.mec_fw_gpu_addr, in gfx_v10_0_mec_init()
4343 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); in gfx_v10_0_mec_init()
4344 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); in gfx_v10_0_mec_init()
4456 adev->gfx.funcs = &gfx_v10_0_gfx_funcs; in gfx_v10_0_gpu_early_init()
4462 adev->gfx.config.max_hw_contexts = 8; in gfx_v10_0_gpu_early_init()
4463 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v10_0_gpu_early_init()
4464 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v10_0_gpu_early_init()
4465 adev->gfx.config.sc_hiz_tile_fifo_size = 0; in gfx_v10_0_gpu_early_init()
4466 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v10_0_gpu_early_init()
4477 adev->gfx.config.max_hw_contexts = 8; in gfx_v10_0_gpu_early_init()
4478 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v10_0_gpu_early_init()
4479 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v10_0_gpu_early_init()
4480 adev->gfx.config.sc_hiz_tile_fifo_size = 0; in gfx_v10_0_gpu_early_init()
4481 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v10_0_gpu_early_init()
4483 adev->gfx.config.gb_addr_config_fields.num_pkrs = in gfx_v10_0_gpu_early_init()
4488 adev->gfx.config.max_hw_contexts = 8; in gfx_v10_0_gpu_early_init()
4489 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v10_0_gpu_early_init()
4490 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v10_0_gpu_early_init()
4491 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v10_0_gpu_early_init()
4492 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v10_0_gpu_early_init()
4500 adev->gfx.config.gb_addr_config = gb_addr_config; in gfx_v10_0_gpu_early_init()
4502 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << in gfx_v10_0_gpu_early_init()
4503 REG_GET_FIELD(adev->gfx.config.gb_addr_config, in gfx_v10_0_gpu_early_init()
4506 adev->gfx.config.max_tile_pipes = in gfx_v10_0_gpu_early_init()
4507 adev->gfx.config.gb_addr_config_fields.num_pipes; in gfx_v10_0_gpu_early_init()
4509 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << in gfx_v10_0_gpu_early_init()
4510 REG_GET_FIELD(adev->gfx.config.gb_addr_config, in gfx_v10_0_gpu_early_init()
4512 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << in gfx_v10_0_gpu_early_init()
4513 REG_GET_FIELD(adev->gfx.config.gb_addr_config, in gfx_v10_0_gpu_early_init()
4515 adev->gfx.config.gb_addr_config_fields.num_se = 1 << in gfx_v10_0_gpu_early_init()
4516 REG_GET_FIELD(adev->gfx.config.gb_addr_config, in gfx_v10_0_gpu_early_init()
4518 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + in gfx_v10_0_gpu_early_init()
4519 REG_GET_FIELD(adev->gfx.config.gb_addr_config, in gfx_v10_0_gpu_early_init()
4530 ring = &adev->gfx.gfx_ring[ring_id]; in gfx_v10_0_gfx_ring_init()
4548 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, in gfx_v10_0_gfx_ring_init()
4559 ring = &adev->gfx.compute_ring[ring_id]; in gfx_v10_0_compute_ring_init()
4569 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr in gfx_v10_0_compute_ring_init()
4574 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) in gfx_v10_0_compute_ring_init()
4579 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, in gfx_v10_0_compute_ring_init()
4595 adev->gfx.me.num_me = 1; in gfx_v10_0_sw_init()
4596 adev->gfx.me.num_pipe_per_me = 1; in gfx_v10_0_sw_init()
4597 adev->gfx.me.num_queue_per_pipe = 1; in gfx_v10_0_sw_init()
4598 adev->gfx.mec.num_mec = 2; in gfx_v10_0_sw_init()
4599 adev->gfx.mec.num_pipe_per_mec = 4; in gfx_v10_0_sw_init()
4600 adev->gfx.mec.num_queue_per_pipe = 8; in gfx_v10_0_sw_init()
4610 adev->gfx.me.num_me = 1; in gfx_v10_0_sw_init()
4611 adev->gfx.me.num_pipe_per_me = 1; in gfx_v10_0_sw_init()
4612 adev->gfx.me.num_queue_per_pipe = 1; in gfx_v10_0_sw_init()
4613 adev->gfx.mec.num_mec = 2; in gfx_v10_0_sw_init()
4614 adev->gfx.mec.num_pipe_per_mec = 4; in gfx_v10_0_sw_init()
4615 adev->gfx.mec.num_queue_per_pipe = 4; in gfx_v10_0_sw_init()
4618 adev->gfx.me.num_me = 1; in gfx_v10_0_sw_init()
4619 adev->gfx.me.num_pipe_per_me = 1; in gfx_v10_0_sw_init()
4620 adev->gfx.me.num_queue_per_pipe = 1; in gfx_v10_0_sw_init()
4621 adev->gfx.mec.num_mec = 1; in gfx_v10_0_sw_init()
4622 adev->gfx.mec.num_pipe_per_mec = 4; in gfx_v10_0_sw_init()
4623 adev->gfx.mec.num_queue_per_pipe = 8; in gfx_v10_0_sw_init()
4630 &adev->gfx.kiq.irq); in gfx_v10_0_sw_init()
4637 &adev->gfx.eop_irq); in gfx_v10_0_sw_init()
4643 &adev->gfx.priv_reg_irq); in gfx_v10_0_sw_init()
4649 &adev->gfx.priv_inst_irq); in gfx_v10_0_sw_init()
4653 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; in gfx_v10_0_sw_init()
4659 if (adev->gfx.rlc.funcs) { in gfx_v10_0_sw_init()
4660 if (adev->gfx.rlc.funcs->init) { in gfx_v10_0_sw_init()
4661 r = adev->gfx.rlc.funcs->init(adev); in gfx_v10_0_sw_init()
4675 /* set up the gfx ring */ in gfx_v10_0_sw_init()
4676 for (i = 0; i < adev->gfx.me.num_me; i++) { in gfx_v10_0_sw_init()
4677 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) { in gfx_v10_0_sw_init()
4678 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) { in gfx_v10_0_sw_init()
4693 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { in gfx_v10_0_sw_init()
4694 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { in gfx_v10_0_sw_init()
4695 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { in gfx_v10_0_sw_init()
4717 kiq = &adev->gfx.kiq; in gfx_v10_0_sw_init()
4734 adev->gfx.ce_ram_size = F32_CE_PROGRAM_RAM_SIZE; in gfx_v10_0_sw_init()
4743 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj, in gfx_v10_0_pfp_fini()
4744 &adev->gfx.pfp.pfp_fw_gpu_addr, in gfx_v10_0_pfp_fini()
4745 (void **)&adev->gfx.pfp.pfp_fw_ptr); in gfx_v10_0_pfp_fini()
4750 amdgpu_bo_free_kernel(&adev->gfx.ce.ce_fw_obj, in gfx_v10_0_ce_fini()
4751 &adev->gfx.ce.ce_fw_gpu_addr, in gfx_v10_0_ce_fini()
4752 (void **)&adev->gfx.ce.ce_fw_ptr); in gfx_v10_0_ce_fini()
4757 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj, in gfx_v10_0_me_fini()
4758 &adev->gfx.me.me_fw_gpu_addr, in gfx_v10_0_me_fini()
4759 (void **)&adev->gfx.me.me_fw_ptr); in gfx_v10_0_me_fini()
4767 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v10_0_sw_fini()
4768 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); in gfx_v10_0_sw_fini()
4769 for (i = 0; i < adev->gfx.num_compute_rings; i++) in gfx_v10_0_sw_fini()
4770 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); in gfx_v10_0_sw_fini()
4775 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); in gfx_v10_0_sw_fini()
4830 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / in gfx_v10_0_get_rb_active_bitmap()
4831 adev->gfx.config.max_sh_per_se); in gfx_v10_0_get_rb_active_bitmap()
4842 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / in gfx_v10_0_setup_rb()
4843 adev->gfx.config.max_sh_per_se; in gfx_v10_0_setup_rb()
4846 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v10_0_setup_rb()
4847 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v10_0_setup_rb()
4848 bitmap = i * adev->gfx.config.max_sh_per_se + j; in gfx_v10_0_setup_rb()
4856 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * in gfx_v10_0_setup_rb()
4863 adev->gfx.config.backend_enable_mask = active_rbs; in gfx_v10_0_setup_rb()
4864 adev->gfx.config.num_rbs = hweight32(active_rbs); in gfx_v10_0_setup_rb()
4876 /* for ASICs that integrates GFX v10.3 in gfx_v10_0_init_pa_sc_tile_steering_override()
4882 num_sc = adev->gfx.config.max_shader_engines * adev->gfx.config.max_sh_per_se * in gfx_v10_0_init_pa_sc_tile_steering_override()
4883 adev->gfx.config.num_sc_per_sh; in gfx_v10_0_init_pa_sc_tile_steering_override()
4887 num_rb_per_sc = enabled_rb_per_sh / adev->gfx.config.num_sc_per_sh; in gfx_v10_0_init_pa_sc_tile_steering_override()
4889 num_packer_per_sc = adev->gfx.config.num_packer_per_sc; in gfx_v10_0_init_pa_sc_tile_steering_override()
4945 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA in gfx_v10_0_init_gds_vmid()
4962 int max_wgp_per_sh = adev->gfx.config.max_cu_per_sh >> 1; in gfx_v10_0_tcp_harvest()
4987 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v10_0_tcp_harvest()
4988 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v10_0_tcp_harvest()
5039 adev->gfx.config.tcc_disabled_mask = in gfx_v10_0_get_tcc_info()
5052 gfx_v10_0_get_cu_info(adev, &adev->gfx.cu_info); in gfx_v10_0_constants_init()
5054 adev->gfx.config.pa_sc_tile_steering_override = in gfx_v10_0_constants_init()
5105 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); in gfx_v10_0_init_csb()
5110 adev->gfx.rlc.clear_state_gpu_addr >> 32); in gfx_v10_0_init_csb()
5112 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); in gfx_v10_0_init_csb()
5113 WREG32_SOC15_RLC(GC, 0, mmRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size); in gfx_v10_0_init_csb()
5116 adev->gfx.rlc.clear_state_gpu_addr >> 32); in gfx_v10_0_init_csb()
5118 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); in gfx_v10_0_init_csb()
5119 WREG32_SOC15(GC, 0, mmRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size); in gfx_v10_0_init_csb()
5190 if (!adev->gfx.rlc_fw) in gfx_v10_0_rlc_load_microcode()
5193 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; in gfx_v10_0_rlc_load_microcode()
5196 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + in gfx_v10_0_rlc_load_microcode()
5207 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); in gfx_v10_0_rlc_load_microcode()
5233 adev->gfx.rlc.funcs->stop(adev); in gfx_v10_0_rlc_resume()
5255 adev->gfx.rlc.funcs->start(adev); in gfx_v10_0_rlc_resume()
5279 &adev->gfx.rlc.rlc_toc_bo, in gfx_v10_0_parse_rlc_toc()
5280 &adev->gfx.rlc.rlc_toc_gpu_addr, in gfx_v10_0_parse_rlc_toc()
5281 (void **)&adev->gfx.rlc.rlc_toc_buf); in gfx_v10_0_parse_rlc_toc()
5288 memcpy(adev->gfx.rlc.rlc_toc_buf, adev->psp.toc.start_addr, adev->psp.toc.size_bytes); in gfx_v10_0_parse_rlc_toc()
5290 rlc_toc = (RLC_TABLE_OF_CONTENT *)adev->gfx.rlc.rlc_toc_buf; in gfx_v10_0_parse_rlc_toc()
5341 &adev->gfx.rlc.rlc_autoload_bo, in gfx_v10_0_rlc_backdoor_autoload_buffer_init()
5342 &adev->gfx.rlc.rlc_autoload_gpu_addr, in gfx_v10_0_rlc_backdoor_autoload_buffer_init()
5343 (void **)&adev->gfx.rlc.rlc_autoload_ptr); in gfx_v10_0_rlc_backdoor_autoload_buffer_init()
5354 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_toc_bo, in gfx_v10_0_rlc_backdoor_autoload_buffer_fini()
5355 &adev->gfx.rlc.rlc_toc_gpu_addr, in gfx_v10_0_rlc_backdoor_autoload_buffer_fini()
5356 (void **)&adev->gfx.rlc.rlc_toc_buf); in gfx_v10_0_rlc_backdoor_autoload_buffer_fini()
5357 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo, in gfx_v10_0_rlc_backdoor_autoload_buffer_fini()
5358 &adev->gfx.rlc.rlc_autoload_gpu_addr, in gfx_v10_0_rlc_backdoor_autoload_buffer_fini()
5359 (void **)&adev->gfx.rlc.rlc_autoload_ptr); in gfx_v10_0_rlc_backdoor_autoload_buffer_fini()
5369 char *ptr = adev->gfx.rlc.rlc_autoload_ptr; in gfx_v10_0_rlc_backdoor_autoload_copy_ucode()
5394 data = adev->gfx.rlc.rlc_toc_buf; in gfx_v10_0_rlc_backdoor_autoload_copy_toc_ucode()
5411 adev->gfx.pfp_fw->data; in gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode()
5412 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + in gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode()
5421 adev->gfx.ce_fw->data; in gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode()
5422 fw_data = (const __le32 *)(adev->gfx.ce_fw->data + in gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode()
5431 adev->gfx.me_fw->data; in gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode()
5432 fw_data = (const __le32 *)(adev->gfx.me_fw->data + in gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode()
5441 adev->gfx.rlc_fw->data; in gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode()
5442 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + in gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode()
5451 adev->gfx.mec_fw->data; in gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode()
5452 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + in gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode()
5508 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset; in gfx_v10_0_rlc_backdoor_autoload_enable()
5557 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + in gfx_v10_0_rlc_backdoor_autoload_config_me_cache()
5594 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + in gfx_v10_0_rlc_backdoor_autoload_config_ce_cache()
5631 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + in gfx_v10_0_rlc_backdoor_autoload_config_pfp_cache()
5668 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + in gfx_v10_0_rlc_backdoor_autoload_config_mec_cache()
5746 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt"); in gfx_v10_0_cp_gfx_enable()
5761 adev->gfx.pfp_fw->data; in gfx_v10_0_cp_gfx_load_pfp_microcode()
5765 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + in gfx_v10_0_cp_gfx_load_pfp_microcode()
5771 &adev->gfx.pfp.pfp_fw_obj, in gfx_v10_0_cp_gfx_load_pfp_microcode()
5772 &adev->gfx.pfp.pfp_fw_gpu_addr, in gfx_v10_0_cp_gfx_load_pfp_microcode()
5773 (void **)&adev->gfx.pfp.pfp_fw_ptr); in gfx_v10_0_cp_gfx_load_pfp_microcode()
5780 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size); in gfx_v10_0_cp_gfx_load_pfp_microcode()
5782 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj); in gfx_v10_0_cp_gfx_load_pfp_microcode()
5783 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj); in gfx_v10_0_cp_gfx_load_pfp_microcode()
5814 adev->gfx.pfp.pfp_fw_gpu_addr & 0xFFFFF000); in gfx_v10_0_cp_gfx_load_pfp_microcode()
5816 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr)); in gfx_v10_0_cp_gfx_load_pfp_microcode()
5824 WREG32_SOC15(GC, 0, mmCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); in gfx_v10_0_cp_gfx_load_pfp_microcode()
5839 adev->gfx.ce_fw->data; in gfx_v10_0_cp_gfx_load_ce_microcode()
5843 fw_data = (const __le32 *)(adev->gfx.ce_fw->data + in gfx_v10_0_cp_gfx_load_ce_microcode()
5849 &adev->gfx.ce.ce_fw_obj, in gfx_v10_0_cp_gfx_load_ce_microcode()
5850 &adev->gfx.ce.ce_fw_gpu_addr, in gfx_v10_0_cp_gfx_load_ce_microcode()
5851 (void **)&adev->gfx.ce.ce_fw_ptr); in gfx_v10_0_cp_gfx_load_ce_microcode()
5858 memcpy(adev->gfx.ce.ce_fw_ptr, fw_data, fw_size); in gfx_v10_0_cp_gfx_load_ce_microcode()
5860 amdgpu_bo_kunmap(adev->gfx.ce.ce_fw_obj); in gfx_v10_0_cp_gfx_load_ce_microcode()
5861 amdgpu_bo_unreserve(adev->gfx.ce.ce_fw_obj); in gfx_v10_0_cp_gfx_load_ce_microcode()
5891 adev->gfx.ce.ce_fw_gpu_addr & 0xFFFFF000); in gfx_v10_0_cp_gfx_load_ce_microcode()
5893 upper_32_bits(adev->gfx.ce.ce_fw_gpu_addr)); in gfx_v10_0_cp_gfx_load_ce_microcode()
5901 WREG32_SOC15(GC, 0, mmCP_HYP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); in gfx_v10_0_cp_gfx_load_ce_microcode()
5916 adev->gfx.me_fw->data; in gfx_v10_0_cp_gfx_load_me_microcode()
5920 fw_data = (const __le32 *)(adev->gfx.me_fw->data + in gfx_v10_0_cp_gfx_load_me_microcode()
5926 &adev->gfx.me.me_fw_obj, in gfx_v10_0_cp_gfx_load_me_microcode()
5927 &adev->gfx.me.me_fw_gpu_addr, in gfx_v10_0_cp_gfx_load_me_microcode()
5928 (void **)&adev->gfx.me.me_fw_ptr); in gfx_v10_0_cp_gfx_load_me_microcode()
5935 memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size); in gfx_v10_0_cp_gfx_load_me_microcode()
5937 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj); in gfx_v10_0_cp_gfx_load_me_microcode()
5938 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj); in gfx_v10_0_cp_gfx_load_me_microcode()
5968 adev->gfx.me.me_fw_gpu_addr & 0xFFFFF000); in gfx_v10_0_cp_gfx_load_me_microcode()
5970 upper_32_bits(adev->gfx.me.me_fw_gpu_addr)); in gfx_v10_0_cp_gfx_load_me_microcode()
5978 WREG32_SOC15(GC, 0, mmCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version); in gfx_v10_0_cp_gfx_load_me_microcode()
5987 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) in gfx_v10_0_cp_gfx_load_microcode()
6023 adev->gfx.config.max_hw_contexts - 1); in gfx_v10_0_cp_gfx_start()
6028 ring = &adev->gfx.gfx_ring[0]; in gfx_v10_0_cp_gfx_start()
6060 amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override); in gfx_v10_0_cp_gfx_start()
6076 if (adev->gfx.num_gfx_rings > 1) { in gfx_v10_0_cp_gfx_start()
6077 /* maximum supported gfx ring is 2 */ in gfx_v10_0_cp_gfx_start()
6078 ring = &adev->gfx.gfx_ring[1]; in gfx_v10_0_cp_gfx_start()
6163 /* Init gfx ring 0 for pipe 0 */ in gfx_v10_0_cp_gfx_resume()
6168 ring = &adev->gfx.gfx_ring[0]; in gfx_v10_0_cp_gfx_resume()
6206 /* Init gfx ring 1 for pipe 1 */ in gfx_v10_0_cp_gfx_resume()
6207 if (adev->gfx.num_gfx_rings > 1) { in gfx_v10_0_cp_gfx_resume()
6210 /* maximum supported gfx ring is 2 */ in gfx_v10_0_cp_gfx_resume()
6211 ring = &adev->gfx.gfx_ring[1]; in gfx_v10_0_cp_gfx_resume()
6250 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v10_0_cp_gfx_resume()
6251 ring = &adev->gfx.gfx_ring[i]; in gfx_v10_0_cp_gfx_resume()
6296 adev->gfx.kiq.ring.sched.ready = false; in gfx_v10_0_cp_compute_enable()
6309 if (!adev->gfx.mec_fw) in gfx_v10_0_cp_compute_load_microcode()
6314 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; in gfx_v10_0_cp_compute_load_microcode()
6318 (adev->gfx.mec_fw->data + in gfx_v10_0_cp_compute_load_microcode()
6349 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr & in gfx_v10_0_cp_compute_load_microcode()
6352 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); in gfx_v10_0_cp_compute_load_microcode()
6361 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version); in gfx_v10_0_cp_compute_load_microcode()
6430 /* set up gfx hqd wptr */ in gfx_v10_0_gfx_mqd_init()
6450 /* set up gfx queue priority */ in gfx_v10_0_gfx_mqd_init()
6458 /* set up gfx hqd base. this is similar as CP_RB_BASE */ in gfx_v10_0_gfx_mqd_init()
6558 int mqd_idx = ring - &adev->gfx.gfx_ring[0]; in gfx_v10_0_gfx_init_queue()
6567 * if there are 2 gfx rings, set the lower doorbell in gfx_v10_0_gfx_init_queue()
6579 if (adev->gfx.me.mqd_backup[mqd_idx]) in gfx_v10_0_gfx_init_queue()
6580 memcpy(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); in gfx_v10_0_gfx_init_queue()
6583 if (adev->gfx.me.mqd_backup[mqd_idx]) in gfx_v10_0_gfx_init_queue()
6584 memcpy(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd)); in gfx_v10_0_gfx_init_queue()
6606 struct amdgpu_kiq *kiq = &adev->gfx.kiq; in gfx_v10_0_kiq_enable_kgq()
6607 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; in gfx_v10_0_kiq_enable_kgq()
6614 adev->gfx.num_gfx_rings); in gfx_v10_0_kiq_enable_kgq()
6620 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v10_0_kiq_enable_kgq()
6621 kiq->pmf->kiq_map_queues(kiq_ring, &adev->gfx.gfx_ring[i]); in gfx_v10_0_kiq_enable_kgq()
6632 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v10_0_cp_async_gfx_ring_resume()
6633 ring = &adev->gfx.gfx_ring[i]; in gfx_v10_0_cp_async_gfx_ring_resume()
6658 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v10_0_cp_async_gfx_ring_resume()
6659 ring = &adev->gfx.gfx_ring[i]; in gfx_v10_0_cp_async_gfx_ring_resume()
6900 if (adev->gfx.mec.mqd_backup[mqd_idx]) in gfx_v10_0_kiq_init_queue()
6901 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd)); in gfx_v10_0_kiq_init_queue()
6921 if (adev->gfx.mec.mqd_backup[mqd_idx]) in gfx_v10_0_kiq_init_queue()
6922 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); in gfx_v10_0_kiq_init_queue()
6932 int mqd_idx = ring - &adev->gfx.compute_ring[0]; in gfx_v10_0_kcq_init_queue()
6942 if (adev->gfx.mec.mqd_backup[mqd_idx]) in gfx_v10_0_kcq_init_queue()
6943 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); in gfx_v10_0_kcq_init_queue()
6946 if (adev->gfx.mec.mqd_backup[mqd_idx]) in gfx_v10_0_kcq_init_queue()
6947 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd)); in gfx_v10_0_kcq_init_queue()
6965 ring = &adev->gfx.kiq.ring; in gfx_v10_0_kiq_resume()
6990 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v10_0_kcq_resume()
6991 ring = &adev->gfx.compute_ring[i]; in gfx_v10_0_kcq_resume()
7052 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v10_0_cp_resume()
7053 ring = &adev->gfx.gfx_ring[i]; in gfx_v10_0_cp_resume()
7059 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v10_0_cp_resume()
7060 ring = &adev->gfx.compute_ring[i]; in gfx_v10_0_cp_resume()
7276 * For gfx 10, rlc firmware loading relies on smu firmware is in gfx_v10_0_hw_init()
7323 struct amdgpu_kiq *kiq = &adev->gfx.kiq; in gfx_v10_0_kiq_disable_kgq()
7331 adev->gfx.num_gfx_rings)) in gfx_v10_0_kiq_disable_kgq()
7334 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v10_0_kiq_disable_kgq()
7335 kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.gfx_ring[i], in gfx_v10_0_kiq_disable_kgq()
7350 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); in gfx_v10_0_hw_fini()
7351 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); in gfx_v10_0_hw_fini()
7481 /* Disable GFX parsing/prefetching */ in gfx_v10_0_soft_reset()
7602 adev->gfx.num_gfx_rings = GFX10_NUM_GFX_RINGS_NV1X; in gfx_v10_0_early_init()
7612 adev->gfx.num_gfx_rings = GFX10_NUM_GFX_RINGS_Sienna_Cichlid; in gfx_v10_0_early_init()
7618 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), in gfx_v10_0_early_init()
7639 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); in gfx_v10_0_late_init()
7643 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); in gfx_v10_0_late_init()
7750 /* MGLS is a global flag to control all MGLS in GFX */ in gfx_v10_0_update_medium_grain_clock_gating()
8056 /* === CGCG /CGLS for GFX 3D Only === */ in gfx_v10_0_update_gfx_clock_gating()
8070 /* === CGCG /CGLS for GFX 3D Only === */ in gfx_v10_0_update_gfx_clock_gating()
8677 /* set load_per_context_state & load_gfx_sh_regs for GFX */ in gfx_v10_0_ring_emit_cntxcntl()
8727 struct amdgpu_kiq *kiq = &adev->gfx.kiq; in gfx_v10_0_ring_preempt_ib()
8782 gfx[0].gfx_meta_data) + in gfx_v10_0_ring_emit_ce_meta()
8820 gfx[0].gfx_meta_data) + in gfx_v10_0_ring_emit_de_meta()
8828 gfx[0].gds_backup) + in gfx_v10_0_ring_emit_de_meta()
8924 fw_version_ok = adev->gfx.cp_fw_write_wait; in gfx_v10_0_ring_emit_reg_write_reg_wait()
9115 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); in gfx_v10_0_eop_irq()
9117 amdgpu_fence_process(&adev->gfx.gfx_ring[1]); in gfx_v10_0_eop_irq()
9121 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v10_0_eop_irq()
9122 ring = &adev->gfx.compute_ring[i]; in gfx_v10_0_eop_irq()
9190 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v10_0_handle_priv_fault()
9191 ring = &adev->gfx.gfx_ring[i]; in gfx_v10_0_handle_priv_fault()
9192 /* we only enabled 1 gfx queue per pipe for now */ in gfx_v10_0_handle_priv_fault()
9199 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v10_0_handle_priv_fault()
9200 ring = &adev->gfx.compute_ring[i]; in gfx_v10_0_handle_priv_fault()
9235 struct amdgpu_ring *ring = &(adev->gfx.kiq.ring); in gfx_v10_0_kiq_set_interrupt_state()
9279 struct amdgpu_ring *ring = &(adev->gfx.kiq.ring); in gfx_v10_0_kiq_irq()
9459 adev->gfx.kiq.ring.funcs = &gfx_v10_0_ring_funcs_kiq; in gfx_v10_0_set_ring_funcs()
9461 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v10_0_set_ring_funcs()
9462 adev->gfx.gfx_ring[i].funcs = &gfx_v10_0_ring_funcs_gfx; in gfx_v10_0_set_ring_funcs()
9464 for (i = 0; i < adev->gfx.num_compute_rings; i++) in gfx_v10_0_set_ring_funcs()
9465 adev->gfx.compute_ring[i].funcs = &gfx_v10_0_ring_funcs_compute; in gfx_v10_0_set_ring_funcs()
9490 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; in gfx_v10_0_set_irq_funcs()
9491 adev->gfx.eop_irq.funcs = &gfx_v10_0_eop_irq_funcs; in gfx_v10_0_set_irq_funcs()
9493 adev->gfx.kiq.irq.num_types = AMDGPU_CP_KIQ_IRQ_LAST; in gfx_v10_0_set_irq_funcs()
9494 adev->gfx.kiq.irq.funcs = &gfx_v10_0_kiq_irq_funcs; in gfx_v10_0_set_irq_funcs()
9496 adev->gfx.priv_reg_irq.num_types = 1; in gfx_v10_0_set_irq_funcs()
9497 adev->gfx.priv_reg_irq.funcs = &gfx_v10_0_priv_reg_irq_funcs; in gfx_v10_0_set_irq_funcs()
9499 adev->gfx.priv_inst_irq.num_types = 1; in gfx_v10_0_set_irq_funcs()
9500 adev->gfx.priv_inst_irq.funcs = &gfx_v10_0_priv_inst_irq_funcs; in gfx_v10_0_set_irq_funcs()
9517 adev->gfx.rlc.funcs = &gfx_v10_0_rlc_funcs; in gfx_v10_0_set_rlc_funcs()
9521 adev->gfx.rlc.funcs = &gfx_v10_0_rlc_funcs_sriov; in gfx_v10_0_set_rlc_funcs()
9530 unsigned total_cu = adev->gfx.config.max_cu_per_sh * in gfx_v10_0_set_gds_init()
9531 adev->gfx.config.max_sh_per_se * in gfx_v10_0_set_gds_init()
9532 adev->gfx.config.max_shader_engines; in gfx_v10_0_set_gds_init()
9542 /* set gfx eng mqd */ in gfx_v10_0_set_mqd_funcs()
9571 ~amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1); in gfx_v10_0_get_wgp_active_bitmap_per_sh()
9619 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v10_0_get_cu_info()
9620 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v10_0_get_cu_info()
9621 bitmap = i * adev->gfx.config.max_sh_per_se + j; in gfx_v10_0_get_cu_info()
9638 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) { in gfx_v10_0_get_cu_info()
9640 if (counter < adev->gfx.config.max_cu_per_sh) in gfx_v10_0_get_cu_info()
9674 max_sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se * in gfx_v10_3_get_disabled_sa()
9675 adev->gfx.config.max_shader_engines); in gfx_v10_3_get_disabled_sa()
9689 max_sa_per_se = adev->gfx.config.max_sh_per_se; in gfx_v10_3_program_pbb_mode()
9691 max_shader_engines = adev->gfx.config.max_shader_engines; in gfx_v10_3_program_pbb_mode()