800 	adev->gfx.scratch.num_reg = 8;  in gfx_v9_0_scratch_init()
801 	adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);  in gfx_v9_0_scratch_init()
802 	adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;  in gfx_v9_0_scratch_init()
934 	release_firmware(adev->gfx.pfp_fw);  in gfx_v9_0_free_microcode()
935 	adev->gfx.pfp_fw = NULL;  in gfx_v9_0_free_microcode()
936 	release_firmware(adev->gfx.me_fw);  in gfx_v9_0_free_microcode()
937 	adev->gfx.me_fw = NULL;  in gfx_v9_0_free_microcode()
938 	release_firmware(adev->gfx.ce_fw);  in gfx_v9_0_free_microcode()
939 	adev->gfx.ce_fw = NULL;  in gfx_v9_0_free_microcode()
940 	release_firmware(adev->gfx.rlc_fw);  in gfx_v9_0_free_microcode()
941 	adev->gfx.rlc_fw = NULL;  in gfx_v9_0_free_microcode()
942 	release_firmware(adev->gfx.mec_fw);  in gfx_v9_0_free_microcode()
943 	adev->gfx.mec_fw = NULL;  in gfx_v9_0_free_microcode()
944 	release_firmware(adev->gfx.mec2_fw);  in gfx_v9_0_free_microcode()
945 	adev->gfx.mec2_fw = NULL;  in gfx_v9_0_free_microcode()
947 	kfree(adev->gfx.rlc.register_list_format);  in gfx_v9_0_free_microcode()
954 	rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;  in gfx_v9_0_init_rlc_ext_microcode()
955 	adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);  in gfx_v9_0_init_rlc_ext_microcode()
956 	adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);  in gfx_v9_0_init_rlc_ext_microcode()
957 …adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size…  in gfx_v9_0_init_rlc_ext_microcode()
958 …adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl…  in gfx_v9_0_init_rlc_ext_microcode()
959 	adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);  in gfx_v9_0_init_rlc_ext_microcode()
960 	adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);  in gfx_v9_0_init_rlc_ext_microcode()
961 …adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_b…  in gfx_v9_0_init_rlc_ext_microcode()
962 …adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_o…  in gfx_v9_0_init_rlc_ext_microcode()
963 	adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);  in gfx_v9_0_init_rlc_ext_microcode()
964 	adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);  in gfx_v9_0_init_rlc_ext_microcode()
965 …adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_b…  in gfx_v9_0_init_rlc_ext_microcode()
966 …adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_o…  in gfx_v9_0_init_rlc_ext_microcode()
967 	adev->gfx.rlc.reg_list_format_direct_reg_list_length =  in gfx_v9_0_init_rlc_ext_microcode()
973 	adev->gfx.me_fw_write_wait = false;  in gfx_v9_0_check_fw_write_wait()
974 	adev->gfx.mec_fw_write_wait = false;  in gfx_v9_0_check_fw_write_wait()
976 	if ((adev->gfx.mec_fw_version < 0x000001a5) ||  in gfx_v9_0_check_fw_write_wait()
977 	    (adev->gfx.mec_feature_version < 46) ||  in gfx_v9_0_check_fw_write_wait()
978 	    (adev->gfx.pfp_fw_version < 0x000000b7) ||  in gfx_v9_0_check_fw_write_wait()
979 	    (adev->gfx.pfp_feature_version < 46))  in gfx_v9_0_check_fw_write_wait()
985 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&  in gfx_v9_0_check_fw_write_wait()
986 		    (adev->gfx.me_feature_version >= 42) &&  in gfx_v9_0_check_fw_write_wait()
987 		    (adev->gfx.pfp_fw_version >=  0x000000b1) &&  in gfx_v9_0_check_fw_write_wait()
988 		    (adev->gfx.pfp_feature_version >= 42))  in gfx_v9_0_check_fw_write_wait()
989 			adev->gfx.me_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
991 		if ((adev->gfx.mec_fw_version >=  0x00000193) &&  in gfx_v9_0_check_fw_write_wait()
992 		    (adev->gfx.mec_feature_version >= 42))  in gfx_v9_0_check_fw_write_wait()
993 			adev->gfx.mec_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
996 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&  in gfx_v9_0_check_fw_write_wait()
997 		    (adev->gfx.me_feature_version >= 44) &&  in gfx_v9_0_check_fw_write_wait()
998 		    (adev->gfx.pfp_fw_version >=  0x000000b2) &&  in gfx_v9_0_check_fw_write_wait()
999 		    (adev->gfx.pfp_feature_version >= 44))  in gfx_v9_0_check_fw_write_wait()
1000 			adev->gfx.me_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1002 		if ((adev->gfx.mec_fw_version >=  0x00000196) &&  in gfx_v9_0_check_fw_write_wait()
1003 		    (adev->gfx.mec_feature_version >= 44))  in gfx_v9_0_check_fw_write_wait()
1004 			adev->gfx.mec_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1007 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&  in gfx_v9_0_check_fw_write_wait()
1008 		    (adev->gfx.me_feature_version >= 44) &&  in gfx_v9_0_check_fw_write_wait()
1009 		    (adev->gfx.pfp_fw_version >=  0x000000b2) &&  in gfx_v9_0_check_fw_write_wait()
1010 		    (adev->gfx.pfp_feature_version >= 44))  in gfx_v9_0_check_fw_write_wait()
1011 			adev->gfx.me_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1013 		if ((adev->gfx.mec_fw_version >=  0x00000197) &&  in gfx_v9_0_check_fw_write_wait()
1014 		    (adev->gfx.mec_feature_version >= 44))  in gfx_v9_0_check_fw_write_wait()
1015 			adev->gfx.mec_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1018 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&  in gfx_v9_0_check_fw_write_wait()
1019 		    (adev->gfx.me_feature_version >= 42) &&  in gfx_v9_0_check_fw_write_wait()
1020 		    (adev->gfx.pfp_fw_version >=  0x000000b1) &&  in gfx_v9_0_check_fw_write_wait()
1021 		    (adev->gfx.pfp_feature_version >= 42))  in gfx_v9_0_check_fw_write_wait()
1022 			adev->gfx.me_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1024 		if ((adev->gfx.mec_fw_version >=  0x00000192) &&  in gfx_v9_0_check_fw_write_wait()
1025 		    (adev->gfx.mec_feature_version >= 42))  in gfx_v9_0_check_fw_write_wait()
1026 			adev->gfx.mec_fw_write_wait = true;  in gfx_v9_0_check_fw_write_wait()
1047 			 &&((adev->gfx.rlc_fw_version != 106 &&  in gfx_v9_0_check_if_need_gfxoff()
1048 			     adev->gfx.rlc_fw_version < 531) ||  in gfx_v9_0_check_if_need_gfxoff()
1049 			    (adev->gfx.rlc_fw_version == 53815) ||  in gfx_v9_0_check_if_need_gfxoff()
1050 			    (adev->gfx.rlc_feature_version < 1) ||  in gfx_v9_0_check_if_need_gfxoff()
1051 			    !adev->gfx.rlc.is_rlc_v2_1))  in gfx_v9_0_check_if_need_gfxoff()
1080 	err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);  in gfx_v9_0_init_cp_gfx_microcode()
1083 	err = amdgpu_ucode_validate(adev->gfx.pfp_fw);  in gfx_v9_0_init_cp_gfx_microcode()
1086 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;  in gfx_v9_0_init_cp_gfx_microcode()
1087 	adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);  in gfx_v9_0_init_cp_gfx_microcode()
1088 	adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);  in gfx_v9_0_init_cp_gfx_microcode()
1091 	err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);  in gfx_v9_0_init_cp_gfx_microcode()
1094 	err = amdgpu_ucode_validate(adev->gfx.me_fw);  in gfx_v9_0_init_cp_gfx_microcode()
1097 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;  in gfx_v9_0_init_cp_gfx_microcode()
1098 	adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);  in gfx_v9_0_init_cp_gfx_microcode()
1099 	adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);  in gfx_v9_0_init_cp_gfx_microcode()
1102 	err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);  in gfx_v9_0_init_cp_gfx_microcode()
1105 	err = amdgpu_ucode_validate(adev->gfx.ce_fw);  in gfx_v9_0_init_cp_gfx_microcode()
1108 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;  in gfx_v9_0_init_cp_gfx_microcode()
1109 	adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);  in gfx_v9_0_init_cp_gfx_microcode()
1110 	adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);  in gfx_v9_0_init_cp_gfx_microcode()
1115 		info->fw = adev->gfx.pfp_fw;  in gfx_v9_0_init_cp_gfx_microcode()
1122 		info->fw = adev->gfx.me_fw;  in gfx_v9_0_init_cp_gfx_microcode()
1129 		info->fw = adev->gfx.ce_fw;  in gfx_v9_0_init_cp_gfx_microcode()
1140 		release_firmware(adev->gfx.pfp_fw);  in gfx_v9_0_init_cp_gfx_microcode()
1141 		adev->gfx.pfp_fw = NULL;  in gfx_v9_0_init_cp_gfx_microcode()
1142 		release_firmware(adev->gfx.me_fw);  in gfx_v9_0_init_cp_gfx_microcode()
1143 		adev->gfx.me_fw = NULL;  in gfx_v9_0_init_cp_gfx_microcode()
1144 		release_firmware(adev->gfx.ce_fw);  in gfx_v9_0_init_cp_gfx_microcode()
1145 		adev->gfx.ce_fw = NULL;  in gfx_v9_0_init_cp_gfx_microcode()
1184 	err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);  in gfx_v9_0_init_rlc_microcode()
1187 	err = amdgpu_ucode_validate(adev->gfx.rlc_fw);  in gfx_v9_0_init_rlc_microcode()
1188 	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;  in gfx_v9_0_init_rlc_microcode()
1193 		adev->gfx.rlc.is_rlc_v2_1 = true;  in gfx_v9_0_init_rlc_microcode()
1195 	adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);  in gfx_v9_0_init_rlc_microcode()
1196 	adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);  in gfx_v9_0_init_rlc_microcode()
1197 	adev->gfx.rlc.save_and_restore_offset =  in gfx_v9_0_init_rlc_microcode()
1199 	adev->gfx.rlc.clear_state_descriptor_offset =  in gfx_v9_0_init_rlc_microcode()
1201 	adev->gfx.rlc.avail_scratch_ram_locations =  in gfx_v9_0_init_rlc_microcode()
1203 	adev->gfx.rlc.reg_restore_list_size =  in gfx_v9_0_init_rlc_microcode()
1205 	adev->gfx.rlc.reg_list_format_start =  in gfx_v9_0_init_rlc_microcode()
1207 	adev->gfx.rlc.reg_list_format_separate_start =  in gfx_v9_0_init_rlc_microcode()
1209 	adev->gfx.rlc.starting_offsets_start =  in gfx_v9_0_init_rlc_microcode()
1211 	adev->gfx.rlc.reg_list_format_size_bytes =  in gfx_v9_0_init_rlc_microcode()
1213 	adev->gfx.rlc.reg_list_size_bytes =  in gfx_v9_0_init_rlc_microcode()
1215 	adev->gfx.rlc.register_list_format =  in gfx_v9_0_init_rlc_microcode()
1216 			kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +  in gfx_v9_0_init_rlc_microcode()
1217 				adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);  in gfx_v9_0_init_rlc_microcode()
1218 	if (!adev->gfx.rlc.register_list_format) {  in gfx_v9_0_init_rlc_microcode()
1225 	for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)  in gfx_v9_0_init_rlc_microcode()
1226 		adev->gfx.rlc.register_list_format[i] =	le32_to_cpu(tmp[i]);  in gfx_v9_0_init_rlc_microcode()
1228 	adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;  in gfx_v9_0_init_rlc_microcode()
1232 	for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)  in gfx_v9_0_init_rlc_microcode()
1233 		adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);  in gfx_v9_0_init_rlc_microcode()
1235 	if (adev->gfx.rlc.is_rlc_v2_1)  in gfx_v9_0_init_rlc_microcode()
1241 		info->fw = adev->gfx.rlc_fw;  in gfx_v9_0_init_rlc_microcode()
1246 		if (adev->gfx.rlc.is_rlc_v2_1 &&  in gfx_v9_0_init_rlc_microcode()
1247 		    adev->gfx.rlc.save_restore_list_cntl_size_bytes &&  in gfx_v9_0_init_rlc_microcode()
1248 		    adev->gfx.rlc.save_restore_list_gpm_size_bytes &&  in gfx_v9_0_init_rlc_microcode()
1249 		    adev->gfx.rlc.save_restore_list_srm_size_bytes) {  in gfx_v9_0_init_rlc_microcode()
1252 			info->fw = adev->gfx.rlc_fw;  in gfx_v9_0_init_rlc_microcode()
1254 				ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);  in gfx_v9_0_init_rlc_microcode()
1258 			info->fw = adev->gfx.rlc_fw;  in gfx_v9_0_init_rlc_microcode()
1260 				ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);  in gfx_v9_0_init_rlc_microcode()
1264 			info->fw = adev->gfx.rlc_fw;  in gfx_v9_0_init_rlc_microcode()
1266 				ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);  in gfx_v9_0_init_rlc_microcode()
1275 		release_firmware(adev->gfx.rlc_fw);  in gfx_v9_0_init_rlc_microcode()
1276 		adev->gfx.rlc_fw = NULL;  in gfx_v9_0_init_rlc_microcode()
1291 	err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);  in gfx_v9_0_init_cp_compute_microcode()
1294 	err = amdgpu_ucode_validate(adev->gfx.mec_fw);  in gfx_v9_0_init_cp_compute_microcode()
1297 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;  in gfx_v9_0_init_cp_compute_microcode()
1298 	adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);  in gfx_v9_0_init_cp_compute_microcode()
1299 	adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);  in gfx_v9_0_init_cp_compute_microcode()
1303 	err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);  in gfx_v9_0_init_cp_compute_microcode()
1305 		err = amdgpu_ucode_validate(adev->gfx.mec2_fw);  in gfx_v9_0_init_cp_compute_microcode()
1309 		adev->gfx.mec2_fw->data;  in gfx_v9_0_init_cp_compute_microcode()
1310 		adev->gfx.mec2_fw_version =  in gfx_v9_0_init_cp_compute_microcode()
1312 		adev->gfx.mec2_feature_version =  in gfx_v9_0_init_cp_compute_microcode()
1316 		adev->gfx.mec2_fw = NULL;  in gfx_v9_0_init_cp_compute_microcode()
1322 		info->fw = adev->gfx.mec_fw;  in gfx_v9_0_init_cp_compute_microcode()
1330 		info->fw = adev->gfx.mec_fw;  in gfx_v9_0_init_cp_compute_microcode()
1334 		if (adev->gfx.mec2_fw) {  in gfx_v9_0_init_cp_compute_microcode()
1337 			info->fw = adev->gfx.mec2_fw;  in gfx_v9_0_init_cp_compute_microcode()
1348 				info->fw = adev->gfx.mec2_fw;  in gfx_v9_0_init_cp_compute_microcode()
1363 		release_firmware(adev->gfx.mec_fw);  in gfx_v9_0_init_cp_compute_microcode()
1364 		adev->gfx.mec_fw = NULL;  in gfx_v9_0_init_cp_compute_microcode()
1365 		release_firmware(adev->gfx.mec2_fw);  in gfx_v9_0_init_cp_compute_microcode()
1366 		adev->gfx.mec2_fw = NULL;  in gfx_v9_0_init_cp_compute_microcode()
1459 	if (adev->gfx.rlc.cs_data == NULL)  in gfx_v9_0_get_csb_buffer()
1471 	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {  in gfx_v9_0_get_csb_buffer()
1495 	struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;  in gfx_v9_0_init_always_on_cu_mask()
1509 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {  in gfx_v9_0_init_always_on_cu_mask()
1510 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {  in gfx_v9_0_init_always_on_cu_mask()
1516 			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {  in gfx_v9_0_init_always_on_cu_mask()
1650 	adev->gfx.rlc.cs_data = gfx9_cs_data;  in gfx_v9_0_rlc_init()
1652 	cs_data = adev->gfx.rlc.cs_data;  in gfx_v9_0_rlc_init()
1663 		adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */  in gfx_v9_0_rlc_init()
1687 	r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);  in gfx_v9_0_csb_vram_pin()
1691 	r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj,  in gfx_v9_0_csb_vram_pin()
1694 		adev->gfx.rlc.clear_state_gpu_addr =  in gfx_v9_0_csb_vram_pin()
1695 			amdgpu_bo_gpu_offset(adev->gfx.rlc.clear_state_obj);  in gfx_v9_0_csb_vram_pin()
1697 	amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);  in gfx_v9_0_csb_vram_pin()
1706 	if (!adev->gfx.rlc.clear_state_obj)  in gfx_v9_0_csb_vram_unpin()
1709 	r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);  in gfx_v9_0_csb_vram_unpin()
1711 		amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);  in gfx_v9_0_csb_vram_unpin()
1712 		amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);  in gfx_v9_0_csb_vram_unpin()
1718 	amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);  in gfx_v9_0_mec_fini()
1719 	amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);  in gfx_v9_0_mec_fini()
1733 	bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);  in gfx_v9_0_mec_init()
1737 	mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;  in gfx_v9_0_mec_init()
1741 				      &adev->gfx.mec.hpd_eop_obj,  in gfx_v9_0_mec_init()
1742 				      &adev->gfx.mec.hpd_eop_gpu_addr,  in gfx_v9_0_mec_init()
1750 	memset(hpd, 0, adev->gfx.mec.hpd_eop_obj->tbo.mem.size);  in gfx_v9_0_mec_init()
1752 	amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);  in gfx_v9_0_mec_init()
1753 	amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);  in gfx_v9_0_mec_init()
1755 	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;  in gfx_v9_0_mec_init()
1758 		(adev->gfx.mec_fw->data +  in gfx_v9_0_mec_init()
1764 				      &adev->gfx.mec.mec_fw_obj,  in gfx_v9_0_mec_init()
1765 				      &adev->gfx.mec.mec_fw_gpu_addr,  in gfx_v9_0_mec_init()
1775 	amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);  in gfx_v9_0_mec_init()
1776 	amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);  in gfx_v9_0_mec_init()
1867 	adev->gfx.funcs = &gfx_v9_0_gfx_funcs;  in gfx_v9_0_gpu_early_init()
1871 		adev->gfx.config.max_hw_contexts = 8;  in gfx_v9_0_gpu_early_init()
1872 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
1873 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
1874 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
1875 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
1879 		adev->gfx.config.max_hw_contexts = 8;  in gfx_v9_0_gpu_early_init()
1880 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
1881 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
1882 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
1883 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
1888 		adev->gfx.config.max_hw_contexts = 8;  in gfx_v9_0_gpu_early_init()
1889 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
1890 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
1891 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
1892 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
1902 		adev->gfx.config.max_hw_contexts = 8;  in gfx_v9_0_gpu_early_init()
1903 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
1904 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
1905 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
1906 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
1913 		adev->gfx.config.max_hw_contexts = 8;  in gfx_v9_0_gpu_early_init()
1914 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
1915 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
1916 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;  in gfx_v9_0_gpu_early_init()
1917 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
1923 		adev->gfx.config.max_hw_contexts = 8;  in gfx_v9_0_gpu_early_init()
1924 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;  in gfx_v9_0_gpu_early_init()
1925 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;  in gfx_v9_0_gpu_early_init()
1926 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;  in gfx_v9_0_gpu_early_init()
1927 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;  in gfx_v9_0_gpu_early_init()
1937 	adev->gfx.config.gb_addr_config = gb_addr_config;  in gfx_v9_0_gpu_early_init()
1939 	adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<  in gfx_v9_0_gpu_early_init()
1941 					adev->gfx.config.gb_addr_config,  in gfx_v9_0_gpu_early_init()
1945 	adev->gfx.config.max_tile_pipes =  in gfx_v9_0_gpu_early_init()
1946 		adev->gfx.config.gb_addr_config_fields.num_pipes;  in gfx_v9_0_gpu_early_init()
1948 	adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<  in gfx_v9_0_gpu_early_init()
1950 					adev->gfx.config.gb_addr_config,  in gfx_v9_0_gpu_early_init()
1953 	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<  in gfx_v9_0_gpu_early_init()
1955 					adev->gfx.config.gb_addr_config,  in gfx_v9_0_gpu_early_init()
1958 	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<  in gfx_v9_0_gpu_early_init()
1960 					adev->gfx.config.gb_addr_config,  in gfx_v9_0_gpu_early_init()
1963 	adev->gfx.config.gb_addr_config_fields.num_se = 1 <<  in gfx_v9_0_gpu_early_init()
1965 					adev->gfx.config.gb_addr_config,  in gfx_v9_0_gpu_early_init()
1968 	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +  in gfx_v9_0_gpu_early_init()
1970 					adev->gfx.config.gb_addr_config,  in gfx_v9_0_gpu_early_init()
1990 	ngg_buf->size = size_se * adev->gfx.config.max_shader_engines;  in gfx_v9_0_ngg_create_buf()
2010 		amdgpu_bo_free_kernel(&adev->gfx.ngg.buf[i].bo,  in gfx_v9_0_ngg_fini()
2011 				      &adev->gfx.ngg.buf[i].gpu_addr,  in gfx_v9_0_ngg_fini()
2014 	memset(&adev->gfx.ngg.buf[0], 0,  in gfx_v9_0_ngg_fini()
2017 	adev->gfx.ngg.init = false;  in gfx_v9_0_ngg_fini()
2026 	if (!amdgpu_ngg || adev->gfx.ngg.init == true)  in gfx_v9_0_ngg_init()
2030 	adev->gfx.ngg.gds_reserve_size = ALIGN(5 * 4, 0x40);  in gfx_v9_0_ngg_init()
2031 	adev->gds.gds_size -= adev->gfx.ngg.gds_reserve_size;  in gfx_v9_0_ngg_init()
2032 	adev->gfx.ngg.gds_reserve_addr = RREG32_SOC15(GC, 0, mmGDS_VMID0_BASE);  in gfx_v9_0_ngg_init()
2033 	adev->gfx.ngg.gds_reserve_addr += RREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE);  in gfx_v9_0_ngg_init()
2036 	r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[NGG_PRIM],  in gfx_v9_0_ngg_init()
2045 	r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[NGG_POS],  in gfx_v9_0_ngg_init()
2054 	r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[NGG_CNTL],  in gfx_v9_0_ngg_init()
2066 	r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[NGG_PARAM],  in gfx_v9_0_ngg_init()
2075 	adev->gfx.ngg.init = true;  in gfx_v9_0_ngg_init()
2084 	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];  in gfx_v9_0_ngg_en()
2093 			     adev->gfx.ngg.buf[NGG_PRIM].size >> 8);  in gfx_v9_0_ngg_en()
2095 			     adev->gfx.ngg.buf[NGG_POS].size >> 8);  in gfx_v9_0_ngg_en()
2099 			     adev->gfx.ngg.buf[NGG_CNTL].size >> 8);  in gfx_v9_0_ngg_en()
2101 			     adev->gfx.ngg.buf[NGG_PARAM].size >> 10);  in gfx_v9_0_ngg_en()
2105 	base = lower_32_bits(adev->gfx.ngg.buf[NGG_PRIM].gpu_addr);  in gfx_v9_0_ngg_en()
2109 	base = upper_32_bits(adev->gfx.ngg.buf[NGG_PRIM].gpu_addr);  in gfx_v9_0_ngg_en()
2113 	base = lower_32_bits(adev->gfx.ngg.buf[NGG_POS].gpu_addr);  in gfx_v9_0_ngg_en()
2117 	base = upper_32_bits(adev->gfx.ngg.buf[NGG_POS].gpu_addr);  in gfx_v9_0_ngg_en()
2121 	base = lower_32_bits(adev->gfx.ngg.buf[NGG_CNTL].gpu_addr);  in gfx_v9_0_ngg_en()
2125 	base = upper_32_bits(adev->gfx.ngg.buf[NGG_CNTL].gpu_addr);  in gfx_v9_0_ngg_en()
2140 				    adev->gfx.ngg.gds_reserve_size));  in gfx_v9_0_ngg_en()
2148 	amdgpu_ring_write(ring, adev->gfx.ngg.gds_reserve_addr);  in gfx_v9_0_ngg_en()
2151 				adev->gfx.ngg.gds_reserve_size);  in gfx_v9_0_ngg_en()
2166 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];  in gfx_v9_0_compute_ring_init()
2168 	ring = &adev->gfx.compute_ring[ring_id];  in gfx_v9_0_compute_ring_init()
2178 	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr  in gfx_v9_0_compute_ring_init()
2183 		+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)  in gfx_v9_0_compute_ring_init()
2188 			     &adev->gfx.eop_irq, irq_type);  in gfx_v9_0_compute_ring_init()
2210 		adev->gfx.mec.num_mec = 2;  in gfx_v9_0_sw_init()
2213 		adev->gfx.mec.num_mec = 1;  in gfx_v9_0_sw_init()
2217 	adev->gfx.mec.num_pipe_per_mec = 4;  in gfx_v9_0_sw_init()
2218 	adev->gfx.mec.num_queue_per_pipe = 8;  in gfx_v9_0_sw_init()
2221 …_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);  in gfx_v9_0_sw_init()
2227 			      &adev->gfx.priv_reg_irq);  in gfx_v9_0_sw_init()
2233 			      &adev->gfx.priv_inst_irq);  in gfx_v9_0_sw_init()
2239 			      &adev->gfx.cp_ecc_error_irq);  in gfx_v9_0_sw_init()
2245 			      &adev->gfx.cp_ecc_error_irq);  in gfx_v9_0_sw_init()
2249 	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;  in gfx_v9_0_sw_init()
2259 	r = adev->gfx.rlc.funcs->init(adev);  in gfx_v9_0_sw_init()
2272 	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {  in gfx_v9_0_sw_init()
2273 		ring = &adev->gfx.gfx_ring[i];  in gfx_v9_0_sw_init()
2282 				     &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP);  in gfx_v9_0_sw_init()
2289 	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {  in gfx_v9_0_sw_init()
2290 		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {  in gfx_v9_0_sw_init()
2291 			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {  in gfx_v9_0_sw_init()
2312 	kiq = &adev->gfx.kiq;  in gfx_v9_0_sw_init()
2322 	adev->gfx.ce_ram_size = 0x8000;  in gfx_v9_0_sw_init()
2342 			adev->gfx.ras_if) {  in gfx_v9_0_sw_fini()
2343 		struct ras_common_if *ras_if = adev->gfx.ras_if;  in gfx_v9_0_sw_fini()
2355 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)  in gfx_v9_0_sw_fini()
2356 		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);  in gfx_v9_0_sw_fini()
2357 	for (i = 0; i < adev->gfx.num_compute_rings; i++)  in gfx_v9_0_sw_fini()
2358 		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);  in gfx_v9_0_sw_fini()
2361 	amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);  in gfx_v9_0_sw_fini()
2366 	amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);  in gfx_v9_0_sw_fini()
2368 		amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,  in gfx_v9_0_sw_fini()
2369 				&adev->gfx.rlc.cp_table_gpu_addr,  in gfx_v9_0_sw_fini()
2370 				(void **)&adev->gfx.rlc.cp_table_ptr);  in gfx_v9_0_sw_fini()
2415 	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /  in gfx_v9_0_get_rb_active_bitmap()
2416 					 adev->gfx.config.max_sh_per_se);  in gfx_v9_0_get_rb_active_bitmap()
2426 	u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /  in gfx_v9_0_setup_rb()
2427 					adev->gfx.config.max_sh_per_se;  in gfx_v9_0_setup_rb()
2430 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {  in gfx_v9_0_setup_rb()
2431 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {  in gfx_v9_0_setup_rb()
2434 			active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *  in gfx_v9_0_setup_rb()
2441 	adev->gfx.config.backend_enable_mask = active_rbs;  in gfx_v9_0_setup_rb()
2442 	adev->gfx.config.num_rbs = hweight32(active_rbs);  in gfx_v9_0_setup_rb()
2514 	gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);  in gfx_v9_0_constants_init()
2515 	adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);  in gfx_v9_0_constants_init()
2557 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {  in gfx_v9_0_wait_for_rlc_serdes()
2558 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {  in gfx_v9_0_wait_for_rlc_serdes()
2606 			adev->gfx.rlc.clear_state_gpu_addr >> 32);  in gfx_v9_0_init_csb()
2608 			adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);  in gfx_v9_0_init_csb()
2610 			adev->gfx.rlc.clear_state_size);  in gfx_v9_0_init_csb()
2663 		kmemdup(adev->gfx.rlc.register_list_format,  in gfx_v9_1_init_rlc_save_restore_list()
2664 			adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);  in gfx_v9_1_init_rlc_save_restore_list()
2671 				    adev->gfx.rlc.reg_list_format_direct_reg_list_length,  in gfx_v9_1_init_rlc_save_restore_list()
2672 				    adev->gfx.rlc.reg_list_format_size_bytes >> 2,  in gfx_v9_1_init_rlc_save_restore_list()
2687 	for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)  in gfx_v9_1_init_rlc_save_restore_list()
2689 			adev->gfx.rlc.register_restore[i]);  in gfx_v9_1_init_rlc_save_restore_list()
2693 		adev->gfx.rlc.reg_list_format_start);  in gfx_v9_1_init_rlc_save_restore_list()
2696 	for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)  in gfx_v9_1_init_rlc_save_restore_list()
2701 	while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {  in gfx_v9_1_init_rlc_save_restore_list()
2723 	list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;  in gfx_v9_1_init_rlc_save_restore_list()
2726 		adev->gfx.rlc.reg_restore_list_size);  in gfx_v9_1_init_rlc_save_restore_list()
2731 		adev->gfx.rlc.starting_offsets_start);  in gfx_v9_1_init_rlc_save_restore_list()
2932 	if (adev->gfx.rlc.is_rlc_v2_1) {  in gfx_v9_0_init_pg()
2944 		       adev->gfx.rlc.cp_table_gpu_addr >> 8);  in gfx_v9_0_init_pg()
2984 				rlc_ucode_ver, adev->gfx.rlc_fw_version);  in gfx_v9_0_rlc_start()
3002 	if (!adev->gfx.rlc_fw)  in gfx_v9_0_rlc_load_microcode()
3005 	hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;  in gfx_v9_0_rlc_load_microcode()
3008 	fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +  in gfx_v9_0_rlc_load_microcode()
3016 	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);  in gfx_v9_0_rlc_load_microcode()
3030 	adev->gfx.rlc.funcs->stop(adev);  in gfx_v9_0_rlc_resume()
3061 	adev->gfx.rlc.funcs->start(adev);  in gfx_v9_0_rlc_resume()
3075 		for (i = 0; i < adev->gfx.num_gfx_rings; i++)  in gfx_v9_0_cp_gfx_enable()
3076 			adev->gfx.gfx_ring[i].sched.ready = false;  in gfx_v9_0_cp_gfx_enable()
3090 	if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)  in gfx_v9_0_cp_gfx_load_microcode()
3094 		adev->gfx.pfp_fw->data;  in gfx_v9_0_cp_gfx_load_microcode()
3096 		adev->gfx.ce_fw->data;  in gfx_v9_0_cp_gfx_load_microcode()
3098 		adev->gfx.me_fw->data;  in gfx_v9_0_cp_gfx_load_microcode()
3108 		(adev->gfx.pfp_fw->data +  in gfx_v9_0_cp_gfx_load_microcode()
3114 	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);  in gfx_v9_0_cp_gfx_load_microcode()
3118 		(adev->gfx.ce_fw->data +  in gfx_v9_0_cp_gfx_load_microcode()
3124 	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);  in gfx_v9_0_cp_gfx_load_microcode()
3128 		(adev->gfx.me_fw->data +  in gfx_v9_0_cp_gfx_load_microcode()
3134 	WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);  in gfx_v9_0_cp_gfx_load_microcode()
3141 	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];  in gfx_v9_0_cp_gfx_start()
3147 	WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);  in gfx_v9_0_cp_gfx_start()
3215 	ring = &adev->gfx.gfx_ring[0];  in gfx_v9_0_cp_gfx_resume()
3280 		for (i = 0; i < adev->gfx.num_compute_rings; i++)  in gfx_v9_0_cp_compute_enable()
3281 			adev->gfx.compute_ring[i].sched.ready = false;  in gfx_v9_0_cp_compute_enable()
3282 		adev->gfx.kiq.ring.sched.ready = false;  in gfx_v9_0_cp_compute_enable()
3294 	if (!adev->gfx.mec_fw)  in gfx_v9_0_cp_compute_load_microcode()
3299 	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;  in gfx_v9_0_cp_compute_load_microcode()
3303 		(adev->gfx.mec_fw->data +  in gfx_v9_0_cp_compute_load_microcode()
3311 		adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);  in gfx_v9_0_cp_compute_load_microcode()
3313 		upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));  in gfx_v9_0_cp_compute_load_microcode()
3323 			adev->gfx.mec_fw_version);  in gfx_v9_0_cp_compute_load_microcode()
3346 	struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;  in gfx_v9_0_kiq_kcq_enable()
3351 		if (!test_bit(i, adev->gfx.mec.queue_bitmap))  in gfx_v9_0_kiq_kcq_enable()
3365 	r = amdgpu_ring_alloc(kiq_ring, (7 * adev->gfx.num_compute_rings) + 8);  in gfx_v9_0_kiq_kcq_enable()
3381 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_kiq_kcq_enable()
3382 		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];  in gfx_v9_0_kiq_kcq_enable()
3708 		if (adev->gfx.mec.mqd_backup[mqd_idx])  in gfx_v9_0_kiq_init_queue()
3709 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));  in gfx_v9_0_kiq_init_queue()
3731 		if (adev->gfx.mec.mqd_backup[mqd_idx])  in gfx_v9_0_kiq_init_queue()
3732 			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));  in gfx_v9_0_kiq_init_queue()
3742 	int mqd_idx = ring - &adev->gfx.compute_ring[0];  in gfx_v9_0_kcq_init_queue()
3754 		if (adev->gfx.mec.mqd_backup[mqd_idx])  in gfx_v9_0_kcq_init_queue()
3755 			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));  in gfx_v9_0_kcq_init_queue()
3758 		if (adev->gfx.mec.mqd_backup[mqd_idx])  in gfx_v9_0_kcq_init_queue()
3759 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));  in gfx_v9_0_kcq_init_queue()
3776 	ring = &adev->gfx.kiq.ring;  in gfx_v9_0_kiq_resume()
3801 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_kcq_resume()
3802 		ring = &adev->gfx.compute_ring[i];  in gfx_v9_0_kcq_resume()
3859 		ring = &adev->gfx.gfx_ring[0];  in gfx_v9_0_cp_resume()
3865 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_cp_resume()
3866 		ring = &adev->gfx.compute_ring[i];  in gfx_v9_0_cp_resume()
3896 	r = adev->gfx.rlc.funcs->resume(adev);  in gfx_v9_0_hw_init()
3916 	struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;  in gfx_v9_0_kcq_disable()
3918 	r = amdgpu_ring_alloc(kiq_ring, 6 * adev->gfx.num_compute_rings);  in gfx_v9_0_kcq_disable()
3922 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_kcq_disable()
3923 		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];  in gfx_v9_0_kcq_disable()
3947 	amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);  in gfx_v9_0_hw_fini()
3948 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);  in gfx_v9_0_hw_fini()
3949 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);  in gfx_v9_0_hw_fini()
3970 		soc15_grbm_select(adev, adev->gfx.kiq.ring.me,  in gfx_v9_0_hw_fini()
3971 				adev->gfx.kiq.ring.pipe,  in gfx_v9_0_hw_fini()
3972 				adev->gfx.kiq.ring.queue, 0);  in gfx_v9_0_hw_fini()
3973 		gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring);  in gfx_v9_0_hw_fini()
3979 	adev->gfx.rlc.funcs->stop(adev);  in gfx_v9_0_hw_fini()
4054 		adev->gfx.rlc.funcs->stop(adev);  in gfx_v9_0_soft_reset()
4087 	mutex_lock(&adev->gfx.gpu_clock_mutex);  in gfx_v9_0_get_gpu_clock_counter()
4091 	mutex_unlock(&adev->gfx.gpu_clock_mutex);  in gfx_v9_0_get_gpu_clock_counter()
4221 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];  in gfx_v9_0_do_edc_gds_workarounds()
4264 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];  in gfx_v9_0_do_edc_gpr_workarounds()
4402 		adev->gfx.num_gfx_rings = 0;  in gfx_v9_0_early_init()
4404 		adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;  in gfx_v9_0_early_init()
4405 	adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS;  in gfx_v9_0_early_init()
4421 	struct ras_common_if **ras_if = &adev->gfx.ras_if;  in gfx_v9_0_ecc_late_init()
4501 	r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);  in gfx_v9_0_ecc_late_init()
4524 	r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);  in gfx_v9_0_late_init()
4528 	r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);  in gfx_v9_0_late_init()
4843 			cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work);  in gfx_v9_0_set_powergating_state()
4872 			cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work);  in gfx_v9_0_set_powergating_state()
5166 	pipe_num = ring->me * adev->gfx.mec.num_pipe_per_mec + ring->pipe;  in gfx_v9_0_ring_set_pipe_percent()
5186 	mutex_lock(&adev->gfx.pipe_reserve_mutex);  in gfx_v9_0_pipe_reserve_resources()
5189 		set_bit(pipe, adev->gfx.pipe_reserve_bitmap);  in gfx_v9_0_pipe_reserve_resources()
5191 		clear_bit(pipe, adev->gfx.pipe_reserve_bitmap);  in gfx_v9_0_pipe_reserve_resources()
5193 	if (!bitmap_weight(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)) {  in gfx_v9_0_pipe_reserve_resources()
5195 		for (i = 0; i < adev->gfx.num_gfx_rings; ++i)  in gfx_v9_0_pipe_reserve_resources()
5196 			gfx_v9_0_ring_set_pipe_percent(&adev->gfx.gfx_ring[i],  in gfx_v9_0_pipe_reserve_resources()
5199 		for (i = 0; i < adev->gfx.num_compute_rings; ++i)  in gfx_v9_0_pipe_reserve_resources()
5200 			gfx_v9_0_ring_set_pipe_percent(&adev->gfx.compute_ring[i],  in gfx_v9_0_pipe_reserve_resources()
5204 		for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {  in gfx_v9_0_pipe_reserve_resources()
5205 			iring = &adev->gfx.gfx_ring[i];  in gfx_v9_0_pipe_reserve_resources()
5210 			reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);  in gfx_v9_0_pipe_reserve_resources()
5214 		for (i = 0; i < adev->gfx.num_compute_rings; ++i) {  in gfx_v9_0_pipe_reserve_resources()
5215 			iring = &adev->gfx.compute_ring[i];  in gfx_v9_0_pipe_reserve_resources()
5220 			reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);  in gfx_v9_0_pipe_reserve_resources()
5225 	mutex_unlock(&adev->gfx.pipe_reserve_mutex);  in gfx_v9_0_pipe_reserve_resources()
5462 		adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;  in gfx_v9_0_ring_emit_reg_write_reg_wait()
5682 		amdgpu_fence_process(&adev->gfx.gfx_ring[0]);  in gfx_v9_0_eop_irq()
5686 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_eop_irq()
5687 			ring = &adev->gfx.compute_ring[i];  in gfx_v9_0_eop_irq()
5712 		drm_sched_fault(&adev->gfx.gfx_ring[0].sched);  in gfx_v9_0_fault()
5716 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in gfx_v9_0_fault()
5717 			ring = &adev->gfx.compute_ring[i];  in gfx_v9_0_fault()
5750 	if (adev->gfx.funcs->query_ras_error_count)  in gfx_v9_0_process_ras_data_cb()
5751 		adev->gfx.funcs->query_ras_error_count(adev, err_data);  in gfx_v9_0_process_ras_data_cb()
6115 	for (se_id = 0; se_id < adev->gfx.config.max_shader_engines; se_id++) {  in gfx_v9_0_query_ras_error_count()
6168 	struct ras_common_if *ras_if = adev->gfx.ras_if;  in gfx_v9_0_cp_ecc_error_irq()
6320 	adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;  in gfx_v9_0_set_ring_funcs()
6322 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)  in gfx_v9_0_set_ring_funcs()
6323 		adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;  in gfx_v9_0_set_ring_funcs()
6325 	for (i = 0; i < adev->gfx.num_compute_rings; i++)  in gfx_v9_0_set_ring_funcs()
6326 		adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;  in gfx_v9_0_set_ring_funcs()
6352 	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;  in gfx_v9_0_set_irq_funcs()
6353 	adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;  in gfx_v9_0_set_irq_funcs()
6355 	adev->gfx.priv_reg_irq.num_types = 1;  in gfx_v9_0_set_irq_funcs()
6356 	adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;  in gfx_v9_0_set_irq_funcs()
6358 	adev->gfx.priv_inst_irq.num_types = 1;  in gfx_v9_0_set_irq_funcs()
6359 	adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;  in gfx_v9_0_set_irq_funcs()
6361 	adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/  in gfx_v9_0_set_irq_funcs()
6362 	adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;  in gfx_v9_0_set_irq_funcs()
6374 		adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;  in gfx_v9_0_set_rlc_funcs()
6450 	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);  in gfx_v9_0_get_cu_active_bitmap()
6468 	if (adev->gfx.config.max_shader_engines *  in gfx_v9_0_get_cu_info()
6469 		adev->gfx.config.max_sh_per_se > 16)  in gfx_v9_0_get_cu_info()
6473 				    adev->gfx.config.max_shader_engines,  in gfx_v9_0_get_cu_info()
6474 				    adev->gfx.config.max_sh_per_se);  in gfx_v9_0_get_cu_info()
6477 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {  in gfx_v9_0_get_cu_info()
6478 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {  in gfx_v9_0_get_cu_info()
6484 				adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);  in gfx_v9_0_get_cu_info()
6501 			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {  in gfx_v9_0_get_cu_info()
6503 					if (counter < adev->gfx.config.max_cu_per_sh)  in gfx_v9_0_get_cu_info()