47 	bit += mec * adev->gfx.mec.num_pipe_per_mec  in amdgpu_gfx_mec_queue_to_bit()
48 		* adev->gfx.mec.num_queue_per_pipe;  in amdgpu_gfx_mec_queue_to_bit()
49 	bit += pipe * adev->gfx.mec.num_queue_per_pipe;  in amdgpu_gfx_mec_queue_to_bit()
58 	*queue = bit % adev->gfx.mec.num_queue_per_pipe;  in amdgpu_queue_mask_bit_to_mec_queue()
59 	*pipe = (bit / adev->gfx.mec.num_queue_per_pipe)  in amdgpu_queue_mask_bit_to_mec_queue()
60 		% adev->gfx.mec.num_pipe_per_mec;  in amdgpu_queue_mask_bit_to_mec_queue()
61 	*mec = (bit / adev->gfx.mec.num_queue_per_pipe)  in amdgpu_queue_mask_bit_to_mec_queue()
62 	       / adev->gfx.mec.num_pipe_per_mec;  in amdgpu_queue_mask_bit_to_mec_queue()
70 			adev->gfx.mec_bitmap[xcc_id].queue_bitmap);  in amdgpu_gfx_is_mec_queue_enabled()
78 	bit += me * adev->gfx.me.num_pipe_per_me  in amdgpu_gfx_me_queue_to_bit()
79 		* adev->gfx.me.num_queue_per_pipe;  in amdgpu_gfx_me_queue_to_bit()
80 	bit += pipe * adev->gfx.me.num_queue_per_pipe;  in amdgpu_gfx_me_queue_to_bit()
89 	*queue = bit % adev->gfx.me.num_queue_per_pipe;  in amdgpu_gfx_bit_to_me_queue()
90 	*pipe = (bit / adev->gfx.me.num_queue_per_pipe)  in amdgpu_gfx_bit_to_me_queue()
91 		% adev->gfx.me.num_pipe_per_me;  in amdgpu_gfx_bit_to_me_queue()
92 	*me = (bit / adev->gfx.me.num_queue_per_pipe)  in amdgpu_gfx_bit_to_me_queue()
93 		/ adev->gfx.me.num_pipe_per_me;  in amdgpu_gfx_bit_to_me_queue()
100 			adev->gfx.me.queue_bitmap);  in amdgpu_gfx_is_me_queue_enabled()
150 	return amdgpu_async_gfx_ring && adev->gfx.me.num_pipe_per_me > 1;  in amdgpu_gfx_is_graphics_multipipe_capable()
169 	return adev->gfx.mec.num_mec > 1;  in amdgpu_gfx_is_compute_multipipe_capable()
182 	    adev->gfx.num_gfx_rings > 1 && pipe == 1 && queue == 0) {  in amdgpu_gfx_is_high_priority_graphics_queue()
187 		if (ring == &adev->gfx.gfx_ring[bit])  in amdgpu_gfx_is_high_priority_graphics_queue()
200 	if (adev->gfx.num_compute_rings > 1 &&  in amdgpu_gfx_is_high_priority_compute_queue()
201 	    ring == &adev->gfx.compute_ring[0])  in amdgpu_gfx_is_high_priority_compute_queue()
211 	int max_queues_per_mec = min(adev->gfx.mec.num_pipe_per_mec *  in amdgpu_gfx_compute_queue_acquire()
212 				     adev->gfx.mec.num_queue_per_pipe,  in amdgpu_gfx_compute_queue_acquire()
213 				     adev->gfx.num_compute_rings);  in amdgpu_gfx_compute_queue_acquire()
214 	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;  in amdgpu_gfx_compute_queue_acquire()
221 				pipe = i % adev->gfx.mec.num_pipe_per_mec;  in amdgpu_gfx_compute_queue_acquire()
222 				queue = (i / adev->gfx.mec.num_pipe_per_mec) %  in amdgpu_gfx_compute_queue_acquire()
223 					 adev->gfx.mec.num_queue_per_pipe;  in amdgpu_gfx_compute_queue_acquire()
225 				set_bit(pipe * adev->gfx.mec.num_queue_per_pipe + queue,  in amdgpu_gfx_compute_queue_acquire()
226 					adev->gfx.mec_bitmap[j].queue_bitmap);  in amdgpu_gfx_compute_queue_acquire()
233 				set_bit(i, adev->gfx.mec_bitmap[j].queue_bitmap);  in amdgpu_gfx_compute_queue_acquire()
239 			bitmap_weight(adev->gfx.mec_bitmap[j].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES));  in amdgpu_gfx_compute_queue_acquire()
247 	int max_queues_per_me = adev->gfx.me.num_pipe_per_me *  in amdgpu_gfx_graphics_queue_acquire()
248 					adev->gfx.me.num_queue_per_pipe;  in amdgpu_gfx_graphics_queue_acquire()
254 			pipe = i % adev->gfx.me.num_pipe_per_me;  in amdgpu_gfx_graphics_queue_acquire()
255 			queue = (i / adev->gfx.me.num_pipe_per_me) %  in amdgpu_gfx_graphics_queue_acquire()
256 				adev->gfx.me.num_queue_per_pipe;  in amdgpu_gfx_graphics_queue_acquire()
258 			set_bit(pipe * adev->gfx.me.num_queue_per_pipe + queue,  in amdgpu_gfx_graphics_queue_acquire()
259 				adev->gfx.me.queue_bitmap);  in amdgpu_gfx_graphics_queue_acquire()
263 			set_bit(i, adev->gfx.me.queue_bitmap);  in amdgpu_gfx_graphics_queue_acquire()
267 	adev->gfx.num_gfx_rings =  in amdgpu_gfx_graphics_queue_acquire()
268 		bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);  in amdgpu_gfx_graphics_queue_acquire()
277 	queue_bit = adev->gfx.mec.num_mec  in amdgpu_gfx_kiq_acquire()
278 		    * adev->gfx.mec.num_pipe_per_mec  in amdgpu_gfx_kiq_acquire()
279 		    * adev->gfx.mec.num_queue_per_pipe;  in amdgpu_gfx_kiq_acquire()
282 		if (test_bit(queue_bit, adev->gfx.mec_bitmap[xcc_id].queue_bitmap))  in amdgpu_gfx_kiq_acquire()
310 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_kiq_init_ring()
347 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_kiq_fini()
357 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_kiq_init()
383 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_mqd_sw_init()
420 		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {  in amdgpu_gfx_mqd_sw_init()
421 			ring = &adev->gfx.gfx_ring[i];  in amdgpu_gfx_mqd_sw_init()
433 				adev->gfx.me.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL);  in amdgpu_gfx_mqd_sw_init()
434 				if (!adev->gfx.me.mqd_backup[i]) {  in amdgpu_gfx_mqd_sw_init()
443 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in amdgpu_gfx_mqd_sw_init()
444 		j = i + xcc_id * adev->gfx.num_compute_rings;  in amdgpu_gfx_mqd_sw_init()
445 		ring = &adev->gfx.compute_ring[j];  in amdgpu_gfx_mqd_sw_init()
457 			adev->gfx.mec.mqd_backup[j] = kmalloc(mqd_size, GFP_KERNEL);  in amdgpu_gfx_mqd_sw_init()
458 			if (!adev->gfx.mec.mqd_backup[j]) {  in amdgpu_gfx_mqd_sw_init()
472 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_mqd_sw_fini()
475 		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {  in amdgpu_gfx_mqd_sw_fini()
476 			ring = &adev->gfx.gfx_ring[i];  in amdgpu_gfx_mqd_sw_fini()
477 			kfree(adev->gfx.me.mqd_backup[i]);  in amdgpu_gfx_mqd_sw_fini()
484 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in amdgpu_gfx_mqd_sw_fini()
485 		j = i + xcc_id * adev->gfx.num_compute_rings;  in amdgpu_gfx_mqd_sw_fini()
486 		ring = &adev->gfx.compute_ring[j];  in amdgpu_gfx_mqd_sw_fini()
487 		kfree(adev->gfx.mec.mqd_backup[j]);  in amdgpu_gfx_mqd_sw_fini()
502 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_disable_kcq()
512 					adev->gfx.num_compute_rings)) {  in amdgpu_gfx_disable_kcq()
517 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in amdgpu_gfx_disable_kcq()
518 		j = i + xcc_id * adev->gfx.num_compute_rings;  in amdgpu_gfx_disable_kcq()
520 					   &adev->gfx.compute_ring[j],  in amdgpu_gfx_disable_kcq()
533 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_disable_kgq()
544 						adev->gfx.num_gfx_rings)) {  in amdgpu_gfx_disable_kgq()
549 		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {  in amdgpu_gfx_disable_kgq()
550 			j = i + xcc_id * adev->gfx.num_gfx_rings;  in amdgpu_gfx_disable_kgq()
552 						   &adev->gfx.gfx_ring[j],  in amdgpu_gfx_disable_kgq()
557 	if (adev->gfx.kiq[0].ring.sched.ready && !adev->job_hang)  in amdgpu_gfx_disable_kgq()
579 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_enable_kcq()
588 		if (!test_bit(i, adev->gfx.mec_bitmap[xcc_id].queue_bitmap))  in amdgpu_gfx_enable_kcq()
608 					adev->gfx.num_compute_rings +  in amdgpu_gfx_enable_kcq()
620 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in amdgpu_gfx_enable_kcq()
621 		j = i + xcc_id * adev->gfx.num_compute_rings;  in amdgpu_gfx_enable_kcq()
623 						 &adev->gfx.compute_ring[j]);  in amdgpu_gfx_enable_kcq()
636 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_enable_kgq()
649 						adev->gfx.num_gfx_rings);  in amdgpu_gfx_enable_kgq()
656 		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {  in amdgpu_gfx_enable_kgq()
657 			j = i + xcc_id * adev->gfx.num_gfx_rings;  in amdgpu_gfx_enable_kgq()
659 						 &adev->gfx.gfx_ring[j]);  in amdgpu_gfx_enable_kgq()
689 	mutex_lock(&adev->gfx.gfx_off_mutex);  in amdgpu_gfx_off_ctrl()
696 		if (WARN_ON_ONCE(adev->gfx.gfx_off_req_count == 0))  in amdgpu_gfx_off_ctrl()
699 		adev->gfx.gfx_off_req_count--;  in amdgpu_gfx_off_ctrl()
701 		if (adev->gfx.gfx_off_req_count == 0 &&  in amdgpu_gfx_off_ctrl()
702 		    !adev->gfx.gfx_off_state) {  in amdgpu_gfx_off_ctrl()
703 			schedule_delayed_work(&adev->gfx.gfx_off_delay_work,  in amdgpu_gfx_off_ctrl()
707 		if (adev->gfx.gfx_off_req_count == 0) {  in amdgpu_gfx_off_ctrl()
708 			cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work);  in amdgpu_gfx_off_ctrl()
710 			if (adev->gfx.gfx_off_state &&  in amdgpu_gfx_off_ctrl()
712 				adev->gfx.gfx_off_state = false;  in amdgpu_gfx_off_ctrl()
714 				if (adev->gfx.funcs->init_spm_golden) {  in amdgpu_gfx_off_ctrl()
722 		adev->gfx.gfx_off_req_count++;  in amdgpu_gfx_off_ctrl()
726 	mutex_unlock(&adev->gfx.gfx_off_mutex);  in amdgpu_gfx_off_ctrl()
733 	mutex_lock(&adev->gfx.gfx_off_mutex);  in amdgpu_set_gfx_off_residency()
737 	mutex_unlock(&adev->gfx.gfx_off_mutex);  in amdgpu_set_gfx_off_residency()
746 	mutex_lock(&adev->gfx.gfx_off_mutex);  in amdgpu_get_gfx_off_residency()
750 	mutex_unlock(&adev->gfx.gfx_off_mutex);  in amdgpu_get_gfx_off_residency()
759 	mutex_lock(&adev->gfx.gfx_off_mutex);  in amdgpu_get_gfx_off_entrycount()
763 	mutex_unlock(&adev->gfx.gfx_off_mutex);  in amdgpu_get_gfx_off_entrycount()
773 	mutex_lock(&adev->gfx.gfx_off_mutex);  in amdgpu_get_gfx_off_status()
777 	mutex_unlock(&adev->gfx.gfx_off_mutex);  in amdgpu_get_gfx_off_status()
794 		if (adev->gfx.cp_ecc_error_irq.funcs) {  in amdgpu_gfx_ras_late_init()
795 			r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);  in amdgpu_gfx_ras_late_init()
817 	if (!adev->gfx.ras)  in amdgpu_gfx_ras_sw_init()
820 	ras = adev->gfx.ras;  in amdgpu_gfx_ras_sw_init()
831 	adev->gfx.ras_if = &ras->ras_block.ras_comm;  in amdgpu_gfx_ras_sw_init()
847 	if (adev->gfx.ras && adev->gfx.ras->poison_consumption_handler)  in amdgpu_gfx_poison_consumption_handler()
848 		return adev->gfx.ras->poison_consumption_handler(adev, entry);  in amdgpu_gfx_poison_consumption_handler()
865 		if (adev->gfx.ras && adev->gfx.ras->ras_block.hw_ops &&  in amdgpu_gfx_process_ras_data_cb()
866 		    adev->gfx.ras->ras_block.hw_ops->query_ras_error_count)  in amdgpu_gfx_process_ras_data_cb()
867 			adev->gfx.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);  in amdgpu_gfx_process_ras_data_cb()
877 	struct ras_common_if *ras_if = adev->gfx.ras_if;  in amdgpu_gfx_cp_ecc_error_irq()
898 	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;  in amdgpu_gfx_ras_error_func()
916 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];  in amdgpu_kiq_rreg()
984 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];  in amdgpu_kiq_wreg()
1062 			adev->gfx.pfp_fw->data;  in amdgpu_gfx_cp_init_microcode()
1063 		adev->gfx.pfp_fw_version =  in amdgpu_gfx_cp_init_microcode()
1065 		adev->gfx.pfp_feature_version =  in amdgpu_gfx_cp_init_microcode()
1067 		ucode_fw = adev->gfx.pfp_fw;  in amdgpu_gfx_cp_init_microcode()
1072 			adev->gfx.pfp_fw->data;  in amdgpu_gfx_cp_init_microcode()
1073 		adev->gfx.pfp_fw_version =  in amdgpu_gfx_cp_init_microcode()
1075 		adev->gfx.pfp_feature_version =  in amdgpu_gfx_cp_init_microcode()
1077 		ucode_fw = adev->gfx.pfp_fw;  in amdgpu_gfx_cp_init_microcode()
1083 			adev->gfx.pfp_fw->data;  in amdgpu_gfx_cp_init_microcode()
1084 		ucode_fw = adev->gfx.pfp_fw;  in amdgpu_gfx_cp_init_microcode()
1089 			adev->gfx.me_fw->data;  in amdgpu_gfx_cp_init_microcode()
1090 		adev->gfx.me_fw_version =  in amdgpu_gfx_cp_init_microcode()
1092 		adev->gfx.me_feature_version =  in amdgpu_gfx_cp_init_microcode()
1094 		ucode_fw = adev->gfx.me_fw;  in amdgpu_gfx_cp_init_microcode()
1099 			adev->gfx.me_fw->data;  in amdgpu_gfx_cp_init_microcode()
1100 		adev->gfx.me_fw_version =  in amdgpu_gfx_cp_init_microcode()
1102 		adev->gfx.me_feature_version =  in amdgpu_gfx_cp_init_microcode()
1104 		ucode_fw = adev->gfx.me_fw;  in amdgpu_gfx_cp_init_microcode()
1110 			adev->gfx.me_fw->data;  in amdgpu_gfx_cp_init_microcode()
1111 		ucode_fw = adev->gfx.me_fw;  in amdgpu_gfx_cp_init_microcode()
1116 			adev->gfx.ce_fw->data;  in amdgpu_gfx_cp_init_microcode()
1117 		adev->gfx.ce_fw_version =  in amdgpu_gfx_cp_init_microcode()
1119 		adev->gfx.ce_feature_version =  in amdgpu_gfx_cp_init_microcode()
1121 		ucode_fw = adev->gfx.ce_fw;  in amdgpu_gfx_cp_init_microcode()
1126 			adev->gfx.mec_fw->data;  in amdgpu_gfx_cp_init_microcode()
1127 		adev->gfx.mec_fw_version =  in amdgpu_gfx_cp_init_microcode()
1129 		adev->gfx.mec_feature_version =  in amdgpu_gfx_cp_init_microcode()
1131 		ucode_fw = adev->gfx.mec_fw;  in amdgpu_gfx_cp_init_microcode()
1137 			adev->gfx.mec_fw->data;  in amdgpu_gfx_cp_init_microcode()
1138 		ucode_fw = adev->gfx.mec_fw;  in amdgpu_gfx_cp_init_microcode()
1143 			adev->gfx.mec2_fw->data;  in amdgpu_gfx_cp_init_microcode()
1144 		adev->gfx.mec2_fw_version =  in amdgpu_gfx_cp_init_microcode()
1146 		adev->gfx.mec2_feature_version =  in amdgpu_gfx_cp_init_microcode()
1148 		ucode_fw = adev->gfx.mec2_fw;  in amdgpu_gfx_cp_init_microcode()
1154 			adev->gfx.mec2_fw->data;  in amdgpu_gfx_cp_init_microcode()
1155 		ucode_fw = adev->gfx.mec2_fw;  in amdgpu_gfx_cp_init_microcode()
1160 			adev->gfx.mec_fw->data;  in amdgpu_gfx_cp_init_microcode()
1161 		adev->gfx.mec_fw_version =  in amdgpu_gfx_cp_init_microcode()
1163 		adev->gfx.mec_feature_version =  in amdgpu_gfx_cp_init_microcode()
1165 		ucode_fw = adev->gfx.mec_fw;  in amdgpu_gfx_cp_init_microcode()
1173 			adev->gfx.mec_fw->data;  in amdgpu_gfx_cp_init_microcode()
1174 		ucode_fw = adev->gfx.mec_fw;  in amdgpu_gfx_cp_init_microcode()
1191 	return !(xcc_id % (adev->gfx.num_xcc_per_xcp ?  in amdgpu_gfx_is_master_xcc()
1192 			adev->gfx.num_xcc_per_xcp : 1));  in amdgpu_gfx_is_master_xcc()
1218 	num_xcc = NUM_XCC(adev->gfx.xcc_mask);  in amdgpu_gfx_set_compute_partition()
1263 	switch (NUM_XCC(adev->gfx.xcc_mask)) {  in amdgpu_gfx_get_available_compute_partition()