32 /* delay 0.1 second to enable gfx off feature */
38  * GPU GFX IP block helpers function.
46 	bit += mec * adev->gfx.mec.num_pipe_per_mec  in amdgpu_gfx_mec_queue_to_bit()
47 		* adev->gfx.mec.num_queue_per_pipe;  in amdgpu_gfx_mec_queue_to_bit()
48 	bit += pipe * adev->gfx.mec.num_queue_per_pipe;  in amdgpu_gfx_mec_queue_to_bit()
57 	*queue = bit % adev->gfx.mec.num_queue_per_pipe;  in amdgpu_queue_mask_bit_to_mec_queue()
58 	*pipe = (bit / adev->gfx.mec.num_queue_per_pipe)  in amdgpu_queue_mask_bit_to_mec_queue()
59 		% adev->gfx.mec.num_pipe_per_mec;  in amdgpu_queue_mask_bit_to_mec_queue()
60 	*mec = (bit / adev->gfx.mec.num_queue_per_pipe)  in amdgpu_queue_mask_bit_to_mec_queue()
61 	       / adev->gfx.mec.num_pipe_per_mec;  in amdgpu_queue_mask_bit_to_mec_queue()
69 			adev->gfx.mec.queue_bitmap);  in amdgpu_gfx_is_mec_queue_enabled()
77 	bit += me * adev->gfx.me.num_pipe_per_me  in amdgpu_gfx_me_queue_to_bit()
78 		* adev->gfx.me.num_queue_per_pipe;  in amdgpu_gfx_me_queue_to_bit()
79 	bit += pipe * adev->gfx.me.num_queue_per_pipe;  in amdgpu_gfx_me_queue_to_bit()
88 	*queue = bit % adev->gfx.me.num_queue_per_pipe;  in amdgpu_gfx_bit_to_me_queue()
89 	*pipe = (bit / adev->gfx.me.num_queue_per_pipe)  in amdgpu_gfx_bit_to_me_queue()
90 		% adev->gfx.me.num_pipe_per_me;  in amdgpu_gfx_bit_to_me_queue()
91 	*me = (bit / adev->gfx.me.num_queue_per_pipe)  in amdgpu_gfx_bit_to_me_queue()
92 		/ adev->gfx.me.num_pipe_per_me;  in amdgpu_gfx_bit_to_me_queue()
99 			adev->gfx.me.queue_bitmap);  in amdgpu_gfx_is_me_queue_enabled()
148 	return amdgpu_async_gfx_ring && adev->gfx.me.num_pipe_per_me > 1;  in amdgpu_gfx_is_graphics_multipipe_capable()
164 	return adev->gfx.mec.num_mec > 1;  in amdgpu_gfx_is_compute_multipipe_capable()
174 	 * have more than one gfx pipe.  in amdgpu_gfx_is_high_priority_graphics_queue()
177 	    adev->gfx.num_gfx_rings > 1 && pipe == 1 && queue == 0) {  in amdgpu_gfx_is_high_priority_graphics_queue()
182 		if (ring == &adev->gfx.gfx_ring[bit])  in amdgpu_gfx_is_high_priority_graphics_queue()
195 	if (adev->gfx.num_compute_rings > 1 &&  in amdgpu_gfx_is_high_priority_compute_queue()
196 	    ring == &adev->gfx.compute_ring[0])  in amdgpu_gfx_is_high_priority_compute_queue()
206 	int max_queues_per_mec = min(adev->gfx.mec.num_pipe_per_mec *  in amdgpu_gfx_compute_queue_acquire()
207 				     adev->gfx.mec.num_queue_per_pipe,  in amdgpu_gfx_compute_queue_acquire()
208 				     adev->gfx.num_compute_rings);  in amdgpu_gfx_compute_queue_acquire()
213 			pipe = i % adev->gfx.mec.num_pipe_per_mec;  in amdgpu_gfx_compute_queue_acquire()
214 			queue = (i / adev->gfx.mec.num_pipe_per_mec) %  in amdgpu_gfx_compute_queue_acquire()
215 				adev->gfx.mec.num_queue_per_pipe;  in amdgpu_gfx_compute_queue_acquire()
217 			set_bit(pipe * adev->gfx.mec.num_queue_per_pipe + queue,  in amdgpu_gfx_compute_queue_acquire()
218 					adev->gfx.mec.queue_bitmap);  in amdgpu_gfx_compute_queue_acquire()
223 			set_bit(i, adev->gfx.mec.queue_bitmap);  in amdgpu_gfx_compute_queue_acquire()
226 …dev_dbg(adev->dev, "mec queue bitmap weight=%d\n", bitmap_weight(adev->gfx.mec.queue_bitmap, AMDGP…  in amdgpu_gfx_compute_queue_acquire()
233 	int max_queues_per_me = adev->gfx.me.num_pipe_per_me *  in amdgpu_gfx_graphics_queue_acquire()
234 					adev->gfx.me.num_queue_per_pipe;  in amdgpu_gfx_graphics_queue_acquire()
240 			pipe = i % adev->gfx.me.num_pipe_per_me;  in amdgpu_gfx_graphics_queue_acquire()
241 			queue = (i / adev->gfx.me.num_pipe_per_me) %  in amdgpu_gfx_graphics_queue_acquire()
242 				adev->gfx.me.num_queue_per_pipe;  in amdgpu_gfx_graphics_queue_acquire()
244 			set_bit(pipe * adev->gfx.me.num_queue_per_pipe + queue,  in amdgpu_gfx_graphics_queue_acquire()
245 				adev->gfx.me.queue_bitmap);  in amdgpu_gfx_graphics_queue_acquire()
249 			set_bit(i, adev->gfx.me.queue_bitmap);  in amdgpu_gfx_graphics_queue_acquire()
253 	adev->gfx.num_gfx_rings =  in amdgpu_gfx_graphics_queue_acquire()
254 		bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);  in amdgpu_gfx_graphics_queue_acquire()
263 	queue_bit = adev->gfx.mec.num_mec  in amdgpu_gfx_kiq_acquire()
264 		    * adev->gfx.mec.num_pipe_per_mec  in amdgpu_gfx_kiq_acquire()
265 		    * adev->gfx.mec.num_queue_per_pipe;  in amdgpu_gfx_kiq_acquire()
268 		if (test_bit(queue_bit, adev->gfx.mec.queue_bitmap))  in amdgpu_gfx_kiq_acquire()
296 	struct amdgpu_kiq *kiq = &adev->gfx.kiq;  in amdgpu_gfx_kiq_init_ring()
328 	struct amdgpu_kiq *kiq = &adev->gfx.kiq;  in amdgpu_gfx_kiq_fini()
338 	struct amdgpu_kiq *kiq = &adev->gfx.kiq;  in amdgpu_gfx_kiq_init()
359 /* create MQD for each compute/gfx queue */
367 	ring = &adev->gfx.kiq.ring;  in amdgpu_gfx_mqd_sw_init()
383 		adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS] = kmalloc(mqd_size, GFP_KERNEL);  in amdgpu_gfx_mqd_sw_init()
384 		if (!adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS])  in amdgpu_gfx_mqd_sw_init()
390 		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {  in amdgpu_gfx_mqd_sw_init()
391 			ring = &adev->gfx.gfx_ring[i];  in amdgpu_gfx_mqd_sw_init()
402 				adev->gfx.me.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL);  in amdgpu_gfx_mqd_sw_init()
403 				if (!adev->gfx.me.mqd_backup[i])  in amdgpu_gfx_mqd_sw_init()
410 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in amdgpu_gfx_mqd_sw_init()
411 		ring = &adev->gfx.compute_ring[i];  in amdgpu_gfx_mqd_sw_init()
422 			adev->gfx.mec.mqd_backup[i] = kmalloc(mqd_size, GFP_KERNEL);  in amdgpu_gfx_mqd_sw_init()
423 			if (!adev->gfx.mec.mqd_backup[i])  in amdgpu_gfx_mqd_sw_init()
437 		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {  in amdgpu_gfx_mqd_sw_fini()
438 			ring = &adev->gfx.gfx_ring[i];  in amdgpu_gfx_mqd_sw_fini()
439 			kfree(adev->gfx.me.mqd_backup[i]);  in amdgpu_gfx_mqd_sw_fini()
446 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {  in amdgpu_gfx_mqd_sw_fini()
447 		ring = &adev->gfx.compute_ring[i];  in amdgpu_gfx_mqd_sw_fini()
448 		kfree(adev->gfx.mec.mqd_backup[i]);  in amdgpu_gfx_mqd_sw_fini()
454 	ring = &adev->gfx.kiq.ring;  in amdgpu_gfx_mqd_sw_fini()
455 	kfree(adev->gfx.mec.mqd_backup[AMDGPU_MAX_COMPUTE_RINGS]);  in amdgpu_gfx_mqd_sw_fini()
463 	struct amdgpu_kiq *kiq = &adev->gfx.kiq;  in amdgpu_gfx_disable_kcq()
470 	spin_lock(&adev->gfx.kiq.ring_lock);  in amdgpu_gfx_disable_kcq()
472 					adev->gfx.num_compute_rings)) {  in amdgpu_gfx_disable_kcq()
473 		spin_unlock(&adev->gfx.kiq.ring_lock);  in amdgpu_gfx_disable_kcq()
477 	for (i = 0; i < adev->gfx.num_compute_rings; i++)  in amdgpu_gfx_disable_kcq()
478 		kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.compute_ring[i],  in amdgpu_gfx_disable_kcq()
481 	if (adev->gfx.kiq.ring.sched.ready && !adev->job_hang)  in amdgpu_gfx_disable_kcq()
483 	spin_unlock(&adev->gfx.kiq.ring_lock);  in amdgpu_gfx_disable_kcq()
503 	struct amdgpu_kiq *kiq = &adev->gfx.kiq;  in amdgpu_gfx_enable_kcq()
504 	struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;  in amdgpu_gfx_enable_kcq()
512 		if (!test_bit(i, adev->gfx.mec.queue_bitmap))  in amdgpu_gfx_enable_kcq()
528 	spin_lock(&adev->gfx.kiq.ring_lock);  in amdgpu_gfx_enable_kcq()
530 					adev->gfx.num_compute_rings +  in amdgpu_gfx_enable_kcq()
534 		spin_unlock(&adev->gfx.kiq.ring_lock);  in amdgpu_gfx_enable_kcq()
542 	for (i = 0; i < adev->gfx.num_compute_rings; i++)  in amdgpu_gfx_enable_kcq()
543 		kiq->pmf->kiq_map_queues(kiq_ring, &adev->gfx.compute_ring[i]);  in amdgpu_gfx_enable_kcq()
546 	spin_unlock(&adev->gfx.kiq.ring_lock);  in amdgpu_gfx_enable_kcq()
553 /* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable
556  * @bool enable true: enable gfx off feature, false: disable gfx off feature
558  * 1. gfx off feature will be enabled by gfx ip after gfx cg gp enabled.
559  * 2. other client can send request to disable gfx off feature, the request should be honored.
560  * 3. other client can cancel their request of disable gfx off feature
561  * 4. other client should not send request to enable gfx off feature before disable gfx off feature.
571 	mutex_lock(&adev->gfx.gfx_off_mutex);  in amdgpu_gfx_off_ctrl()
578 		if (WARN_ON_ONCE(adev->gfx.gfx_off_req_count == 0))  in amdgpu_gfx_off_ctrl()
581 		adev->gfx.gfx_off_req_count--;  in amdgpu_gfx_off_ctrl()
583 		if (adev->gfx.gfx_off_req_count == 0 &&  in amdgpu_gfx_off_ctrl()
584 		    !adev->gfx.gfx_off_state) {  in amdgpu_gfx_off_ctrl()
588 			schedule_delayed_work(&adev->gfx.gfx_off_delay_work,  in amdgpu_gfx_off_ctrl()
592 		if (adev->gfx.gfx_off_req_count == 0) {  in amdgpu_gfx_off_ctrl()
593 			cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work);  in amdgpu_gfx_off_ctrl()
595 			if (adev->gfx.gfx_off_state &&  in amdgpu_gfx_off_ctrl()
597 				adev->gfx.gfx_off_state = false;  in amdgpu_gfx_off_ctrl()
599 				if (adev->gfx.funcs->init_spm_golden) {  in amdgpu_gfx_off_ctrl()
607 		adev->gfx.gfx_off_req_count++;  in amdgpu_gfx_off_ctrl()
611 	mutex_unlock(&adev->gfx.gfx_off_mutex);  in amdgpu_gfx_off_ctrl()
618 	mutex_lock(&adev->gfx.gfx_off_mutex);  in amdgpu_set_gfx_off_residency()
622 	mutex_unlock(&adev->gfx.gfx_off_mutex);  in amdgpu_set_gfx_off_residency()
631 	mutex_lock(&adev->gfx.gfx_off_mutex);  in amdgpu_get_gfx_off_residency()
635 	mutex_unlock(&adev->gfx.gfx_off_mutex);  in amdgpu_get_gfx_off_residency()
644 	mutex_lock(&adev->gfx.gfx_off_mutex);  in amdgpu_get_gfx_off_entrycount()
648 	mutex_unlock(&adev->gfx.gfx_off_mutex);  in amdgpu_get_gfx_off_entrycount()
658 	mutex_lock(&adev->gfx.gfx_off_mutex);  in amdgpu_get_gfx_off_status()
662 	mutex_unlock(&adev->gfx.gfx_off_mutex);  in amdgpu_get_gfx_off_status()
679 		r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);  in amdgpu_gfx_ras_late_init()
704 		if (adev->gfx.ras && adev->gfx.ras->ras_block.hw_ops &&  in amdgpu_gfx_process_ras_data_cb()
705 		    adev->gfx.ras->ras_block.hw_ops->query_ras_error_count)  in amdgpu_gfx_process_ras_data_cb()
706 			adev->gfx.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);  in amdgpu_gfx_process_ras_data_cb()
716 	struct ras_common_if *ras_if = adev->gfx.ras_if;  in amdgpu_gfx_cp_ecc_error_irq()
736 	struct amdgpu_kiq *kiq = &adev->gfx.kiq;  in amdgpu_kiq_rreg()
804 	struct amdgpu_kiq *kiq = &adev->gfx.kiq;  in amdgpu_kiq_wreg()
882 			adev->gfx.pfp_fw->data;  in amdgpu_gfx_cp_init_microcode()
883 		adev->gfx.pfp_fw_version =  in amdgpu_gfx_cp_init_microcode()
885 		adev->gfx.pfp_feature_version =  in amdgpu_gfx_cp_init_microcode()
887 		ucode_fw = adev->gfx.pfp_fw;  in amdgpu_gfx_cp_init_microcode()
892 			adev->gfx.pfp_fw->data;  in amdgpu_gfx_cp_init_microcode()
893 		adev->gfx.pfp_fw_version =  in amdgpu_gfx_cp_init_microcode()
895 		adev->gfx.pfp_feature_version =  in amdgpu_gfx_cp_init_microcode()
897 		ucode_fw = adev->gfx.pfp_fw;  in amdgpu_gfx_cp_init_microcode()
903 			adev->gfx.pfp_fw->data;  in amdgpu_gfx_cp_init_microcode()
904 		ucode_fw = adev->gfx.pfp_fw;  in amdgpu_gfx_cp_init_microcode()
909 			adev->gfx.me_fw->data;  in amdgpu_gfx_cp_init_microcode()
910 		adev->gfx.me_fw_version =  in amdgpu_gfx_cp_init_microcode()
912 		adev->gfx.me_feature_version =  in amdgpu_gfx_cp_init_microcode()
914 		ucode_fw = adev->gfx.me_fw;  in amdgpu_gfx_cp_init_microcode()
919 			adev->gfx.me_fw->data;  in amdgpu_gfx_cp_init_microcode()
920 		adev->gfx.me_fw_version =  in amdgpu_gfx_cp_init_microcode()
922 		adev->gfx.me_feature_version =  in amdgpu_gfx_cp_init_microcode()
924 		ucode_fw = adev->gfx.me_fw;  in amdgpu_gfx_cp_init_microcode()
930 			adev->gfx.me_fw->data;  in amdgpu_gfx_cp_init_microcode()
931 		ucode_fw = adev->gfx.me_fw;  in amdgpu_gfx_cp_init_microcode()
936 			adev->gfx.ce_fw->data;  in amdgpu_gfx_cp_init_microcode()
937 		adev->gfx.ce_fw_version =  in amdgpu_gfx_cp_init_microcode()
939 		adev->gfx.ce_feature_version =  in amdgpu_gfx_cp_init_microcode()
941 		ucode_fw = adev->gfx.ce_fw;  in amdgpu_gfx_cp_init_microcode()
946 			adev->gfx.mec_fw->data;  in amdgpu_gfx_cp_init_microcode()
947 		adev->gfx.mec_fw_version =  in amdgpu_gfx_cp_init_microcode()
949 		adev->gfx.mec_feature_version =  in amdgpu_gfx_cp_init_microcode()
951 		ucode_fw = adev->gfx.mec_fw;  in amdgpu_gfx_cp_init_microcode()
957 			adev->gfx.mec_fw->data;  in amdgpu_gfx_cp_init_microcode()
958 		ucode_fw = adev->gfx.mec_fw;  in amdgpu_gfx_cp_init_microcode()
963 			adev->gfx.mec2_fw->data;  in amdgpu_gfx_cp_init_microcode()
964 		adev->gfx.mec2_fw_version =  in amdgpu_gfx_cp_init_microcode()
966 		adev->gfx.mec2_feature_version =  in amdgpu_gfx_cp_init_microcode()
968 		ucode_fw = adev->gfx.mec2_fw;  in amdgpu_gfx_cp_init_microcode()
974 			adev->gfx.mec2_fw->data;  in amdgpu_gfx_cp_init_microcode()
975 		ucode_fw = adev->gfx.mec2_fw;  in amdgpu_gfx_cp_init_microcode()
980 			adev->gfx.mec_fw->data;  in amdgpu_gfx_cp_init_microcode()
981 		adev->gfx.mec_fw_version =  in amdgpu_gfx_cp_init_microcode()
983 		adev->gfx.mec_feature_version =  in amdgpu_gfx_cp_init_microcode()
985 		ucode_fw = adev->gfx.mec_fw;  in amdgpu_gfx_cp_init_microcode()
993 			adev->gfx.mec_fw->data;  in amdgpu_gfx_cp_init_microcode()
994 		ucode_fw = adev->gfx.mec_fw;  in amdgpu_gfx_cp_init_microcode()