90 static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,  in mes_v11_0_submit_pkt_and_poll_completion()  argument
98 	struct amdgpu_device *adev = mes->adev;  in mes_v11_0_submit_pkt_and_poll_completion()
99 	struct amdgpu_ring *ring = &mes->ring;  in mes_v11_0_submit_pkt_and_poll_completion()
111 	spin_lock_irqsave(&mes->ring_lock, flags);  in mes_v11_0_submit_pkt_and_poll_completion()
113 		spin_unlock_irqrestore(&mes->ring_lock, flags);  in mes_v11_0_submit_pkt_and_poll_completion()
118 	api_status->api_completion_fence_addr = mes->ring.fence_drv.gpu_addr;  in mes_v11_0_submit_pkt_and_poll_completion()
119 	api_status->api_completion_fence_value = ++mes->ring.fence_drv.sync_seq;  in mes_v11_0_submit_pkt_and_poll_completion()
123 	spin_unlock_irqrestore(&mes->ring_lock, flags);  in mes_v11_0_submit_pkt_and_poll_completion()
151 static int mes_v11_0_add_hw_queue(struct amdgpu_mes *mes,  in mes_v11_0_add_hw_queue()  argument
154 	struct amdgpu_device *adev = mes->adev;  in mes_v11_0_add_hw_queue()
180 	if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>  in mes_v11_0_add_hw_queue()
201 	if (!(((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 4) &&  in mes_v11_0_add_hw_queue()
210 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_add_hw_queue()
215 static int mes_v11_0_remove_hw_queue(struct amdgpu_mes *mes,  in mes_v11_0_remove_hw_queue()  argument
229 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_remove_hw_queue()
234 static int mes_v11_0_unmap_legacy_queue(struct amdgpu_mes *mes,  in mes_v11_0_unmap_legacy_queue()  argument
262 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_unmap_legacy_queue()
267 static int mes_v11_0_suspend_gang(struct amdgpu_mes *mes,  in mes_v11_0_suspend_gang()  argument
273 static int mes_v11_0_resume_gang(struct amdgpu_mes *mes,  in mes_v11_0_resume_gang()  argument
279 static int mes_v11_0_query_sched_status(struct amdgpu_mes *mes)  in mes_v11_0_query_sched_status()  argument
289 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_query_sched_status()
294 static int mes_v11_0_misc_op(struct amdgpu_mes *mes,  in mes_v11_0_misc_op()  argument
337 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_misc_op()
342 static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)  in mes_v11_0_set_hw_resources()  argument
345 	struct amdgpu_device *adev = mes->adev;  in mes_v11_0_set_hw_resources()
354 	mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;  in mes_v11_0_set_hw_resources()
355 	mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;  in mes_v11_0_set_hw_resources()
358 	mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr = mes->sch_ctx_gpu_addr;  in mes_v11_0_set_hw_resources()
360 		mes->query_status_fence_gpu_addr;  in mes_v11_0_set_hw_resources()
364 			mes->compute_hqd_mask[i];  in mes_v11_0_set_hw_resources()
367 		mes_set_hw_res_pkt.gfx_hqd_mask[i] = mes->gfx_hqd_mask[i];  in mes_v11_0_set_hw_resources()
370 		mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i];  in mes_v11_0_set_hw_resources()
374 			mes->aggregated_doorbells[i];  in mes_v11_0_set_hw_resources()
389 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_set_hw_resources()
394 static void mes_v11_0_init_aggregated_doorbell(struct amdgpu_mes *mes)  in mes_v11_0_init_aggregated_doorbell()  argument
396 	struct amdgpu_device *adev = mes->adev;  in mes_v11_0_init_aggregated_doorbell()
403 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<  in mes_v11_0_init_aggregated_doorbell()
412 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<  in mes_v11_0_init_aggregated_doorbell()
421 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<  in mes_v11_0_init_aggregated_doorbell()
430 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<  in mes_v11_0_init_aggregated_doorbell()
439 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<  in mes_v11_0_init_aggregated_doorbell()
475 	err = request_firmware(&adev->mes.fw[pipe], fw_name, adev->dev);  in mes_v11_0_init_microcode()
479 	err = amdgpu_ucode_validate(adev->mes.fw[pipe]);  in mes_v11_0_init_microcode()
481 		release_firmware(adev->mes.fw[pipe]);  in mes_v11_0_init_microcode()
482 		adev->mes.fw[pipe] = NULL;  in mes_v11_0_init_microcode()
487 		adev->mes.fw[pipe]->data;  in mes_v11_0_init_microcode()
488 	adev->mes.ucode_fw_version[pipe] =  in mes_v11_0_init_microcode()
490 	adev->mes.ucode_fw_version[pipe] =  in mes_v11_0_init_microcode()
492 	adev->mes.uc_start_addr[pipe] =  in mes_v11_0_init_microcode()
495 	adev->mes.data_start_addr[pipe] =  in mes_v11_0_init_microcode()
512 		info->fw = adev->mes.fw[pipe];  in mes_v11_0_init_microcode()
519 		info->fw = adev->mes.fw[pipe];  in mes_v11_0_init_microcode()
531 	release_firmware(adev->mes.fw[pipe]);  in mes_v11_0_free_microcode()
532 	adev->mes.fw[pipe] = NULL;  in mes_v11_0_free_microcode()
544 		adev->mes.fw[pipe]->data;  in mes_v11_0_allocate_ucode_buffer()
546 	fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +  in mes_v11_0_allocate_ucode_buffer()
552 				      &adev->mes.ucode_fw_obj[pipe],  in mes_v11_0_allocate_ucode_buffer()
553 				      &adev->mes.ucode_fw_gpu_addr[pipe],  in mes_v11_0_allocate_ucode_buffer()
554 				      (void **)&adev->mes.ucode_fw_ptr[pipe]);  in mes_v11_0_allocate_ucode_buffer()
560 	memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size);  in mes_v11_0_allocate_ucode_buffer()
562 	amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);  in mes_v11_0_allocate_ucode_buffer()
563 	amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);  in mes_v11_0_allocate_ucode_buffer()
577 		adev->mes.fw[pipe]->data;  in mes_v11_0_allocate_ucode_data_buffer()
579 	fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +  in mes_v11_0_allocate_ucode_data_buffer()
585 				      &adev->mes.data_fw_obj[pipe],  in mes_v11_0_allocate_ucode_data_buffer()
586 				      &adev->mes.data_fw_gpu_addr[pipe],  in mes_v11_0_allocate_ucode_data_buffer()
587 				      (void **)&adev->mes.data_fw_ptr[pipe]);  in mes_v11_0_allocate_ucode_data_buffer()
593 	memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size);  in mes_v11_0_allocate_ucode_data_buffer()
595 	amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);  in mes_v11_0_allocate_ucode_data_buffer()
596 	amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);  in mes_v11_0_allocate_ucode_data_buffer()
604 	amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],  in mes_v11_0_free_ucode_buffers()
605 			      &adev->mes.data_fw_gpu_addr[pipe],  in mes_v11_0_free_ucode_buffers()
606 			      (void **)&adev->mes.data_fw_ptr[pipe]);  in mes_v11_0_free_ucode_buffers()
608 	amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],  in mes_v11_0_free_ucode_buffers()
609 			      &adev->mes.ucode_fw_gpu_addr[pipe],  in mes_v11_0_free_ucode_buffers()
610 			      (void **)&adev->mes.ucode_fw_ptr[pipe]);  in mes_v11_0_free_ucode_buffers()
633 			ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;  in mes_v11_0_enable()
676 	if (!adev->mes.fw[pipe])  in mes_v11_0_load_microcode()
696 	ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;  in mes_v11_0_load_microcode()
704 		     lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));  in mes_v11_0_load_microcode()
706 		     upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));  in mes_v11_0_load_microcode()
713 		     lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]));  in mes_v11_0_load_microcode()
715 		     upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]));  in mes_v11_0_load_microcode()
747 			      &adev->mes.eop_gpu_obj[pipe],  in mes_v11_0_allocate_eop_buf()
748 			      &adev->mes.eop_gpu_addr[pipe],  in mes_v11_0_allocate_eop_buf()
756 	       adev->mes.eop_gpu_obj[pipe]->tbo.base.size);  in mes_v11_0_allocate_eop_buf()
758 	amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);  in mes_v11_0_allocate_eop_buf()
759 	amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);  in mes_v11_0_allocate_eop_buf()
943 	kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring);  in mes_v11_0_kiq_enable_queue()
962 		ring = &adev->mes.ring;  in mes_v11_0_queue_init()
990 		adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);  in mes_v11_0_queue_init()
992 		adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);  in mes_v11_0_queue_init()
1004 	ring = &adev->mes.ring;  in mes_v11_0_ring_init()
1015 	ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_SCHED_PIPE];  in mes_v11_0_ring_init()
1039 	ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];  in mes_v11_0_kiq_ring_init()
1057 		ring = &adev->mes.ring;  in mes_v11_0_mqd_sw_init()
1075 	adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL);  in mes_v11_0_mqd_sw_init()
1076 	if (!adev->mes.mqd_backup[pipe])  in mes_v11_0_mqd_sw_init()
1089 	adev->mes.adev = adev;  in mes_v11_0_sw_init()
1090 	adev->mes.funcs = &mes_v11_0_funcs;  in mes_v11_0_sw_init()
1091 	adev->mes.kiq_hw_init = &mes_v11_0_kiq_hw_init;  in mes_v11_0_sw_init()
1092 	adev->mes.kiq_hw_fini = &mes_v11_0_kiq_hw_fini;  in mes_v11_0_sw_init()
1133 	amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);  in mes_v11_0_sw_fini()
1134 	amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);  in mes_v11_0_sw_fini()
1137 		kfree(adev->mes.mqd_backup[pipe]);  in mes_v11_0_sw_fini()
1139 		amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],  in mes_v11_0_sw_fini()
1140 				      &adev->mes.eop_gpu_addr[pipe],  in mes_v11_0_sw_fini()
1150 	amdgpu_bo_free_kernel(&adev->mes.ring.mqd_obj,  in mes_v11_0_sw_fini()
1151 			      &adev->mes.ring.mqd_gpu_addr,  in mes_v11_0_sw_fini()
1152 			      &adev->mes.ring.mqd_ptr);  in mes_v11_0_sw_fini()
1155 	amdgpu_ring_fini(&adev->mes.ring);  in mes_v11_0_sw_fini()
1199 	adev->mes.ring.sched.ready = false;  in mes_v11_0_kiq_dequeue_sched()
1253 	if (adev->mes.ring.sched.ready)  in mes_v11_0_kiq_hw_fini()
1282 	r = mes_v11_0_set_hw_resources(&adev->mes);  in mes_v11_0_hw_init()
1286 	mes_v11_0_init_aggregated_doorbell(&adev->mes);  in mes_v11_0_hw_init()
1288 	r = mes_v11_0_query_sched_status(&adev->mes);  in mes_v11_0_hw_init()
1300 	adev->mes.ring.sched.ready = true;  in mes_v11_0_hw_init()