97 static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,  in mes_v11_0_submit_pkt_and_poll_completion()  argument
105 	struct amdgpu_device *adev = mes->adev;  in mes_v11_0_submit_pkt_and_poll_completion()
106 	struct amdgpu_ring *ring = &mes->ring;  in mes_v11_0_submit_pkt_and_poll_completion()
118 	spin_lock_irqsave(&mes->ring_lock, flags);  in mes_v11_0_submit_pkt_and_poll_completion()
120 		spin_unlock_irqrestore(&mes->ring_lock, flags);  in mes_v11_0_submit_pkt_and_poll_completion()
125 	api_status->api_completion_fence_addr = mes->ring.fence_drv.gpu_addr;  in mes_v11_0_submit_pkt_and_poll_completion()
126 	api_status->api_completion_fence_value = ++mes->ring.fence_drv.sync_seq;  in mes_v11_0_submit_pkt_and_poll_completion()
130 	spin_unlock_irqrestore(&mes->ring_lock, flags);  in mes_v11_0_submit_pkt_and_poll_completion()
162 static int mes_v11_0_add_hw_queue(struct amdgpu_mes *mes,  in mes_v11_0_add_hw_queue()  argument
165 	struct amdgpu_device *adev = mes->adev;  in mes_v11_0_add_hw_queue()
191 	if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>  in mes_v11_0_add_hw_queue()
215 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_add_hw_queue()
220 static int mes_v11_0_remove_hw_queue(struct amdgpu_mes *mes,  in mes_v11_0_remove_hw_queue()  argument
234 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_remove_hw_queue()
239 static int mes_v11_0_unmap_legacy_queue(struct amdgpu_mes *mes,  in mes_v11_0_unmap_legacy_queue()  argument
267 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_unmap_legacy_queue()
272 static int mes_v11_0_suspend_gang(struct amdgpu_mes *mes,  in mes_v11_0_suspend_gang()  argument
278 static int mes_v11_0_resume_gang(struct amdgpu_mes *mes,  in mes_v11_0_resume_gang()  argument
284 static int mes_v11_0_query_sched_status(struct amdgpu_mes *mes)  in mes_v11_0_query_sched_status()  argument
294 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_query_sched_status()
299 static int mes_v11_0_misc_op(struct amdgpu_mes *mes,  in mes_v11_0_misc_op()  argument
355 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_misc_op()
360 static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)  in mes_v11_0_set_hw_resources()  argument
363 	struct amdgpu_device *adev = mes->adev;  in mes_v11_0_set_hw_resources()
372 	mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;  in mes_v11_0_set_hw_resources()
373 	mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;  in mes_v11_0_set_hw_resources()
376 	mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr = mes->sch_ctx_gpu_addr;  in mes_v11_0_set_hw_resources()
378 		mes->query_status_fence_gpu_addr;  in mes_v11_0_set_hw_resources()
382 			mes->compute_hqd_mask[i];  in mes_v11_0_set_hw_resources()
385 		mes_set_hw_res_pkt.gfx_hqd_mask[i] = mes->gfx_hqd_mask[i];  in mes_v11_0_set_hw_resources()
388 		mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i];  in mes_v11_0_set_hw_resources()
392 			mes->aggregated_doorbells[i];  in mes_v11_0_set_hw_resources()
408 	return mes_v11_0_submit_pkt_and_poll_completion(mes,  in mes_v11_0_set_hw_resources()
413 static void mes_v11_0_init_aggregated_doorbell(struct amdgpu_mes *mes)  in mes_v11_0_init_aggregated_doorbell()  argument
415 	struct amdgpu_device *adev = mes->adev;  in mes_v11_0_init_aggregated_doorbell()
422 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<  in mes_v11_0_init_aggregated_doorbell()
431 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<  in mes_v11_0_init_aggregated_doorbell()
440 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<  in mes_v11_0_init_aggregated_doorbell()
449 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<  in mes_v11_0_init_aggregated_doorbell()
458 	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<  in mes_v11_0_init_aggregated_doorbell()
485 		adev->mes.fw[pipe]->data;  in mes_v11_0_allocate_ucode_buffer()
487 	fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +  in mes_v11_0_allocate_ucode_buffer()
495 				      &adev->mes.ucode_fw_obj[pipe],  in mes_v11_0_allocate_ucode_buffer()
496 				      &adev->mes.ucode_fw_gpu_addr[pipe],  in mes_v11_0_allocate_ucode_buffer()
497 				      (void **)&adev->mes.ucode_fw_ptr[pipe]);  in mes_v11_0_allocate_ucode_buffer()
503 	memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size);  in mes_v11_0_allocate_ucode_buffer()
505 	amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);  in mes_v11_0_allocate_ucode_buffer()
506 	amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);  in mes_v11_0_allocate_ucode_buffer()
520 		adev->mes.fw[pipe]->data;  in mes_v11_0_allocate_ucode_data_buffer()
522 	fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +  in mes_v11_0_allocate_ucode_data_buffer()
530 				      &adev->mes.data_fw_obj[pipe],  in mes_v11_0_allocate_ucode_data_buffer()
531 				      &adev->mes.data_fw_gpu_addr[pipe],  in mes_v11_0_allocate_ucode_data_buffer()
532 				      (void **)&adev->mes.data_fw_ptr[pipe]);  in mes_v11_0_allocate_ucode_data_buffer()
538 	memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size);  in mes_v11_0_allocate_ucode_data_buffer()
540 	amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);  in mes_v11_0_allocate_ucode_data_buffer()
541 	amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);  in mes_v11_0_allocate_ucode_data_buffer()
549 	amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],  in mes_v11_0_free_ucode_buffers()
550 			      &adev->mes.data_fw_gpu_addr[pipe],  in mes_v11_0_free_ucode_buffers()
551 			      (void **)&adev->mes.data_fw_ptr[pipe]);  in mes_v11_0_free_ucode_buffers()
553 	amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],  in mes_v11_0_free_ucode_buffers()
554 			      &adev->mes.ucode_fw_gpu_addr[pipe],  in mes_v11_0_free_ucode_buffers()
555 			      (void **)&adev->mes.ucode_fw_ptr[pipe]);  in mes_v11_0_free_ucode_buffers()
578 			ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;  in mes_v11_0_enable()
621 	if (!adev->mes.fw[pipe])  in mes_v11_0_load_microcode()
641 	ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;  in mes_v11_0_load_microcode()
649 		     lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));  in mes_v11_0_load_microcode()
651 		     upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));  in mes_v11_0_load_microcode()
658 		     lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]));  in mes_v11_0_load_microcode()
660 		     upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]));  in mes_v11_0_load_microcode()
692 			      &adev->mes.eop_gpu_obj[pipe],  in mes_v11_0_allocate_eop_buf()
693 			      &adev->mes.eop_gpu_addr[pipe],  in mes_v11_0_allocate_eop_buf()
701 	       adev->mes.eop_gpu_obj[pipe]->tbo.base.size);  in mes_v11_0_allocate_eop_buf()
703 	amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);  in mes_v11_0_allocate_eop_buf()
704 	amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);  in mes_v11_0_allocate_eop_buf()
890 	kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring);  in mes_v11_0_kiq_enable_queue()
904 		ring = &adev->mes.ring;  in mes_v11_0_queue_init()
932 		adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);  in mes_v11_0_queue_init()
934 		adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);  in mes_v11_0_queue_init()
946 	ring = &adev->mes.ring;  in mes_v11_0_ring_init()
957 	ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_SCHED_PIPE];  in mes_v11_0_ring_init()
981 	ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];  in mes_v11_0_kiq_ring_init()
999 		ring = &adev->mes.ring;  in mes_v11_0_mqd_sw_init()
1018 	adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL);  in mes_v11_0_mqd_sw_init()
1019 	if (!adev->mes.mqd_backup[pipe]) {  in mes_v11_0_mqd_sw_init()
1034 	adev->mes.funcs = &mes_v11_0_funcs;  in mes_v11_0_sw_init()
1035 	adev->mes.kiq_hw_init = &mes_v11_0_kiq_hw_init;  in mes_v11_0_sw_init()
1036 	adev->mes.kiq_hw_fini = &mes_v11_0_kiq_hw_fini;  in mes_v11_0_sw_init()
1073 	amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);  in mes_v11_0_sw_fini()
1074 	amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);  in mes_v11_0_sw_fini()
1077 		kfree(adev->mes.mqd_backup[pipe]);  in mes_v11_0_sw_fini()
1079 		amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],  in mes_v11_0_sw_fini()
1080 				      &adev->mes.eop_gpu_addr[pipe],  in mes_v11_0_sw_fini()
1082 		amdgpu_ucode_release(&adev->mes.fw[pipe]);  in mes_v11_0_sw_fini()
1089 	amdgpu_bo_free_kernel(&adev->mes.ring.mqd_obj,  in mes_v11_0_sw_fini()
1090 			      &adev->mes.ring.mqd_gpu_addr,  in mes_v11_0_sw_fini()
1091 			      &adev->mes.ring.mqd_ptr);  in mes_v11_0_sw_fini()
1094 	amdgpu_ring_fini(&adev->mes.ring);  in mes_v11_0_sw_fini()
1201 	if (adev->mes.ring.sched.ready) {  in mes_v11_0_kiq_hw_fini()
1202 		mes_v11_0_kiq_dequeue(&adev->mes.ring);  in mes_v11_0_kiq_hw_fini()
1203 		adev->mes.ring.sched.ready = false;  in mes_v11_0_kiq_hw_fini()
1238 	r = mes_v11_0_set_hw_resources(&adev->mes);  in mes_v11_0_hw_init()
1242 	mes_v11_0_init_aggregated_doorbell(&adev->mes);  in mes_v11_0_hw_init()
1244 	r = mes_v11_0_query_sched_status(&adev->mes);  in mes_v11_0_hw_init()
1256 	adev->mes.ring.sched.ready = true;  in mes_v11_0_hw_init()