42 	int r = ida_simple_get(&adev->mes.doorbell_ida, 2,  in amdgpu_mes_alloc_process_doorbells()
43 			       adev->mes.max_doorbell_slices,  in amdgpu_mes_alloc_process_doorbells()
55 		ida_simple_remove(&adev->mes.doorbell_ida, doorbell_index);  in amdgpu_mes_free_process_doorbells()
141 	adev->mes.doorbell_id_offset = doorbell_start_offset / sizeof(u32);  in amdgpu_mes_doorbell_init()
142 	adev->mes.max_doorbell_slices = doorbell_process_limit;  in amdgpu_mes_doorbell_init()
146 		adev->mes.aggregated_doorbells[i] =  in amdgpu_mes_doorbell_init()
157 	adev->mes.adev = adev;  in amdgpu_mes_init()
159 	idr_init(&adev->mes.pasid_idr);  in amdgpu_mes_init()
160 	idr_init(&adev->mes.gang_id_idr);  in amdgpu_mes_init()
161 	idr_init(&adev->mes.queue_id_idr);  in amdgpu_mes_init()
162 	ida_init(&adev->mes.doorbell_ida);  in amdgpu_mes_init()
163 	spin_lock_init(&adev->mes.queue_id_lock);  in amdgpu_mes_init()
164 	spin_lock_init(&adev->mes.ring_lock);  in amdgpu_mes_init()
165 	mutex_init(&adev->mes.mutex_hidden);  in amdgpu_mes_init()
167 	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;  in amdgpu_mes_init()
168 	adev->mes.vmid_mask_mmhub = 0xffffff00;  in amdgpu_mes_init()
169 	adev->mes.vmid_mask_gfxhub = 0xffffff00;  in amdgpu_mes_init()
175 		adev->mes.compute_hqd_mask[i] = 0xc;  in amdgpu_mes_init()
179 		adev->mes.gfx_hqd_mask[i] = i ? 0 : 0xfffffffe;  in amdgpu_mes_init()
183 			adev->mes.sdma_hqd_mask[i] = i ? 0 : 0x3fc;  in amdgpu_mes_init()
186 			adev->mes.sdma_hqd_mask[i] = i ? 0 : 0xfc;  in amdgpu_mes_init()
188 			adev->mes.sdma_hqd_mask[i] = 0xfc;  in amdgpu_mes_init()
191 	r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs);  in amdgpu_mes_init()
197 	adev->mes.sch_ctx_gpu_addr =  in amdgpu_mes_init()
198 		adev->wb.gpu_addr + (adev->mes.sch_ctx_offs * 4);  in amdgpu_mes_init()
199 	adev->mes.sch_ctx_ptr =  in amdgpu_mes_init()
200 		(uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs];  in amdgpu_mes_init()
202 	r = amdgpu_device_wb_get(adev, &adev->mes.query_status_fence_offs);  in amdgpu_mes_init()
204 		amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);  in amdgpu_mes_init()
209 	adev->mes.query_status_fence_gpu_addr =  in amdgpu_mes_init()
210 		adev->wb.gpu_addr + (adev->mes.query_status_fence_offs * 4);  in amdgpu_mes_init()
211 	adev->mes.query_status_fence_ptr =  in amdgpu_mes_init()
212 		(uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs];  in amdgpu_mes_init()
214 	r = amdgpu_device_wb_get(adev, &adev->mes.read_val_offs);  in amdgpu_mes_init()
216 		amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);  in amdgpu_mes_init()
217 		amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);  in amdgpu_mes_init()
222 	adev->mes.read_val_gpu_addr =  in amdgpu_mes_init()
223 		adev->wb.gpu_addr + (adev->mes.read_val_offs * 4);  in amdgpu_mes_init()
224 	adev->mes.read_val_ptr =  in amdgpu_mes_init()
225 		(uint32_t *)&adev->wb.wb[adev->mes.read_val_offs];  in amdgpu_mes_init()
234 	amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);  in amdgpu_mes_init()
235 	amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);  in amdgpu_mes_init()
236 	amdgpu_device_wb_free(adev, adev->mes.read_val_offs);  in amdgpu_mes_init()
238 	idr_destroy(&adev->mes.pasid_idr);  in amdgpu_mes_init()
239 	idr_destroy(&adev->mes.gang_id_idr);  in amdgpu_mes_init()
240 	idr_destroy(&adev->mes.queue_id_idr);  in amdgpu_mes_init()
241 	ida_destroy(&adev->mes.doorbell_ida);  in amdgpu_mes_init()
242 	mutex_destroy(&adev->mes.mutex_hidden);  in amdgpu_mes_init()
248 	amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);  in amdgpu_mes_fini()
249 	amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);  in amdgpu_mes_fini()
250 	amdgpu_device_wb_free(adev, adev->mes.read_val_offs);  in amdgpu_mes_fini()
252 	idr_destroy(&adev->mes.pasid_idr);  in amdgpu_mes_fini()
253 	idr_destroy(&adev->mes.gang_id_idr);  in amdgpu_mes_fini()
254 	idr_destroy(&adev->mes.queue_id_idr);  in amdgpu_mes_fini()
255 	ida_destroy(&adev->mes.doorbell_ida);  in amdgpu_mes_fini()
256 	mutex_destroy(&adev->mes.mutex_hidden);  in amdgpu_mes_fini()
304 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_create_process()
307 	r = idr_alloc(&adev->mes.pasid_idr, process, pasid, pasid + 1,  in amdgpu_mes_create_process()
326 	process->process_quantum = adev->mes.default_process_quantum;  in amdgpu_mes_create_process()
329 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_create_process()
333 	idr_remove(&adev->mes.pasid_idr, pasid);  in amdgpu_mes_create_process()
334 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_create_process()
358 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_destroy_process()
360 	process = idr_find(&adev->mes.pasid_idr, pasid);  in amdgpu_mes_destroy_process()
363 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_destroy_process()
370 			spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_destroy_process()
371 			idr_remove(&adev->mes.queue_id_idr, queue->queue_id);  in amdgpu_mes_destroy_process()
372 			spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_destroy_process()
377 			r = adev->mes.funcs->remove_hw_queue(&adev->mes,  in amdgpu_mes_destroy_process()
383 		idr_remove(&adev->mes.gang_id_idr, gang->gang_id);  in amdgpu_mes_destroy_process()
387 	idr_remove(&adev->mes.pasid_idr, pasid);  in amdgpu_mes_destroy_process()
388 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_destroy_process()
442 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_add_gang()
444 	process = idr_find(&adev->mes.pasid_idr, pasid);  in amdgpu_mes_add_gang()
452 	r = idr_alloc(&adev->mes.gang_id_idr, gang, 1, 0,  in amdgpu_mes_add_gang()
466 		gprops->gang_quantum : adev->mes.default_gang_quantum;  in amdgpu_mes_add_gang()
471 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_gang()
475 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_gang()
492 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_remove_gang()
494 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);  in amdgpu_mes_remove_gang()
497 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_gang()
503 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_gang()
507 	idr_remove(&adev->mes.gang_id_idr, gang->gang_id);  in amdgpu_mes_remove_gang()
509 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_gang()
532 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_suspend()
534 	idp = &adev->mes.pasid_idr;  in amdgpu_mes_suspend()
538 			r = adev->mes.funcs->suspend_gang(&adev->mes, &input);  in amdgpu_mes_suspend()
545 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_suspend()
561 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_resume()
563 	idp = &adev->mes.pasid_idr;  in amdgpu_mes_resume()
567 			r = adev->mes.funcs->resume_gang(&adev->mes, &input);  in amdgpu_mes_resume()
574 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_resume()
659 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_add_hw_queue()
661 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);  in amdgpu_mes_add_hw_queue()
669 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
670 	r = idr_alloc(&adev->mes.queue_id_idr, queue, 1, 0,  in amdgpu_mes_add_hw_queue()
673 		spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
676 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
713 	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);  in amdgpu_mes_add_hw_queue()
734 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_hw_queue()
741 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
742 	idr_remove(&adev->mes.queue_id_idr, queue->queue_id);  in amdgpu_mes_add_hw_queue()
743 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
745 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_hw_queue()
764 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_remove_hw_queue()
767 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_remove_hw_queue()
769 	queue = idr_find(&adev->mes.queue_id_idr, queue_id);  in amdgpu_mes_remove_hw_queue()
771 		spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_remove_hw_queue()
772 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_hw_queue()
777 	idr_remove(&adev->mes.queue_id_idr, queue_id);  in amdgpu_mes_remove_hw_queue()
778 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_remove_hw_queue()
787 	r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);  in amdgpu_mes_remove_hw_queue()
795 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_hw_queue()
818 	r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input);  in amdgpu_mes_unmap_legacy_queue()
832 	op_input.read_reg.buffer_addr = adev->mes.read_val_gpu_addr;  in amdgpu_mes_rreg()
834 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_rreg()
839 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_rreg()
843 		val = *(adev->mes.read_val_ptr);  in amdgpu_mes_rreg()
859 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_wreg()
865 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_wreg()
886 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_reg_write_reg_wait()
892 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_reg_write_reg_wait()
911 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_reg_wait()
917 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_reg_wait()
994 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_add_ring()
995 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);  in amdgpu_mes_add_ring()
998 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_ring()
1005 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_ring()
1046 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_ring()
1073 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_ring()
1091 	return adev->mes.aggregated_doorbells[prio];  in amdgpu_mes_get_aggregated_doorbell_index()
1264 	gprops.gang_quantum = adev->mes.default_gang_quantum;  in amdgpu_mes_test_create_gang_and_queues()