310 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_kiq_init_ring()  local
313 	spin_lock_init(&kiq->ring_lock);  in amdgpu_gfx_kiq_init_ring()
321 		(adev->doorbell_index.kiq +  in amdgpu_gfx_kiq_init_ring()
329 	ring->eop_gpu_addr = kiq->eop_gpu_addr;  in amdgpu_gfx_kiq_init_ring()
347 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_kiq_fini()  local
349 	amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL);  in amdgpu_gfx_kiq_fini()
357 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_kiq_init()  local
360 				    AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj,  in amdgpu_gfx_kiq_init()
361 				    &kiq->eop_gpu_addr, (void **)&hpd);  in amdgpu_gfx_kiq_init()
369 	r = amdgpu_bo_reserve(kiq->eop_obj, true);  in amdgpu_gfx_kiq_init()
372 	amdgpu_bo_kunmap(kiq->eop_obj);  in amdgpu_gfx_kiq_init()
373 	amdgpu_bo_unreserve(kiq->eop_obj);  in amdgpu_gfx_kiq_init()
383 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_mqd_sw_init()  local
384 	struct amdgpu_ring *ring = &kiq->ring;  in amdgpu_gfx_mqd_sw_init()
410 		kiq->mqd_backup = kmalloc(mqd_size, GFP_KERNEL);  in amdgpu_gfx_mqd_sw_init()
411 		if (!kiq->mqd_backup) {  in amdgpu_gfx_mqd_sw_init()
472 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_mqd_sw_fini()  local
493 	ring = &kiq->ring;  in amdgpu_gfx_mqd_sw_fini()
494 	kfree(kiq->mqd_backup);  in amdgpu_gfx_mqd_sw_fini()
502 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_disable_kcq()  local
503 	struct amdgpu_ring *kiq_ring = &kiq->ring;  in amdgpu_gfx_disable_kcq()
507 	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)  in amdgpu_gfx_disable_kcq()
510 	spin_lock(&kiq->ring_lock);  in amdgpu_gfx_disable_kcq()
511 	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *  in amdgpu_gfx_disable_kcq()
513 		spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_disable_kcq()
519 		kiq->pmf->kiq_unmap_queues(kiq_ring,  in amdgpu_gfx_disable_kcq()
526 	spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_disable_kcq()
533 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_disable_kgq()  local
534 	struct amdgpu_ring *kiq_ring = &kiq->ring;  in amdgpu_gfx_disable_kgq()
538 	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)  in amdgpu_gfx_disable_kgq()
541 	spin_lock(&kiq->ring_lock);  in amdgpu_gfx_disable_kgq()
543 		if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *  in amdgpu_gfx_disable_kgq()
545 			spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_disable_kgq()
551 			kiq->pmf->kiq_unmap_queues(kiq_ring,  in amdgpu_gfx_disable_kgq()
557 	if (adev->gfx.kiq[0].ring.sched.ready && !adev->job_hang)  in amdgpu_gfx_disable_kgq()
559 	spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_disable_kgq()
579 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_enable_kcq()  local
580 	struct amdgpu_ring *kiq_ring = &kiq->ring;  in amdgpu_gfx_enable_kcq()
584 	if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources)  in amdgpu_gfx_enable_kcq()
606 	spin_lock(&kiq->ring_lock);  in amdgpu_gfx_enable_kcq()
607 	r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *  in amdgpu_gfx_enable_kcq()
609 					kiq->pmf->set_resources_size);  in amdgpu_gfx_enable_kcq()
612 		spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_enable_kcq()
619 	kiq->pmf->kiq_set_resources(kiq_ring, queue_mask);  in amdgpu_gfx_enable_kcq()
622 			kiq->pmf->kiq_map_queues(kiq_ring,  in amdgpu_gfx_enable_kcq()
627 	spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_enable_kcq()
636 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];  in amdgpu_gfx_enable_kgq()  local
637 	struct amdgpu_ring *kiq_ring = &kiq->ring;  in amdgpu_gfx_enable_kgq()
640 	if (!kiq->pmf || !kiq->pmf->kiq_map_queues)  in amdgpu_gfx_enable_kgq()
645 	spin_lock(&kiq->ring_lock);  in amdgpu_gfx_enable_kgq()
648 		r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *  in amdgpu_gfx_enable_kgq()
652 			spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_enable_kgq()
658 			kiq->pmf->kiq_map_queues(kiq_ring,  in amdgpu_gfx_enable_kgq()
664 	spin_unlock(&kiq->ring_lock);  in amdgpu_gfx_enable_kgq()
916 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];  in amdgpu_kiq_rreg()  local
917 	struct amdgpu_ring *ring = &kiq->ring;  in amdgpu_kiq_rreg()
927 	spin_lock_irqsave(&kiq->ring_lock, flags);  in amdgpu_kiq_rreg()
939 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in amdgpu_kiq_rreg()
971 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in amdgpu_kiq_rreg()
984 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];  in amdgpu_kiq_wreg()  local
985 	struct amdgpu_ring *ring = &kiq->ring;  in amdgpu_kiq_wreg()
997 	spin_lock_irqsave(&kiq->ring_lock, flags);  in amdgpu_kiq_wreg()
1005 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in amdgpu_kiq_wreg()
1034 	spin_unlock_irqrestore(&kiq->ring_lock, flags);  in amdgpu_kiq_wreg()