99 			struct kfd_process_device *pdd)  in kfd_process_free_gpuvm()  argument
101 	struct kfd_dev *dev = pdd->dev;  in kfd_process_free_gpuvm()
103 	dev->kfd2kgd->unmap_memory_to_gpu(dev->kgd, mem, pdd->vm);  in kfd_process_free_gpuvm()
113 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,  in kfd_process_alloc_gpuvm()  argument
117 	struct kfd_dev *kdev = pdd->dev;  in kfd_process_alloc_gpuvm()
123 						 pdd->vm, &mem, NULL, flags);  in kfd_process_alloc_gpuvm()
127 	err = kdev->kfd2kgd->map_memory_to_gpu(kdev->kgd, mem, pdd->vm);  in kfd_process_alloc_gpuvm()
142 	handle = kfd_process_device_create_obj_handle(pdd, mem);  in kfd_process_alloc_gpuvm()
161 	kfd_process_device_remove_obj_handle(pdd, handle);  in kfd_process_alloc_gpuvm()
164 	kfd_process_free_gpuvm(mem, pdd);  in kfd_process_alloc_gpuvm()
180 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)  in kfd_process_device_reserve_ib_mem()  argument
182 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_reserve_ib_mem()
194 	ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,  in kfd_process_device_reserve_ib_mem()
282 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)  in kfd_process_device_free_bos()  argument
284 	struct kfd_process *p = pdd->process;  in kfd_process_device_free_bos()
292 	idr_for_each_entry(&pdd->alloc_idr, mem, id) {  in kfd_process_device_free_bos()
303 		pdd->dev->kfd2kgd->free_memory_of_gpu(pdd->dev->kgd, mem);  in kfd_process_device_free_bos()
304 		kfd_process_device_remove_obj_handle(pdd, id);  in kfd_process_device_free_bos()
310 	struct kfd_process_device *pdd;  in kfd_process_free_outstanding_kfd_bos()  local
312 	list_for_each_entry(pdd, &p->per_device_data, per_device_list)  in kfd_process_free_outstanding_kfd_bos()
313 		kfd_process_device_free_bos(pdd);  in kfd_process_free_outstanding_kfd_bos()
318 	struct kfd_process_device *pdd, *temp;  in kfd_process_destroy_pdds()  local
320 	list_for_each_entry_safe(pdd, temp, &p->per_device_data,  in kfd_process_destroy_pdds()
323 				pdd->dev->id, p->pasid);  in kfd_process_destroy_pdds()
325 		if (pdd->drm_file)  in kfd_process_destroy_pdds()
326 			fput(pdd->drm_file);  in kfd_process_destroy_pdds()
327 		else if (pdd->vm)  in kfd_process_destroy_pdds()
328 			pdd->dev->kfd2kgd->destroy_process_vm(  in kfd_process_destroy_pdds()
329 				pdd->dev->kgd, pdd->vm);  in kfd_process_destroy_pdds()
331 		list_del(&pdd->per_device_list);  in kfd_process_destroy_pdds()
333 		if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)  in kfd_process_destroy_pdds()
334 			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,  in kfd_process_destroy_pdds()
337 		kfree(pdd->qpd.doorbell_bitmap);  in kfd_process_destroy_pdds()
338 		idr_destroy(&pdd->alloc_idr);  in kfd_process_destroy_pdds()
340 		kfree(pdd);  in kfd_process_destroy_pdds()
392 	struct kfd_process_device *pdd = NULL;  in kfd_process_notifier_release()  local
416 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {  in kfd_process_notifier_release()
417 		struct kfd_dev *dev = pdd->dev;  in kfd_process_notifier_release()
448 	struct kfd_process_device *pdd;  in kfd_process_init_cwsr_apu()  local
450 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {  in kfd_process_init_cwsr_apu()
451 		struct kfd_dev *dev = pdd->dev;  in kfd_process_init_cwsr_apu()
452 		struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_init_cwsr_apu()
482 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)  in kfd_process_device_init_cwsr_dgpu()  argument
484 	struct kfd_dev *dev = pdd->dev;  in kfd_process_device_init_cwsr_dgpu()
485 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_init_cwsr_dgpu()
495 	ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,  in kfd_process_device_init_cwsr_dgpu()
620 	struct kfd_process_device *pdd = NULL;  in kfd_get_process_device_data()  local
622 	list_for_each_entry(pdd, &p->per_device_data, per_device_list)  in kfd_get_process_device_data()
623 		if (pdd->dev == dev)  in kfd_get_process_device_data()
624 			return pdd;  in kfd_get_process_device_data()
632 	struct kfd_process_device *pdd = NULL;  in kfd_create_process_device_data()  local
634 	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);  in kfd_create_process_device_data()
635 	if (!pdd)  in kfd_create_process_device_data()
638 	if (init_doorbell_bitmap(&pdd->qpd, dev)) {  in kfd_create_process_device_data()
640 		kfree(pdd);  in kfd_create_process_device_data()
644 	pdd->dev = dev;  in kfd_create_process_device_data()
645 	INIT_LIST_HEAD(&pdd->qpd.queues_list);  in kfd_create_process_device_data()
646 	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);  in kfd_create_process_device_data()
647 	pdd->qpd.dqm = dev->dqm;  in kfd_create_process_device_data()
648 	pdd->qpd.pqm = &p->pqm;  in kfd_create_process_device_data()
649 	pdd->qpd.evicted = 0;  in kfd_create_process_device_data()
650 	pdd->process = p;  in kfd_create_process_device_data()
651 	pdd->bound = PDD_UNBOUND;  in kfd_create_process_device_data()
652 	pdd->already_dequeued = false;  in kfd_create_process_device_data()
653 	list_add(&pdd->per_device_list, &p->per_device_data);  in kfd_create_process_device_data()
656 	idr_init(&pdd->alloc_idr);  in kfd_create_process_device_data()
658 	return pdd;  in kfd_create_process_device_data()
675 int kfd_process_device_init_vm(struct kfd_process_device *pdd,  in kfd_process_device_init_vm()  argument
682 	if (pdd->vm)  in kfd_process_device_init_vm()
685 	p = pdd->process;  in kfd_process_device_init_vm()
686 	dev = pdd->dev;  in kfd_process_device_init_vm()
691 			&pdd->vm, &p->kgd_process_info, &p->ef);  in kfd_process_device_init_vm()
694 			dev->kgd, &pdd->vm, &p->kgd_process_info, &p->ef);  in kfd_process_device_init_vm()
700 	ret = kfd_process_device_reserve_ib_mem(pdd);  in kfd_process_device_init_vm()
703 	ret = kfd_process_device_init_cwsr_dgpu(pdd);  in kfd_process_device_init_vm()
707 	pdd->drm_file = drm_file;  in kfd_process_device_init_vm()
713 	kfd_process_device_free_bos(pdd);  in kfd_process_device_init_vm()
715 		dev->kfd2kgd->destroy_process_vm(dev->kgd, pdd->vm);  in kfd_process_device_init_vm()
716 	pdd->vm = NULL;  in kfd_process_device_init_vm()
731 	struct kfd_process_device *pdd;  in kfd_bind_process_to_device()  local
734 	pdd = kfd_get_process_device_data(dev, p);  in kfd_bind_process_to_device()
735 	if (!pdd) {  in kfd_bind_process_to_device()
740 	err = kfd_iommu_bind_process_to_device(pdd);  in kfd_bind_process_to_device()
744 	err = kfd_process_device_init_vm(pdd, NULL);  in kfd_bind_process_to_device()
748 	return pdd;  in kfd_bind_process_to_device()
761 						struct kfd_process_device *pdd)  in kfd_get_next_process_device_data()  argument
763 	if (list_is_last(&pdd->per_device_list, &p->per_device_data))  in kfd_get_next_process_device_data()
765 	return list_next_entry(pdd, per_device_list);  in kfd_get_next_process_device_data()
776 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,  in kfd_process_device_create_obj_handle()  argument
779 	return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);  in kfd_process_device_create_obj_handle()
785 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,  in kfd_process_device_translate_handle()  argument
791 	return idr_find(&pdd->alloc_idr, handle);  in kfd_process_device_translate_handle()
797 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,  in kfd_process_device_remove_obj_handle()  argument
801 		idr_remove(&pdd->alloc_idr, handle);  in kfd_process_device_remove_obj_handle()
848 	struct kfd_process_device *pdd;  in kfd_process_evict_queues()  local
852 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {  in kfd_process_evict_queues()
853 		r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,  in kfd_process_evict_queues()
854 							    &pdd->qpd);  in kfd_process_evict_queues()
868 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {  in kfd_process_evict_queues()
871 		if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,  in kfd_process_evict_queues()
872 							      &pdd->qpd))  in kfd_process_evict_queues()
884 	struct kfd_process_device *pdd;  in kfd_process_restore_queues()  local
887 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {  in kfd_process_restore_queues()
888 		r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,  in kfd_process_restore_queues()
889 							      &pdd->qpd);  in kfd_process_restore_queues()
941 	struct kfd_process_device *pdd;  in restore_process_worker()  local
956 	pdd = list_first_entry(&p->per_device_data,  in restore_process_worker()
973 	ret = pdd->dev->kfd2kgd->restore_process_bos(p->kgd_process_info,  in restore_process_worker()
1030 	struct kfd_process_device *pdd;  in kfd_reserved_mem_mmap()  local
1038 	pdd = kfd_get_process_device_data(dev, process);  in kfd_reserved_mem_mmap()
1039 	if (!pdd)  in kfd_reserved_mem_mmap()
1041 	qpd = &pdd->qpd;  in kfd_reserved_mem_mmap()
1058 void kfd_flush_tlb(struct kfd_process_device *pdd)  in kfd_flush_tlb()  argument
1060 	struct kfd_dev *dev = pdd->dev;  in kfd_flush_tlb()
1067 		if (pdd->qpd.vmid)  in kfd_flush_tlb()
1068 			f2g->invalidate_tlbs_vmid(dev->kgd, pdd->qpd.vmid);  in kfd_flush_tlb()
1070 		f2g->invalidate_tlbs(dev->kgd, pdd->process->pasid);  in kfd_flush_tlb()