85 	struct amdgpu_vm *vm;  member
147 				   struct amdgpu_vm *vm,  in amdgpu_vm_bo_base_init()  argument
150 	base->vm = vm;  in amdgpu_vm_bo_base_init()
160 		list_move(&base->vm_status, &vm->relocated);  in amdgpu_vm_bo_base_init()
162 	if (bo->tbo.resv != vm->root.base.bo->tbo.resv)  in amdgpu_vm_bo_base_init()
174 	list_move_tail(&base->vm_status, &vm->evicted);  in amdgpu_vm_bo_base_init()
259 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,  in amdgpu_vm_get_pd_bo()  argument
263 	entry->robj = vm->root.base.bo;  in amdgpu_vm_get_pd_bo()
284 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,  in amdgpu_vm_validate_pt_bos()  argument
292 	list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {  in amdgpu_vm_validate_pt_bos()
308 			spin_lock(&vm->moved_lock);  in amdgpu_vm_validate_pt_bos()
309 			list_move(&bo_base->vm_status, &vm->moved);  in amdgpu_vm_validate_pt_bos()
310 			spin_unlock(&vm->moved_lock);  in amdgpu_vm_validate_pt_bos()
312 			list_move(&bo_base->vm_status, &vm->relocated);  in amdgpu_vm_validate_pt_bos()
317 	list_for_each_entry(bo_base, &vm->idle, vm_status) {  in amdgpu_vm_validate_pt_bos()
342 bool amdgpu_vm_ready(struct amdgpu_vm *vm)  in amdgpu_vm_ready()  argument
344 	return list_empty(&vm->evicted);  in amdgpu_vm_ready()
362 			      struct amdgpu_vm *vm, struct amdgpu_bo *bo,  in amdgpu_vm_clear_bo()  argument
390 	ring = container_of(vm->entity.rq->sched, struct amdgpu_ring, sched);  in amdgpu_vm_clear_bo()
429 	r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_UNDEFINED,  in amdgpu_vm_clear_bo()
438 		return amdgpu_vm_clear_bo(adev, vm, bo->shadow,  in amdgpu_vm_clear_bo()
467 				  struct amdgpu_vm *vm,  in amdgpu_vm_alloc_levels()  argument
499 	if (vm->root.base.bo->shadow)  in amdgpu_vm_alloc_levels()
501 	if (vm->use_cpu_for_update)  in amdgpu_vm_alloc_levels()
508 		struct reservation_object *resv = vm->root.base.bo->tbo.resv;  in amdgpu_vm_alloc_levels()
526 			r = amdgpu_vm_clear_bo(adev, vm, pt, level, ats);  in amdgpu_vm_alloc_levels()
533 			if (vm->use_cpu_for_update) {  in amdgpu_vm_alloc_levels()
547 			amdgpu_vm_bo_base_init(&entry->base, vm, pt);  in amdgpu_vm_alloc_levels()
554 			r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr,  in amdgpu_vm_alloc_levels()
578 			struct amdgpu_vm *vm,  in amdgpu_vm_alloc_pts()  argument
590 	if (vm->pte_support_ats)  in amdgpu_vm_alloc_pts()
602 	return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr,  in amdgpu_vm_alloc_pts()
799 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,  in amdgpu_vm_bo_find()  argument
805 		if (bo_va->base.vm == vm) {  in amdgpu_vm_bo_find()
946 static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,  in amdgpu_vm_wait_pd()  argument
953 	amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner, false);  in amdgpu_vm_wait_pd()
971 				 struct amdgpu_vm *vm,  in amdgpu_vm_update_pde()  argument
1007 				       struct amdgpu_vm *vm,  in amdgpu_vm_invalidate_level()  argument
1025 			list_move(&entry->base.vm_status, &vm->relocated);  in amdgpu_vm_invalidate_level()
1026 		amdgpu_vm_invalidate_level(adev, vm, entry, level + 1);  in amdgpu_vm_invalidate_level()
1042 				 struct amdgpu_vm *vm)  in amdgpu_vm_update_directories()  argument
1049 	if (list_empty(&vm->relocated))  in amdgpu_vm_update_directories()
1056 	if (vm->use_cpu_for_update) {  in amdgpu_vm_update_directories()
1059 		list_for_each_entry(bo_base, &vm->relocated, vm_status) {  in amdgpu_vm_update_directories()
1065 		r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);  in amdgpu_vm_update_directories()
1080 	while (!list_empty(&vm->relocated)) {  in amdgpu_vm_update_directories()
1085 		bo_base = list_first_entry(&vm->relocated,  in amdgpu_vm_update_directories()
1100 		amdgpu_vm_update_pde(&params, vm, pt, entry);  in amdgpu_vm_update_directories()
1102 		if (!vm->use_cpu_for_update &&  in amdgpu_vm_update_directories()
1107 	if (vm->use_cpu_for_update) {  in amdgpu_vm_update_directories()
1114 		struct amdgpu_bo *root = vm->root.base.bo;  in amdgpu_vm_update_directories()
1118 		ring = container_of(vm->entity.rq->sched, struct amdgpu_ring,  in amdgpu_vm_update_directories()
1125 		r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_VM,  in amdgpu_vm_update_directories()
1131 		dma_fence_put(vm->last_update);  in amdgpu_vm_update_directories()
1132 		vm->last_update = fence;  in amdgpu_vm_update_directories()
1135 	if (!list_empty(&vm->relocated))  in amdgpu_vm_update_directories()
1141 	amdgpu_vm_invalidate_level(adev, vm, &vm->root,  in amdgpu_vm_update_directories()
1164 	*entry = &p->vm->root;  in amdgpu_vm_get_entry()
1208 			list_move(&entry->base.vm_status, &p->vm->relocated);  in amdgpu_vm_handle_huge_pages()
1369 				       struct amdgpu_vm *vm,  in amdgpu_vm_bo_update_mapping()  argument
1384 	params.vm = vm;  in amdgpu_vm_bo_update_mapping()
1390 	if (vm->use_cpu_for_update) {  in amdgpu_vm_bo_update_mapping()
1398 		r = amdgpu_vm_wait_pd(adev, vm, owner);  in amdgpu_vm_bo_update_mapping()
1408 	ring = container_of(vm->entity.rq->sched, struct amdgpu_ring, sched);  in amdgpu_vm_bo_update_mapping()
1418 	if (vm->root.base.bo->shadow)  in amdgpu_vm_bo_update_mapping()
1440 		if (vm->root.base.bo->shadow)  in amdgpu_vm_bo_update_mapping()
1475 	r = amdgpu_sync_resv(adev, &job->sync, vm->root.base.bo->tbo.resv,  in amdgpu_vm_bo_update_mapping()
1480 	r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv);  in amdgpu_vm_bo_update_mapping()
1490 	r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_VM, &f);  in amdgpu_vm_bo_update_mapping()
1494 	amdgpu_bo_fence(vm->root.base.bo, f, true);  in amdgpu_vm_bo_update_mapping()
1525 				      struct amdgpu_vm *vm,  in amdgpu_vm_bo_split_mapping()  argument
1607 		r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm,  in amdgpu_vm_bo_split_mapping()
1642 	struct amdgpu_vm *vm = bo_va->base.vm;  in amdgpu_vm_bo_update()  local
1672 	if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv))  in amdgpu_vm_bo_update()
1673 		last_update = &vm->last_update;  in amdgpu_vm_bo_update()
1686 		r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,  in amdgpu_vm_bo_update()
1693 	if (vm->use_cpu_for_update) {  in amdgpu_vm_bo_update()
1699 	spin_lock(&vm->moved_lock);  in amdgpu_vm_bo_update()
1701 	spin_unlock(&vm->moved_lock);  in amdgpu_vm_bo_update()
1707 	if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {  in amdgpu_vm_bo_update()
1711 			list_add_tail(&bo_va->base.vm_status, &vm->evicted);  in amdgpu_vm_bo_update()
1713 			list_add(&bo_va->base.vm_status, &vm->idle);  in amdgpu_vm_bo_update()
1822 				   struct amdgpu_vm *vm,  in amdgpu_vm_free_mapping()  argument
1839 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)  in amdgpu_vm_prt_fini()  argument
1841 	struct reservation_object *resv = vm->root.base.bo->tbo.resv;  in amdgpu_vm_prt_fini()
1885 			  struct amdgpu_vm *vm,  in amdgpu_vm_clear_freed()  argument
1893 	while (!list_empty(&vm->freed)) {  in amdgpu_vm_clear_freed()
1894 		mapping = list_first_entry(&vm->freed,  in amdgpu_vm_clear_freed()
1898 		if (vm->pte_support_ats && mapping->start < AMDGPU_VA_HOLE_START)  in amdgpu_vm_clear_freed()
1901 		r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,  in amdgpu_vm_clear_freed()
1904 		amdgpu_vm_free_mapping(adev, vm, mapping, f);  in amdgpu_vm_clear_freed()
1936 			   struct amdgpu_vm *vm)  in amdgpu_vm_handle_moved()  argument
1944 	spin_lock(&vm->moved_lock);  in amdgpu_vm_handle_moved()
1945 	list_splice_init(&vm->moved, &moved);  in amdgpu_vm_handle_moved()
1946 	spin_unlock(&vm->moved_lock);  in amdgpu_vm_handle_moved()
1952 		if (resv == vm->root.base.bo->tbo.resv)  in amdgpu_vm_handle_moved()
1963 			spin_lock(&vm->moved_lock);  in amdgpu_vm_handle_moved()
1964 			list_splice(&moved, &vm->moved);  in amdgpu_vm_handle_moved()
1965 			spin_unlock(&vm->moved_lock);  in amdgpu_vm_handle_moved()
1969 		if (!clear && resv != vm->root.base.bo->tbo.resv)  in amdgpu_vm_handle_moved()
1993 				      struct amdgpu_vm *vm,  in amdgpu_vm_bo_add()  argument
2002 	amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);  in amdgpu_vm_bo_add()
2025 	struct amdgpu_vm *vm = bo_va->base.vm;  in amdgpu_vm_bo_insert_map()  local
2030 	amdgpu_vm_it_insert(mapping, &vm->va);  in amdgpu_vm_bo_insert_map()
2035 	if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv &&  in amdgpu_vm_bo_insert_map()
2037 		spin_lock(&vm->moved_lock);  in amdgpu_vm_bo_insert_map()
2038 		list_move(&bo_va->base.vm_status, &vm->moved);  in amdgpu_vm_bo_insert_map()
2039 		spin_unlock(&vm->moved_lock);  in amdgpu_vm_bo_insert_map()
2068 	struct amdgpu_vm *vm = bo_va->base.vm;  in amdgpu_vm_bo_map()  local
2085 	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);  in amdgpu_vm_bo_map()
2152 	r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);  in amdgpu_vm_bo_replace_map()
2190 	struct amdgpu_vm *vm = bo_va->base.vm;  in amdgpu_vm_bo_unmap()  local
2213 	amdgpu_vm_it_remove(mapping, &vm->va);  in amdgpu_vm_bo_unmap()
2218 		list_add(&mapping->list, &vm->freed);  in amdgpu_vm_bo_unmap()
2220 		amdgpu_vm_free_mapping(adev, vm, mapping,  in amdgpu_vm_bo_unmap()
2240 				struct amdgpu_vm *vm,  in amdgpu_vm_bo_clear_mappings()  argument
2265 	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);  in amdgpu_vm_bo_clear_mappings()
2296 		amdgpu_vm_it_remove(tmp, &vm->va);  in amdgpu_vm_bo_clear_mappings()
2305 		list_add(&tmp->list, &vm->freed);  in amdgpu_vm_bo_clear_mappings()
2311 		amdgpu_vm_it_insert(before, &vm->va);  in amdgpu_vm_bo_clear_mappings()
2320 		amdgpu_vm_it_insert(after, &vm->va);  in amdgpu_vm_bo_clear_mappings()
2342 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,  in amdgpu_vm_bo_lookup_mapping()  argument
2345 	return amdgpu_vm_it_iter_first(&vm->va, addr, addr);  in amdgpu_vm_bo_lookup_mapping()
2356 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)  in amdgpu_vm_bo_trace_cs()  argument
2363 	for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;  in amdgpu_vm_bo_trace_cs()
2391 	struct amdgpu_vm *vm = bo_va->base.vm;  in amdgpu_vm_bo_rmv()  local
2395 	spin_lock(&vm->moved_lock);  in amdgpu_vm_bo_rmv()
2397 	spin_unlock(&vm->moved_lock);  in amdgpu_vm_bo_rmv()
2401 		amdgpu_vm_it_remove(mapping, &vm->va);  in amdgpu_vm_bo_rmv()
2404 		list_add(&mapping->list, &vm->freed);  in amdgpu_vm_bo_rmv()
2408 		amdgpu_vm_it_remove(mapping, &vm->va);  in amdgpu_vm_bo_rmv()
2409 		amdgpu_vm_free_mapping(adev, vm, mapping,  in amdgpu_vm_bo_rmv()
2436 		struct amdgpu_vm *vm = bo_base->vm;  in amdgpu_vm_bo_invalidate()  local
2440 		if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {  in amdgpu_vm_bo_invalidate()
2442 				list_move(&bo_base->vm_status, &vm->evicted);  in amdgpu_vm_bo_invalidate()
2445 					       &vm->evicted);  in amdgpu_vm_bo_invalidate()
2453 			list_move(&bo_base->vm_status, &vm->relocated);  in amdgpu_vm_bo_invalidate()
2455 			spin_lock(&bo_base->vm->moved_lock);  in amdgpu_vm_bo_invalidate()
2456 			list_move(&bo_base->vm_status, &vm->moved);  in amdgpu_vm_bo_invalidate()
2457 			spin_unlock(&bo_base->vm->moved_lock);  in amdgpu_vm_bo_invalidate()
2590 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,  in amdgpu_vm_init()  argument
2604 	vm->va = RB_ROOT_CACHED;  in amdgpu_vm_init()
2606 		vm->reserved_vmid[i] = NULL;  in amdgpu_vm_init()
2607 	INIT_LIST_HEAD(&vm->evicted);  in amdgpu_vm_init()
2608 	INIT_LIST_HEAD(&vm->relocated);  in amdgpu_vm_init()
2609 	spin_lock_init(&vm->moved_lock);  in amdgpu_vm_init()
2610 	INIT_LIST_HEAD(&vm->moved);  in amdgpu_vm_init()
2611 	INIT_LIST_HEAD(&vm->idle);  in amdgpu_vm_init()
2612 	INIT_LIST_HEAD(&vm->freed);  in amdgpu_vm_init()
2620 	r = drm_sched_entity_init(&vm->entity, &rq, 1, NULL);  in amdgpu_vm_init()
2624 	vm->pte_support_ats = false;  in amdgpu_vm_init()
2627 		vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &  in amdgpu_vm_init()
2631 			vm->pte_support_ats = true;  in amdgpu_vm_init()
2633 		vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &  in amdgpu_vm_init()
2637 			 vm->use_cpu_for_update ? "CPU" : "SDMA");  in amdgpu_vm_init()
2638 	WARN_ONCE((vm->use_cpu_for_update & !amdgpu_gmc_vram_full_visible(&adev->gmc)),  in amdgpu_vm_init()
2640 	vm->last_update = NULL;  in amdgpu_vm_init()
2643 	if (vm->use_cpu_for_update)  in amdgpu_vm_init()
2664 	r = amdgpu_vm_clear_bo(adev, vm, root,  in amdgpu_vm_init()
2666 			       vm->pte_support_ats);  in amdgpu_vm_init()
2670 	amdgpu_vm_bo_base_init(&vm->root.base, vm, root);  in amdgpu_vm_init()
2671 	amdgpu_bo_unreserve(vm->root.base.bo);  in amdgpu_vm_init()
2677 		r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,  in amdgpu_vm_init()
2683 		vm->pasid = pasid;  in amdgpu_vm_init()
2686 	INIT_KFIFO(vm->faults);  in amdgpu_vm_init()
2687 	vm->fault_credit = 16;  in amdgpu_vm_init()
2692 	amdgpu_bo_unreserve(vm->root.base.bo);  in amdgpu_vm_init()
2695 	amdgpu_bo_unref(&vm->root.base.bo->shadow);  in amdgpu_vm_init()
2696 	amdgpu_bo_unref(&vm->root.base.bo);  in amdgpu_vm_init()
2697 	vm->root.base.bo = NULL;  in amdgpu_vm_init()
2700 	drm_sched_entity_destroy(&vm->entity);  in amdgpu_vm_init()
2725 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)  in amdgpu_vm_make_compute()  argument
2730 	r = amdgpu_bo_reserve(vm->root.base.bo, true);  in amdgpu_vm_make_compute()
2735 	if (!RB_EMPTY_ROOT(&vm->va.rb_root) || vm->root.entries) {  in amdgpu_vm_make_compute()
2743 	if (pte_support_ats != vm->pte_support_ats) {  in amdgpu_vm_make_compute()
2744 		r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo,  in amdgpu_vm_make_compute()
2752 	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &  in amdgpu_vm_make_compute()
2754 	vm->pte_support_ats = pte_support_ats;  in amdgpu_vm_make_compute()
2756 			 vm->use_cpu_for_update ? "CPU" : "SDMA");  in amdgpu_vm_make_compute()
2757 	WARN_ONCE((vm->use_cpu_for_update & !amdgpu_gmc_vram_full_visible(&adev->gmc)),  in amdgpu_vm_make_compute()
2760 	if (vm->pasid) {  in amdgpu_vm_make_compute()
2764 		idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);  in amdgpu_vm_make_compute()
2767 		vm->pasid = 0;  in amdgpu_vm_make_compute()
2771 	amdgpu_bo_unref(&vm->root.base.bo->shadow);  in amdgpu_vm_make_compute()
2774 	amdgpu_bo_unreserve(vm->root.base.bo);  in amdgpu_vm_make_compute()
2817 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)  in amdgpu_vm_fini()  argument
2825 	amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);  in amdgpu_vm_fini()
2828 	while (kfifo_get(&vm->faults, &fault))  in amdgpu_vm_fini()
2831 	if (vm->pasid) {  in amdgpu_vm_fini()
2835 		idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);  in amdgpu_vm_fini()
2839 	drm_sched_entity_destroy(&vm->entity);  in amdgpu_vm_fini()
2841 	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {  in amdgpu_vm_fini()
2845 					     &vm->va.rb_root, rb) {  in amdgpu_vm_fini()
2847 		amdgpu_vm_it_remove(mapping, &vm->va);  in amdgpu_vm_fini()
2850 	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {  in amdgpu_vm_fini()
2852 			amdgpu_vm_prt_fini(adev, vm);  in amdgpu_vm_fini()
2857 		amdgpu_vm_free_mapping(adev, vm, mapping, NULL);  in amdgpu_vm_fini()
2860 	root = amdgpu_bo_ref(vm->root.base.bo);  in amdgpu_vm_fini()
2865 		amdgpu_vm_free_levels(adev, &vm->root,  in amdgpu_vm_fini()
2870 	dma_fence_put(vm->last_update);  in amdgpu_vm_fini()
2872 		amdgpu_vmid_free_reserved(adev, vm, i);  in amdgpu_vm_fini()
2889 	struct amdgpu_vm *vm;  in amdgpu_vm_pasid_fault_credit()  local
2892 	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);  in amdgpu_vm_pasid_fault_credit()
2893 	if (!vm) {  in amdgpu_vm_pasid_fault_credit()
2900 	if (!vm->fault_credit) {  in amdgpu_vm_pasid_fault_credit()
2906 	vm->fault_credit--;  in amdgpu_vm_pasid_fault_credit()
2988 		r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);  in amdgpu_vm_ioctl()
2993 		amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);  in amdgpu_vm_ioctl()
3012 	struct amdgpu_vm *vm;  in amdgpu_vm_get_task_info()  local
3016 	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);  in amdgpu_vm_get_task_info()
3017 	if (vm)  in amdgpu_vm_get_task_info()
3018 		*task_info = vm->task_info;  in amdgpu_vm_get_task_info()
3028 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)  in amdgpu_vm_set_task_info()  argument
3030 	if (!vm->task_info.pid) {  in amdgpu_vm_set_task_info()
3031 		vm->task_info.pid = current->pid;  in amdgpu_vm_set_task_info()
3032 		get_task_comm(vm->task_info.task_name, current);  in amdgpu_vm_set_task_info()
3035 			vm->task_info.tgid = current->group_leader->pid;  in amdgpu_vm_set_task_info()
3036 			get_task_comm(vm->task_info.process_name, current->group_leader);  in amdgpu_vm_set_task_info()