210 void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo)  in amdgpu_amdkfd_unreserve_memory_limit()  argument
212 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_amdkfd_unreserve_memory_limit()
213 	u32 domain = bo->preferred_domains;  in amdgpu_amdkfd_unreserve_memory_limit()
214 	bool sg = (bo->preferred_domains == AMDGPU_GEM_DOMAIN_CPU);  in amdgpu_amdkfd_unreserve_memory_limit()
216 	if (bo->flags & AMDGPU_AMDKFD_CREATE_USERPTR_BO) {  in amdgpu_amdkfd_unreserve_memory_limit()
221 	unreserve_mem_limit(adev, amdgpu_bo_size(bo), domain, sg);  in amdgpu_amdkfd_unreserve_memory_limit()
234 static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,  in amdgpu_amdkfd_remove_eviction_fence()  argument
237 	struct dma_resv *resv = bo->tbo.base.resv;  in amdgpu_amdkfd_remove_eviction_fence()
287 int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo)  in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()  argument
289 	struct amdgpu_bo *root = bo;  in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
315 	BUG_ON(!dma_resv_trylock(bo->tbo.base.resv));  in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
316 	ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef);  in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
317 	dma_resv_unlock(bo->tbo.base.resv);  in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
323 static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,  in amdgpu_amdkfd_bo_validate()  argument
329 	if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),  in amdgpu_amdkfd_bo_validate()
333 	amdgpu_bo_placement_from_domain(bo, domain);  in amdgpu_amdkfd_bo_validate()
335 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in amdgpu_amdkfd_bo_validate()
339 		amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);  in amdgpu_amdkfd_bo_validate()
345 static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo)  in amdgpu_amdkfd_validate_vm_bo()  argument
347 	return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false);  in amdgpu_amdkfd_validate_vm_bo()
359 	struct amdgpu_bo *pd = vm->root.bo;  in vm_validate_pt_pd_bos()
375 	vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo);  in vm_validate_pt_pd_bos()
390 	struct amdgpu_bo *pd = vm->root.bo;  in vm_update_pds()
403 	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);  in get_pte_flags()
473 	struct amdgpu_bo *bo = attachment->bo_va->base.bo;  in kfd_mem_dmamap_userptr()  local
475 	struct ttm_tt *src_ttm = mem->bo->tbo.ttm;  in kfd_mem_dmamap_userptr()
476 	struct ttm_tt *ttm = bo->tbo.ttm;  in kfd_mem_dmamap_userptr()
501 	amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);  in kfd_mem_dmamap_userptr()
502 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmamap_userptr()
523 	struct amdgpu_bo *bo = attachment->bo_va->base.bo;  in kfd_mem_dmamap_dmabuf()  local
525 	amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);  in kfd_mem_dmamap_dmabuf()
526 	return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmamap_dmabuf()
554 	struct amdgpu_bo *bo = attachment->bo_va->base.bo;  in kfd_mem_dmaunmap_userptr()  local
556 	struct ttm_tt *ttm = bo->tbo.ttm;  in kfd_mem_dmaunmap_userptr()
561 	amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);  in kfd_mem_dmaunmap_userptr()
562 	ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmaunmap_userptr()
574 	struct amdgpu_bo *bo = attachment->bo_va->base.bo;  in kfd_mem_dmaunmap_dmabuf()  local
576 	amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);  in kfd_mem_dmaunmap_dmabuf()
577 	ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmaunmap_dmabuf()
600 		       struct amdgpu_bo **bo)  in kfd_mem_attach_userptr()  argument
602 	unsigned long bo_size = mem->bo->tbo.base.size;  in kfd_mem_attach_userptr()
606 	ret = amdgpu_bo_reserve(mem->bo, false);  in kfd_mem_attach_userptr()
613 				       ttm_bo_type_sg, mem->bo->tbo.base.resv,  in kfd_mem_attach_userptr()
615 	amdgpu_bo_unreserve(mem->bo);  in kfd_mem_attach_userptr()
619 	*bo = gem_to_amdgpu_bo(gobj);  in kfd_mem_attach_userptr()
620 	(*bo)->parent = amdgpu_bo_ref(mem->bo);  in kfd_mem_attach_userptr()
627 		      struct amdgpu_bo **bo)  in kfd_mem_attach_dmabuf()  argument
633 		mem->dmabuf = amdgpu_gem_prime_export(&mem->bo->tbo.base,  in kfd_mem_attach_dmabuf()
653 	*bo = gem_to_amdgpu_bo(gobj);  in kfd_mem_attach_dmabuf()
654 	(*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE;  in kfd_mem_attach_dmabuf()
655 	(*bo)->parent = amdgpu_bo_ref(mem->bo);  in kfd_mem_attach_dmabuf()
676 	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);  in kfd_mem_attach()
677 	unsigned long bo_size = mem->bo->tbo.base.size;  in kfd_mem_attach()
680 	struct amdgpu_bo *bo[2] = {NULL, NULL};  in kfd_mem_attach()  local
704 			bo[i] = mem->bo;  in kfd_mem_attach()
705 			drm_gem_object_get(&bo[i]->tbo.base);  in kfd_mem_attach()
709 			bo[i] = bo[0];  in kfd_mem_attach()
710 			drm_gem_object_get(&bo[i]->tbo.base);  in kfd_mem_attach()
711 		} else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {  in kfd_mem_attach()
714 			ret = kfd_mem_attach_userptr(adev, mem, &bo[i]);  in kfd_mem_attach()
718 			   mem->bo->tbo.type != ttm_bo_type_sg) {  in kfd_mem_attach()
724 			ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);  in kfd_mem_attach()
732 			bo[i] = mem->bo;  in kfd_mem_attach()
733 			drm_gem_object_get(&bo[i]->tbo.base);  in kfd_mem_attach()
737 		attachment[i]->bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]);  in kfd_mem_attach()
763 		if (bo[i])  in kfd_mem_attach()
764 			drm_gem_object_put(&bo[i]->tbo.base);  in kfd_mem_attach()
772 	struct amdgpu_bo *bo = attachment->bo_va->base.bo;  in kfd_mem_detach()  local
777 	drm_gem_object_put(&bo->tbo.base);  in kfd_mem_detach()
787 	struct amdgpu_bo *bo = mem->bo;  in add_kgd_mem_to_kfd_bo_list()  local
791 	entry->bo = &bo->tbo;  in add_kgd_mem_to_kfd_bo_list()
826 	struct amdgpu_bo *bo = mem->bo;  in init_user_pages()  local
832 	ret = amdgpu_ttm_tt_set_userptr(&bo->tbo, user_addr, 0);  in init_user_pages()
838 	ret = amdgpu_mn_register(bo, user_addr);  in init_user_pages()
845 	ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);  in init_user_pages()
851 	ret = amdgpu_bo_reserve(bo, true);  in init_user_pages()
856 	amdgpu_bo_placement_from_domain(bo, mem->domain);  in init_user_pages()
857 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in init_user_pages()
860 	amdgpu_bo_unreserve(bo);  in init_user_pages()
863 	amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);  in init_user_pages()
866 		amdgpu_mn_unregister(bo);  in init_user_pages()
903 	struct amdgpu_bo *bo = mem->bo;  in reserve_bo_and_vm()  local
920 	ctx->kfd_bo.tv.bo = &bo->tbo;  in reserve_bo_and_vm()
953 	struct amdgpu_bo *bo = mem->bo;  in reserve_bo_and_cond_vms()  local
983 	ctx->kfd_bo.tv.bo = &bo->tbo;  in reserve_bo_and_cond_vms()
1092 			       amdgpu_bo_size(entry->bo_va->base.bo),  in map_bo_to_gpuvm()
1157 		struct amdgpu_bo *pd = peer_vm->root.bo;  in process_sync_pds_resv()
1224 	ret = amdgpu_bo_reserve(vm->root.bo, true);  in init_kfd_vm()
1232 	ret = amdgpu_bo_sync_wait(vm->root.bo,  in init_kfd_vm()
1236 	ret = dma_resv_reserve_shared(vm->root.bo->tbo.base.resv, 1);  in init_kfd_vm()
1239 	amdgpu_bo_fence(vm->root.bo,  in init_kfd_vm()
1241 	amdgpu_bo_unreserve(vm->root.bo);  in init_kfd_vm()
1255 	amdgpu_bo_unreserve(vm->root.bo);  in init_kfd_vm()
1321 	struct amdgpu_bo *pd = vm->root.bo;  in amdgpu_amdkfd_gpuvm_destroy_cb()
1377 	struct amdgpu_bo *pd = avm->root.bo;  in amdgpu_amdkfd_gpuvm_get_process_page_dir()
1395 	struct amdgpu_bo *bo;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()  local
1475 	bo = gem_to_amdgpu_bo(gobj);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1477 		bo->tbo.sg = sg;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1478 		bo->tbo.ttm->sg = sg;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1480 	bo->kfd_bo = *mem;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1481 	(*mem)->bo = bo;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1483 		bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1498 		*offset = amdgpu_bo_mmap_offset(bo);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1506 	amdgpu_bo_unref(&bo);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1527 	unsigned long bo_size = mem->bo->tbo.base.size;  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1556 	amdgpu_mn_unregister(mem->bo);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1566 	amdgpu_amdkfd_remove_eviction_fence(mem->bo,  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1583 	if (mem->bo->tbo.sg) {  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1584 		sg_free_table(mem->bo->tbo.sg);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1585 		kfree(mem->bo->tbo.sg);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1592 		if ((mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM) &&  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1600 	drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1603 	drm_gem_object_put(&mem->bo->tbo.base);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1617 	struct amdgpu_bo *bo;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()  local
1624 	bo = mem->bo;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1625 	if (!bo) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1640 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1649 	bo_size = bo->tbo.base.size;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1671 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) &&  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1672 	    bo->tbo.resource->mem_type == TTM_PL_SYSTEM)  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1680 	    !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1685 		ret = amdgpu_amdkfd_bo_validate(bo, domain, true);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1718 	if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->tbo.pin_count)  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1719 		amdgpu_bo_fence(bo,  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1745 	unsigned long bo_size = mem->bo->tbo.base.size;  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1789 	    !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) &&  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1790 	    !mem->bo->tbo.pin_count)  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1791 		amdgpu_amdkfd_remove_eviction_fence(mem->bo,  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1822 	struct amdgpu_bo *bo = mem->bo;  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()  local
1824 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1834 	ret = amdgpu_bo_reserve(bo, true);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1840 	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1846 	ret = amdgpu_bo_kmap(bo, kptr);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1853 		bo, mem->process_info->eviction_fence);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1857 		*size = amdgpu_bo_size(bo);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1859 	amdgpu_bo_unreserve(bo);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1865 	amdgpu_bo_unpin(bo);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1867 	amdgpu_bo_unreserve(bo);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1897 	struct amdgpu_bo *bo;  in amdgpu_amdkfd_gpuvm_import_dmabuf()  local
1909 	bo = gem_to_amdgpu_bo(obj);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1910 	if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1926 		*size = amdgpu_bo_size(bo);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1929 		*mmap_offset = amdgpu_bo_mmap_offset(bo);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1935 		((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1940 	drm_gem_object_get(&bo->tbo.base);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1941 	(*mem)->bo = bo;  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1943 	(*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1996 	struct amdgpu_bo *bo;  in update_invalid_user_pages()  local
2009 		bo = mem->bo;  in update_invalid_user_pages()
2011 		if (amdgpu_bo_reserve(bo, true))  in update_invalid_user_pages()
2013 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);  in update_invalid_user_pages()
2014 		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in update_invalid_user_pages()
2015 		amdgpu_bo_unreserve(bo);  in update_invalid_user_pages()
2039 		bo = mem->bo;  in update_invalid_user_pages()
2042 		ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);  in update_invalid_user_pages()
2055 		amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);  in update_invalid_user_pages()
2082 	struct amdgpu_bo *bo;  in validate_invalid_user_pages()  local
2108 		mem->resv_list.bo = mem->validate_list.bo;  in validate_invalid_user_pages()
2130 		bo = mem->bo;  in validate_invalid_user_pages()
2133 		if (bo->tbo.ttm->pages[0]) {  in validate_invalid_user_pages()
2134 			amdgpu_bo_placement_from_domain(bo, mem->domain);  in validate_invalid_user_pages()
2135 			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in validate_invalid_user_pages()
2307 		mem->resv_list.bo = mem->validate_list.bo;  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2335 		struct amdgpu_bo *bo = mem->bo;  in amdgpu_amdkfd_gpuvm_restore_process_bos()  local
2339 		total_size += amdgpu_bo_size(bo);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2341 		ret = amdgpu_amdkfd_bo_validate(bo, domain, false);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2344 			failed_size += amdgpu_bo_size(bo);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2345 			ret = amdgpu_amdkfd_bo_validate(bo,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2352 		ret = amdgpu_sync_fence(&sync_obj, bo->tbo.moving);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2403 		amdgpu_bo_fence(mem->bo,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2409 		struct amdgpu_bo *bo = peer_vm->root.bo;  in amdgpu_amdkfd_gpuvm_restore_process_bos()  local
2411 		amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2438 	(*mem)->bo = amdgpu_bo_ref(gws_bo);  in amdgpu_amdkfd_add_gws_to_process()
2490 	struct amdgpu_bo *gws_bo = kgd_mem->bo;  in amdgpu_amdkfd_remove_gws_from_process()