194 void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo)  in amdgpu_amdkfd_unreserve_memory_limit()  argument
196 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_amdkfd_unreserve_memory_limit()
197 	u32 domain = bo->preferred_domains;  in amdgpu_amdkfd_unreserve_memory_limit()
198 	bool sg = (bo->preferred_domains == AMDGPU_GEM_DOMAIN_CPU);  in amdgpu_amdkfd_unreserve_memory_limit()
200 	if (bo->flags & AMDGPU_AMDKFD_USERPTR_BO) {  in amdgpu_amdkfd_unreserve_memory_limit()
205 	unreserve_mem_limit(adev, amdgpu_bo_size(bo), domain, sg);  in amdgpu_amdkfd_unreserve_memory_limit()
218 static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,  in amdgpu_amdkfd_remove_eviction_fence()  argument
221 	struct dma_resv *resv = bo->tbo.base.resv;  in amdgpu_amdkfd_remove_eviction_fence()
274 static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,  in amdgpu_amdkfd_bo_validate()  argument
280 	if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),  in amdgpu_amdkfd_bo_validate()
284 	amdgpu_bo_placement_from_domain(bo, domain);  in amdgpu_amdkfd_bo_validate()
286 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in amdgpu_amdkfd_bo_validate()
290 		amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);  in amdgpu_amdkfd_bo_validate()
296 static int amdgpu_amdkfd_validate(void *param, struct amdgpu_bo *bo)  in amdgpu_amdkfd_validate()  argument
300 	return amdgpu_amdkfd_bo_validate(bo, p->domain, p->wait);  in amdgpu_amdkfd_validate()
312 	struct amdgpu_bo *pd = vm->root.base.bo;  in vm_validate_pt_pd_bos()
333 	vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.base.bo);  in vm_validate_pt_pd_bos()
348 	struct amdgpu_bo *pd = vm->root.base.bo;  in vm_update_pds()
377 	struct amdgpu_bo *bo = mem->bo;  in add_bo_to_vm()  local
380 	unsigned long bo_size = bo->tbo.mem.size;  in add_bo_to_vm()
398 	bo_va_entry->bo_va = amdgpu_vm_bo_add(adev, vm, bo);  in add_bo_to_vm()
448 	struct amdgpu_bo *bo = mem->bo;  in add_kgd_mem_to_kfd_bo_list()  local
452 	entry->bo = &bo->tbo;  in add_kgd_mem_to_kfd_bo_list()
488 	struct amdgpu_bo *bo = mem->bo;  in init_user_pages()  local
494 	ret = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, user_addr, 0);  in init_user_pages()
500 	ret = amdgpu_mn_register(bo, user_addr);  in init_user_pages()
507 	ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);  in init_user_pages()
513 	ret = amdgpu_bo_reserve(bo, true);  in init_user_pages()
518 	amdgpu_bo_placement_from_domain(bo, mem->domain);  in init_user_pages()
519 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in init_user_pages()
522 	amdgpu_bo_unreserve(bo);  in init_user_pages()
525 	amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);  in init_user_pages()
528 		amdgpu_mn_unregister(bo);  in init_user_pages()
565 	struct amdgpu_bo *bo = mem->bo;  in reserve_bo_and_vm()  local
582 	ctx->kfd_bo.tv.bo = &bo->tbo;  in reserve_bo_and_vm()
615 	struct amdgpu_bo *bo = mem->bo;  in reserve_bo_and_cond_vms()  local
645 	ctx->kfd_bo.tv.bo = &bo->tbo;  in reserve_bo_and_cond_vms()
747 			       amdgpu_bo_size(entry->bo_va->base.bo),  in map_bo_to_gpuvm()
812 		struct amdgpu_bo *pd = peer_vm->root.base.bo;  in process_sync_pds_resv()
878 	ret = amdgpu_bo_reserve(vm->root.base.bo, true);  in init_kfd_vm()
886 	ret = amdgpu_bo_sync_wait(vm->root.base.bo,  in init_kfd_vm()
890 	ret = dma_resv_reserve_shared(vm->root.base.bo->tbo.base.resv, 1);  in init_kfd_vm()
893 	amdgpu_bo_fence(vm->root.base.bo,  in init_kfd_vm()
895 	amdgpu_bo_unreserve(vm->root.base.bo);  in init_kfd_vm()
909 	amdgpu_bo_unreserve(vm->root.base.bo);  in init_kfd_vm()
995 	struct amdgpu_bo *pd = vm->root.base.bo;  in amdgpu_amdkfd_gpuvm_destroy_cb()
1062 	struct amdgpu_bo *pd = avm->root.base.bo;  in amdgpu_amdkfd_gpuvm_get_process_page_dir()
1080 	struct amdgpu_bo *bo;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()  local
1176 	ret = amdgpu_bo_create(adev, &bp, &bo);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1183 		bo->tbo.sg = sg;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1184 		bo->tbo.ttm->sg = sg;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1186 	bo->kfd_bo = *mem;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1187 	(*mem)->bo = bo;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1189 		bo->flags |= AMDGPU_AMDKFD_USERPTR_BO;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1204 		*offset = amdgpu_bo_mmap_offset(bo);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1210 	amdgpu_bo_unref(&bo);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1230 	unsigned long bo_size = mem->bo->tbo.mem.size;  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1251 	amdgpu_mn_unregister(mem->bo);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1267 	amdgpu_amdkfd_remove_eviction_fence(mem->bo,  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1285 	if (mem->bo->tbo.sg) {  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1286 		sg_free_table(mem->bo->tbo.sg);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1287 		kfree(mem->bo->tbo.sg);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1291 	amdgpu_bo_unref(&mem->bo);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1304 	struct amdgpu_bo *bo;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()  local
1313 	bo = mem->bo;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1314 	if (!bo) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1329 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1338 	bo_size = bo->tbo.mem.size;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1354 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) &&  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1355 	    bo->tbo.mem.mem_type == TTM_PL_SYSTEM)  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1376 	    !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1381 		ret = amdgpu_amdkfd_bo_validate(bo, domain, true);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1414 	if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->pin_count)  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1415 		amdgpu_bo_fence(bo,  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1442 	unsigned long bo_size = mem->bo->tbo.mem.size;  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1493 	    !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && !mem->bo->pin_count)  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1494 		amdgpu_amdkfd_remove_eviction_fence(mem->bo,  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1525 	struct amdgpu_bo *bo = mem->bo;  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()  local
1527 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1537 	ret = amdgpu_bo_reserve(bo, true);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1543 	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1549 	ret = amdgpu_bo_kmap(bo, kptr);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1556 		bo, mem->process_info->eviction_fence);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1560 		*size = amdgpu_bo_size(bo);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1562 	amdgpu_bo_unreserve(bo);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1568 	amdgpu_bo_unpin(bo);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1570 	amdgpu_bo_unreserve(bo);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1599 	struct amdgpu_bo *bo;  in amdgpu_amdkfd_gpuvm_import_dmabuf()  local
1611 	bo = gem_to_amdgpu_bo(obj);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1612 	if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1622 		*size = amdgpu_bo_size(bo);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1625 		*mmap_offset = amdgpu_bo_mmap_offset(bo);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1633 	(*mem)->bo = amdgpu_bo_ref(bo);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1635 	(*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?  in amdgpu_amdkfd_gpuvm_import_dmabuf()
1687 	struct amdgpu_bo *bo;  in update_invalid_user_pages()  local
1700 		bo = mem->bo;  in update_invalid_user_pages()
1702 		if (amdgpu_bo_reserve(bo, true))  in update_invalid_user_pages()
1704 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);  in update_invalid_user_pages()
1705 		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in update_invalid_user_pages()
1706 		amdgpu_bo_unreserve(bo);  in update_invalid_user_pages()
1730 		bo = mem->bo;  in update_invalid_user_pages()
1733 		ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);  in update_invalid_user_pages()
1742 		amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);  in update_invalid_user_pages()
1769 	struct amdgpu_bo *bo;  in validate_invalid_user_pages()  local
1795 		mem->resv_list.bo = mem->validate_list.bo;  in validate_invalid_user_pages()
1818 		bo = mem->bo;  in validate_invalid_user_pages()
1821 		if (bo->tbo.ttm->pages[0]) {  in validate_invalid_user_pages()
1822 			amdgpu_bo_placement_from_domain(bo, mem->domain);  in validate_invalid_user_pages()
1823 			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in validate_invalid_user_pages()
1994 		mem->resv_list.bo = mem->validate_list.bo;  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2022 		struct amdgpu_bo *bo = mem->bo;  in amdgpu_amdkfd_gpuvm_restore_process_bos()  local
2026 		ret = amdgpu_amdkfd_bo_validate(bo, domain, false);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2031 		ret = amdgpu_sync_fence(NULL, &sync_obj, bo->tbo.moving, false);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2078 		amdgpu_bo_fence(mem->bo,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2084 		struct amdgpu_bo *bo = peer_vm->root.base.bo;  in amdgpu_amdkfd_gpuvm_restore_process_bos()  local
2086 		amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2112 	(*mem)->bo = amdgpu_bo_ref(gws_bo);  in amdgpu_amdkfd_add_gws_to_process()
2164 	struct amdgpu_bo *gws_bo = kgd_mem->bo;  in amdgpu_amdkfd_remove_gws_from_process()