230 void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)  in amdgpu_amdkfd_release_notify()  argument
232 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_amdkfd_release_notify()
233 	u32 alloc_flags = bo->kfd_bo->alloc_flags;  in amdgpu_amdkfd_release_notify()
234 	u64 size = amdgpu_bo_size(bo);  in amdgpu_amdkfd_release_notify()
238 	kfree(bo->kfd_bo);  in amdgpu_amdkfd_release_notify()
256 	ret = amdgpu_bo_reserve(mem->bo, false);  in create_dmamap_sg_bo()
261 	ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,  in create_dmamap_sg_bo()
263 			ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);  in create_dmamap_sg_bo()
265 	amdgpu_bo_unreserve(mem->bo);  in create_dmamap_sg_bo()
273 	(*bo_out)->parent = amdgpu_bo_ref(mem->bo);  in create_dmamap_sg_bo()
286 static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,  in amdgpu_amdkfd_remove_eviction_fence()  argument
298 	dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context,  in amdgpu_amdkfd_remove_eviction_fence()
304 int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo)  in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()  argument
306 	struct amdgpu_bo *root = bo;  in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
332 	BUG_ON(!dma_resv_trylock(bo->tbo.base.resv));  in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
333 	ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef);  in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
334 	dma_resv_unlock(bo->tbo.base.resv);  in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
340 static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,  in amdgpu_amdkfd_bo_validate()  argument
346 	if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),  in amdgpu_amdkfd_bo_validate()
350 	amdgpu_bo_placement_from_domain(bo, domain);  in amdgpu_amdkfd_bo_validate()
352 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in amdgpu_amdkfd_bo_validate()
356 		amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);  in amdgpu_amdkfd_bo_validate()
362 static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo)  in amdgpu_amdkfd_validate_vm_bo()  argument
364 	return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false);  in amdgpu_amdkfd_validate_vm_bo()
376 	struct amdgpu_bo *pd = vm->root.bo;  in vm_validate_pt_pd_bos()
386 	vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo);  in vm_validate_pt_pd_bos()
393 	struct amdgpu_bo *pd = vm->root.bo;  in vm_update_pds()
406 	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);  in get_pte_flags()
505 	struct amdgpu_bo *bo = attachment->bo_va->base.bo;  in kfd_mem_dmamap_userptr()  local
507 	struct ttm_tt *src_ttm = mem->bo->tbo.ttm;  in kfd_mem_dmamap_userptr()
508 	struct ttm_tt *ttm = bo->tbo.ttm;  in kfd_mem_dmamap_userptr()
533 	amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);  in kfd_mem_dmamap_userptr()
534 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmamap_userptr()
555 	struct amdgpu_bo *bo = attachment->bo_va->base.bo;  in kfd_mem_dmamap_dmabuf()  local
557 	amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);  in kfd_mem_dmamap_dmabuf()
558 	return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmamap_dmabuf()
592 	struct amdgpu_bo *bo = attachment->bo_va->base.bo;  in kfd_mem_dmamap_sg_bo()  local
594 	struct ttm_tt *ttm = bo->tbo.ttm;  in kfd_mem_dmamap_sg_bo()
609 	dma_addr = mem->bo->tbo.sg->sgl->dma_address;  in kfd_mem_dmamap_sg_bo()
610 	pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);  in kfd_mem_dmamap_sg_bo()
613 			mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);  in kfd_mem_dmamap_sg_bo()
619 	ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);  in kfd_mem_dmamap_sg_bo()
625 	amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);  in kfd_mem_dmamap_sg_bo()
626 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmamap_sg_bo()
637 	dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,  in kfd_mem_dmamap_sg_bo()
669 	struct amdgpu_bo *bo = attachment->bo_va->base.bo;  in kfd_mem_dmaunmap_userptr()  local
671 	struct ttm_tt *ttm = bo->tbo.ttm;  in kfd_mem_dmaunmap_userptr()
676 	amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);  in kfd_mem_dmaunmap_userptr()
677 	ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmaunmap_userptr()
689 	struct amdgpu_bo *bo = attachment->bo_va->base.bo;  in kfd_mem_dmaunmap_dmabuf()  local
691 	amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);  in kfd_mem_dmaunmap_dmabuf()
692 	ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmaunmap_dmabuf()
716 	struct amdgpu_bo *bo = attachment->bo_va->base.bo;  in kfd_mem_dmaunmap_sg_bo()  local
718 	struct ttm_tt *ttm = bo->tbo.ttm;  in kfd_mem_dmaunmap_sg_bo()
726 	amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);  in kfd_mem_dmaunmap_sg_bo()
727 	ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmaunmap_sg_bo()
736 	bo->tbo.sg = NULL;  in kfd_mem_dmaunmap_sg_bo()
762 		      struct amdgpu_bo **bo)  in kfd_mem_attach_dmabuf()  argument
768 		mem->dmabuf = amdgpu_gem_prime_export(&mem->bo->tbo.base,  in kfd_mem_attach_dmabuf()
782 	*bo = gem_to_amdgpu_bo(gobj);  in kfd_mem_attach_dmabuf()
783 	(*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE;  in kfd_mem_attach_dmabuf()
804 	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);  in kfd_mem_attach()
805 	unsigned long bo_size = mem->bo->tbo.base.size;  in kfd_mem_attach()
808 	struct amdgpu_bo *bo[2] = {NULL, NULL};  in kfd_mem_attach()  local
846 		    (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||  in kfd_mem_attach()
853 			bo[i] = mem->bo;  in kfd_mem_attach()
854 			drm_gem_object_get(&bo[i]->tbo.base);  in kfd_mem_attach()
858 			bo[i] = bo[0];  in kfd_mem_attach()
859 			drm_gem_object_get(&bo[i]->tbo.base);  in kfd_mem_attach()
860 		} else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {  in kfd_mem_attach()
863 			ret = create_dmamap_sg_bo(adev, mem, &bo[i]);  in kfd_mem_attach()
867 		} else if (mem->bo->tbo.type == ttm_bo_type_sg) {  in kfd_mem_attach()
872 			ret = create_dmamap_sg_bo(adev, mem, &bo[i]);  in kfd_mem_attach()
879 			ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);  in kfd_mem_attach()
890 		ret = amdgpu_bo_reserve(bo[i], false);  in kfd_mem_attach()
895 		attachment[i]->bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]);  in kfd_mem_attach()
896 		amdgpu_bo_unreserve(bo[i]);  in kfd_mem_attach()
918 			amdgpu_bo_reserve(bo[i], true);  in kfd_mem_attach()
920 			amdgpu_bo_unreserve(bo[i]);  in kfd_mem_attach()
923 		if (bo[i])  in kfd_mem_attach()
924 			drm_gem_object_put(&bo[i]->tbo.base);  in kfd_mem_attach()
932 	struct amdgpu_bo *bo = attachment->bo_va->base.bo;  in kfd_mem_detach()  local
937 	drm_gem_object_put(&bo->tbo.base);  in kfd_mem_detach()
947 	struct amdgpu_bo *bo = mem->bo;  in add_kgd_mem_to_kfd_bo_list()  local
951 	entry->bo = &bo->tbo;  in add_kgd_mem_to_kfd_bo_list()
987 	struct amdgpu_bo *bo = mem->bo;  in init_user_pages()  local
994 	ret = amdgpu_ttm_tt_set_userptr(&bo->tbo, user_addr, 0);  in init_user_pages()
1000 	ret = amdgpu_mn_register(bo, user_addr);  in init_user_pages()
1019 	ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages, &range);  in init_user_pages()
1025 	ret = amdgpu_bo_reserve(bo, true);  in init_user_pages()
1030 	amdgpu_bo_placement_from_domain(bo, mem->domain);  in init_user_pages()
1031 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in init_user_pages()
1034 	amdgpu_bo_unreserve(bo);  in init_user_pages()
1037 	amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range);  in init_user_pages()
1040 		amdgpu_mn_unregister(bo);  in init_user_pages()
1077 	struct amdgpu_bo *bo = mem->bo;  in reserve_bo_and_vm()  local
1094 	ctx->kfd_bo.tv.bo = &bo->tbo;  in reserve_bo_and_vm()
1127 	struct amdgpu_bo *bo = mem->bo;  in reserve_bo_and_cond_vms()  local
1157 	ctx->kfd_bo.tv.bo = &bo->tbo;  in reserve_bo_and_cond_vms()
1264 			       amdgpu_bo_size(entry->bo_va->base.bo),  in map_bo_to_gpuvm()
1311 		struct amdgpu_bo *pd = peer_vm->root.bo;  in process_sync_pds_resv()
1378 	ret = amdgpu_bo_reserve(vm->root.bo, true);  in init_kfd_vm()
1386 	ret = amdgpu_bo_sync_wait(vm->root.bo,  in init_kfd_vm()
1390 	ret = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1);  in init_kfd_vm()
1393 	dma_resv_add_fence(vm->root.bo->tbo.base.resv,  in init_kfd_vm()
1396 	amdgpu_bo_unreserve(vm->root.bo);  in init_kfd_vm()
1410 	amdgpu_bo_unreserve(vm->root.bo);  in init_kfd_vm()
1438 static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)  in amdgpu_amdkfd_gpuvm_pin_bo()  argument
1442 	ret = amdgpu_bo_reserve(bo, false);  in amdgpu_amdkfd_gpuvm_pin_bo()
1446 	ret = amdgpu_bo_pin_restricted(bo, domain, 0, 0);  in amdgpu_amdkfd_gpuvm_pin_bo()
1450 	amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);  in amdgpu_amdkfd_gpuvm_pin_bo()
1451 	amdgpu_bo_unreserve(bo);  in amdgpu_amdkfd_gpuvm_pin_bo()
1464 static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)  in amdgpu_amdkfd_gpuvm_unpin_bo()  argument
1468 	ret = amdgpu_bo_reserve(bo, false);  in amdgpu_amdkfd_gpuvm_unpin_bo()
1472 	amdgpu_bo_unpin(bo);  in amdgpu_amdkfd_gpuvm_unpin_bo()
1473 	amdgpu_bo_unreserve(bo);  in amdgpu_amdkfd_gpuvm_unpin_bo()
1574 	struct amdgpu_bo *pd = avm->root.bo;  in amdgpu_amdkfd_gpuvm_get_process_page_dir()
1636 	struct amdgpu_bo *bo;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()  local
1716 	bo = gem_to_amdgpu_bo(gobj);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1718 		bo->tbo.sg = sg;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1719 		bo->tbo.ttm->sg = sg;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1721 	bo->kfd_bo = *mem;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1722 	(*mem)->bo = bo;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1724 		bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1739 		ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1744 		bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1745 		bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1749 		*offset = amdgpu_bo_mmap_offset(bo);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1781 	unsigned long bo_size = mem->bo->tbo.base.size;  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1782 	bool use_release_notifier = (mem->bo->kfd_bo == mem);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1796 		amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1819 	amdgpu_mn_unregister(mem->bo);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1829 	amdgpu_amdkfd_remove_eviction_fence(mem->bo,  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1846 	if (mem->bo->tbo.sg) {  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1847 		sg_free_table(mem->bo->tbo.sg);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1848 		kfree(mem->bo->tbo.sg);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1855 		if ((mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM) &&  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1863 	drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1872 	drm_gem_object_put(&mem->bo->tbo.base);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1890 	struct amdgpu_bo *bo;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()  local
1897 	bo = mem->bo;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1898 	if (!bo) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1913 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1922 	bo_size = bo->tbo.base.size;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1944 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) &&  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1945 	    bo->tbo.resource->mem_type == TTM_PL_SYSTEM)  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1953 	    !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1958 		ret = amdgpu_amdkfd_bo_validate(bo, domain, true);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1991 	if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->tbo.pin_count)  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1992 		dma_resv_add_fence(bo->tbo.base.resv,  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
2012 	unsigned long bo_size = mem->bo->tbo.base.size;  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
2056 	    !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) &&  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
2057 	    !mem->bo->tbo.pin_count)  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
2058 		amdgpu_amdkfd_remove_eviction_fence(mem->bo,  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
2093 int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_device *adev, struct amdgpu_bo *bo)  in amdgpu_amdkfd_map_gtt_bo_to_gart()  argument
2097 	ret = amdgpu_bo_reserve(bo, true);  in amdgpu_amdkfd_map_gtt_bo_to_gart()
2103 	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);  in amdgpu_amdkfd_map_gtt_bo_to_gart()
2109 	ret = amdgpu_ttm_alloc_gart(&bo->tbo);  in amdgpu_amdkfd_map_gtt_bo_to_gart()
2116 		bo, bo->kfd_bo->process_info->eviction_fence);  in amdgpu_amdkfd_map_gtt_bo_to_gart()
2118 	amdgpu_bo_unreserve(bo);  in amdgpu_amdkfd_map_gtt_bo_to_gart()
2120 	bo = amdgpu_bo_ref(bo);  in amdgpu_amdkfd_map_gtt_bo_to_gart()
2125 	amdgpu_bo_unpin(bo);  in amdgpu_amdkfd_map_gtt_bo_to_gart()
2127 	amdgpu_bo_unreserve(bo);  in amdgpu_amdkfd_map_gtt_bo_to_gart()
2150 	struct amdgpu_bo *bo = mem->bo;  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()  local
2152 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2159 	ret = amdgpu_bo_reserve(bo, true);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2165 	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2171 	ret = amdgpu_bo_kmap(bo, kptr);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2178 		bo, mem->process_info->eviction_fence);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2181 		*size = amdgpu_bo_size(bo);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2183 	amdgpu_bo_unreserve(bo);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2189 	amdgpu_bo_unpin(bo);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2191 	amdgpu_bo_unreserve(bo);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2208 	struct amdgpu_bo *bo = mem->bo;  in amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel()  local
2210 	amdgpu_bo_reserve(bo, true);  in amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel()
2211 	amdgpu_bo_kunmap(bo);  in amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel()
2212 	amdgpu_bo_unpin(bo);  in amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel()
2213 	amdgpu_bo_unreserve(bo);  in amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel()
2235 	struct amdgpu_bo *bo;  in amdgpu_amdkfd_gpuvm_import_dmabuf()  local
2247 	bo = gem_to_amdgpu_bo(obj);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
2248 	if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |  in amdgpu_amdkfd_gpuvm_import_dmabuf()
2264 		*size = amdgpu_bo_size(bo);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
2267 		*mmap_offset = amdgpu_bo_mmap_offset(bo);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
2273 		((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?  in amdgpu_amdkfd_gpuvm_import_dmabuf()
2278 	drm_gem_object_get(&bo->tbo.base);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
2279 	(*mem)->bo = bo;  in amdgpu_amdkfd_gpuvm_import_dmabuf()
2281 	(*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?  in amdgpu_amdkfd_gpuvm_import_dmabuf()
2338 	struct amdgpu_bo *bo;  in update_invalid_user_pages()  local
2351 		bo = mem->bo;  in update_invalid_user_pages()
2353 		if (amdgpu_bo_reserve(bo, true))  in update_invalid_user_pages()
2355 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);  in update_invalid_user_pages()
2356 		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in update_invalid_user_pages()
2357 		amdgpu_bo_unreserve(bo);  in update_invalid_user_pages()
2383 		bo = mem->bo;  in update_invalid_user_pages()
2386 		ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages,  in update_invalid_user_pages()
2406 			amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range);  in update_invalid_user_pages()
2434 	struct amdgpu_bo *bo;  in validate_invalid_user_pages()  local
2460 		mem->resv_list.bo = mem->validate_list.bo;  in validate_invalid_user_pages()
2482 		bo = mem->bo;  in validate_invalid_user_pages()
2485 		if (bo->tbo.ttm->pages[0]) {  in validate_invalid_user_pages()
2486 			amdgpu_bo_placement_from_domain(bo, mem->domain);  in validate_invalid_user_pages()
2487 			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in validate_invalid_user_pages()
2662 		mem->resv_list.bo = mem->validate_list.bo;  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2690 		struct amdgpu_bo *bo = mem->bo;  in amdgpu_amdkfd_gpuvm_restore_process_bos()  local
2696 		total_size += amdgpu_bo_size(bo);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2698 		ret = amdgpu_amdkfd_bo_validate(bo, domain, false);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2701 			failed_size += amdgpu_bo_size(bo);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2702 			ret = amdgpu_amdkfd_bo_validate(bo,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2709 		dma_resv_for_each_fence(&cursor, bo->tbo.base.resv,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2763 		if (mem->bo->tbo.pin_count)  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2766 		dma_resv_add_fence(mem->bo->tbo.base.resv,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2773 		struct amdgpu_bo *bo = peer_vm->root.bo;  in amdgpu_amdkfd_gpuvm_restore_process_bos()  local
2775 		dma_resv_add_fence(bo->tbo.base.resv,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2804 	(*mem)->bo = amdgpu_bo_ref(gws_bo);  in amdgpu_amdkfd_add_gws_to_process()
2858 	struct amdgpu_bo *gws_bo = kgd_mem->bo;  in amdgpu_amdkfd_remove_gws_from_process()