232 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_amdkfd_release_notify()
261 	ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,  in create_dmamap_sg_bo()
263 			ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);  in create_dmamap_sg_bo()
298 	dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context,  in amdgpu_amdkfd_remove_eviction_fence()
332 	BUG_ON(!dma_resv_trylock(bo->tbo.base.resv));  in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
334 	dma_resv_unlock(bo->tbo.base.resv);  in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
346 	if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),  in amdgpu_amdkfd_bo_validate()
352 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in amdgpu_amdkfd_bo_validate()
377 	struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);  in vm_validate_pt_pd_bos()
394 	struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);  in vm_update_pds()
406 	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);  in get_pte_flags()
507 	struct ttm_tt *src_ttm = mem->bo->tbo.ttm;  in kfd_mem_dmamap_userptr()
508 	struct ttm_tt *ttm = bo->tbo.ttm;  in kfd_mem_dmamap_userptr()
534 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmamap_userptr()
558 	return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmamap_dmabuf()
594 	struct ttm_tt *ttm = bo->tbo.ttm;  in kfd_mem_dmamap_sg_bo()
609 	dma_addr = mem->bo->tbo.sg->sgl->dma_address;  in kfd_mem_dmamap_sg_bo()
610 	pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);  in kfd_mem_dmamap_sg_bo()
613 			mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);  in kfd_mem_dmamap_sg_bo()
619 	ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);  in kfd_mem_dmamap_sg_bo()
626 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmamap_sg_bo()
637 	dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,  in kfd_mem_dmamap_sg_bo()
671 	struct ttm_tt *ttm = bo->tbo.ttm;  in kfd_mem_dmaunmap_userptr()
677 	ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmaunmap_userptr()
692 	ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmaunmap_dmabuf()
718 	struct ttm_tt *ttm = bo->tbo.ttm;  in kfd_mem_dmaunmap_sg_bo()
727 	ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in kfd_mem_dmaunmap_sg_bo()
736 	bo->tbo.sg = NULL;  in kfd_mem_dmaunmap_sg_bo()
768 		mem->dmabuf = amdgpu_gem_prime_export(&mem->bo->tbo.base,  in kfd_mem_attach_dmabuf()
804 	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev);  in kfd_mem_attach()
805 	unsigned long bo_size = mem->bo->tbo.base.size;  in kfd_mem_attach()
846 		    (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||  in kfd_mem_attach()
854 			drm_gem_object_get(&bo[i]->tbo.base);  in kfd_mem_attach()
859 			drm_gem_object_get(&bo[i]->tbo.base);  in kfd_mem_attach()
860 		} else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {  in kfd_mem_attach()
867 		} else if (mem->bo->tbo.type == ttm_bo_type_sg) {  in kfd_mem_attach()
924 			drm_gem_object_put(&bo[i]->tbo.base);  in kfd_mem_attach()
937 	drm_gem_object_put(&bo->tbo.base);  in kfd_mem_detach()
951 	entry->bo = &bo->tbo;  in add_kgd_mem_to_kfd_bo_list()
994 	ret = amdgpu_ttm_tt_set_userptr(&bo->tbo, user_addr, 0);  in init_user_pages()
1019 	ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages, &range);  in init_user_pages()
1031 	ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in init_user_pages()
1037 	amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range);  in init_user_pages()
1094 	ctx->kfd_bo.tv.bo = &bo->tbo;  in reserve_bo_and_vm()
1157 	ctx->kfd_bo.tv.bo = &bo->tbo;  in reserve_bo_and_cond_vms()
1313 		ret = amdgpu_sync_resv(NULL, sync, pd->tbo.base.resv,  in process_sync_pds_resv()
1390 	ret = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1);  in init_kfd_vm()
1393 	dma_resv_add_fence(vm->root.bo->tbo.base.resv,  in init_kfd_vm()
1575 	struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);  in amdgpu_amdkfd_gpuvm_get_process_page_dir()
1718 		bo->tbo.sg = sg;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1719 		bo->tbo.ttm->sg = sg;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1781 	unsigned long bo_size = mem->bo->tbo.base.size;  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1846 	if (mem->bo->tbo.sg) {  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1847 		sg_free_table(mem->bo->tbo.sg);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1848 		kfree(mem->bo->tbo.sg);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1863 	drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1872 	drm_gem_object_put(&mem->bo->tbo.base);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1913 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1922 	bo_size = bo->tbo.base.size;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1944 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) &&  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1945 	    bo->tbo.resource->mem_type == TTM_PL_SYSTEM)  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1953 	    !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1991 	if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->tbo.pin_count)  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1992 		dma_resv_add_fence(bo->tbo.base.resv,  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
2012 	unsigned long bo_size = mem->bo->tbo.base.size;  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
2056 	    !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) &&  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
2057 	    !mem->bo->tbo.pin_count)  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
2109 	ret = amdgpu_ttm_alloc_gart(&bo->tbo);  in amdgpu_amdkfd_map_gtt_bo_to_gart()
2152 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2278 	drm_gem_object_get(&bo->tbo.base);  in amdgpu_amdkfd_gpuvm_import_dmabuf()
2356 		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in update_invalid_user_pages()
2386 		ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages,  in update_invalid_user_pages()
2406 			amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range);  in update_invalid_user_pages()
2485 		if (bo->tbo.ttm->pages[0]) {  in validate_invalid_user_pages()
2487 			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in validate_invalid_user_pages()
2709 		dma_resv_for_each_fence(&cursor, bo->tbo.base.resv,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2763 		if (mem->bo->tbo.pin_count)  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2766 		dma_resv_add_fence(mem->bo->tbo.base.resv,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2775 		dma_resv_add_fence(bo->tbo.base.resv,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2828 	ret = dma_resv_reserve_fences(gws_bo->tbo.base.resv, 1);  in amdgpu_amdkfd_add_gws_to_process()
2831 	dma_resv_add_fence(gws_bo->tbo.base.resv,  in amdgpu_amdkfd_add_gws_to_process()