58 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);  in amdgpu_bo_destroy()  local
60 	amdgpu_bo_kunmap(bo);  in amdgpu_bo_destroy()
62 	if (bo->tbo.base.import_attach)  in amdgpu_bo_destroy()
63 		drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg);  in amdgpu_bo_destroy()
64 	drm_gem_object_release(&bo->tbo.base);  in amdgpu_bo_destroy()
65 	amdgpu_bo_unref(&bo->parent);  in amdgpu_bo_destroy()
66 	kvfree(bo);  in amdgpu_bo_destroy()
71 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);  in amdgpu_bo_user_destroy()  local
74 	ubo = to_amdgpu_bo_user(bo);  in amdgpu_bo_user_destroy()
82 	struct amdgpu_bo *shadow_bo = ttm_to_amdgpu_bo(tbo), *bo;  in amdgpu_bo_vm_destroy()  local
85 	bo = shadow_bo->parent;  in amdgpu_bo_vm_destroy()
86 	vmbo = to_amdgpu_bo_vm(bo);  in amdgpu_bo_vm_destroy()
107 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)  in amdgpu_bo_is_amdgpu_bo()  argument
109 	if (bo->destroy == &amdgpu_bo_destroy ||  in amdgpu_bo_is_amdgpu_bo()
110 	    bo->destroy == &amdgpu_bo_user_destroy ||  in amdgpu_bo_is_amdgpu_bo()
111 	    bo->destroy == &amdgpu_bo_vm_destroy)  in amdgpu_bo_is_amdgpu_bo()
438 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,  in amdgpu_bo_free_kernel()  argument
441 	if (*bo == NULL)  in amdgpu_bo_free_kernel()
444 	WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend);  in amdgpu_bo_free_kernel()
446 	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {  in amdgpu_bo_free_kernel()
448 			amdgpu_bo_kunmap(*bo);  in amdgpu_bo_free_kernel()
450 		amdgpu_bo_unpin(*bo);  in amdgpu_bo_free_kernel()
451 		amdgpu_bo_unreserve(*bo);  in amdgpu_bo_free_kernel()
453 	amdgpu_bo_unref(bo);  in amdgpu_bo_free_kernel()
555 	struct amdgpu_bo *bo;  in amdgpu_bo_create()  local
581 	bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL);  in amdgpu_bo_create()
582 	if (bo == NULL)  in amdgpu_bo_create()
584 	drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size);  in amdgpu_bo_create()
585 	bo->vm_bo = NULL;  in amdgpu_bo_create()
586 	bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :  in amdgpu_bo_create()
588 	bo->allowed_domains = bo->preferred_domains;  in amdgpu_bo_create()
591 	    bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)  in amdgpu_bo_create()
592 		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;  in amdgpu_bo_create()
594 	bo->flags = bp->flags;  in amdgpu_bo_create()
598 		bo->xcp_id = bp->xcp_id_plus1 - 1;  in amdgpu_bo_create()
601 		bo->xcp_id = 0;  in amdgpu_bo_create()
603 	if (!amdgpu_bo_support_uswc(bo->flags))  in amdgpu_bo_create()
604 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;  in amdgpu_bo_create()
607 		bo->flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;  in amdgpu_bo_create()
609 	bo->tbo.bdev = &adev->mman.bdev;  in amdgpu_bo_create()
612 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);  in amdgpu_bo_create()
614 		amdgpu_bo_placement_from_domain(bo, bp->domain);  in amdgpu_bo_create()
616 		bo->tbo.priority = 1;  in amdgpu_bo_create()
621 	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type,  in amdgpu_bo_create()
622 				 &bo->placement, page_align, &ctx,  NULL,  in amdgpu_bo_create()
628 	    bo->tbo.resource->mem_type == TTM_PL_VRAM &&  in amdgpu_bo_create()
629 	    amdgpu_bo_in_cpu_visible_vram(bo))  in amdgpu_bo_create()
636 	    bo->tbo.resource->mem_type == TTM_PL_VRAM) {  in amdgpu_bo_create()
639 		r = amdgpu_fill_buffer(bo, 0, bo->tbo.base.resv, &fence, true);  in amdgpu_bo_create()
643 		dma_resv_add_fence(bo->tbo.base.resv, fence,  in amdgpu_bo_create()
648 		amdgpu_bo_unreserve(bo);  in amdgpu_bo_create()
649 	*bo_ptr = bo;  in amdgpu_bo_create()
651 	trace_amdgpu_bo_create(bo);  in amdgpu_bo_create()
655 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;  in amdgpu_bo_create()
661 		dma_resv_unlock(bo->tbo.base.resv);  in amdgpu_bo_create()
662 	amdgpu_bo_unref(&bo);  in amdgpu_bo_create()
735 	struct amdgpu_device *adev = amdgpu_ttm_adev(vmbo->bo.tbo.bdev);  in amdgpu_bo_add_to_shadow_list()
739 	vmbo->shadow->parent = amdgpu_bo_ref(&vmbo->bo);  in amdgpu_bo_add_to_shadow_list()
783 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)  in amdgpu_bo_kmap()  argument
788 	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)  in amdgpu_bo_kmap()
791 	r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL,  in amdgpu_bo_kmap()
796 	kptr = amdgpu_bo_kptr(bo);  in amdgpu_bo_kmap()
803 	r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap);  in amdgpu_bo_kmap()
808 		*ptr = amdgpu_bo_kptr(bo);  in amdgpu_bo_kmap()
822 void *amdgpu_bo_kptr(struct amdgpu_bo *bo)  in amdgpu_bo_kptr()  argument
826 	return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);  in amdgpu_bo_kptr()
835 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)  in amdgpu_bo_kunmap()  argument
837 	if (bo->kmap.bo)  in amdgpu_bo_kunmap()
838 		ttm_bo_kunmap(&bo->kmap);  in amdgpu_bo_kunmap()
850 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)  in amdgpu_bo_ref()  argument
852 	if (bo == NULL)  in amdgpu_bo_ref()
855 	ttm_bo_get(&bo->tbo);  in amdgpu_bo_ref()
856 	return bo;  in amdgpu_bo_ref()
865 void amdgpu_bo_unref(struct amdgpu_bo **bo)  in amdgpu_bo_unref()  argument
869 	if ((*bo) == NULL)  in amdgpu_bo_unref()
872 	tbo = &((*bo)->tbo);  in amdgpu_bo_unref()
874 	*bo = NULL;  in amdgpu_bo_unref()
899 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,  in amdgpu_bo_pin_restricted()  argument
902 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_pin_restricted()
906 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))  in amdgpu_bo_pin_restricted()
913 	if (bo->preferred_domains & domain)  in amdgpu_bo_pin_restricted()
914 		domain = bo->preferred_domains & domain;  in amdgpu_bo_pin_restricted()
917 	if (bo->tbo.base.import_attach) {  in amdgpu_bo_pin_restricted()
924 	if (bo->tbo.pin_count) {  in amdgpu_bo_pin_restricted()
925 		uint32_t mem_type = bo->tbo.resource->mem_type;  in amdgpu_bo_pin_restricted()
926 		uint32_t mem_flags = bo->tbo.resource->placement;  in amdgpu_bo_pin_restricted()
932 		    (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) &&  in amdgpu_bo_pin_restricted()
936 		ttm_bo_pin(&bo->tbo);  in amdgpu_bo_pin_restricted()
942 				     (amdgpu_bo_gpu_offset(bo) - domain_start));  in amdgpu_bo_pin_restricted()
953 	if (bo->tbo.base.import_attach)  in amdgpu_bo_pin_restricted()
954 		dma_buf_pin(bo->tbo.base.import_attach);  in amdgpu_bo_pin_restricted()
957 	if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))  in amdgpu_bo_pin_restricted()
958 		bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;  in amdgpu_bo_pin_restricted()
959 	amdgpu_bo_placement_from_domain(bo, domain);  in amdgpu_bo_pin_restricted()
960 	for (i = 0; i < bo->placement.num_placement; i++) {  in amdgpu_bo_pin_restricted()
966 		if (fpfn > bo->placements[i].fpfn)  in amdgpu_bo_pin_restricted()
967 			bo->placements[i].fpfn = fpfn;  in amdgpu_bo_pin_restricted()
968 		if (!bo->placements[i].lpfn ||  in amdgpu_bo_pin_restricted()
969 		    (lpfn && lpfn < bo->placements[i].lpfn))  in amdgpu_bo_pin_restricted()
970 			bo->placements[i].lpfn = lpfn;  in amdgpu_bo_pin_restricted()
973 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in amdgpu_bo_pin_restricted()
975 		dev_err(adev->dev, "%p pin failed\n", bo);  in amdgpu_bo_pin_restricted()
979 	ttm_bo_pin(&bo->tbo);  in amdgpu_bo_pin_restricted()
981 	domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);  in amdgpu_bo_pin_restricted()
983 		atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);  in amdgpu_bo_pin_restricted()
984 		atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),  in amdgpu_bo_pin_restricted()
987 		atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);  in amdgpu_bo_pin_restricted()
1006 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)  in amdgpu_bo_pin()  argument
1008 	bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;  in amdgpu_bo_pin()
1009 	return amdgpu_bo_pin_restricted(bo, domain, 0, 0);  in amdgpu_bo_pin()
1022 void amdgpu_bo_unpin(struct amdgpu_bo *bo)  in amdgpu_bo_unpin()  argument
1024 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_unpin()
1026 	ttm_bo_unpin(&bo->tbo);  in amdgpu_bo_unpin()
1027 	if (bo->tbo.pin_count)  in amdgpu_bo_unpin()
1030 	if (bo->tbo.base.import_attach)  in amdgpu_bo_unpin()
1031 		dma_buf_unpin(bo->tbo.base.import_attach);  in amdgpu_bo_unpin()
1033 	if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {  in amdgpu_bo_unpin()
1034 		atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);  in amdgpu_bo_unpin()
1035 		atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),  in amdgpu_bo_unpin()
1037 	} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {  in amdgpu_bo_unpin()
1038 		atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);  in amdgpu_bo_unpin()
1126 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)  in amdgpu_bo_set_tiling_flags()  argument
1128 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_set_tiling_flags()
1131 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);  in amdgpu_bo_set_tiling_flags()
1136 	ubo = to_amdgpu_bo_user(bo);  in amdgpu_bo_set_tiling_flags()
1149 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)  in amdgpu_bo_get_tiling_flags()  argument
1153 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);  in amdgpu_bo_get_tiling_flags()
1154 	dma_resv_assert_held(bo->tbo.base.resv);  in amdgpu_bo_get_tiling_flags()
1155 	ubo = to_amdgpu_bo_user(bo);  in amdgpu_bo_get_tiling_flags()
1174 int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata,  in amdgpu_bo_set_metadata()  argument
1180 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);  in amdgpu_bo_set_metadata()
1181 	ubo = to_amdgpu_bo_user(bo);  in amdgpu_bo_set_metadata()
1221 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,  in amdgpu_bo_get_metadata()  argument
1230 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);  in amdgpu_bo_get_metadata()
1231 	ubo = to_amdgpu_bo_user(bo);  in amdgpu_bo_get_metadata()
1259 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,  in amdgpu_bo_move_notify()  argument
1263 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);  in amdgpu_bo_move_notify()
1265 	struct ttm_resource *old_mem = bo->resource;  in amdgpu_bo_move_notify()
1267 	if (!amdgpu_bo_is_amdgpu_bo(bo))  in amdgpu_bo_move_notify()
1270 	abo = ttm_to_amdgpu_bo(bo);  in amdgpu_bo_move_notify()
1276 	    bo->resource->mem_type != TTM_PL_SYSTEM)  in amdgpu_bo_move_notify()
1291 void amdgpu_bo_get_memory(struct amdgpu_bo *bo,  in amdgpu_bo_get_memory()  argument
1294 	uint64_t size = amdgpu_bo_size(bo);  in amdgpu_bo_get_memory()
1298 	if (!bo->tbo.resource)  in amdgpu_bo_get_memory()
1301 	domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);  in amdgpu_bo_get_memory()
1305 		if (amdgpu_bo_in_cpu_visible_vram(bo))  in amdgpu_bo_get_memory()
1317 	if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) {  in amdgpu_bo_get_memory()
1319 		if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)  in amdgpu_bo_get_memory()
1324 			if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)  in amdgpu_bo_get_memory()
1327 	} else if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_GTT) {  in amdgpu_bo_get_memory()
1339 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)  in amdgpu_bo_release_notify()  argument
1341 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);  in amdgpu_bo_release_notify()
1346 	if (!amdgpu_bo_is_amdgpu_bo(bo))  in amdgpu_bo_release_notify()
1349 	abo = ttm_to_amdgpu_bo(bo);  in amdgpu_bo_release_notify()
1355 	WARN_ON_ONCE(bo->type == ttm_bo_type_kernel  in amdgpu_bo_release_notify()
1356 			&& bo->base.resv != &bo->base._resv);  in amdgpu_bo_release_notify()
1357 	if (bo->base.resv == &bo->base._resv)  in amdgpu_bo_release_notify()
1360 	if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM ||  in amdgpu_bo_release_notify()
1365 	if (WARN_ON_ONCE(!dma_resv_trylock(bo->base.resv)))  in amdgpu_bo_release_notify()
1368 	r = amdgpu_fill_buffer(abo, AMDGPU_POISON, bo->base.resv, &fence, true);  in amdgpu_bo_release_notify()
1374 	dma_resv_unlock(bo->base.resv);  in amdgpu_bo_release_notify()
1388 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)  in amdgpu_bo_fault_reserve_notify()  argument
1390 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);  in amdgpu_bo_fault_reserve_notify()
1392 	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);  in amdgpu_bo_fault_reserve_notify()
1398 	if (bo->resource->mem_type != TTM_PL_VRAM)  in amdgpu_bo_fault_reserve_notify()
1417 	r = ttm_bo_validate(bo, &abo->placement, &ctx);  in amdgpu_bo_fault_reserve_notify()
1424 	if (bo->resource->mem_type == TTM_PL_VRAM &&  in amdgpu_bo_fault_reserve_notify()
1428 	ttm_bo_move_to_lru_tail_unlocked(bo);  in amdgpu_bo_fault_reserve_notify()
1440 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,  in amdgpu_bo_fence()  argument
1443 	struct dma_resv *resv = bo->tbo.base.resv;  in amdgpu_bo_fence()
1495 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr)  in amdgpu_bo_sync_wait()  argument
1497 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_sync_wait()
1499 	return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv,  in amdgpu_bo_sync_wait()
1513 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)  in amdgpu_bo_gpu_offset()  argument
1515 	WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM);  in amdgpu_bo_gpu_offset()
1516 	WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) &&  in amdgpu_bo_gpu_offset()
1517 		     !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel);  in amdgpu_bo_gpu_offset()
1518 	WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET);  in amdgpu_bo_gpu_offset()
1519 	WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM &&  in amdgpu_bo_gpu_offset()
1520 		     !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));  in amdgpu_bo_gpu_offset()
1522 	return amdgpu_bo_gpu_offset_no_check(bo);  in amdgpu_bo_gpu_offset()
1532 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo)  in amdgpu_bo_gpu_offset_no_check()  argument
1534 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_gpu_offset_no_check()
1537 	offset = (bo->tbo.resource->start << PAGE_SHIFT) +  in amdgpu_bo_gpu_offset_no_check()
1538 		 amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type);  in amdgpu_bo_gpu_offset_no_check()
1564 #define amdgpu_bo_print_flag(m, bo, flag)		        \  argument
1566 		if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) {	\
1583 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)  in amdgpu_bo_print_info()  argument
1591 	if (dma_resv_trylock(bo->tbo.base.resv)) {  in amdgpu_bo_print_info()
1593 		domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);  in amdgpu_bo_print_info()
1596 			if (amdgpu_bo_in_cpu_visible_vram(bo))  in amdgpu_bo_print_info()
1609 		dma_resv_unlock(bo->tbo.base.resv);  in amdgpu_bo_print_info()
1614 	size = amdgpu_bo_size(bo);  in amdgpu_bo_print_info()
1618 	pin_count = READ_ONCE(bo->tbo.pin_count);  in amdgpu_bo_print_info()
1622 	dma_buf = READ_ONCE(bo->tbo.base.dma_buf);  in amdgpu_bo_print_info()
1623 	attachment = READ_ONCE(bo->tbo.base.import_attach);  in amdgpu_bo_print_info()
1630 	amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);  in amdgpu_bo_print_info()
1631 	amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);  in amdgpu_bo_print_info()
1632 	amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC);  in amdgpu_bo_print_info()
1633 	amdgpu_bo_print_flag(m, bo, VRAM_CLEARED);  in amdgpu_bo_print_info()
1634 	amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS);  in amdgpu_bo_print_info()
1635 	amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID);  in amdgpu_bo_print_info()
1636 	amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC);  in amdgpu_bo_print_info()