74 static void amdgpu_bo_subtract_pin_size(struct amdgpu_bo *bo)  in amdgpu_bo_subtract_pin_size()  argument
76 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_subtract_pin_size()
78 	if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {  in amdgpu_bo_subtract_pin_size()
79 		atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);  in amdgpu_bo_subtract_pin_size()
80 		atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),  in amdgpu_bo_subtract_pin_size()
82 	} else if (bo->tbo.mem.mem_type == TTM_PL_TT) {  in amdgpu_bo_subtract_pin_size()
83 		atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);  in amdgpu_bo_subtract_pin_size()
90 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);  in amdgpu_bo_destroy()  local
92 	if (bo->pin_count > 0)  in amdgpu_bo_destroy()
93 		amdgpu_bo_subtract_pin_size(bo);  in amdgpu_bo_destroy()
95 	if (bo->kfd_bo)  in amdgpu_bo_destroy()
96 		amdgpu_amdkfd_unreserve_system_memory_limit(bo);  in amdgpu_bo_destroy()
98 	amdgpu_bo_kunmap(bo);  in amdgpu_bo_destroy()
100 	if (bo->gem_base.import_attach)  in amdgpu_bo_destroy()
101 		drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg);  in amdgpu_bo_destroy()
102 	drm_gem_object_release(&bo->gem_base);  in amdgpu_bo_destroy()
103 	amdgpu_bo_unref(&bo->parent);  in amdgpu_bo_destroy()
104 	if (!list_empty(&bo->shadow_list)) {  in amdgpu_bo_destroy()
106 		list_del_init(&bo->shadow_list);  in amdgpu_bo_destroy()
109 	kfree(bo->metadata);  in amdgpu_bo_destroy()
110 	kfree(bo);  in amdgpu_bo_destroy()
123 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)  in amdgpu_bo_is_amdgpu_bo()  argument
125 	if (bo->destroy == &amdgpu_bo_destroy)  in amdgpu_bo_is_amdgpu_bo()
363 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,  in amdgpu_bo_free_kernel()  argument
366 	if (*bo == NULL)  in amdgpu_bo_free_kernel()
369 	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {  in amdgpu_bo_free_kernel()
371 			amdgpu_bo_kunmap(*bo);  in amdgpu_bo_free_kernel()
373 		amdgpu_bo_unpin(*bo);  in amdgpu_bo_free_kernel()
374 		amdgpu_bo_unreserve(*bo);  in amdgpu_bo_free_kernel()
376 	amdgpu_bo_unref(bo);  in amdgpu_bo_free_kernel()
433 	struct amdgpu_bo *bo;  in amdgpu_bo_do_create()  local
449 	bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL);  in amdgpu_bo_do_create()
450 	if (bo == NULL)  in amdgpu_bo_do_create()
452 	drm_gem_private_object_init(adev->ddev, &bo->gem_base, size);  in amdgpu_bo_do_create()
453 	INIT_LIST_HEAD(&bo->shadow_list);  in amdgpu_bo_do_create()
454 	INIT_LIST_HEAD(&bo->va);  in amdgpu_bo_do_create()
455 	bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :  in amdgpu_bo_do_create()
457 	bo->allowed_domains = bo->preferred_domains;  in amdgpu_bo_do_create()
459 	    bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)  in amdgpu_bo_do_create()
460 		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;  in amdgpu_bo_do_create()
462 	bo->flags = bp->flags;  in amdgpu_bo_do_create()
468 	bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;  in amdgpu_bo_do_create()
480 	if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)  in amdgpu_bo_do_create()
483 	bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;  in amdgpu_bo_do_create()
489 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;  in amdgpu_bo_do_create()
492 	bo->tbo.bdev = &adev->mman.bdev;  in amdgpu_bo_do_create()
493 	amdgpu_bo_placement_from_domain(bo, bp->domain);  in amdgpu_bo_do_create()
495 		bo->tbo.priority = 1;  in amdgpu_bo_do_create()
497 	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type,  in amdgpu_bo_do_create()
498 				 &bo->placement, page_align, &ctx, acc_size,  in amdgpu_bo_do_create()
504 	    bo->tbo.mem.mem_type == TTM_PL_VRAM &&  in amdgpu_bo_do_create()
505 	    bo->tbo.mem.start < adev->gmc.visible_vram_size >> PAGE_SHIFT)  in amdgpu_bo_do_create()
512 	    bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {  in amdgpu_bo_do_create()
515 		r = amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence);  in amdgpu_bo_do_create()
519 		amdgpu_bo_fence(bo, fence, false);  in amdgpu_bo_do_create()
520 		dma_fence_put(bo->tbo.moving);  in amdgpu_bo_do_create()
521 		bo->tbo.moving = dma_fence_get(fence);  in amdgpu_bo_do_create()
525 		amdgpu_bo_unreserve(bo);  in amdgpu_bo_do_create()
526 	*bo_ptr = bo;  in amdgpu_bo_do_create()
528 	trace_amdgpu_bo_create(bo);  in amdgpu_bo_do_create()
532 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;  in amdgpu_bo_do_create()
538 		ww_mutex_unlock(&bo->tbo.resv->lock);  in amdgpu_bo_do_create()
539 	amdgpu_bo_unref(&bo);  in amdgpu_bo_do_create()
545 				   struct amdgpu_bo *bo)  in amdgpu_bo_create_shadow()  argument
550 	if (bo->shadow)  in amdgpu_bo_create_shadow()
560 	bp.resv = bo->tbo.resv;  in amdgpu_bo_create_shadow()
562 	r = amdgpu_bo_do_create(adev, &bp, &bo->shadow);  in amdgpu_bo_create_shadow()
564 		bo->shadow->parent = amdgpu_bo_ref(bo);  in amdgpu_bo_create_shadow()
566 		list_add_tail(&bo->shadow_list, &adev->shadow_list);  in amdgpu_bo_create_shadow()
633 			       struct amdgpu_bo *bo,  in amdgpu_bo_backup_to_shadow()  argument
639 	struct amdgpu_bo *shadow = bo->shadow;  in amdgpu_bo_backup_to_shadow()
646 	bo_addr = amdgpu_bo_gpu_offset(bo);  in amdgpu_bo_backup_to_shadow()
647 	shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);  in amdgpu_bo_backup_to_shadow()
649 	r = reservation_object_reserve_shared(bo->tbo.resv);  in amdgpu_bo_backup_to_shadow()
654 			       amdgpu_bo_size(bo), resv, fence,  in amdgpu_bo_backup_to_shadow()
657 		amdgpu_bo_fence(bo, *fence, true);  in amdgpu_bo_backup_to_shadow()
675 int amdgpu_bo_validate(struct amdgpu_bo *bo)  in amdgpu_bo_validate()  argument
681 	if (bo->pin_count)  in amdgpu_bo_validate()
684 	domain = bo->preferred_domains;  in amdgpu_bo_validate()
687 	amdgpu_bo_placement_from_domain(bo, domain);  in amdgpu_bo_validate()
688 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in amdgpu_bo_validate()
689 	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {  in amdgpu_bo_validate()
690 		domain = bo->allowed_domains;  in amdgpu_bo_validate()
715 				  struct amdgpu_bo *bo,  in amdgpu_bo_restore_from_shadow()  argument
721 	struct amdgpu_bo *shadow = bo->shadow;  in amdgpu_bo_restore_from_shadow()
728 	bo_addr = amdgpu_bo_gpu_offset(bo);  in amdgpu_bo_restore_from_shadow()
729 	shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);  in amdgpu_bo_restore_from_shadow()
731 	r = reservation_object_reserve_shared(bo->tbo.resv);  in amdgpu_bo_restore_from_shadow()
736 			       amdgpu_bo_size(bo), resv, fence,  in amdgpu_bo_restore_from_shadow()
739 		amdgpu_bo_fence(bo, *fence, true);  in amdgpu_bo_restore_from_shadow()
756 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)  in amdgpu_bo_kmap()  argument
761 	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)  in amdgpu_bo_kmap()
764 	kptr = amdgpu_bo_kptr(bo);  in amdgpu_bo_kmap()
771 	r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false,  in amdgpu_bo_kmap()
776 	r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);  in amdgpu_bo_kmap()
781 		*ptr = amdgpu_bo_kptr(bo);  in amdgpu_bo_kmap()
795 void *amdgpu_bo_kptr(struct amdgpu_bo *bo)  in amdgpu_bo_kptr()  argument
799 	return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);  in amdgpu_bo_kptr()
808 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)  in amdgpu_bo_kunmap()  argument
810 	if (bo->kmap.bo)  in amdgpu_bo_kunmap()
811 		ttm_bo_kunmap(&bo->kmap);  in amdgpu_bo_kunmap()
823 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)  in amdgpu_bo_ref()  argument
825 	if (bo == NULL)  in amdgpu_bo_ref()
828 	ttm_bo_get(&bo->tbo);  in amdgpu_bo_ref()
829 	return bo;  in amdgpu_bo_ref()
838 void amdgpu_bo_unref(struct amdgpu_bo **bo)  in amdgpu_bo_unref()  argument
842 	if ((*bo) == NULL)  in amdgpu_bo_unref()
845 	tbo = &((*bo)->tbo);  in amdgpu_bo_unref()
847 	*bo = NULL;  in amdgpu_bo_unref()
872 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,  in amdgpu_bo_pin_restricted()  argument
875 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_pin_restricted()
879 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))  in amdgpu_bo_pin_restricted()
886 	if (bo->prime_shared_count) {  in amdgpu_bo_pin_restricted()
898 	if (bo->pin_count) {  in amdgpu_bo_pin_restricted()
899 		uint32_t mem_type = bo->tbo.mem.mem_type;  in amdgpu_bo_pin_restricted()
904 		bo->pin_count++;  in amdgpu_bo_pin_restricted()
907 			u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset;  in amdgpu_bo_pin_restricted()
909 				     (amdgpu_bo_gpu_offset(bo) - domain_start));  in amdgpu_bo_pin_restricted()
915 	bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;  in amdgpu_bo_pin_restricted()
917 	if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))  in amdgpu_bo_pin_restricted()
918 		bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;  in amdgpu_bo_pin_restricted()
919 	amdgpu_bo_placement_from_domain(bo, domain);  in amdgpu_bo_pin_restricted()
920 	for (i = 0; i < bo->placement.num_placement; i++) {  in amdgpu_bo_pin_restricted()
926 		if (fpfn > bo->placements[i].fpfn)  in amdgpu_bo_pin_restricted()
927 			bo->placements[i].fpfn = fpfn;  in amdgpu_bo_pin_restricted()
928 		if (!bo->placements[i].lpfn ||  in amdgpu_bo_pin_restricted()
929 		    (lpfn && lpfn < bo->placements[i].lpfn))  in amdgpu_bo_pin_restricted()
930 			bo->placements[i].lpfn = lpfn;  in amdgpu_bo_pin_restricted()
931 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;  in amdgpu_bo_pin_restricted()
934 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in amdgpu_bo_pin_restricted()
936 		dev_err(adev->dev, "%p pin failed\n", bo);  in amdgpu_bo_pin_restricted()
940 	bo->pin_count = 1;  in amdgpu_bo_pin_restricted()
942 	domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);  in amdgpu_bo_pin_restricted()
944 		atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);  in amdgpu_bo_pin_restricted()
945 		atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),  in amdgpu_bo_pin_restricted()
948 		atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);  in amdgpu_bo_pin_restricted()
967 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)  in amdgpu_bo_pin()  argument
969 	return amdgpu_bo_pin_restricted(bo, domain, 0, 0);  in amdgpu_bo_pin()
982 int amdgpu_bo_unpin(struct amdgpu_bo *bo)  in amdgpu_bo_unpin()  argument
984 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_unpin()
988 	if (!bo->pin_count) {  in amdgpu_bo_unpin()
989 		dev_warn(adev->dev, "%p unpin not necessary\n", bo);  in amdgpu_bo_unpin()
992 	bo->pin_count--;  in amdgpu_bo_unpin()
993 	if (bo->pin_count)  in amdgpu_bo_unpin()
996 	amdgpu_bo_subtract_pin_size(bo);  in amdgpu_bo_unpin()
998 	for (i = 0; i < bo->placement.num_placement; i++) {  in amdgpu_bo_unpin()
999 		bo->placements[i].lpfn = 0;  in amdgpu_bo_unpin()
1000 		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;  in amdgpu_bo_unpin()
1002 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in amdgpu_bo_unpin()
1004 		dev_err(adev->dev, "%p validate failed for unpin\n", bo);  in amdgpu_bo_unpin()
1107 int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo,  in amdgpu_bo_fbdev_mmap()  argument
1110 	return ttm_fbdev_mmap(vma, &bo->tbo);  in amdgpu_bo_fbdev_mmap()
1124 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)  in amdgpu_bo_set_tiling_flags()  argument
1126 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_set_tiling_flags()
1132 	bo->tiling_flags = tiling_flags;  in amdgpu_bo_set_tiling_flags()
1144 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)  in amdgpu_bo_get_tiling_flags()  argument
1146 	lockdep_assert_held(&bo->tbo.resv->lock.base);  in amdgpu_bo_get_tiling_flags()
1149 		*tiling_flags = bo->tiling_flags;  in amdgpu_bo_get_tiling_flags()
1165 int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,  in amdgpu_bo_set_metadata()  argument
1171 		if (bo->metadata_size) {  in amdgpu_bo_set_metadata()
1172 			kfree(bo->metadata);  in amdgpu_bo_set_metadata()
1173 			bo->metadata = NULL;  in amdgpu_bo_set_metadata()
1174 			bo->metadata_size = 0;  in amdgpu_bo_set_metadata()
1186 	kfree(bo->metadata);  in amdgpu_bo_set_metadata()
1187 	bo->metadata_flags = flags;  in amdgpu_bo_set_metadata()
1188 	bo->metadata = buffer;  in amdgpu_bo_set_metadata()
1189 	bo->metadata_size = metadata_size;  in amdgpu_bo_set_metadata()
1209 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,  in amdgpu_bo_get_metadata()  argument
1217 		if (buffer_size < bo->metadata_size)  in amdgpu_bo_get_metadata()
1220 		if (bo->metadata_size)  in amdgpu_bo_get_metadata()
1221 			memcpy(buffer, bo->metadata, bo->metadata_size);  in amdgpu_bo_get_metadata()
1225 		*metadata_size = bo->metadata_size;  in amdgpu_bo_get_metadata()
1227 		*flags = bo->metadata_flags;  in amdgpu_bo_get_metadata()
1242 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,  in amdgpu_bo_move_notify()  argument
1246 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);  in amdgpu_bo_move_notify()
1248 	struct ttm_mem_reg *old_mem = &bo->mem;  in amdgpu_bo_move_notify()
1250 	if (!amdgpu_bo_is_amdgpu_bo(bo))  in amdgpu_bo_move_notify()
1253 	abo = ttm_to_amdgpu_bo(bo);  in amdgpu_bo_move_notify()
1281 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)  in amdgpu_bo_fault_reserve_notify()  argument
1283 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);  in amdgpu_bo_fault_reserve_notify()
1289 	if (!amdgpu_bo_is_amdgpu_bo(bo))  in amdgpu_bo_fault_reserve_notify()
1292 	abo = ttm_to_amdgpu_bo(bo);  in amdgpu_bo_fault_reserve_notify()
1297 	if (bo->mem.mem_type != TTM_PL_VRAM)  in amdgpu_bo_fault_reserve_notify()
1300 	size = bo->mem.num_pages << PAGE_SHIFT;  in amdgpu_bo_fault_reserve_notify()
1301 	offset = bo->mem.start << PAGE_SHIFT;  in amdgpu_bo_fault_reserve_notify()
1318 	r = ttm_bo_validate(bo, &abo->placement, &ctx);  in amdgpu_bo_fault_reserve_notify()
1322 	offset = bo->mem.start << PAGE_SHIFT;  in amdgpu_bo_fault_reserve_notify()
1324 	if (bo->mem.mem_type == TTM_PL_VRAM &&  in amdgpu_bo_fault_reserve_notify()
1339 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,  in amdgpu_bo_fence()  argument
1342 	struct reservation_object *resv = bo->tbo.resv;  in amdgpu_bo_fence()
1360 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)  in amdgpu_bo_gpu_offset()  argument
1362 	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);  in amdgpu_bo_gpu_offset()
1363 	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT &&  in amdgpu_bo_gpu_offset()
1364 		     !amdgpu_gtt_mgr_has_gart_addr(&bo->tbo.mem));  in amdgpu_bo_gpu_offset()
1365 	WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&  in amdgpu_bo_gpu_offset()
1366 		     !bo->pin_count);  in amdgpu_bo_gpu_offset()
1367 	WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);  in amdgpu_bo_gpu_offset()
1368 	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM &&  in amdgpu_bo_gpu_offset()
1369 		     !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));  in amdgpu_bo_gpu_offset()
1371 	return bo->tbo.offset;  in amdgpu_bo_gpu_offset()