62 static void amdgpu_bo_subtract_pin_size(struct amdgpu_bo *bo)  in amdgpu_bo_subtract_pin_size()  argument
64 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_subtract_pin_size()
66 	if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {  in amdgpu_bo_subtract_pin_size()
67 		atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);  in amdgpu_bo_subtract_pin_size()
68 		atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),  in amdgpu_bo_subtract_pin_size()
70 	} else if (bo->tbo.mem.mem_type == TTM_PL_TT) {  in amdgpu_bo_subtract_pin_size()
71 		atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);  in amdgpu_bo_subtract_pin_size()
78 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);  in amdgpu_bo_destroy()  local
80 	if (bo->pin_count > 0)  in amdgpu_bo_destroy()
81 		amdgpu_bo_subtract_pin_size(bo);  in amdgpu_bo_destroy()
83 	amdgpu_bo_kunmap(bo);  in amdgpu_bo_destroy()
85 	if (bo->tbo.base.import_attach)  in amdgpu_bo_destroy()
86 		drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg);  in amdgpu_bo_destroy()
87 	drm_gem_object_release(&bo->tbo.base);  in amdgpu_bo_destroy()
89 	if (!list_empty(&bo->shadow_list)) {  in amdgpu_bo_destroy()
91 		list_del_init(&bo->shadow_list);  in amdgpu_bo_destroy()
94 	amdgpu_bo_unref(&bo->parent);  in amdgpu_bo_destroy()
96 	kfree(bo->metadata);  in amdgpu_bo_destroy()
97 	kfree(bo);  in amdgpu_bo_destroy()
110 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)  in amdgpu_bo_is_amdgpu_bo()  argument
112 	if (bo->destroy == &amdgpu_bo_destroy)  in amdgpu_bo_is_amdgpu_bo()
354 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,  in amdgpu_bo_free_kernel()  argument
357 	if (*bo == NULL)  in amdgpu_bo_free_kernel()
360 	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {  in amdgpu_bo_free_kernel()
362 			amdgpu_bo_kunmap(*bo);  in amdgpu_bo_free_kernel()
364 		amdgpu_bo_unpin(*bo);  in amdgpu_bo_free_kernel()
365 		amdgpu_bo_unreserve(*bo);  in amdgpu_bo_free_kernel()
367 	amdgpu_bo_unref(bo);  in amdgpu_bo_free_kernel()
459 	struct amdgpu_bo *bo;  in amdgpu_bo_do_create()  local
487 	bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL);  in amdgpu_bo_do_create()
488 	if (bo == NULL)  in amdgpu_bo_do_create()
490 	drm_gem_private_object_init(adev->ddev, &bo->tbo.base, size);  in amdgpu_bo_do_create()
491 	INIT_LIST_HEAD(&bo->shadow_list);  in amdgpu_bo_do_create()
492 	bo->vm_bo = NULL;  in amdgpu_bo_do_create()
493 	bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :  in amdgpu_bo_do_create()
495 	bo->allowed_domains = bo->preferred_domains;  in amdgpu_bo_do_create()
497 	    bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)  in amdgpu_bo_do_create()
498 		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;  in amdgpu_bo_do_create()
500 	bo->flags = bp->flags;  in amdgpu_bo_do_create()
502 	if (!amdgpu_bo_support_uswc(bo->flags))  in amdgpu_bo_do_create()
503 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;  in amdgpu_bo_do_create()
505 	bo->tbo.bdev = &adev->mman.bdev;  in amdgpu_bo_do_create()
508 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);  in amdgpu_bo_do_create()
510 		amdgpu_bo_placement_from_domain(bo, bp->domain);  in amdgpu_bo_do_create()
512 		bo->tbo.priority = 1;  in amdgpu_bo_do_create()
514 	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type,  in amdgpu_bo_do_create()
515 				 &bo->placement, page_align, &ctx, acc_size,  in amdgpu_bo_do_create()
521 	    bo->tbo.mem.mem_type == TTM_PL_VRAM &&  in amdgpu_bo_do_create()
522 	    bo->tbo.mem.start < adev->gmc.visible_vram_size >> PAGE_SHIFT)  in amdgpu_bo_do_create()
529 	    bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {  in amdgpu_bo_do_create()
532 		r = amdgpu_fill_buffer(bo, 0, bo->tbo.base.resv, &fence);  in amdgpu_bo_do_create()
536 		amdgpu_bo_fence(bo, fence, false);  in amdgpu_bo_do_create()
537 		dma_fence_put(bo->tbo.moving);  in amdgpu_bo_do_create()
538 		bo->tbo.moving = dma_fence_get(fence);  in amdgpu_bo_do_create()
542 		amdgpu_bo_unreserve(bo);  in amdgpu_bo_do_create()
543 	*bo_ptr = bo;  in amdgpu_bo_do_create()
545 	trace_amdgpu_bo_create(bo);  in amdgpu_bo_do_create()
549 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;  in amdgpu_bo_do_create()
555 		dma_resv_unlock(bo->tbo.base.resv);  in amdgpu_bo_do_create()
556 	amdgpu_bo_unref(&bo);  in amdgpu_bo_do_create()
562 				   struct amdgpu_bo *bo)  in amdgpu_bo_create_shadow()  argument
567 	if (bo->shadow)  in amdgpu_bo_create_shadow()
576 	bp.resv = bo->tbo.base.resv;  in amdgpu_bo_create_shadow()
578 	r = amdgpu_bo_do_create(adev, &bp, &bo->shadow);  in amdgpu_bo_create_shadow()
580 		bo->shadow->parent = amdgpu_bo_ref(bo);  in amdgpu_bo_create_shadow()
582 		list_add_tail(&bo->shadow->shadow_list, &adev->shadow_list);  in amdgpu_bo_create_shadow()
644 int amdgpu_bo_validate(struct amdgpu_bo *bo)  in amdgpu_bo_validate()  argument
650 	if (bo->pin_count)  in amdgpu_bo_validate()
653 	domain = bo->preferred_domains;  in amdgpu_bo_validate()
656 	amdgpu_bo_placement_from_domain(bo, domain);  in amdgpu_bo_validate()
657 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in amdgpu_bo_validate()
658 	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {  in amdgpu_bo_validate()
659 		domain = bo->allowed_domains;  in amdgpu_bo_validate()
705 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)  in amdgpu_bo_kmap()  argument
710 	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)  in amdgpu_bo_kmap()
713 	kptr = amdgpu_bo_kptr(bo);  in amdgpu_bo_kmap()
720 	r = dma_resv_wait_timeout_rcu(bo->tbo.base.resv, false, false,  in amdgpu_bo_kmap()
725 	r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);  in amdgpu_bo_kmap()
730 		*ptr = amdgpu_bo_kptr(bo);  in amdgpu_bo_kmap()
744 void *amdgpu_bo_kptr(struct amdgpu_bo *bo)  in amdgpu_bo_kptr()  argument
748 	return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);  in amdgpu_bo_kptr()
757 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)  in amdgpu_bo_kunmap()  argument
759 	if (bo->kmap.bo)  in amdgpu_bo_kunmap()
760 		ttm_bo_kunmap(&bo->kmap);  in amdgpu_bo_kunmap()
772 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)  in amdgpu_bo_ref()  argument
774 	if (bo == NULL)  in amdgpu_bo_ref()
777 	ttm_bo_get(&bo->tbo);  in amdgpu_bo_ref()
778 	return bo;  in amdgpu_bo_ref()
787 void amdgpu_bo_unref(struct amdgpu_bo **bo)  in amdgpu_bo_unref()  argument
791 	if ((*bo) == NULL)  in amdgpu_bo_unref()
794 	tbo = &((*bo)->tbo);  in amdgpu_bo_unref()
796 	*bo = NULL;  in amdgpu_bo_unref()
821 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,  in amdgpu_bo_pin_restricted()  argument
824 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_pin_restricted()
828 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))  in amdgpu_bo_pin_restricted()
835 	if (bo->prime_shared_count) {  in amdgpu_bo_pin_restricted()
847 	if (bo->pin_count) {  in amdgpu_bo_pin_restricted()
848 		uint32_t mem_type = bo->tbo.mem.mem_type;  in amdgpu_bo_pin_restricted()
853 		bo->pin_count++;  in amdgpu_bo_pin_restricted()
856 			u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset;  in amdgpu_bo_pin_restricted()
858 				     (amdgpu_bo_gpu_offset(bo) - domain_start));  in amdgpu_bo_pin_restricted()
864 	bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;  in amdgpu_bo_pin_restricted()
866 	if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))  in amdgpu_bo_pin_restricted()
867 		bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;  in amdgpu_bo_pin_restricted()
868 	amdgpu_bo_placement_from_domain(bo, domain);  in amdgpu_bo_pin_restricted()
869 	for (i = 0; i < bo->placement.num_placement; i++) {  in amdgpu_bo_pin_restricted()
875 		if (fpfn > bo->placements[i].fpfn)  in amdgpu_bo_pin_restricted()
876 			bo->placements[i].fpfn = fpfn;  in amdgpu_bo_pin_restricted()
877 		if (!bo->placements[i].lpfn ||  in amdgpu_bo_pin_restricted()
878 		    (lpfn && lpfn < bo->placements[i].lpfn))  in amdgpu_bo_pin_restricted()
879 			bo->placements[i].lpfn = lpfn;  in amdgpu_bo_pin_restricted()
880 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;  in amdgpu_bo_pin_restricted()
883 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in amdgpu_bo_pin_restricted()
885 		dev_err(adev->dev, "%p pin failed\n", bo);  in amdgpu_bo_pin_restricted()
889 	bo->pin_count = 1;  in amdgpu_bo_pin_restricted()
891 	domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);  in amdgpu_bo_pin_restricted()
893 		atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);  in amdgpu_bo_pin_restricted()
894 		atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),  in amdgpu_bo_pin_restricted()
897 		atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);  in amdgpu_bo_pin_restricted()
916 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)  in amdgpu_bo_pin()  argument
918 	return amdgpu_bo_pin_restricted(bo, domain, 0, 0);  in amdgpu_bo_pin()
931 int amdgpu_bo_unpin(struct amdgpu_bo *bo)  in amdgpu_bo_unpin()  argument
933 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_unpin()
937 	if (WARN_ON_ONCE(!bo->pin_count)) {  in amdgpu_bo_unpin()
938 		dev_warn(adev->dev, "%p unpin not necessary\n", bo);  in amdgpu_bo_unpin()
941 	bo->pin_count--;  in amdgpu_bo_unpin()
942 	if (bo->pin_count)  in amdgpu_bo_unpin()
945 	amdgpu_bo_subtract_pin_size(bo);  in amdgpu_bo_unpin()
947 	for (i = 0; i < bo->placement.num_placement; i++) {  in amdgpu_bo_unpin()
948 		bo->placements[i].lpfn = 0;  in amdgpu_bo_unpin()
949 		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;  in amdgpu_bo_unpin()
951 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in amdgpu_bo_unpin()
953 		dev_err(adev->dev, "%p validate failed for unpin\n", bo);  in amdgpu_bo_unpin()
1059 int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo,  in amdgpu_bo_fbdev_mmap()  argument
1062 	return ttm_fbdev_mmap(vma, &bo->tbo);  in amdgpu_bo_fbdev_mmap()
1076 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)  in amdgpu_bo_set_tiling_flags()  argument
1078 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_set_tiling_flags()
1084 	bo->tiling_flags = tiling_flags;  in amdgpu_bo_set_tiling_flags()
1096 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)  in amdgpu_bo_get_tiling_flags()  argument
1098 	dma_resv_assert_held(bo->tbo.base.resv);  in amdgpu_bo_get_tiling_flags()
1101 		*tiling_flags = bo->tiling_flags;  in amdgpu_bo_get_tiling_flags()
1117 int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,  in amdgpu_bo_set_metadata()  argument
1123 		if (bo->metadata_size) {  in amdgpu_bo_set_metadata()
1124 			kfree(bo->metadata);  in amdgpu_bo_set_metadata()
1125 			bo->metadata = NULL;  in amdgpu_bo_set_metadata()
1126 			bo->metadata_size = 0;  in amdgpu_bo_set_metadata()
1138 	kfree(bo->metadata);  in amdgpu_bo_set_metadata()
1139 	bo->metadata_flags = flags;  in amdgpu_bo_set_metadata()
1140 	bo->metadata = buffer;  in amdgpu_bo_set_metadata()
1141 	bo->metadata_size = metadata_size;  in amdgpu_bo_set_metadata()
1161 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,  in amdgpu_bo_get_metadata()  argument
1169 		if (buffer_size < bo->metadata_size)  in amdgpu_bo_get_metadata()
1172 		if (bo->metadata_size)  in amdgpu_bo_get_metadata()
1173 			memcpy(buffer, bo->metadata, bo->metadata_size);  in amdgpu_bo_get_metadata()
1177 		*metadata_size = bo->metadata_size;  in amdgpu_bo_get_metadata()
1179 		*flags = bo->metadata_flags;  in amdgpu_bo_get_metadata()
1194 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,  in amdgpu_bo_move_notify()  argument
1198 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);  in amdgpu_bo_move_notify()
1200 	struct ttm_mem_reg *old_mem = &bo->mem;  in amdgpu_bo_move_notify()
1202 	if (!amdgpu_bo_is_amdgpu_bo(bo))  in amdgpu_bo_move_notify()
1205 	abo = ttm_to_amdgpu_bo(bo);  in amdgpu_bo_move_notify()
1229 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)  in amdgpu_bo_release_notify()  argument
1235 	if (!amdgpu_bo_is_amdgpu_bo(bo))  in amdgpu_bo_release_notify()
1238 	abo = ttm_to_amdgpu_bo(bo);  in amdgpu_bo_release_notify()
1243 	if (bo->mem.mem_type != TTM_PL_VRAM || !bo->mem.mm_node ||  in amdgpu_bo_release_notify()
1247 	dma_resv_lock(bo->base.resv, NULL);  in amdgpu_bo_release_notify()
1249 	r = amdgpu_fill_buffer(abo, AMDGPU_POISON, bo->base.resv, &fence);  in amdgpu_bo_release_notify()
1255 	dma_resv_unlock(bo->base.resv);  in amdgpu_bo_release_notify()
1269 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)  in amdgpu_bo_fault_reserve_notify()  argument
1271 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);  in amdgpu_bo_fault_reserve_notify()
1277 	if (!amdgpu_bo_is_amdgpu_bo(bo))  in amdgpu_bo_fault_reserve_notify()
1280 	abo = ttm_to_amdgpu_bo(bo);  in amdgpu_bo_fault_reserve_notify()
1285 	if (bo->mem.mem_type != TTM_PL_VRAM)  in amdgpu_bo_fault_reserve_notify()
1288 	size = bo->mem.num_pages << PAGE_SHIFT;  in amdgpu_bo_fault_reserve_notify()
1289 	offset = bo->mem.start << PAGE_SHIFT;  in amdgpu_bo_fault_reserve_notify()
1306 	r = ttm_bo_validate(bo, &abo->placement, &ctx);  in amdgpu_bo_fault_reserve_notify()
1310 	offset = bo->mem.start << PAGE_SHIFT;  in amdgpu_bo_fault_reserve_notify()
1312 	if (bo->mem.mem_type == TTM_PL_VRAM &&  in amdgpu_bo_fault_reserve_notify()
1327 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,  in amdgpu_bo_fence()  argument
1330 	struct dma_resv *resv = bo->tbo.base.resv;  in amdgpu_bo_fence()
1348 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr)  in amdgpu_bo_sync_wait()  argument
1350 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);  in amdgpu_bo_sync_wait()
1355 	amdgpu_sync_resv(adev, &sync, bo->tbo.base.resv, owner, false);  in amdgpu_bo_sync_wait()
1372 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)  in amdgpu_bo_gpu_offset()  argument
1374 	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);  in amdgpu_bo_gpu_offset()
1375 	WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) &&  in amdgpu_bo_gpu_offset()
1376 		     !bo->pin_count && bo->tbo.type != ttm_bo_type_kernel);  in amdgpu_bo_gpu_offset()
1377 	WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);  in amdgpu_bo_gpu_offset()
1378 	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM &&  in amdgpu_bo_gpu_offset()
1379 		     !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));  in amdgpu_bo_gpu_offset()
1381 	return amdgpu_gmc_sign_extend(bo->tbo.offset);  in amdgpu_bo_gpu_offset()