43 static void radeon_bo_clear_surface_reg(struct radeon_bo *bo);
50 static void radeon_update_memory_usage(struct radeon_bo *bo,  in radeon_update_memory_usage()  argument
53 	struct radeon_device *rdev = bo->rdev;  in radeon_update_memory_usage()
54 	u64 size = (u64)bo->tbo.num_pages << PAGE_SHIFT;  in radeon_update_memory_usage()
74 	struct radeon_bo *bo;  in radeon_ttm_bo_destroy()  local
76 	bo = container_of(tbo, struct radeon_bo, tbo);  in radeon_ttm_bo_destroy()
78 	radeon_update_memory_usage(bo, bo->tbo.mem.mem_type, -1);  in radeon_ttm_bo_destroy()
80 	mutex_lock(&bo->rdev->gem.mutex);  in radeon_ttm_bo_destroy()
81 	list_del_init(&bo->list);  in radeon_ttm_bo_destroy()
82 	mutex_unlock(&bo->rdev->gem.mutex);  in radeon_ttm_bo_destroy()
83 	radeon_bo_clear_surface_reg(bo);  in radeon_ttm_bo_destroy()
84 	WARN_ON_ONCE(!list_empty(&bo->va));  in radeon_ttm_bo_destroy()
85 	if (bo->gem_base.import_attach)  in radeon_ttm_bo_destroy()
86 		drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg);  in radeon_ttm_bo_destroy()
87 	drm_gem_object_release(&bo->gem_base);  in radeon_ttm_bo_destroy()
88 	kfree(bo);  in radeon_ttm_bo_destroy()
91 bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo)  in radeon_ttm_bo_is_radeon_bo()  argument
93 	if (bo->destroy == &radeon_ttm_bo_destroy)  in radeon_ttm_bo_is_radeon_bo()
186 	struct radeon_bo *bo;  in radeon_bo_create()  local
206 	bo = kzalloc(sizeof(struct radeon_bo), GFP_KERNEL);  in radeon_bo_create()
207 	if (bo == NULL)  in radeon_bo_create()
209 	drm_gem_private_object_init(rdev->ddev, &bo->gem_base, size);  in radeon_bo_create()
210 	bo->rdev = rdev;  in radeon_bo_create()
211 	bo->surface_reg = -1;  in radeon_bo_create()
212 	INIT_LIST_HEAD(&bo->list);  in radeon_bo_create()
213 	INIT_LIST_HEAD(&bo->va);  in radeon_bo_create()
214 	bo->initial_domain = domain & (RADEON_GEM_DOMAIN_VRAM |  in radeon_bo_create()
218 	bo->flags = flags;  in radeon_bo_create()
221 		bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC);  in radeon_bo_create()
227 		bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC);  in radeon_bo_create()
233 	bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC);  in radeon_bo_create()
244 	if (bo->flags & RADEON_GEM_GTT_WC)  in radeon_bo_create()
247 	bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC);  in radeon_bo_create()
253 		bo->flags &= ~RADEON_GEM_GTT_WC;  in radeon_bo_create()
256 	radeon_ttm_placement_from_domain(bo, domain);  in radeon_bo_create()
259 	r = ttm_bo_init(&rdev->mman.bdev, &bo->tbo, size, type,  in radeon_bo_create()
260 			&bo->placement, page_align, !kernel, acc_size,  in radeon_bo_create()
266 	*bo_ptr = bo;  in radeon_bo_create()
268 	trace_radeon_bo_create(bo);  in radeon_bo_create()
273 int radeon_bo_kmap(struct radeon_bo *bo, void **ptr)  in radeon_bo_kmap()  argument
278 	if (bo->kptr) {  in radeon_bo_kmap()
280 			*ptr = bo->kptr;  in radeon_bo_kmap()
284 	r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);  in radeon_bo_kmap()
288 	bo->kptr = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);  in radeon_bo_kmap()
290 		*ptr = bo->kptr;  in radeon_bo_kmap()
292 	radeon_bo_check_tiling(bo, 0, 0);  in radeon_bo_kmap()
296 void radeon_bo_kunmap(struct radeon_bo *bo)  in radeon_bo_kunmap()  argument
298 	if (bo->kptr == NULL)  in radeon_bo_kunmap()
300 	bo->kptr = NULL;  in radeon_bo_kunmap()
301 	radeon_bo_check_tiling(bo, 0, 0);  in radeon_bo_kunmap()
302 	ttm_bo_kunmap(&bo->kmap);  in radeon_bo_kunmap()
305 struct radeon_bo *radeon_bo_ref(struct radeon_bo *bo)  in radeon_bo_ref()  argument
307 	if (bo == NULL)  in radeon_bo_ref()
310 	ttm_bo_get(&bo->tbo);  in radeon_bo_ref()
311 	return bo;  in radeon_bo_ref()
314 void radeon_bo_unref(struct radeon_bo **bo)  in radeon_bo_unref()  argument
319 	if ((*bo) == NULL)  in radeon_bo_unref()
321 	rdev = (*bo)->rdev;  in radeon_bo_unref()
322 	tbo = &((*bo)->tbo);  in radeon_bo_unref()
324 	*bo = NULL;  in radeon_bo_unref()
327 int radeon_bo_pin_restricted(struct radeon_bo *bo, u32 domain, u64 max_offset,  in radeon_bo_pin_restricted()  argument
333 	if (radeon_ttm_tt_has_userptr(bo->tbo.ttm))  in radeon_bo_pin_restricted()
336 	if (bo->pin_count) {  in radeon_bo_pin_restricted()
337 		bo->pin_count++;  in radeon_bo_pin_restricted()
339 			*gpu_addr = radeon_bo_gpu_offset(bo);  in radeon_bo_pin_restricted()
345 				domain_start = bo->rdev->mc.vram_start;  in radeon_bo_pin_restricted()
347 				domain_start = bo->rdev->mc.gtt_start;  in radeon_bo_pin_restricted()
349 				     (radeon_bo_gpu_offset(bo) - domain_start));  in radeon_bo_pin_restricted()
354 	if (bo->prime_shared_count && domain == RADEON_GEM_DOMAIN_VRAM) {  in radeon_bo_pin_restricted()
359 	radeon_ttm_placement_from_domain(bo, domain);  in radeon_bo_pin_restricted()
360 	for (i = 0; i < bo->placement.num_placement; i++) {  in radeon_bo_pin_restricted()
362 		if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&  in radeon_bo_pin_restricted()
363 		    !(bo->flags & RADEON_GEM_NO_CPU_ACCESS) &&  in radeon_bo_pin_restricted()
364 		    (!max_offset || max_offset > bo->rdev->mc.visible_vram_size))  in radeon_bo_pin_restricted()
365 			bo->placements[i].lpfn =  in radeon_bo_pin_restricted()
366 				bo->rdev->mc.visible_vram_size >> PAGE_SHIFT;  in radeon_bo_pin_restricted()
368 			bo->placements[i].lpfn = max_offset >> PAGE_SHIFT;  in radeon_bo_pin_restricted()
370 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;  in radeon_bo_pin_restricted()
373 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in radeon_bo_pin_restricted()
375 		bo->pin_count = 1;  in radeon_bo_pin_restricted()
377 			*gpu_addr = radeon_bo_gpu_offset(bo);  in radeon_bo_pin_restricted()
379 			bo->rdev->vram_pin_size += radeon_bo_size(bo);  in radeon_bo_pin_restricted()
381 			bo->rdev->gart_pin_size += radeon_bo_size(bo);  in radeon_bo_pin_restricted()
383 		dev_err(bo->rdev->dev, "%p pin failed\n", bo);  in radeon_bo_pin_restricted()
388 int radeon_bo_pin(struct radeon_bo *bo, u32 domain, u64 *gpu_addr)  in radeon_bo_pin()  argument
390 	return radeon_bo_pin_restricted(bo, domain, 0, gpu_addr);  in radeon_bo_pin()
393 int radeon_bo_unpin(struct radeon_bo *bo)  in radeon_bo_unpin()  argument
398 	if (!bo->pin_count) {  in radeon_bo_unpin()
399 		dev_warn(bo->rdev->dev, "%p unpin not necessary\n", bo);  in radeon_bo_unpin()
402 	bo->pin_count--;  in radeon_bo_unpin()
403 	if (bo->pin_count)  in radeon_bo_unpin()
405 	for (i = 0; i < bo->placement.num_placement; i++) {  in radeon_bo_unpin()
406 		bo->placements[i].lpfn = 0;  in radeon_bo_unpin()
407 		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;  in radeon_bo_unpin()
409 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in radeon_bo_unpin()
411 		if (bo->tbo.mem.mem_type == TTM_PL_VRAM)  in radeon_bo_unpin()
412 			bo->rdev->vram_pin_size -= radeon_bo_size(bo);  in radeon_bo_unpin()
414 			bo->rdev->gart_pin_size -= radeon_bo_size(bo);  in radeon_bo_unpin()
416 		dev_err(bo->rdev->dev, "%p validate failed for unpin\n", bo);  in radeon_bo_unpin()
434 	struct radeon_bo *bo, *n;  in radeon_bo_force_delete()  local
440 	list_for_each_entry_safe(bo, n, &rdev->gem.objects, list) {  in radeon_bo_force_delete()
442 			&bo->gem_base, bo, (unsigned long)bo->gem_base.size,  in radeon_bo_force_delete()
443 			*((unsigned long *)&bo->gem_base.refcount));  in radeon_bo_force_delete()
444 		mutex_lock(&bo->rdev->gem.mutex);  in radeon_bo_force_delete()
445 		list_del_init(&bo->list);  in radeon_bo_force_delete()
446 		mutex_unlock(&bo->rdev->gem.mutex);  in radeon_bo_force_delete()
448 		drm_gem_object_put_unlocked(&bo->gem_base);  in radeon_bo_force_delete()
548 		struct radeon_bo *bo = lobj->robj;  in radeon_bo_list_validate()  local
549 		if (!bo->pin_count) {  in radeon_bo_list_validate()
553 				radeon_mem_type_to_domain(bo->tbo.mem.mem_type);  in radeon_bo_list_validate()
571 			radeon_ttm_placement_from_domain(bo, domain);  in radeon_bo_list_validate()
573 				radeon_uvd_force_into_uvd_segment(bo, allowed);  in radeon_bo_list_validate()
576 			r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);  in radeon_bo_list_validate()
590 		lobj->gpu_offset = radeon_bo_gpu_offset(bo);  in radeon_bo_list_validate()
591 		lobj->tiling_flags = bo->tiling_flags;  in radeon_bo_list_validate()
602 int radeon_bo_get_surface_reg(struct radeon_bo *bo)  in radeon_bo_get_surface_reg()  argument
604 	struct radeon_device *rdev = bo->rdev;  in radeon_bo_get_surface_reg()
610 	lockdep_assert_held(&bo->tbo.resv->lock.base);  in radeon_bo_get_surface_reg()
612 	if (!bo->tiling_flags)  in radeon_bo_get_surface_reg()
615 	if (bo->surface_reg >= 0) {  in radeon_bo_get_surface_reg()
616 		reg = &rdev->surface_regs[bo->surface_reg];  in radeon_bo_get_surface_reg()
617 		i = bo->surface_reg;  in radeon_bo_get_surface_reg()
625 		if (!reg->bo)  in radeon_bo_get_surface_reg()
628 		old_object = reg->bo;  in radeon_bo_get_surface_reg()
639 		old_object = reg->bo;  in radeon_bo_get_surface_reg()
647 	bo->surface_reg = i;  in radeon_bo_get_surface_reg()
648 	reg->bo = bo;  in radeon_bo_get_surface_reg()
651 	radeon_set_surface_reg(rdev, i, bo->tiling_flags, bo->pitch,  in radeon_bo_get_surface_reg()
652 			       bo->tbo.mem.start << PAGE_SHIFT,  in radeon_bo_get_surface_reg()
653 			       bo->tbo.num_pages << PAGE_SHIFT);  in radeon_bo_get_surface_reg()
657 static void radeon_bo_clear_surface_reg(struct radeon_bo *bo)  in radeon_bo_clear_surface_reg()  argument
659 	struct radeon_device *rdev = bo->rdev;  in radeon_bo_clear_surface_reg()
662 	if (bo->surface_reg == -1)  in radeon_bo_clear_surface_reg()
665 	reg = &rdev->surface_regs[bo->surface_reg];  in radeon_bo_clear_surface_reg()
666 	radeon_clear_surface_reg(rdev, bo->surface_reg);  in radeon_bo_clear_surface_reg()
668 	reg->bo = NULL;  in radeon_bo_clear_surface_reg()
669 	bo->surface_reg = -1;  in radeon_bo_clear_surface_reg()
672 int radeon_bo_set_tiling_flags(struct radeon_bo *bo,  in radeon_bo_set_tiling_flags()  argument
675 	struct radeon_device *rdev = bo->rdev;  in radeon_bo_set_tiling_flags()
723 	r = radeon_bo_reserve(bo, false);  in radeon_bo_set_tiling_flags()
726 	bo->tiling_flags = tiling_flags;  in radeon_bo_set_tiling_flags()
727 	bo->pitch = pitch;  in radeon_bo_set_tiling_flags()
728 	radeon_bo_unreserve(bo);  in radeon_bo_set_tiling_flags()
732 void radeon_bo_get_tiling_flags(struct radeon_bo *bo,  in radeon_bo_get_tiling_flags()  argument
736 	lockdep_assert_held(&bo->tbo.resv->lock.base);  in radeon_bo_get_tiling_flags()
739 		*tiling_flags = bo->tiling_flags;  in radeon_bo_get_tiling_flags()
741 		*pitch = bo->pitch;  in radeon_bo_get_tiling_flags()
744 int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved,  in radeon_bo_check_tiling()  argument
748 		lockdep_assert_held(&bo->tbo.resv->lock.base);  in radeon_bo_check_tiling()
750 	if (!(bo->tiling_flags & RADEON_TILING_SURFACE))  in radeon_bo_check_tiling()
754 		radeon_bo_clear_surface_reg(bo);  in radeon_bo_check_tiling()
758 	if (bo->tbo.mem.mem_type != TTM_PL_VRAM) {  in radeon_bo_check_tiling()
762 		if (bo->surface_reg >= 0)  in radeon_bo_check_tiling()
763 			radeon_bo_clear_surface_reg(bo);  in radeon_bo_check_tiling()
767 	if ((bo->surface_reg >= 0) && !has_moved)  in radeon_bo_check_tiling()
770 	return radeon_bo_get_surface_reg(bo);  in radeon_bo_check_tiling()
773 void radeon_bo_move_notify(struct ttm_buffer_object *bo,  in radeon_bo_move_notify()  argument
779 	if (!radeon_ttm_bo_is_radeon_bo(bo))  in radeon_bo_move_notify()
782 	rbo = container_of(bo, struct radeon_bo, tbo);  in radeon_bo_move_notify()
790 	radeon_update_memory_usage(rbo, bo->mem.mem_type, -1);  in radeon_bo_move_notify()
794 int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo)  in radeon_bo_fault_reserve_notify()  argument
802 	if (!radeon_ttm_bo_is_radeon_bo(bo))  in radeon_bo_fault_reserve_notify()
804 	rbo = container_of(bo, struct radeon_bo, tbo);  in radeon_bo_fault_reserve_notify()
807 	if (bo->mem.mem_type != TTM_PL_VRAM)  in radeon_bo_fault_reserve_notify()
810 	size = bo->mem.num_pages << PAGE_SHIFT;  in radeon_bo_fault_reserve_notify()
811 	offset = bo->mem.start << PAGE_SHIFT;  in radeon_bo_fault_reserve_notify()
828 	r = ttm_bo_validate(bo, &rbo->placement, &ctx);  in radeon_bo_fault_reserve_notify()
831 		return ttm_bo_validate(bo, &rbo->placement, &ctx);  in radeon_bo_fault_reserve_notify()
836 	offset = bo->mem.start << PAGE_SHIFT;  in radeon_bo_fault_reserve_notify()
844 int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type, bool no_wait)  in radeon_bo_wait()  argument
848 	r = ttm_bo_reserve(&bo->tbo, true, no_wait, NULL);  in radeon_bo_wait()
852 		*mem_type = bo->tbo.mem.mem_type;  in radeon_bo_wait()
854 	r = ttm_bo_wait(&bo->tbo, true, no_wait);  in radeon_bo_wait()
855 	ttm_bo_unreserve(&bo->tbo);  in radeon_bo_wait()
867 void radeon_bo_fence(struct radeon_bo *bo, struct radeon_fence *fence,  in radeon_bo_fence()  argument
870 	struct reservation_object *resv = bo->tbo.resv;  in radeon_bo_fence()