45 	struct ttm_buffer_object *bo;  member
48 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)  in ttm_bo_free_old_node()  argument
50 	ttm_resource_free(bo, &bo->mem);  in ttm_bo_free_old_node()
53 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,  in ttm_bo_move_ttm()  argument
57 	struct ttm_tt *ttm = bo->ttm;  in ttm_bo_move_ttm()
58 	struct ttm_resource *old_mem = &bo->mem;  in ttm_bo_move_ttm()
62 		ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);  in ttm_bo_move_ttm()
70 		ttm_bo_tt_unbind(bo);  in ttm_bo_move_ttm()
71 		ttm_bo_free_old_node(bo);  in ttm_bo_move_ttm()
81 		ret = ttm_tt_populate(bo->bdev, ttm, ctx);  in ttm_bo_move_ttm()
85 		ret = ttm_bo_tt_bind(bo, new_mem);  in ttm_bo_move_ttm()
90 	ttm_bo_assign_mem(bo, new_mem);  in ttm_bo_move_ttm()
217 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,  in ttm_bo_move_memcpy()  argument
221 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_move_memcpy()
223 	struct ttm_tt *ttm = bo->ttm;  in ttm_bo_move_memcpy()
224 	struct ttm_resource *old_mem = &bo->mem;  in ttm_bo_move_memcpy()
234 	ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);  in ttm_bo_move_memcpy()
301 	ttm_bo_assign_mem(bo, new_mem);  in ttm_bo_move_memcpy()
304 		ttm_bo_tt_destroy(bo);  in ttm_bo_move_memcpy()
315 		ttm_resource_free(bo, &old_copy);  in ttm_bo_move_memcpy()
320 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)  in ttm_transfered_destroy()  argument
324 	fbo = container_of(bo, struct ttm_transfer_obj, base);  in ttm_transfered_destroy()
325 	ttm_bo_put(fbo->bo);  in ttm_transfered_destroy()
344 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,  in ttm_buffer_object_transfer()  argument
354 	fbo->base = *bo;  in ttm_buffer_object_transfer()
357 	ttm_bo_get(bo);  in ttm_buffer_object_transfer()
358 	fbo->bo = bo;  in ttm_buffer_object_transfer()
375 	if (bo->type != ttm_bo_type_sg)  in ttm_buffer_object_transfer()
413 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,  in ttm_bo_ioremap()  argument
418 	struct ttm_resource *mem = &bo->mem;  in ttm_bo_ioremap()
420 	if (bo->mem.bus.addr) {  in ttm_bo_ioremap()
422 		map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);  in ttm_bo_ioremap()
426 			map->virtual = ioremap_wc(bo->mem.bus.offset + offset,  in ttm_bo_ioremap()
429 			map->virtual = ioremap(bo->mem.bus.offset + offset,  in ttm_bo_ioremap()
435 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,  in ttm_bo_kmap_ttm()  argument
440 	struct ttm_resource *mem = &bo->mem;  in ttm_bo_kmap_ttm()
445 	struct ttm_tt *ttm = bo->ttm;  in ttm_bo_kmap_ttm()
451 	ret = ttm_tt_populate(bo->bdev, ttm, &ctx);  in ttm_bo_kmap_ttm()
477 int ttm_bo_kmap(struct ttm_buffer_object *bo,  in ttm_bo_kmap()  argument
485 	map->bo = bo;  in ttm_bo_kmap()
486 	if (num_pages > bo->num_pages)  in ttm_bo_kmap()
488 	if (start_page > bo->num_pages)  in ttm_bo_kmap()
491 	ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);  in ttm_bo_kmap()
494 	if (!bo->mem.bus.is_iomem) {  in ttm_bo_kmap()
495 		return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);  in ttm_bo_kmap()
499 		return ttm_bo_ioremap(bo, offset, size, map);  in ttm_bo_kmap()
523 	ttm_mem_io_free(map->bo->bdev, &map->bo->mem);  in ttm_bo_kunmap()
529 static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo,  in ttm_bo_wait_free_node()  argument
533 	ret = ttm_bo_wait(bo, false, false);  in ttm_bo_wait_free_node()
538 		ttm_bo_tt_destroy(bo);  in ttm_bo_wait_free_node()
539 	ttm_bo_free_old_node(bo);  in ttm_bo_wait_free_node()
543 static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo,  in ttm_bo_move_to_ghost()  argument
558 	dma_fence_put(bo->moving);  in ttm_bo_move_to_ghost()
559 	bo->moving = dma_fence_get(fence);  in ttm_bo_move_to_ghost()
561 	ret = ttm_buffer_object_transfer(bo, &ghost_obj);  in ttm_bo_move_to_ghost()
576 		bo->ttm = NULL;  in ttm_bo_move_to_ghost()
583 static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo,  in ttm_bo_move_pipeline_evict()  argument
586 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_move_pipeline_evict()
587 	struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type);  in ttm_bo_move_pipeline_evict()
600 	ttm_bo_free_old_node(bo);  in ttm_bo_move_pipeline_evict()
602 	dma_fence_put(bo->moving);  in ttm_bo_move_pipeline_evict()
603 	bo->moving = dma_fence_get(fence);  in ttm_bo_move_pipeline_evict()
606 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,  in ttm_bo_move_accel_cleanup()  argument
612 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_move_accel_cleanup()
613 	struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type);  in ttm_bo_move_accel_cleanup()
617 	dma_resv_add_excl_fence(bo->base.resv, fence);  in ttm_bo_move_accel_cleanup()
619 		ret = ttm_bo_move_to_ghost(bo, fence, man->use_tt);  in ttm_bo_move_accel_cleanup()
621 		ttm_bo_move_pipeline_evict(bo, fence);  in ttm_bo_move_accel_cleanup()
623 		ret = ttm_bo_wait_free_node(bo, man->use_tt);  in ttm_bo_move_accel_cleanup()
628 	ttm_bo_assign_mem(bo, new_mem);  in ttm_bo_move_accel_cleanup()
634 int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)  in ttm_bo_pipeline_gutting()  argument
639 	ret = ttm_buffer_object_transfer(bo, &ghost);  in ttm_bo_pipeline_gutting()
643 	ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv);  in ttm_bo_pipeline_gutting()
646 		ttm_bo_wait(bo, false, false);  in ttm_bo_pipeline_gutting()
648 	memset(&bo->mem, 0, sizeof(bo->mem));  in ttm_bo_pipeline_gutting()
649 	bo->mem.mem_type = TTM_PL_SYSTEM;  in ttm_bo_pipeline_gutting()
650 	bo->ttm = NULL;  in ttm_bo_pipeline_gutting()