79 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, struct drm_printer *p,  in ttm_mem_type_debug()  argument
82 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_mem_type_debug()
111 		ttm_mem_type_debug(bo->bdev, &p, mem_type);  in ttm_bo_mem_space_debug()
151 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_release_list()  local
161 	atomic_dec(&bo->bdev->glob->bo_count);  in ttm_bo_release_list()
167 	ttm_mem_global_free(bdev->glob->mem_glob, acc_size);  in ttm_bo_release_list()
173 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_add_mem_to_lru()  local
184 	man = &bdev->man[mem->mem_type];  in ttm_bo_add_mem_to_lru()
191 		list_add_tail(&bo->swap, &bdev->glob->swap_lru[bo->priority]);  in ttm_bo_add_mem_to_lru()
209 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_del_from_lru()  local
223 	if (notify && bdev->driver->del_from_lru_notify)  in ttm_bo_del_from_lru()
224 		bdev->driver->del_from_lru_notify(bo);  in ttm_bo_del_from_lru()
229 	struct ttm_bo_global *glob = bo->bdev->glob;  in ttm_bo_del_sub_from_lru()
284 		man = &pos->first->bdev->man[TTM_PL_TT];  in ttm_bo_bulk_move_lru_tail()
299 		man = &pos->first->bdev->man[TTM_PL_VRAM];  in ttm_bo_bulk_move_lru_tail()
314 		lru = &pos->first->bdev->glob->swap_lru[i];  in ttm_bo_bulk_move_lru_tail()
324 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_handle_move_mem()  local
325 	bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);  in ttm_bo_handle_move_mem()
326 	bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);  in ttm_bo_handle_move_mem()
327 	struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];  in ttm_bo_handle_move_mem()
328 	struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];  in ttm_bo_handle_move_mem()
363 			if (bdev->driver->move_notify)  in ttm_bo_handle_move_mem()
364 				bdev->driver->move_notify(bo, evict, mem);  in ttm_bo_handle_move_mem()
371 	if (bdev->driver->move_notify)  in ttm_bo_handle_move_mem()
372 		bdev->driver->move_notify(bo, evict, mem);  in ttm_bo_handle_move_mem()
377 	else if (bdev->driver->move)  in ttm_bo_handle_move_mem()
378 		ret = bdev->driver->move(bo, evict, ctx, mem);  in ttm_bo_handle_move_mem()
383 		if (bdev->driver->move_notify) {  in ttm_bo_handle_move_mem()
385 			bdev->driver->move_notify(bo, false, mem);  in ttm_bo_handle_move_mem()
394 		if (bdev->driver->invalidate_caches) {  in ttm_bo_handle_move_mem()
395 			ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);  in ttm_bo_handle_move_mem()
404 		    bdev->man[bo->mem.mem_type].gpu_offset;  in ttm_bo_handle_move_mem()
412 	new_man = &bdev->man[bo->mem.mem_type];  in ttm_bo_handle_move_mem()
431 	if (bo->bdev->driver->move_notify)  in ttm_bo_cleanup_memtype_use()
432 		bo->bdev->driver->move_notify(bo, false, NULL);  in ttm_bo_cleanup_memtype_use()
477 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_cleanup_refs_or_queue()  local
478 	struct ttm_bo_global *glob = bdev->glob;  in ttm_bo_cleanup_refs_or_queue()
525 	list_add_tail(&bo->ddestroy, &bdev->ddestroy);  in ttm_bo_cleanup_refs_or_queue()
528 	schedule_delayed_work(&bdev->wq,  in ttm_bo_cleanup_refs_or_queue()
549 	struct ttm_bo_global *glob = bo->bdev->glob;  in ttm_bo_cleanup_refs()
619 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)  in ttm_bo_delayed_delete()  argument
621 	struct ttm_bo_global *glob = bdev->glob;  in ttm_bo_delayed_delete()
628 	while (!list_empty(&bdev->ddestroy)) {  in ttm_bo_delayed_delete()
631 		bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,  in ttm_bo_delayed_delete()
652 	list_splice_tail(&removed, &bdev->ddestroy);  in ttm_bo_delayed_delete()
653 	empty = list_empty(&bdev->ddestroy);  in ttm_bo_delayed_delete()
661 	struct ttm_bo_device *bdev =  in ttm_bo_delayed_workqueue()  local
664 	if (!ttm_bo_delayed_delete(bdev, false))  in ttm_bo_delayed_workqueue()
665 		schedule_delayed_work(&bdev->wq,  in ttm_bo_delayed_workqueue()
673 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_release()  local
674 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];  in ttm_bo_release()
676 	if (bo->bdev->driver->release_notify)  in ttm_bo_release()
677 		bo->bdev->driver->release_notify(bo);  in ttm_bo_release()
679 	drm_vma_offset_remove(&bdev->vma_manager, &bo->base.vma_node);  in ttm_bo_release()
693 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)  in ttm_bo_lock_delayed_workqueue()  argument
695 	return cancel_delayed_work_sync(&bdev->wq);  in ttm_bo_lock_delayed_workqueue()
699 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)  in ttm_bo_unlock_delayed_workqueue()  argument
702 		schedule_delayed_work(&bdev->wq,  in ttm_bo_unlock_delayed_workqueue()
710 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_evict()  local
719 	bdev->driver->evict_flags(bo, &placement);  in ttm_bo_evict()
838 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,  in ttm_mem_evict_first()  argument
845 	struct ttm_bo_global *glob = bdev->glob;  in ttm_mem_evict_first()
846 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_mem_evict_first()
864 			if (place && !bdev->driver->eviction_valuable(bo,  in ttm_mem_evict_first()
917 	struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];  in ttm_bo_mem_put()
963 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_mem_force_space()  local
964 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];  in ttm_bo_mem_force_space()
975 		ret = ttm_mem_evict_first(bdev, mem->mem_type, place, ctx,  in ttm_bo_mem_force_space()
1044 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_mem_placement()  local
1054 	man = &bdev->man[mem_type];  in ttm_bo_mem_placement()
1072 		spin_lock(&bo->bdev->glob->lru_lock);  in ttm_bo_mem_placement()
1075 		spin_unlock(&bo->bdev->glob->lru_lock);  in ttm_bo_mem_placement()
1094 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_mem_space()  local
1118 		man = &bdev->man[mem->mem_type];  in ttm_bo_mem_space()
1163 		spin_lock(&bo->bdev->glob->lru_lock);  in ttm_bo_mem_space()
1165 		spin_unlock(&bo->bdev->glob->lru_lock);  in ttm_bo_mem_space()
1277 int ttm_bo_init_reserved(struct ttm_bo_device *bdev,  in ttm_bo_init_reserved()  argument
1291 	struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;  in ttm_bo_init_reserved()
1324 	bo->bdev = bdev;  in ttm_bo_init_reserved()
1352 	atomic_inc(&bo->bdev->glob->bo_count);  in ttm_bo_init_reserved()
1360 		ret = drm_vma_offset_add(&bdev->vma_manager, &bo->base.vma_node,  in ttm_bo_init_reserved()
1383 		spin_lock(&bdev->glob->lru_lock);  in ttm_bo_init_reserved()
1385 		spin_unlock(&bdev->glob->lru_lock);  in ttm_bo_init_reserved()
1392 int ttm_bo_init(struct ttm_bo_device *bdev,  in ttm_bo_init()  argument
1407 	ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,  in ttm_bo_init()
1420 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,  in ttm_bo_acc_size()  argument
1434 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,  in ttm_bo_dma_acc_size()  argument
1448 int ttm_bo_create(struct ttm_bo_device *bdev,  in ttm_bo_create()  argument
1464 	acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));  in ttm_bo_create()
1465 	ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,  in ttm_bo_create()
1475 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,  in ttm_bo_force_list_clean()  argument
1483 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_bo_force_list_clean()
1484 	struct ttm_bo_global *glob = bdev->glob;  in ttm_bo_force_list_clean()
1497 			ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx,  in ttm_bo_force_list_clean()
1520 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)  in ttm_bo_clean_mm()  argument
1529 	man = &bdev->man[mem_type];  in ttm_bo_clean_mm()
1542 		ret = ttm_bo_force_list_clean(bdev, mem_type);  in ttm_bo_clean_mm()
1558 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)  in ttm_bo_evict_mm()  argument
1560 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_bo_evict_mm()
1572 	return ttm_bo_force_list_clean(bdev, mem_type);  in ttm_bo_evict_mm()
1576 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,  in ttm_bo_init_mm()  argument
1584 	man = &bdev->man[type];  in ttm_bo_init_mm()
1592 	ret = bdev->driver->init_mem_type(bdev, type, man);  in ttm_bo_init_mm()
1595 	man->bdev = bdev;  in ttm_bo_init_mm()
1676 int ttm_bo_device_release(struct ttm_bo_device *bdev)  in ttm_bo_device_release()  argument
1681 	struct ttm_bo_global *glob = bdev->glob;  in ttm_bo_device_release()
1684 		man = &bdev->man[i];  in ttm_bo_device_release()
1687 			if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {  in ttm_bo_device_release()
1697 	list_del(&bdev->device_list);  in ttm_bo_device_release()
1700 	cancel_delayed_work_sync(&bdev->wq);  in ttm_bo_device_release()
1702 	if (ttm_bo_delayed_delete(bdev, true))  in ttm_bo_device_release()
1707 		if (list_empty(&bdev->man[0].lru[0]))  in ttm_bo_device_release()
1711 	drm_vma_offset_manager_destroy(&bdev->vma_manager);  in ttm_bo_device_release()
1720 int ttm_bo_device_init(struct ttm_bo_device *bdev,  in ttm_bo_device_init()  argument
1732 	bdev->driver = driver;  in ttm_bo_device_init()
1734 	memset(bdev->man, 0, sizeof(bdev->man));  in ttm_bo_device_init()
1740 	ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);  in ttm_bo_device_init()
1744 	drm_vma_offset_manager_init(&bdev->vma_manager,  in ttm_bo_device_init()
1747 	INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);  in ttm_bo_device_init()
1748 	INIT_LIST_HEAD(&bdev->ddestroy);  in ttm_bo_device_init()
1749 	bdev->dev_mapping = mapping;  in ttm_bo_device_init()
1750 	bdev->glob = glob;  in ttm_bo_device_init()
1751 	bdev->need_dma32 = need_dma32;  in ttm_bo_device_init()
1753 	list_add_tail(&bdev->device_list, &glob->device_list);  in ttm_bo_device_init()
1767 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)  in ttm_mem_reg_is_pci()  argument
1769 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];  in ttm_mem_reg_is_pci()
1786 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_unmap_virtual_locked()  local
1788 	drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);  in ttm_bo_unmap_virtual_locked()
1794 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_unmap_virtual()  local
1795 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];  in ttm_bo_unmap_virtual()
1929 	if (bo->bdev->driver->swap_notify)  in ttm_bo_swapout()
1930 		bo->bdev->driver->swap_notify(bo);  in ttm_bo_swapout()
1947 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)  in ttm_bo_swapout_all()  argument
1954 	while (ttm_bo_swapout(bdev->glob, &ctx) == 0)  in ttm_bo_swapout_all()