72 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)  in ttm_mem_type_debug()  argument
74 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_mem_type_debug()
103 		ttm_mem_type_debug(bo->bdev, mem_type);  in ttm_bo_mem_space_debug()
143 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_release_list()  local
153 	atomic_dec(&bo->bdev->glob->bo_count);  in ttm_bo_release_list()
158 	ttm_mem_global_free(bdev->glob->mem_glob, acc_size);  in ttm_bo_release_list()
163 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_add_to_lru()  local
171 		man = &bdev->man[bo->mem.mem_type];  in ttm_bo_add_to_lru()
178 				      &bdev->glob->swap_lru[bo->priority]);  in ttm_bo_add_to_lru()
209 	struct ttm_bo_global *glob = bo->bdev->glob;  in ttm_bo_del_sub_from_lru()
230 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_handle_move_mem()  local
231 	bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);  in ttm_bo_handle_move_mem()
232 	bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);  in ttm_bo_handle_move_mem()
233 	struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];  in ttm_bo_handle_move_mem()
234 	struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];  in ttm_bo_handle_move_mem()
269 			if (bdev->driver->move_notify)  in ttm_bo_handle_move_mem()
270 				bdev->driver->move_notify(bo, evict, mem);  in ttm_bo_handle_move_mem()
277 	if (bdev->driver->move_notify)  in ttm_bo_handle_move_mem()
278 		bdev->driver->move_notify(bo, evict, mem);  in ttm_bo_handle_move_mem()
283 	else if (bdev->driver->move)  in ttm_bo_handle_move_mem()
284 		ret = bdev->driver->move(bo, evict, ctx, mem);  in ttm_bo_handle_move_mem()
289 		if (bdev->driver->move_notify) {  in ttm_bo_handle_move_mem()
291 			bdev->driver->move_notify(bo, false, mem);  in ttm_bo_handle_move_mem()
300 		if (bdev->driver->invalidate_caches) {  in ttm_bo_handle_move_mem()
301 			ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);  in ttm_bo_handle_move_mem()
310 		    bdev->man[bo->mem.mem_type].gpu_offset;  in ttm_bo_handle_move_mem()
318 	new_man = &bdev->man[bo->mem.mem_type];  in ttm_bo_handle_move_mem()
337 	if (bo->bdev->driver->move_notify)  in ttm_bo_cleanup_memtype_use()
338 		bo->bdev->driver->move_notify(bo, false, NULL);  in ttm_bo_cleanup_memtype_use()
383 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_cleanup_refs_or_queue()  local
384 	struct ttm_bo_global *glob = bdev->glob;  in ttm_bo_cleanup_refs_or_queue()
431 	list_add_tail(&bo->ddestroy, &bdev->ddestroy);  in ttm_bo_cleanup_refs_or_queue()
434 	schedule_delayed_work(&bdev->wq,  in ttm_bo_cleanup_refs_or_queue()
455 	struct ttm_bo_global *glob = bo->bdev->glob;  in ttm_bo_cleanup_refs()
525 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)  in ttm_bo_delayed_delete()  argument
527 	struct ttm_bo_global *glob = bdev->glob;  in ttm_bo_delayed_delete()
534 	while (!list_empty(&bdev->ddestroy)) {  in ttm_bo_delayed_delete()
537 		bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,  in ttm_bo_delayed_delete()
558 	list_splice_tail(&removed, &bdev->ddestroy);  in ttm_bo_delayed_delete()
559 	empty = list_empty(&bdev->ddestroy);  in ttm_bo_delayed_delete()
567 	struct ttm_bo_device *bdev =  in ttm_bo_delayed_workqueue()  local
570 	if (!ttm_bo_delayed_delete(bdev, false))  in ttm_bo_delayed_workqueue()
571 		schedule_delayed_work(&bdev->wq,  in ttm_bo_delayed_workqueue()
579 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_release()  local
580 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];  in ttm_bo_release()
582 	drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node);  in ttm_bo_release()
605 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)  in ttm_bo_lock_delayed_workqueue()  argument
607 	return cancel_delayed_work_sync(&bdev->wq);  in ttm_bo_lock_delayed_workqueue()
611 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)  in ttm_bo_unlock_delayed_workqueue()  argument
614 		schedule_delayed_work(&bdev->wq,  in ttm_bo_unlock_delayed_workqueue()
622 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_evict()  local
631 	bdev->driver->evict_flags(bo, &placement);  in ttm_bo_evict()
711 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,  in ttm_mem_evict_first()  argument
716 	struct ttm_bo_global *glob = bdev->glob;  in ttm_mem_evict_first()
717 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_mem_evict_first()
729 			if (place && !bdev->driver->eviction_valuable(bo,  in ttm_mem_evict_first()
777 	struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];  in ttm_bo_mem_put()
822 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_mem_force_space()  local
823 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_bo_mem_force_space()
832 		ret = ttm_mem_evict_first(bdev, mem_type, place, ctx);  in ttm_bo_mem_force_space()
897 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_mem_space()  local
917 		man = &bdev->man[mem_type];  in ttm_bo_mem_space()
966 		man = &bdev->man[mem_type];  in ttm_bo_mem_space()
1112 int ttm_bo_init_reserved(struct ttm_bo_device *bdev,  in ttm_bo_init_reserved()  argument
1126 	struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;  in ttm_bo_init_reserved()
1159 	bo->bdev = bdev;  in ttm_bo_init_reserved()
1180 	atomic_inc(&bo->bdev->glob->bo_count);  in ttm_bo_init_reserved()
1189 		ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node,  in ttm_bo_init_reserved()
1212 		spin_lock(&bdev->glob->lru_lock);  in ttm_bo_init_reserved()
1214 		spin_unlock(&bdev->glob->lru_lock);  in ttm_bo_init_reserved()
1221 int ttm_bo_init(struct ttm_bo_device *bdev,  in ttm_bo_init()  argument
1236 	ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,  in ttm_bo_init()
1249 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,  in ttm_bo_acc_size()  argument
1263 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,  in ttm_bo_dma_acc_size()  argument
1277 int ttm_bo_create(struct ttm_bo_device *bdev,  in ttm_bo_create()  argument
1293 	acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));  in ttm_bo_create()
1294 	ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,  in ttm_bo_create()
1304 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,  in ttm_bo_force_list_clean()  argument
1312 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_bo_force_list_clean()
1313 	struct ttm_bo_global *glob = bdev->glob;  in ttm_bo_force_list_clean()
1326 			ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx);  in ttm_bo_force_list_clean()
1348 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)  in ttm_bo_clean_mm()  argument
1357 	man = &bdev->man[mem_type];  in ttm_bo_clean_mm()
1370 		ret = ttm_bo_force_list_clean(bdev, mem_type);  in ttm_bo_clean_mm()
1386 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)  in ttm_bo_evict_mm()  argument
1388 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_bo_evict_mm()
1400 	return ttm_bo_force_list_clean(bdev, mem_type);  in ttm_bo_evict_mm()
1404 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,  in ttm_bo_init_mm()  argument
1412 	man = &bdev->man[type];  in ttm_bo_init_mm()
1420 	ret = bdev->driver->init_mem_type(bdev, type, man);  in ttm_bo_init_mm()
1423 	man->bdev = bdev;  in ttm_bo_init_mm()
1496 int ttm_bo_device_release(struct ttm_bo_device *bdev)  in ttm_bo_device_release()  argument
1501 	struct ttm_bo_global *glob = bdev->glob;  in ttm_bo_device_release()
1504 		man = &bdev->man[i];  in ttm_bo_device_release()
1507 			if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {  in ttm_bo_device_release()
1517 	list_del(&bdev->device_list);  in ttm_bo_device_release()
1520 	cancel_delayed_work_sync(&bdev->wq);  in ttm_bo_device_release()
1522 	if (ttm_bo_delayed_delete(bdev, true))  in ttm_bo_device_release()
1527 		if (list_empty(&bdev->man[0].lru[0]))  in ttm_bo_device_release()
1531 	drm_vma_offset_manager_destroy(&bdev->vma_manager);  in ttm_bo_device_release()
1537 int ttm_bo_device_init(struct ttm_bo_device *bdev,  in ttm_bo_device_init()  argument
1546 	bdev->driver = driver;  in ttm_bo_device_init()
1548 	memset(bdev->man, 0, sizeof(bdev->man));  in ttm_bo_device_init()
1554 	ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);  in ttm_bo_device_init()
1558 	drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset,  in ttm_bo_device_init()
1560 	INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);  in ttm_bo_device_init()
1561 	INIT_LIST_HEAD(&bdev->ddestroy);  in ttm_bo_device_init()
1562 	bdev->dev_mapping = mapping;  in ttm_bo_device_init()
1563 	bdev->glob = glob;  in ttm_bo_device_init()
1564 	bdev->need_dma32 = need_dma32;  in ttm_bo_device_init()
1566 	list_add_tail(&bdev->device_list, &glob->device_list);  in ttm_bo_device_init()
1579 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)  in ttm_mem_reg_is_pci()  argument
1581 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];  in ttm_mem_reg_is_pci()
1598 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_unmap_virtual_locked()  local
1600 	drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping);  in ttm_bo_unmap_virtual_locked()
1606 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_unmap_virtual()  local
1607 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];  in ttm_bo_unmap_virtual()
1740 	if (bo->bdev->driver->swap_notify)  in ttm_bo_swapout()
1741 		bo->bdev->driver->swap_notify(bo);  in ttm_bo_swapout()
1758 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)  in ttm_bo_swapout_all()  argument
1765 	while (ttm_bo_swapout(bdev->glob, &ctx) == 0)  in ttm_bo_swapout_all()