74 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_mem_type_debug()  local
77 	pr_err("    has_type: %d\n", man->has_type);  in ttm_mem_type_debug()
78 	pr_err("    use_type: %d\n", man->use_type);  in ttm_mem_type_debug()
79 	pr_err("    flags: 0x%08X\n", man->flags);  in ttm_mem_type_debug()
80 	pr_err("    gpu_offset: 0x%08llX\n", man->gpu_offset);  in ttm_mem_type_debug()
81 	pr_err("    size: %llu\n", man->size);  in ttm_mem_type_debug()
82 	pr_err("    available_caching: 0x%08X\n", man->available_caching);  in ttm_mem_type_debug()
83 	pr_err("    default_caching: 0x%08X\n", man->default_caching);  in ttm_mem_type_debug()
85 		(*man->func->debug)(man, &p);  in ttm_mem_type_debug()
164 	struct ttm_mem_type_manager *man;  in ttm_bo_add_to_lru()  local
171 		man = &bdev->man[bo->mem.mem_type];  in ttm_bo_add_to_lru()
172 		list_add_tail(&bo->lru, &man->lru[bo->priority]);  in ttm_bo_add_to_lru()
233 	struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];  in ttm_bo_handle_move_mem()
234 	struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];  in ttm_bo_handle_move_mem()
310 		    bdev->man[bo->mem.mem_type].gpu_offset;  in ttm_bo_handle_move_mem()
318 	new_man = &bdev->man[bo->mem.mem_type];  in ttm_bo_handle_move_mem()
580 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];  in ttm_bo_release()  local
583 	ttm_mem_io_lock(man, false);  in ttm_bo_release()
585 	ttm_mem_io_unlock(man);  in ttm_bo_release()
717 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_mem_evict_first()  local
725 		list_for_each_entry(bo, &man->lru[i], lru) {  in ttm_mem_evict_first()
739 		if (&bo->lru != &man->lru[i])  in ttm_mem_evict_first()
777 	struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];  in ttm_bo_mem_put()  local
780 		(*man->func->put_node)(man, mem);  in ttm_bo_mem_put()
788 				 struct ttm_mem_type_manager *man,  in ttm_bo_add_move_fence()  argument
794 	spin_lock(&man->move_lock);  in ttm_bo_add_move_fence()
795 	fence = dma_fence_get(man->move);  in ttm_bo_add_move_fence()
796 	spin_unlock(&man->move_lock);  in ttm_bo_add_move_fence()
823 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_bo_mem_force_space()  local
827 		ret = (*man->func->get_node)(man, bo, place, mem);  in ttm_bo_mem_force_space()
837 	return ttm_bo_add_move_fence(bo, man, mem);  in ttm_bo_mem_force_space()
840 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,  in ttm_bo_select_caching()  argument
853 	else if ((man->default_caching & caching) != 0)  in ttm_bo_select_caching()
854 		result |= man->default_caching;  in ttm_bo_select_caching()
865 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,  in ttm_bo_mt_compatible()  argument
875 	if ((place->flags & man->available_caching) == 0)  in ttm_bo_mt_compatible()
878 	cur_flags |= (place->flags & man->available_caching);  in ttm_bo_mt_compatible()
898 	struct ttm_mem_type_manager *man;  in ttm_bo_mem_space()  local
917 		man = &bdev->man[mem_type];  in ttm_bo_mem_space()
918 		if (!man->has_type || !man->use_type)  in ttm_bo_mem_space()
921 		type_ok = ttm_bo_mt_compatible(man, mem_type, place,  in ttm_bo_mem_space()
928 		cur_flags = ttm_bo_select_caching(man, bo->mem.placement,  in ttm_bo_mem_space()
940 		ret = (*man->func->get_node)(man, bo, place, mem);  in ttm_bo_mem_space()
945 			ret = ttm_bo_add_move_fence(bo, man, mem);  in ttm_bo_mem_space()
947 				(*man->func->put_node)(man, mem);  in ttm_bo_mem_space()
966 		man = &bdev->man[mem_type];  in ttm_bo_mem_space()
967 		if (!man->has_type || !man->use_type)  in ttm_bo_mem_space()
969 		if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))  in ttm_bo_mem_space()
973 		cur_flags = ttm_bo_select_caching(man, bo->mem.placement,  in ttm_bo_mem_space()
1312 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_bo_force_list_clean()  local
1324 		while (!list_empty(&man->lru[i])) {  in ttm_bo_force_list_clean()
1334 	spin_lock(&man->move_lock);  in ttm_bo_force_list_clean()
1335 	fence = dma_fence_get(man->move);  in ttm_bo_force_list_clean()
1336 	spin_unlock(&man->move_lock);  in ttm_bo_force_list_clean()
1350 	struct ttm_mem_type_manager *man;  in ttm_bo_clean_mm()  local
1357 	man = &bdev->man[mem_type];  in ttm_bo_clean_mm()
1359 	if (!man->has_type) {  in ttm_bo_clean_mm()
1365 	man->use_type = false;  in ttm_bo_clean_mm()
1366 	man->has_type = false;  in ttm_bo_clean_mm()
1376 		ret = (*man->func->takedown)(man);  in ttm_bo_clean_mm()
1379 	dma_fence_put(man->move);  in ttm_bo_clean_mm()
1380 	man->move = NULL;  in ttm_bo_clean_mm()
1388 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_bo_evict_mm()  local
1395 	if (!man->has_type) {  in ttm_bo_evict_mm()
1408 	struct ttm_mem_type_manager *man;  in ttm_bo_init_mm()  local
1412 	man = &bdev->man[type];  in ttm_bo_init_mm()
1413 	BUG_ON(man->has_type);  in ttm_bo_init_mm()
1414 	man->io_reserve_fastpath = true;  in ttm_bo_init_mm()
1415 	man->use_io_reserve_lru = false;  in ttm_bo_init_mm()
1416 	mutex_init(&man->io_reserve_mutex);  in ttm_bo_init_mm()
1417 	spin_lock_init(&man->move_lock);  in ttm_bo_init_mm()
1418 	INIT_LIST_HEAD(&man->io_reserve_lru);  in ttm_bo_init_mm()
1420 	ret = bdev->driver->init_mem_type(bdev, type, man);  in ttm_bo_init_mm()
1423 	man->bdev = bdev;  in ttm_bo_init_mm()
1426 		ret = (*man->func->init)(man, p_size);  in ttm_bo_init_mm()
1430 	man->has_type = true;  in ttm_bo_init_mm()
1431 	man->use_type = true;  in ttm_bo_init_mm()
1432 	man->size = p_size;  in ttm_bo_init_mm()
1435 		INIT_LIST_HEAD(&man->lru[i]);  in ttm_bo_init_mm()
1436 	man->move = NULL;  in ttm_bo_init_mm()
1500 	struct ttm_mem_type_manager *man;  in ttm_bo_device_release()  local
1504 		man = &bdev->man[i];  in ttm_bo_device_release()
1505 		if (man->has_type) {  in ttm_bo_device_release()
1506 			man->use_type = false;  in ttm_bo_device_release()
1512 			man->has_type = false;  in ttm_bo_device_release()
1527 		if (list_empty(&bdev->man[0].lru[0]))  in ttm_bo_device_release()
1548 	memset(bdev->man, 0, sizeof(bdev->man));  in ttm_bo_device_init()
1581 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];  in ttm_mem_reg_is_pci()  local
1583 	if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {  in ttm_mem_reg_is_pci()
1587 		if (man->flags & TTM_MEMTYPE_FLAG_CMA)  in ttm_mem_reg_is_pci()
1607 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];  in ttm_bo_unmap_virtual()  local
1609 	ttm_mem_io_lock(man, false);  in ttm_bo_unmap_virtual()
1611 	ttm_mem_io_unlock(man);  in ttm_bo_unmap_virtual()