82 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_mem_type_debug()  local
84 	drm_printf(p, "    has_type: %d\n", man->has_type);  in ttm_mem_type_debug()
85 	drm_printf(p, "    use_type: %d\n", man->use_type);  in ttm_mem_type_debug()
86 	drm_printf(p, "    flags: 0x%08X\n", man->flags);  in ttm_mem_type_debug()
87 	drm_printf(p, "    gpu_offset: 0x%08llX\n", man->gpu_offset);  in ttm_mem_type_debug()
88 	drm_printf(p, "    size: %llu\n", man->size);  in ttm_mem_type_debug()
89 	drm_printf(p, "    available_caching: 0x%08X\n", man->available_caching);  in ttm_mem_type_debug()
90 	drm_printf(p, "    default_caching: 0x%08X\n", man->default_caching);  in ttm_mem_type_debug()
92 		(*man->func->debug)(man, p);  in ttm_mem_type_debug()
174 	struct ttm_mem_type_manager *man;  in ttm_bo_add_mem_to_lru()  local
184 	man = &bdev->man[mem->mem_type];  in ttm_bo_add_mem_to_lru()
185 	list_add_tail(&bo->lru, &man->lru[bo->priority]);  in ttm_bo_add_mem_to_lru()
188 	if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm &&  in ttm_bo_add_mem_to_lru()
276 		struct ttm_mem_type_manager *man;  in ttm_bo_bulk_move_lru_tail()  local
284 		man = &pos->first->bdev->man[TTM_PL_TT];  in ttm_bo_bulk_move_lru_tail()
285 		list_bulk_move_tail(&man->lru[i], &pos->first->lru,  in ttm_bo_bulk_move_lru_tail()
291 		struct ttm_mem_type_manager *man;  in ttm_bo_bulk_move_lru_tail()  local
299 		man = &pos->first->bdev->man[TTM_PL_VRAM];  in ttm_bo_bulk_move_lru_tail()
300 		list_bulk_move_tail(&man->lru[i], &pos->first->lru,  in ttm_bo_bulk_move_lru_tail()
327 	struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];  in ttm_bo_handle_move_mem()
328 	struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];  in ttm_bo_handle_move_mem()
404 		    bdev->man[bo->mem.mem_type].gpu_offset;  in ttm_bo_handle_move_mem()
412 	new_man = &bdev->man[bo->mem.mem_type];  in ttm_bo_handle_move_mem()
674 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];  in ttm_bo_release()  local
680 	ttm_mem_io_lock(man, false);  in ttm_bo_release()
682 	ttm_mem_io_unlock(man);  in ttm_bo_release()
846 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_mem_evict_first()  local
853 		list_for_each_entry(bo, &man->lru[i], lru) {  in ttm_mem_evict_first()
874 		if (&bo->lru != &man->lru[i])  in ttm_mem_evict_first()
917 	struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];  in ttm_bo_mem_put()  local
920 		(*man->func->put_node)(man, mem);  in ttm_bo_mem_put()
928 				 struct ttm_mem_type_manager *man,  in ttm_bo_add_move_fence()  argument
934 	spin_lock(&man->move_lock);  in ttm_bo_add_move_fence()
935 	fence = dma_fence_get(man->move);  in ttm_bo_add_move_fence()
936 	spin_unlock(&man->move_lock);  in ttm_bo_add_move_fence()
964 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];  in ttm_bo_mem_force_space()  local
970 		ret = (*man->func->get_node)(man, bo, place, mem);  in ttm_bo_mem_force_space()
981 	return ttm_bo_add_move_fence(bo, man, mem);  in ttm_bo_mem_force_space()
984 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,  in ttm_bo_select_caching()  argument
997 	else if ((man->default_caching & caching) != 0)  in ttm_bo_select_caching()
998 		result |= man->default_caching;  in ttm_bo_select_caching()
1009 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,  in ttm_bo_mt_compatible()  argument
1019 	if ((place->flags & man->available_caching) == 0)  in ttm_bo_mt_compatible()
1022 	cur_flags |= (place->flags & man->available_caching);  in ttm_bo_mt_compatible()
1046 	struct ttm_mem_type_manager *man;  in ttm_bo_mem_placement()  local
1054 	man = &bdev->man[mem_type];  in ttm_bo_mem_placement()
1055 	if (!man->has_type || !man->use_type)  in ttm_bo_mem_placement()
1058 	if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))  in ttm_bo_mem_placement()
1061 	cur_flags = ttm_bo_select_caching(man, bo->mem.placement, cur_flags);  in ttm_bo_mem_placement()
1105 		struct ttm_mem_type_manager *man;  in ttm_bo_mem_space()  local
1118 		man = &bdev->man[mem->mem_type];  in ttm_bo_mem_space()
1119 		ret = (*man->func->get_node)(man, bo, place, mem);  in ttm_bo_mem_space()
1124 			ret = ttm_bo_add_move_fence(bo, man, mem);  in ttm_bo_mem_space()
1126 				(*man->func->put_node)(man, mem);  in ttm_bo_mem_space()
1483 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_bo_force_list_clean()  local
1495 		while (!list_empty(&man->lru[i])) {  in ttm_bo_force_list_clean()
1506 	spin_lock(&man->move_lock);  in ttm_bo_force_list_clean()
1507 	fence = dma_fence_get(man->move);  in ttm_bo_force_list_clean()
1508 	spin_unlock(&man->move_lock);  in ttm_bo_force_list_clean()
1522 	struct ttm_mem_type_manager *man;  in ttm_bo_clean_mm()  local
1529 	man = &bdev->man[mem_type];  in ttm_bo_clean_mm()
1531 	if (!man->has_type) {  in ttm_bo_clean_mm()
1537 	man->use_type = false;  in ttm_bo_clean_mm()
1538 	man->has_type = false;  in ttm_bo_clean_mm()
1548 		ret = (*man->func->takedown)(man);  in ttm_bo_clean_mm()
1551 	dma_fence_put(man->move);  in ttm_bo_clean_mm()
1552 	man->move = NULL;  in ttm_bo_clean_mm()
1560 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_bo_evict_mm()  local
1567 	if (!man->has_type) {  in ttm_bo_evict_mm()
1580 	struct ttm_mem_type_manager *man;  in ttm_bo_init_mm()  local
1584 	man = &bdev->man[type];  in ttm_bo_init_mm()
1585 	BUG_ON(man->has_type);  in ttm_bo_init_mm()
1586 	man->io_reserve_fastpath = true;  in ttm_bo_init_mm()
1587 	man->use_io_reserve_lru = false;  in ttm_bo_init_mm()
1588 	mutex_init(&man->io_reserve_mutex);  in ttm_bo_init_mm()
1589 	spin_lock_init(&man->move_lock);  in ttm_bo_init_mm()
1590 	INIT_LIST_HEAD(&man->io_reserve_lru);  in ttm_bo_init_mm()
1592 	ret = bdev->driver->init_mem_type(bdev, type, man);  in ttm_bo_init_mm()
1595 	man->bdev = bdev;  in ttm_bo_init_mm()
1598 		ret = (*man->func->init)(man, p_size);  in ttm_bo_init_mm()
1602 	man->has_type = true;  in ttm_bo_init_mm()
1603 	man->use_type = true;  in ttm_bo_init_mm()
1604 	man->size = p_size;  in ttm_bo_init_mm()
1607 		INIT_LIST_HEAD(&man->lru[i]);  in ttm_bo_init_mm()
1608 	man->move = NULL;  in ttm_bo_init_mm()
1680 	struct ttm_mem_type_manager *man;  in ttm_bo_device_release()  local
1684 		man = &bdev->man[i];  in ttm_bo_device_release()
1685 		if (man->has_type) {  in ttm_bo_device_release()
1686 			man->use_type = false;  in ttm_bo_device_release()
1692 			man->has_type = false;  in ttm_bo_device_release()
1707 		if (list_empty(&bdev->man[0].lru[0]))  in ttm_bo_device_release()
1734 	memset(bdev->man, 0, sizeof(bdev->man));  in ttm_bo_device_init()
1769 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];  in ttm_mem_reg_is_pci()  local
1771 	if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {  in ttm_mem_reg_is_pci()
1775 		if (man->flags & TTM_MEMTYPE_FLAG_CMA)  in ttm_mem_reg_is_pci()
1795 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];  in ttm_bo_unmap_virtual()  local
1797 	ttm_mem_io_lock(man, false);  in ttm_bo_unmap_virtual()
1799 	ttm_mem_io_unlock(man);  in ttm_bo_unmap_virtual()