70 		bo->ttm = NULL;  in ttm_tt_create()
75 	bo->ttm = bdev->driver->ttm_tt_create(bo, page_flags);  in ttm_tt_create()
76 	if (unlikely(bo->ttm == NULL))  in ttm_tt_create()
85 static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)  in ttm_tt_alloc_page_directory()  argument
87 	ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),  in ttm_tt_alloc_page_directory()
89 	if (!ttm->pages)  in ttm_tt_alloc_page_directory()
94 static int ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm)  in ttm_dma_tt_alloc_page_directory()  argument
96 	ttm->ttm.pages = kvmalloc_array(ttm->ttm.num_pages,  in ttm_dma_tt_alloc_page_directory()
97 					  sizeof(*ttm->ttm.pages) +  in ttm_dma_tt_alloc_page_directory()
98 					  sizeof(*ttm->dma_address),  in ttm_dma_tt_alloc_page_directory()
100 	if (!ttm->ttm.pages)  in ttm_dma_tt_alloc_page_directory()
102 	ttm->dma_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages);  in ttm_dma_tt_alloc_page_directory()
106 static int ttm_sg_tt_alloc_page_directory(struct ttm_dma_tt *ttm)  in ttm_sg_tt_alloc_page_directory()  argument
108 	ttm->dma_address = kvmalloc_array(ttm->ttm.num_pages,  in ttm_sg_tt_alloc_page_directory()
109 					  sizeof(*ttm->dma_address),  in ttm_sg_tt_alloc_page_directory()
111 	if (!ttm->dma_address)  in ttm_sg_tt_alloc_page_directory()
147 static int ttm_tt_set_caching(struct ttm_tt *ttm,  in ttm_tt_set_caching()  argument
154 	if (ttm->caching_state == c_state)  in ttm_tt_set_caching()
157 	if (ttm->state == tt_unpopulated) {  in ttm_tt_set_caching()
159 		ttm->caching_state = c_state;  in ttm_tt_set_caching()
163 	if (ttm->caching_state == tt_cached)  in ttm_tt_set_caching()
164 		drm_clflush_pages(ttm->pages, ttm->num_pages);  in ttm_tt_set_caching()
166 	for (i = 0; i < ttm->num_pages; ++i) {  in ttm_tt_set_caching()
167 		cur_page = ttm->pages[i];  in ttm_tt_set_caching()
170 						      ttm->caching_state,  in ttm_tt_set_caching()
177 	ttm->caching_state = c_state;  in ttm_tt_set_caching()
183 		cur_page = ttm->pages[j];  in ttm_tt_set_caching()
186 						      ttm->caching_state);  in ttm_tt_set_caching()
193 int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement)  in ttm_tt_set_placement_caching()  argument
204 	return ttm_tt_set_caching(ttm, state);  in ttm_tt_set_placement_caching()
208 void ttm_tt_destroy(struct ttm_tt *ttm)  in ttm_tt_destroy()  argument
210 	if (ttm == NULL)  in ttm_tt_destroy()
213 	ttm_tt_unbind(ttm);  in ttm_tt_destroy()
215 	if (ttm->state == tt_unbound)  in ttm_tt_destroy()
216 		ttm_tt_unpopulate(ttm);  in ttm_tt_destroy()
218 	if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) &&  in ttm_tt_destroy()
219 	    ttm->swap_storage)  in ttm_tt_destroy()
220 		fput(ttm->swap_storage);  in ttm_tt_destroy()
222 	ttm->swap_storage = NULL;  in ttm_tt_destroy()
223 	ttm->func->destroy(ttm);  in ttm_tt_destroy()
226 void ttm_tt_init_fields(struct ttm_tt *ttm, struct ttm_buffer_object *bo,  in ttm_tt_init_fields()  argument
229 	ttm->bdev = bo->bdev;  in ttm_tt_init_fields()
230 	ttm->num_pages = bo->num_pages;  in ttm_tt_init_fields()
231 	ttm->caching_state = tt_cached;  in ttm_tt_init_fields()
232 	ttm->page_flags = page_flags;  in ttm_tt_init_fields()
233 	ttm->state = tt_unpopulated;  in ttm_tt_init_fields()
234 	ttm->swap_storage = NULL;  in ttm_tt_init_fields()
235 	ttm->sg = bo->sg;  in ttm_tt_init_fields()
238 int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo,  in ttm_tt_init()  argument
241 	ttm_tt_init_fields(ttm, bo, page_flags);  in ttm_tt_init()
243 	if (ttm_tt_alloc_page_directory(ttm)) {  in ttm_tt_init()
244 		ttm_tt_destroy(ttm);  in ttm_tt_init()
252 void ttm_tt_fini(struct ttm_tt *ttm)  in ttm_tt_fini()  argument
254 	kvfree(ttm->pages);  in ttm_tt_fini()
255 	ttm->pages = NULL;  in ttm_tt_fini()
262 	struct ttm_tt *ttm = &ttm_dma->ttm;  in ttm_dma_tt_init()  local
264 	ttm_tt_init_fields(ttm, bo, page_flags);  in ttm_dma_tt_init()
268 		ttm_tt_destroy(ttm);  in ttm_dma_tt_init()
279 	struct ttm_tt *ttm = &ttm_dma->ttm;  in ttm_sg_tt_init()  local
282 	ttm_tt_init_fields(ttm, bo, page_flags);  in ttm_sg_tt_init()
290 		ttm_tt_destroy(ttm);  in ttm_sg_tt_init()
300 	struct ttm_tt *ttm = &ttm_dma->ttm;  in ttm_dma_tt_fini()  local
302 	if (ttm->pages)  in ttm_dma_tt_fini()
303 		kvfree(ttm->pages);  in ttm_dma_tt_fini()
306 	ttm->pages = NULL;  in ttm_dma_tt_fini()
311 void ttm_tt_unbind(struct ttm_tt *ttm)  in ttm_tt_unbind()  argument
315 	if (ttm->state == tt_bound) {  in ttm_tt_unbind()
316 		ret = ttm->func->unbind(ttm);  in ttm_tt_unbind()
318 		ttm->state = tt_unbound;  in ttm_tt_unbind()
322 int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem,  in ttm_tt_bind()  argument
327 	if (!ttm)  in ttm_tt_bind()
330 	if (ttm->state == tt_bound)  in ttm_tt_bind()
333 	ret = ttm_tt_populate(ttm, ctx);  in ttm_tt_bind()
337 	ret = ttm->func->bind(ttm, bo_mem);  in ttm_tt_bind()
341 	ttm->state = tt_bound;  in ttm_tt_bind()
347 int ttm_tt_swapin(struct ttm_tt *ttm)  in ttm_tt_swapin()  argument
356 	swap_storage = ttm->swap_storage;  in ttm_tt_swapin()
361 	for (i = 0; i < ttm->num_pages; ++i) {  in ttm_tt_swapin()
364 		gfp_mask |= (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY ? __GFP_RETRY_MAYFAIL : 0);  in ttm_tt_swapin()
371 		to_page = ttm->pages[i];  in ttm_tt_swapin()
379 	if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP))  in ttm_tt_swapin()
381 	ttm->swap_storage = NULL;  in ttm_tt_swapin()
382 	ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;  in ttm_tt_swapin()
389 int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage)  in ttm_tt_swapout()  argument
398 	BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated);  in ttm_tt_swapout()
399 	BUG_ON(ttm->caching_state != tt_cached);  in ttm_tt_swapout()
403 						ttm->num_pages << PAGE_SHIFT,  in ttm_tt_swapout()
415 	for (i = 0; i < ttm->num_pages; ++i) {  in ttm_tt_swapout()
418 		gfp_mask |= (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY ? __GFP_RETRY_MAYFAIL : 0);  in ttm_tt_swapout()
420 		from_page = ttm->pages[i];  in ttm_tt_swapout()
435 	ttm_tt_unpopulate(ttm);  in ttm_tt_swapout()
436 	ttm->swap_storage = swap_storage;  in ttm_tt_swapout()
437 	ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;  in ttm_tt_swapout()
439 		ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP;  in ttm_tt_swapout()
449 static void ttm_tt_add_mapping(struct ttm_tt *ttm)  in ttm_tt_add_mapping()  argument
453 	if (ttm->page_flags & TTM_PAGE_FLAG_SG)  in ttm_tt_add_mapping()
456 	for (i = 0; i < ttm->num_pages; ++i)  in ttm_tt_add_mapping()
457 		ttm->pages[i]->mapping = ttm->bdev->dev_mapping;  in ttm_tt_add_mapping()
460 int ttm_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)  in ttm_tt_populate()  argument
464 	if (ttm->state != tt_unpopulated)  in ttm_tt_populate()
467 	if (ttm->bdev->driver->ttm_tt_populate)  in ttm_tt_populate()
468 		ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx);  in ttm_tt_populate()
470 		ret = ttm_pool_populate(ttm, ctx);  in ttm_tt_populate()
472 		ttm_tt_add_mapping(ttm);  in ttm_tt_populate()
476 static void ttm_tt_clear_mapping(struct ttm_tt *ttm)  in ttm_tt_clear_mapping()  argument
479 	struct page **page = ttm->pages;  in ttm_tt_clear_mapping()
481 	if (ttm->page_flags & TTM_PAGE_FLAG_SG)  in ttm_tt_clear_mapping()
484 	for (i = 0; i < ttm->num_pages; ++i) {  in ttm_tt_clear_mapping()
490 void ttm_tt_unpopulate(struct ttm_tt *ttm)  in ttm_tt_unpopulate()  argument
492 	if (ttm->state == tt_unpopulated)  in ttm_tt_unpopulate()
495 	ttm_tt_clear_mapping(ttm);  in ttm_tt_unpopulate()
496 	if (ttm->bdev->driver->ttm_tt_unpopulate)  in ttm_tt_unpopulate()
497 		ttm->bdev->driver->ttm_tt_unpopulate(ttm);  in ttm_tt_unpopulate()
499 		ttm_pool_unpopulate(ttm);  in ttm_tt_unpopulate()