50 	struct ttm_tt ttm;  member
186 				      struct ttm_tt *ttm,  in i915_ttm_tt_shmem_populate()  argument
191 	struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm);  in i915_ttm_tt_shmem_populate()
193 	const size_t size = (size_t)ttm->num_pages << PAGE_SHIFT;  in i915_ttm_tt_shmem_populate()
231 		ttm->pages[i++] = page;  in i915_ttm_tt_shmem_populate()
233 	if (ttm->page_flags & TTM_TT_FLAG_SWAPPED)  in i915_ttm_tt_shmem_populate()
234 		ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED;  in i915_ttm_tt_shmem_populate()
244 static void i915_ttm_tt_shmem_unpopulate(struct ttm_tt *ttm)  in i915_ttm_tt_shmem_unpopulate()  argument
246 	struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm);  in i915_ttm_tt_shmem_unpopulate()
247 	bool backup = ttm->page_flags & TTM_TT_FLAG_SWAPPED;  in i915_ttm_tt_shmem_unpopulate()
305 	ret = ttm_tt_init(&i915_tt->ttm, bo, page_flags, caching, ccs_pages);  in i915_ttm_tt_create()
314 	return &i915_tt->ttm;  in i915_ttm_tt_create()
322 				struct ttm_tt *ttm,  in i915_ttm_tt_populate()  argument
325 	struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm);  in i915_ttm_tt_populate()
328 		return i915_ttm_tt_shmem_populate(bdev, ttm, ctx);  in i915_ttm_tt_populate()
330 	return ttm_pool_alloc(&bdev->pool, ttm, ctx);  in i915_ttm_tt_populate()
333 static void i915_ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm)  in i915_ttm_tt_unpopulate()  argument
335 	struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm);  in i915_ttm_tt_unpopulate()
342 		i915_ttm_tt_shmem_unpopulate(ttm);  in i915_ttm_tt_unpopulate()
345 		ttm_pool_free(&bdev->pool, ttm);  in i915_ttm_tt_unpopulate()
349 static void i915_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *ttm)  in i915_ttm_tt_destroy()  argument
351 	struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm);  in i915_ttm_tt_destroy()
356 	ttm_tt_fini(ttm);  in i915_ttm_tt_destroy()
373 	if (bo->ttm && bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL)  in i915_ttm_eviction_valuable()
401 	if (!obj->ttm.cached_io_rsgt)  in i915_ttm_free_cached_io_rsgt()
405 	radix_tree_for_each_slot(slot, &obj->ttm.get_io_page.radix, &iter, 0)  in i915_ttm_free_cached_io_rsgt()
406 		radix_tree_delete(&obj->ttm.get_io_page.radix, iter.index);  in i915_ttm_free_cached_io_rsgt()
409 	i915_refct_sgt_put(obj->ttm.cached_io_rsgt);  in i915_ttm_free_cached_io_rsgt()
410 	obj->ttm.cached_io_rsgt = NULL;  in i915_ttm_free_cached_io_rsgt()
426 		container_of(bo->ttm, typeof(*i915_tt), ttm);  in i915_ttm_purge()
441 	if (bo->ttm && i915_tt->filp) {  in i915_ttm_purge()
466 		container_of(bo->ttm, typeof(*i915_tt), ttm);  in i915_ttm_shrink()
474 	if (!bo->ttm || bo->resource->mem_type != TTM_PL_SYSTEM)  in i915_ttm_shrink()
493 	if (bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED)  in i915_ttm_shrink()
496 	bo->ttm->page_flags |= TTM_TT_FLAG_SWAPPED;  in i915_ttm_shrink()
499 		bo->ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED;  in i915_ttm_shrink()
528 static struct i915_refct_sgt *i915_ttm_tt_get_st(struct ttm_tt *ttm)  in i915_ttm_tt_get_st()  argument
530 	struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm);  in i915_ttm_tt_get_st()
539 			ttm->pages, ttm->num_pages,  in i915_ttm_tt_get_st()
540 			0, (unsigned long)ttm->num_pages << PAGE_SHIFT,  in i915_ttm_tt_get_st()
578 		return i915_ttm_tt_get_st(bo->ttm);  in i915_ttm_resource_get_st()
590 		if (!obj->ttm.cached_io_rsgt) {  in i915_ttm_resource_get_st()
599 			obj->ttm.cached_io_rsgt = rsgt;  in i915_ttm_resource_get_st()
601 		return i915_refct_sgt_get(obj->ttm.cached_io_rsgt);  in i915_ttm_resource_get_st()
636 	GEM_WARN_ON(obj->ttm.cached_io_rsgt);  in i915_ttm_swap_notify()
698 	GEM_WARN_ON(bo->ttm);  in i915_ttm_io_mem_pfn()
701 	sg = __i915_gem_object_get_sg(obj, &obj->ttm.get_io_page, page_offset, &ofs, true);  in i915_ttm_io_mem_pfn()
768 	if (bo->ttm && !ttm_tt_is_populated(bo->ttm)) {  in __i915_ttm_get_pages()
769 		ret = ttm_tt_populate(bo->bdev, bo->ttm, &ctx);  in __i915_ttm_get_pages()
790 	GEM_BUG_ON(bo->ttm && ((obj->base.size >> PAGE_SHIFT) < bo->ttm->num_pages));  in __i915_ttm_get_pages()
884 		container_of(bo->ttm, typeof(*i915_tt), ttm);  in i915_ttm_adjust_lru()
886 		bo->ttm && i915_tt->filp && ttm_tt_is_populated(bo->ttm);  in i915_ttm_adjust_lru()
987 	GEM_BUG_ON(!obj->ttm.created);  in i915_ttm_delayed_free()
1158 	mutex_destroy(&obj->ttm.get_io_page.lock);  in i915_ttm_bo_destroy()
1160 	if (obj->ttm.created) {  in i915_ttm_bo_destroy()
1218 	INIT_RADIX_TREE(&obj->ttm.get_io_page.radix, GFP_KERNEL | __GFP_NOWARN);  in __i915_gem_ttm_object_init()
1219 	mutex_init(&obj->ttm.get_io_page.lock);  in __i915_gem_ttm_object_init()
1249 	obj->ttm.created = true;  in __i915_gem_ttm_object_init()