67 				   struct ttm_tt *ttm,
175 	if (amdgpu_ttm_tt_get_usermm(bo->ttm))  in amdgpu_verify_access()
288 	flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, mem);  in amdgpu_ttm_map_buffer()
298 		dma = container_of(bo->ttm, struct ttm_dma_tt, ttm);  in amdgpu_ttm_map_buffer()
549 	r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);  in amdgpu_move_vram_ram()
554 	r = ttm_tt_populate(bo->bdev, bo->ttm, ctx);  in amdgpu_move_vram_ram()
559 	r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, &tmp_mem);  in amdgpu_move_vram_ram()
670 	if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {  in amdgpu_bo_move()
814 	struct ttm_dma_tt	ttm;  member
836 	struct ttm_tt *ttm = bo->tbo.ttm;  in amdgpu_ttm_tt_get_user_pages()  local
837 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_user_pages()
868 	if (!amdgpu_ttm_tt_is_readonly(ttm))  in amdgpu_ttm_tt_get_user_pages()
871 	range->hmm_pfns = kvmalloc_array(ttm->num_pages,  in amdgpu_ttm_tt_get_user_pages()
913 	for (i = 0; i < ttm->num_pages; i++)  in amdgpu_ttm_tt_get_user_pages()
938 bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)  in amdgpu_ttm_tt_get_user_pages_done()  argument
940 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_user_pages_done()
947 		gtt->userptr, ttm->num_pages);  in amdgpu_ttm_tt_get_user_pages_done()
975 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)  in amdgpu_ttm_tt_set_user_pages()  argument
979 	for (i = 0; i < ttm->num_pages; ++i)  in amdgpu_ttm_tt_set_user_pages()
980 		ttm->pages[i] = pages ? pages[i] : NULL;  in amdgpu_ttm_tt_set_user_pages()
989 				     struct ttm_tt *ttm)  in amdgpu_ttm_tt_pin_userptr()  argument
992 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_pin_userptr()
1000 	r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,  in amdgpu_ttm_tt_pin_userptr()
1001 				      ttm->num_pages << PAGE_SHIFT,  in amdgpu_ttm_tt_pin_userptr()
1007 	r = dma_map_sgtable(adev->dev, ttm->sg, direction, 0);  in amdgpu_ttm_tt_pin_userptr()
1012 	drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,  in amdgpu_ttm_tt_pin_userptr()
1013 					 gtt->ttm.dma_address, ttm->num_pages);  in amdgpu_ttm_tt_pin_userptr()
1018 	kfree(ttm->sg);  in amdgpu_ttm_tt_pin_userptr()
1019 	ttm->sg = NULL;  in amdgpu_ttm_tt_pin_userptr()
1027 					struct ttm_tt *ttm)  in amdgpu_ttm_tt_unpin_userptr()  argument
1030 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_unpin_userptr()
1037 	if (!ttm->sg->sgl)  in amdgpu_ttm_tt_unpin_userptr()
1041 	dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);  in amdgpu_ttm_tt_unpin_userptr()
1042 	sg_free_table(ttm->sg);  in amdgpu_ttm_tt_unpin_userptr()
1048 		for (i = 0; i < ttm->num_pages; i++) {  in amdgpu_ttm_tt_unpin_userptr()
1049 			if (ttm->pages[i] !=  in amdgpu_ttm_tt_unpin_userptr()
1054 		WARN((i == ttm->num_pages), "Missing get_user_page_done\n");  in amdgpu_ttm_tt_unpin_userptr()
1064 	struct ttm_tt *ttm = tbo->ttm;  in amdgpu_ttm_gart_bind()  local
1065 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_gart_bind()
1075 				ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_gart_bind()
1088 				ttm->num_pages - page_idx,  in amdgpu_ttm_gart_bind()
1089 				&ttm->pages[page_idx],  in amdgpu_ttm_gart_bind()
1090 				&(gtt->ttm.dma_address[page_idx]), flags);  in amdgpu_ttm_gart_bind()
1092 		r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,  in amdgpu_ttm_gart_bind()
1093 				     ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_gart_bind()
1099 			  ttm->num_pages, gtt->offset);  in amdgpu_ttm_gart_bind()
1111 				   struct ttm_tt *ttm,  in amdgpu_ttm_backend_bind()  argument
1115 	struct amdgpu_ttm_tt *gtt = (void*)ttm;  in amdgpu_ttm_backend_bind()
1126 		r = amdgpu_ttm_tt_pin_userptr(bdev, ttm);  in amdgpu_ttm_backend_bind()
1132 	if (!ttm->num_pages) {  in amdgpu_ttm_backend_bind()
1134 		     ttm->num_pages, bo_mem, ttm);  in amdgpu_ttm_backend_bind()
1148 	flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);  in amdgpu_ttm_backend_bind()
1152 	r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,  in amdgpu_ttm_backend_bind()
1153 		ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_backend_bind()
1157 			  ttm->num_pages, gtt->offset);  in amdgpu_ttm_backend_bind()
1174 	struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;  in amdgpu_ttm_alloc_gart()
1206 		flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);  in amdgpu_ttm_alloc_gart()
1235 	if (!tbo->ttm)  in amdgpu_ttm_recover_gart()
1238 	flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem);  in amdgpu_ttm_recover_gart()
1251 				      struct ttm_tt *ttm)  in amdgpu_ttm_backend_unbind()  argument
1254 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_backend_unbind()
1262 		amdgpu_ttm_tt_unpin_userptr(bdev, ttm);  in amdgpu_ttm_backend_unbind()
1268 	r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);  in amdgpu_ttm_backend_unbind()
1271 			  gtt->ttm.ttm.num_pages, gtt->offset);  in amdgpu_ttm_backend_unbind()
1276 				       struct ttm_tt *ttm)  in amdgpu_ttm_backend_destroy()  argument
1278 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_backend_destroy()
1280 	amdgpu_ttm_backend_unbind(bdev, ttm);  in amdgpu_ttm_backend_destroy()
1281 	ttm_tt_destroy_common(bdev, ttm);  in amdgpu_ttm_backend_destroy()
1285 	ttm_dma_tt_fini(&gtt->ttm);  in amdgpu_ttm_backend_destroy()
1308 	if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags)) {  in amdgpu_ttm_tt_create()
1312 	return &gtt->ttm.ttm;  in amdgpu_ttm_tt_create()
1322 				  struct ttm_tt *ttm,  in amdgpu_ttm_tt_populate()  argument
1326 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_populate()
1330 		ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);  in amdgpu_ttm_tt_populate()
1331 		if (!ttm->sg)  in amdgpu_ttm_tt_populate()
1334 		ttm->page_flags |= TTM_PAGE_FLAG_SG;  in amdgpu_ttm_tt_populate()
1335 		ttm_tt_set_populated(ttm);  in amdgpu_ttm_tt_populate()
1339 	if (ttm->page_flags & TTM_PAGE_FLAG_SG) {  in amdgpu_ttm_tt_populate()
1340 		if (!ttm->sg) {  in amdgpu_ttm_tt_populate()
1349 			ttm->sg = sgt;  in amdgpu_ttm_tt_populate()
1352 		drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,  in amdgpu_ttm_tt_populate()
1353 						 gtt->ttm.dma_address,  in amdgpu_ttm_tt_populate()
1354 						 ttm->num_pages);  in amdgpu_ttm_tt_populate()
1355 		ttm_tt_set_populated(ttm);  in amdgpu_ttm_tt_populate()
1361 		return ttm_dma_populate(&gtt->ttm, adev->dev, ctx);  in amdgpu_ttm_tt_populate()
1367 	return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);  in amdgpu_ttm_tt_populate()
1376 static void amdgpu_ttm_tt_unpopulate(struct ttm_bo_device *bdev, struct ttm_tt *ttm)  in amdgpu_ttm_tt_unpopulate()  argument
1378 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_unpopulate()
1382 		amdgpu_ttm_tt_set_user_pages(ttm, NULL);  in amdgpu_ttm_tt_unpopulate()
1383 		kfree(ttm->sg);  in amdgpu_ttm_tt_unpopulate()
1384 		ttm->page_flags &= ~TTM_PAGE_FLAG_SG;  in amdgpu_ttm_tt_unpopulate()
1388 	if (ttm->sg && gtt->gobj->import_attach) {  in amdgpu_ttm_tt_unpopulate()
1392 		dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL);  in amdgpu_ttm_tt_unpopulate()
1393 		ttm->sg = NULL;  in amdgpu_ttm_tt_unpopulate()
1397 	if (ttm->page_flags & TTM_PAGE_FLAG_SG)  in amdgpu_ttm_tt_unpopulate()
1404 		ttm_dma_unpopulate(&gtt->ttm, adev->dev);  in amdgpu_ttm_tt_unpopulate()
1410 	ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);  in amdgpu_ttm_tt_unpopulate()
1429 	if (!bo->ttm) {  in amdgpu_ttm_tt_set_userptr()
1431 		bo->ttm = amdgpu_ttm_tt_create(bo, 0);  in amdgpu_ttm_tt_set_userptr()
1432 		if (bo->ttm == NULL)  in amdgpu_ttm_tt_set_userptr()
1436 	gtt = (void*)bo->ttm;  in amdgpu_ttm_tt_set_userptr()
1451 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)  in amdgpu_ttm_tt_get_usermm()  argument
1453 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_usermm()
1469 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,  in amdgpu_ttm_tt_affect_userptr()  argument
1472 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_affect_userptr()
1481 	size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;  in amdgpu_ttm_tt_affect_userptr()
1491 bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)  in amdgpu_ttm_tt_is_userptr()  argument
1493 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_is_userptr()
1504 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)  in amdgpu_ttm_tt_is_readonly()  argument
1506 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_is_readonly()
1522 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem)  in amdgpu_ttm_tt_pde_flags()  argument
1532 		if (ttm->caching_state == tt_cached)  in amdgpu_ttm_tt_pde_flags()
1547 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,  in amdgpu_ttm_tt_pte_flags()  argument
1550 	uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem);  in amdgpu_ttm_tt_pte_flags()
1555 	if (!amdgpu_ttm_tt_is_readonly(ttm))  in amdgpu_ttm_tt_pte_flags()