309 	if (amdgpu_ttm_tt_get_usermm(bo->ttm))  in amdgpu_verify_access()
573 	r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);  in amdgpu_move_vram_ram()
579 	r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx);  in amdgpu_move_vram_ram()
667 	if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {  in amdgpu_bo_move()
787 	struct ttm_dma_tt	ttm;  member
806 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)  in amdgpu_ttm_tt_get_user_pages()  argument
808 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_user_pages()
827 		unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;  in amdgpu_ttm_tt_get_user_pages()
839 		unsigned num_pages = ttm->num_pages - pinned;  in amdgpu_ttm_tt_get_user_pages()
865 	} while (pinned < ttm->num_pages);  in amdgpu_ttm_tt_get_user_pages()
883 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)  in amdgpu_ttm_tt_set_user_pages()  argument
885 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_set_user_pages()
889 	for (i = 0; i < ttm->num_pages; ++i) {  in amdgpu_ttm_tt_set_user_pages()
890 		if (ttm->pages[i])  in amdgpu_ttm_tt_set_user_pages()
891 			put_page(ttm->pages[i]);  in amdgpu_ttm_tt_set_user_pages()
893 		ttm->pages[i] = pages ? pages[i] : NULL;  in amdgpu_ttm_tt_set_user_pages()
902 void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm)  in amdgpu_ttm_tt_mark_user_pages()  argument
904 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_mark_user_pages()
907 	for (i = 0; i < ttm->num_pages; ++i) {  in amdgpu_ttm_tt_mark_user_pages()
908 		struct page *page = ttm->pages[i];  in amdgpu_ttm_tt_mark_user_pages()
925 static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)  in amdgpu_ttm_tt_pin_userptr()  argument
927 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);  in amdgpu_ttm_tt_pin_userptr()
928 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_pin_userptr()
937 	r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,  in amdgpu_ttm_tt_pin_userptr()
938 				      ttm->num_pages << PAGE_SHIFT,  in amdgpu_ttm_tt_pin_userptr()
945 	nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);  in amdgpu_ttm_tt_pin_userptr()
946 	if (nents != ttm->sg->nents)  in amdgpu_ttm_tt_pin_userptr()
950 	drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,  in amdgpu_ttm_tt_pin_userptr()
951 					 gtt->ttm.dma_address, ttm->num_pages);  in amdgpu_ttm_tt_pin_userptr()
956 	kfree(ttm->sg);  in amdgpu_ttm_tt_pin_userptr()
963 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)  in amdgpu_ttm_tt_unpin_userptr()  argument
965 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);  in amdgpu_ttm_tt_unpin_userptr()
966 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_unpin_userptr()
973 	if (!ttm->sg->sgl)  in amdgpu_ttm_tt_unpin_userptr()
977 	dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);  in amdgpu_ttm_tt_unpin_userptr()
980 	amdgpu_ttm_tt_mark_user_pages(ttm);  in amdgpu_ttm_tt_unpin_userptr()
982 	sg_free_table(ttm->sg);  in amdgpu_ttm_tt_unpin_userptr()
990 	struct ttm_tt *ttm = tbo->ttm;  in amdgpu_ttm_gart_bind()  local
991 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_gart_bind()
998 				ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_gart_bind()
1008 				ttm->num_pages - page_idx,  in amdgpu_ttm_gart_bind()
1009 				&ttm->pages[page_idx],  in amdgpu_ttm_gart_bind()
1010 				&(gtt->ttm.dma_address[page_idx]), flags);  in amdgpu_ttm_gart_bind()
1012 		r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,  in amdgpu_ttm_gart_bind()
1013 				     ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_gart_bind()
1019 			  ttm->num_pages, gtt->offset);  in amdgpu_ttm_gart_bind()
1030 static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,  in amdgpu_ttm_backend_bind()  argument
1033 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);  in amdgpu_ttm_backend_bind()
1034 	struct amdgpu_ttm_tt *gtt = (void*)ttm;  in amdgpu_ttm_backend_bind()
1039 		r = amdgpu_ttm_tt_pin_userptr(ttm);  in amdgpu_ttm_backend_bind()
1045 	if (!ttm->num_pages) {  in amdgpu_ttm_backend_bind()
1047 		     ttm->num_pages, bo_mem, ttm);  in amdgpu_ttm_backend_bind()
1061 	flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);  in amdgpu_ttm_backend_bind()
1065 	r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,  in amdgpu_ttm_backend_bind()
1066 		ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_backend_bind()
1070 			  ttm->num_pages, gtt->offset);  in amdgpu_ttm_backend_bind()
1081 	struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;  in amdgpu_ttm_alloc_gart()
1109 	flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);  in amdgpu_ttm_alloc_gart()
1139 	if (!tbo->ttm)  in amdgpu_ttm_recover_gart()
1142 	flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem);  in amdgpu_ttm_recover_gart()
1154 static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)  in amdgpu_ttm_backend_unbind()  argument
1156 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);  in amdgpu_ttm_backend_unbind()
1157 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_backend_unbind()
1162 		amdgpu_ttm_tt_unpin_userptr(ttm);  in amdgpu_ttm_backend_unbind()
1168 	r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);  in amdgpu_ttm_backend_unbind()
1171 			  gtt->ttm.ttm.num_pages, gtt->offset);  in amdgpu_ttm_backend_unbind()
1175 static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm)  in amdgpu_ttm_backend_destroy()  argument
1177 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_backend_destroy()
1182 	ttm_dma_tt_fini(&gtt->ttm);  in amdgpu_ttm_backend_destroy()
1211 	gtt->ttm.ttm.func = &amdgpu_backend_func;  in amdgpu_ttm_tt_create()
1214 	if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags)) {  in amdgpu_ttm_tt_create()
1218 	return &gtt->ttm.ttm;  in amdgpu_ttm_tt_create()
1227 static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm,  in amdgpu_ttm_tt_populate()  argument
1230 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);  in amdgpu_ttm_tt_populate()
1231 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_populate()
1232 	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);  in amdgpu_ttm_tt_populate()
1236 		ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);  in amdgpu_ttm_tt_populate()
1237 		if (!ttm->sg)  in amdgpu_ttm_tt_populate()
1240 		ttm->page_flags |= TTM_PAGE_FLAG_SG;  in amdgpu_ttm_tt_populate()
1241 		ttm->state = tt_unbound;  in amdgpu_ttm_tt_populate()
1245 	if (slave && ttm->sg) {  in amdgpu_ttm_tt_populate()
1246 		drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,  in amdgpu_ttm_tt_populate()
1247 						 gtt->ttm.dma_address,  in amdgpu_ttm_tt_populate()
1248 						 ttm->num_pages);  in amdgpu_ttm_tt_populate()
1249 		ttm->state = tt_unbound;  in amdgpu_ttm_tt_populate()
1255 		return ttm_dma_populate(&gtt->ttm, adev->dev, ctx);  in amdgpu_ttm_tt_populate()
1261 	return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);  in amdgpu_ttm_tt_populate()
1270 static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)  in amdgpu_ttm_tt_unpopulate()  argument
1273 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_unpopulate()
1274 	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);  in amdgpu_ttm_tt_unpopulate()
1277 		amdgpu_ttm_tt_set_user_pages(ttm, NULL);  in amdgpu_ttm_tt_unpopulate()
1278 		kfree(ttm->sg);  in amdgpu_ttm_tt_unpopulate()
1279 		ttm->page_flags &= ~TTM_PAGE_FLAG_SG;  in amdgpu_ttm_tt_unpopulate()
1286 	adev = amdgpu_ttm_adev(ttm->bdev);  in amdgpu_ttm_tt_unpopulate()
1290 		ttm_dma_unpopulate(&gtt->ttm, adev->dev);  in amdgpu_ttm_tt_unpopulate()
1296 	ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);  in amdgpu_ttm_tt_unpopulate()
1310 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,  in amdgpu_ttm_tt_set_userptr()  argument
1313 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_set_userptr()
1337 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)  in amdgpu_ttm_tt_get_usermm()  argument
1339 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_usermm()
1355 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,  in amdgpu_ttm_tt_affect_userptr()  argument
1358 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_affect_userptr()
1368 	size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;  in amdgpu_ttm_tt_affect_userptr()
1392 bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,  in amdgpu_ttm_tt_userptr_invalidated()  argument
1395 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_userptr_invalidated()
1406 bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)  in amdgpu_ttm_tt_userptr_needs_pages()  argument
1408 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_userptr_needs_pages()
1419 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)  in amdgpu_ttm_tt_is_readonly()  argument
1421 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_is_readonly()
1435 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,  in amdgpu_ttm_tt_pte_flags()  argument
1446 		if (ttm->caching_state == tt_cached)  in amdgpu_ttm_tt_pte_flags()
1453 	if (!amdgpu_ttm_tt_is_readonly(ttm))  in amdgpu_ttm_tt_pte_flags()
1959 	struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;  in amdgpu_map_buffer()
1961 	struct ttm_tt *ttm = bo->ttm;  in amdgpu_map_buffer()  local
1998 	dma_address = &gtt->ttm.dma_address[offset >> PAGE_SHIFT];  in amdgpu_map_buffer()
1999 	flags = amdgpu_ttm_tt_pte_flags(adev, ttm, mem);  in amdgpu_map_buffer()