228 	if (amdgpu_ttm_tt_get_usermm(bo->ttm))  in amdgpu_verify_access()
513 	r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);  in amdgpu_move_vram_ram()
519 	r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx);  in amdgpu_move_vram_ram()
632 	if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {  in amdgpu_bo_move()
765 	struct ttm_dma_tt	ttm;  member
789 	struct ttm_tt *ttm = bo->tbo.ttm;  in amdgpu_ttm_tt_get_user_pages()  local
790 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_user_pages()
825 	pfns = kvmalloc_array(ttm->num_pages, sizeof(*pfns), GFP_KERNEL);  in amdgpu_ttm_tt_get_user_pages()
833 	range->default_flags |= amdgpu_ttm_tt_is_readonly(ttm) ?  in amdgpu_ttm_tt_get_user_pages()
838 	range->end = start + ttm->num_pages * PAGE_SIZE;  in amdgpu_ttm_tt_get_user_pages()
856 	for (i = 0; i < ttm->num_pages; i++) {  in amdgpu_ttm_tt_get_user_pages()
886 bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)  in amdgpu_ttm_tt_get_user_pages_done()  argument
888 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_user_pages_done()
895 		gtt->userptr, ttm->num_pages);  in amdgpu_ttm_tt_get_user_pages_done()
920 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)  in amdgpu_ttm_tt_set_user_pages()  argument
924 	for (i = 0; i < ttm->num_pages; ++i)  in amdgpu_ttm_tt_set_user_pages()
925 		ttm->pages[i] = pages ? pages[i] : NULL;  in amdgpu_ttm_tt_set_user_pages()
933 static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)  in amdgpu_ttm_tt_pin_userptr()  argument
935 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);  in amdgpu_ttm_tt_pin_userptr()
936 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_pin_userptr()
945 	r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,  in amdgpu_ttm_tt_pin_userptr()
946 				      ttm->num_pages << PAGE_SHIFT,  in amdgpu_ttm_tt_pin_userptr()
953 	nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);  in amdgpu_ttm_tt_pin_userptr()
954 	if (nents != ttm->sg->nents)  in amdgpu_ttm_tt_pin_userptr()
958 	drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,  in amdgpu_ttm_tt_pin_userptr()
959 					 gtt->ttm.dma_address, ttm->num_pages);  in amdgpu_ttm_tt_pin_userptr()
964 	kfree(ttm->sg);  in amdgpu_ttm_tt_pin_userptr()
971 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)  in amdgpu_ttm_tt_unpin_userptr()  argument
973 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);  in amdgpu_ttm_tt_unpin_userptr()
974 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_unpin_userptr()
981 	if (!ttm->sg->sgl)  in amdgpu_ttm_tt_unpin_userptr()
985 	dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);  in amdgpu_ttm_tt_unpin_userptr()
987 	sg_free_table(ttm->sg);  in amdgpu_ttm_tt_unpin_userptr()
991 	    ttm->pages[0] == hmm_device_entry_to_page(gtt->range,  in amdgpu_ttm_tt_unpin_userptr()
1002 	struct ttm_tt *ttm = tbo->ttm;  in amdgpu_ttm_gart_bind()  local
1003 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_gart_bind()
1010 				ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_gart_bind()
1020 				ttm->num_pages - page_idx,  in amdgpu_ttm_gart_bind()
1021 				&ttm->pages[page_idx],  in amdgpu_ttm_gart_bind()
1022 				&(gtt->ttm.dma_address[page_idx]), flags);  in amdgpu_ttm_gart_bind()
1024 		r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,  in amdgpu_ttm_gart_bind()
1025 				     ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_gart_bind()
1031 			  ttm->num_pages, gtt->offset);  in amdgpu_ttm_gart_bind()
1042 static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,  in amdgpu_ttm_backend_bind()  argument
1045 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);  in amdgpu_ttm_backend_bind()
1046 	struct amdgpu_ttm_tt *gtt = (void*)ttm;  in amdgpu_ttm_backend_bind()
1051 		r = amdgpu_ttm_tt_pin_userptr(ttm);  in amdgpu_ttm_backend_bind()
1057 	if (!ttm->num_pages) {  in amdgpu_ttm_backend_bind()
1059 		     ttm->num_pages, bo_mem, ttm);  in amdgpu_ttm_backend_bind()
1073 	flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);  in amdgpu_ttm_backend_bind()
1077 	r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,  in amdgpu_ttm_backend_bind()
1078 		ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_backend_bind()
1082 			  ttm->num_pages, gtt->offset);  in amdgpu_ttm_backend_bind()
1093 	struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;  in amdgpu_ttm_alloc_gart()
1125 		flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);  in amdgpu_ttm_alloc_gart()
1157 	if (!tbo->ttm)  in amdgpu_ttm_recover_gart()
1160 	flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem);  in amdgpu_ttm_recover_gart()
1172 static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)  in amdgpu_ttm_backend_unbind()  argument
1174 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);  in amdgpu_ttm_backend_unbind()
1175 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_backend_unbind()
1180 		amdgpu_ttm_tt_unpin_userptr(ttm);  in amdgpu_ttm_backend_unbind()
1186 	r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);  in amdgpu_ttm_backend_unbind()
1189 			  gtt->ttm.ttm.num_pages, gtt->offset);  in amdgpu_ttm_backend_unbind()
1193 static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm)  in amdgpu_ttm_backend_destroy()  argument
1195 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_backend_destroy()
1200 	ttm_dma_tt_fini(&gtt->ttm);  in amdgpu_ttm_backend_destroy()
1229 	gtt->ttm.ttm.func = &amdgpu_backend_func;  in amdgpu_ttm_tt_create()
1232 	if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags)) {  in amdgpu_ttm_tt_create()
1236 	return &gtt->ttm.ttm;  in amdgpu_ttm_tt_create()
1245 static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm,  in amdgpu_ttm_tt_populate()  argument
1248 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);  in amdgpu_ttm_tt_populate()
1249 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_populate()
1250 	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);  in amdgpu_ttm_tt_populate()
1254 		ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);  in amdgpu_ttm_tt_populate()
1255 		if (!ttm->sg)  in amdgpu_ttm_tt_populate()
1258 		ttm->page_flags |= TTM_PAGE_FLAG_SG;  in amdgpu_ttm_tt_populate()
1259 		ttm->state = tt_unbound;  in amdgpu_ttm_tt_populate()
1263 	if (slave && ttm->sg) {  in amdgpu_ttm_tt_populate()
1264 		drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,  in amdgpu_ttm_tt_populate()
1265 						 gtt->ttm.dma_address,  in amdgpu_ttm_tt_populate()
1266 						 ttm->num_pages);  in amdgpu_ttm_tt_populate()
1267 		ttm->state = tt_unbound;  in amdgpu_ttm_tt_populate()
1273 		return ttm_dma_populate(&gtt->ttm, adev->dev, ctx);  in amdgpu_ttm_tt_populate()
1279 	return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);  in amdgpu_ttm_tt_populate()
1288 static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)  in amdgpu_ttm_tt_unpopulate()  argument
1291 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_unpopulate()
1292 	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);  in amdgpu_ttm_tt_unpopulate()
1295 		amdgpu_ttm_tt_set_user_pages(ttm, NULL);  in amdgpu_ttm_tt_unpopulate()
1296 		kfree(ttm->sg);  in amdgpu_ttm_tt_unpopulate()
1297 		ttm->page_flags &= ~TTM_PAGE_FLAG_SG;  in amdgpu_ttm_tt_unpopulate()
1304 	adev = amdgpu_ttm_adev(ttm->bdev);  in amdgpu_ttm_tt_unpopulate()
1308 		ttm_dma_unpopulate(&gtt->ttm, adev->dev);  in amdgpu_ttm_tt_unpopulate()
1314 	ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);  in amdgpu_ttm_tt_unpopulate()
1328 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,  in amdgpu_ttm_tt_set_userptr()  argument
1331 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_set_userptr()
1350 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)  in amdgpu_ttm_tt_get_usermm()  argument
1352 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_usermm()
1368 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,  in amdgpu_ttm_tt_affect_userptr()  argument
1371 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_affect_userptr()
1380 	size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;  in amdgpu_ttm_tt_affect_userptr()
1390 bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)  in amdgpu_ttm_tt_is_userptr()  argument
1392 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_is_userptr()
1403 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)  in amdgpu_ttm_tt_is_readonly()  argument
1405 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_is_readonly()
1421 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem)  in amdgpu_ttm_tt_pde_flags()  argument
1431 		if (ttm->caching_state == tt_cached)  in amdgpu_ttm_tt_pde_flags()
1446 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,  in amdgpu_ttm_tt_pte_flags()  argument
1449 	uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem);  in amdgpu_ttm_tt_pte_flags()
1454 	if (!amdgpu_ttm_tt_is_readonly(ttm))  in amdgpu_ttm_tt_pte_flags()
1937 	struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;  in amdgpu_map_buffer()
1939 	struct ttm_tt *ttm = bo->ttm;  in amdgpu_map_buffer()  local
1976 	dma_address = &gtt->ttm.dma_address[offset >> PAGE_SHIFT];  in amdgpu_map_buffer()
1977 	flags = amdgpu_ttm_tt_pte_flags(adev, ttm, mem);  in amdgpu_map_buffer()