790 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_user_pages()  local
791 	struct mm_struct *mm = gtt->usertask->mm;  in amdgpu_ttm_tt_get_user_pages()
792 	unsigned long start = gtt->userptr;  in amdgpu_ttm_tt_get_user_pages()
813 	if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&  in amdgpu_ttm_tt_get_user_pages()
867 	gtt->range = range;  in amdgpu_ttm_tt_get_user_pages()
888 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_user_pages_done()  local
891 	if (!gtt || !gtt->userptr)  in amdgpu_ttm_tt_get_user_pages_done()
895 		gtt->userptr, ttm->num_pages);  in amdgpu_ttm_tt_get_user_pages_done()
897 	WARN_ONCE(!gtt->range || !gtt->range->pfns,  in amdgpu_ttm_tt_get_user_pages_done()
900 	if (gtt->range) {  in amdgpu_ttm_tt_get_user_pages_done()
901 		r = hmm_range_valid(gtt->range);  in amdgpu_ttm_tt_get_user_pages_done()
902 		hmm_range_unregister(gtt->range);  in amdgpu_ttm_tt_get_user_pages_done()
904 		kvfree(gtt->range->pfns);  in amdgpu_ttm_tt_get_user_pages_done()
905 		kfree(gtt->range);  in amdgpu_ttm_tt_get_user_pages_done()
906 		gtt->range = NULL;  in amdgpu_ttm_tt_get_user_pages_done()
936 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_pin_userptr()  local
940 	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);  in amdgpu_ttm_tt_pin_userptr()
959 					 gtt->ttm.dma_address, ttm->num_pages);  in amdgpu_ttm_tt_pin_userptr()
974 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_unpin_userptr()  local
976 	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);  in amdgpu_ttm_tt_unpin_userptr()
990 	if (gtt->range &&  in amdgpu_ttm_tt_unpin_userptr()
991 	    ttm->pages[0] == hmm_device_entry_to_page(gtt->range,  in amdgpu_ttm_tt_unpin_userptr()
992 						      gtt->range->pfns[0]))  in amdgpu_ttm_tt_unpin_userptr()
1003 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_gart_bind()  local
1009 		r = amdgpu_gart_bind(adev, gtt->offset, page_idx,  in amdgpu_ttm_gart_bind()
1010 				ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_gart_bind()
1019 				gtt->offset + (page_idx << PAGE_SHIFT),  in amdgpu_ttm_gart_bind()
1022 				&(gtt->ttm.dma_address[page_idx]), flags);  in amdgpu_ttm_gart_bind()
1024 		r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,  in amdgpu_ttm_gart_bind()
1025 				     ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_gart_bind()
1031 			  ttm->num_pages, gtt->offset);  in amdgpu_ttm_gart_bind()
1046 	struct amdgpu_ttm_tt *gtt = (void*)ttm;  in amdgpu_ttm_backend_bind()  local
1050 	if (gtt->userptr) {  in amdgpu_ttm_backend_bind()
1068 		gtt->offset = AMDGPU_BO_INVALID_OFFSET;  in amdgpu_ttm_backend_bind()
1076 	gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;  in amdgpu_ttm_backend_bind()
1077 	r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,  in amdgpu_ttm_backend_bind()
1078 		ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_backend_bind()
1082 			  ttm->num_pages, gtt->offset);  in amdgpu_ttm_backend_bind()
1093 	struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;  in amdgpu_ttm_alloc_gart()  local
1128 		gtt->offset = (u64)tmp.start << PAGE_SHIFT;  in amdgpu_ttm_alloc_gart()
1175 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_backend_unbind()  local
1179 	if (gtt->userptr)  in amdgpu_ttm_backend_unbind()
1182 	if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)  in amdgpu_ttm_backend_unbind()
1186 	r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);  in amdgpu_ttm_backend_unbind()
1189 			  gtt->ttm.ttm.num_pages, gtt->offset);  in amdgpu_ttm_backend_unbind()
1195 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_backend_destroy()  local
1197 	if (gtt->usertask)  in amdgpu_ttm_backend_destroy()
1198 		put_task_struct(gtt->usertask);  in amdgpu_ttm_backend_destroy()
1200 	ttm_dma_tt_fini(&gtt->ttm);  in amdgpu_ttm_backend_destroy()
1201 	kfree(gtt);  in amdgpu_ttm_backend_destroy()
1221 	struct amdgpu_ttm_tt *gtt;  in amdgpu_ttm_tt_create()  local
1225 	gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);  in amdgpu_ttm_tt_create()
1226 	if (gtt == NULL) {  in amdgpu_ttm_tt_create()
1229 	gtt->ttm.ttm.func = &amdgpu_backend_func;  in amdgpu_ttm_tt_create()
1232 	if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags)) {  in amdgpu_ttm_tt_create()
1233 		kfree(gtt);  in amdgpu_ttm_tt_create()
1236 	return &gtt->ttm.ttm;  in amdgpu_ttm_tt_create()
1249 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_populate()  local
1253 	if (gtt && gtt->userptr) {  in amdgpu_ttm_tt_populate()
1265 						 gtt->ttm.dma_address,  in amdgpu_ttm_tt_populate()
1273 		return ttm_dma_populate(&gtt->ttm, adev->dev, ctx);  in amdgpu_ttm_tt_populate()
1279 	return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);  in amdgpu_ttm_tt_populate()
1291 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_unpopulate()  local
1294 	if (gtt && gtt->userptr) {  in amdgpu_ttm_tt_unpopulate()
1308 		ttm_dma_unpopulate(&gtt->ttm, adev->dev);  in amdgpu_ttm_tt_unpopulate()
1314 	ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);  in amdgpu_ttm_tt_unpopulate()
1331 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_set_userptr()  local
1333 	if (gtt == NULL)  in amdgpu_ttm_tt_set_userptr()
1336 	gtt->userptr = addr;  in amdgpu_ttm_tt_set_userptr()
1337 	gtt->userflags = flags;  in amdgpu_ttm_tt_set_userptr()
1339 	if (gtt->usertask)  in amdgpu_ttm_tt_set_userptr()
1340 		put_task_struct(gtt->usertask);  in amdgpu_ttm_tt_set_userptr()
1341 	gtt->usertask = current->group_leader;  in amdgpu_ttm_tt_set_userptr()
1342 	get_task_struct(gtt->usertask);  in amdgpu_ttm_tt_set_userptr()
1352 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_usermm()  local
1354 	if (gtt == NULL)  in amdgpu_ttm_tt_get_usermm()
1357 	if (gtt->usertask == NULL)  in amdgpu_ttm_tt_get_usermm()
1360 	return gtt->usertask->mm;  in amdgpu_ttm_tt_get_usermm()
1371 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_affect_userptr()  local
1374 	if (gtt == NULL || !gtt->userptr)  in amdgpu_ttm_tt_affect_userptr()
1380 	size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;  in amdgpu_ttm_tt_affect_userptr()
1381 	if (gtt->userptr > end || gtt->userptr + size <= start)  in amdgpu_ttm_tt_affect_userptr()
1392 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_is_userptr()  local
1394 	if (gtt == NULL || !gtt->userptr)  in amdgpu_ttm_tt_is_userptr()
1405 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_is_readonly()  local
1407 	if (gtt == NULL)  in amdgpu_ttm_tt_is_readonly()
1410 	return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);  in amdgpu_ttm_tt_is_readonly()
1937 	struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;  in amdgpu_map_buffer()  local
1976 	dma_address = &gtt->ttm.dma_address[offset >> PAGE_SHIFT];  in amdgpu_map_buffer()