837 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_user_pages()  local
838 	unsigned long start = gtt->userptr;  in amdgpu_ttm_tt_get_user_pages()
853 	if (WARN_ON(gtt->range))  in amdgpu_ttm_tt_get_user_pages()
884 	if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&  in amdgpu_ttm_tt_get_user_pages()
916 	gtt->range = range;  in amdgpu_ttm_tt_get_user_pages()
940 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_user_pages_done()  local
943 	if (!gtt || !gtt->userptr)  in amdgpu_ttm_tt_get_user_pages_done()
947 		gtt->userptr, ttm->num_pages);  in amdgpu_ttm_tt_get_user_pages_done()
949 	WARN_ONCE(!gtt->range || !gtt->range->hmm_pfns,  in amdgpu_ttm_tt_get_user_pages_done()
952 	if (gtt->range) {  in amdgpu_ttm_tt_get_user_pages_done()
957 		r = mmu_interval_read_retry(gtt->range->notifier,  in amdgpu_ttm_tt_get_user_pages_done()
958 					 gtt->range->notifier_seq);  in amdgpu_ttm_tt_get_user_pages_done()
959 		kvfree(gtt->range->hmm_pfns);  in amdgpu_ttm_tt_get_user_pages_done()
960 		kfree(gtt->range);  in amdgpu_ttm_tt_get_user_pages_done()
961 		gtt->range = NULL;  in amdgpu_ttm_tt_get_user_pages_done()
992 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_pin_userptr()  local
995 	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);  in amdgpu_ttm_tt_pin_userptr()
1013 					 gtt->ttm.dma_address, ttm->num_pages);  in amdgpu_ttm_tt_pin_userptr()
1030 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_unpin_userptr()  local
1032 	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);  in amdgpu_ttm_tt_unpin_userptr()
1045 	if (gtt->range) {  in amdgpu_ttm_tt_unpin_userptr()
1050 			    hmm_pfn_to_page(gtt->range->hmm_pfns[i]))  in amdgpu_ttm_tt_unpin_userptr()
1065 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_gart_bind()  local
1074 		r = amdgpu_gart_bind(adev, gtt->offset, page_idx,  in amdgpu_ttm_gart_bind()
1075 				ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_gart_bind()
1087 				gtt->offset + (page_idx << PAGE_SHIFT),  in amdgpu_ttm_gart_bind()
1090 				&(gtt->ttm.dma_address[page_idx]), flags);  in amdgpu_ttm_gart_bind()
1092 		r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,  in amdgpu_ttm_gart_bind()
1093 				     ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_gart_bind()
1099 			  ttm->num_pages, gtt->offset);  in amdgpu_ttm_gart_bind()
1115 	struct amdgpu_ttm_tt *gtt = (void*)ttm;  in amdgpu_ttm_backend_bind()  local
1122 	if (gtt->bound)  in amdgpu_ttm_backend_bind()
1125 	if (gtt->userptr) {  in amdgpu_ttm_backend_bind()
1143 		gtt->offset = AMDGPU_BO_INVALID_OFFSET;  in amdgpu_ttm_backend_bind()
1151 	gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;  in amdgpu_ttm_backend_bind()
1152 	r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,  in amdgpu_ttm_backend_bind()
1153 		ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_backend_bind()
1157 			  ttm->num_pages, gtt->offset);  in amdgpu_ttm_backend_bind()
1158 	gtt->bound = true;  in amdgpu_ttm_backend_bind()
1174 	struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;  in amdgpu_ttm_alloc_gart()  local
1209 		gtt->offset = (u64)tmp.start << PAGE_SHIFT;  in amdgpu_ttm_alloc_gart()
1254 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_backend_unbind()  local
1257 	if (!gtt->bound)  in amdgpu_ttm_backend_unbind()
1261 	if (gtt->userptr)  in amdgpu_ttm_backend_unbind()
1264 	if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)  in amdgpu_ttm_backend_unbind()
1268 	r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);  in amdgpu_ttm_backend_unbind()
1271 			  gtt->ttm.ttm.num_pages, gtt->offset);  in amdgpu_ttm_backend_unbind()
1272 	gtt->bound = false;  in amdgpu_ttm_backend_unbind()
1278 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_backend_destroy()  local
1282 	if (gtt->usertask)  in amdgpu_ttm_backend_destroy()
1283 		put_task_struct(gtt->usertask);  in amdgpu_ttm_backend_destroy()
1285 	ttm_dma_tt_fini(&gtt->ttm);  in amdgpu_ttm_backend_destroy()
1286 	kfree(gtt);  in amdgpu_ttm_backend_destroy()
1299 	struct amdgpu_ttm_tt *gtt;  in amdgpu_ttm_tt_create()  local
1301 	gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);  in amdgpu_ttm_tt_create()
1302 	if (gtt == NULL) {  in amdgpu_ttm_tt_create()
1305 	gtt->gobj = &bo->base;  in amdgpu_ttm_tt_create()
1308 	if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags)) {  in amdgpu_ttm_tt_create()
1309 		kfree(gtt);  in amdgpu_ttm_tt_create()
1312 	return &gtt->ttm.ttm;  in amdgpu_ttm_tt_create()
1326 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_populate()  local
1329 	if (gtt && gtt->userptr) {  in amdgpu_ttm_tt_populate()
1344 			attach = gtt->gobj->import_attach;  in amdgpu_ttm_tt_populate()
1353 						 gtt->ttm.dma_address,  in amdgpu_ttm_tt_populate()
1361 		return ttm_dma_populate(&gtt->ttm, adev->dev, ctx);  in amdgpu_ttm_tt_populate()
1367 	return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);  in amdgpu_ttm_tt_populate()
1378 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_unpopulate()  local
1381 	if (gtt && gtt->userptr) {  in amdgpu_ttm_tt_unpopulate()
1388 	if (ttm->sg && gtt->gobj->import_attach) {  in amdgpu_ttm_tt_unpopulate()
1391 		attach = gtt->gobj->import_attach;  in amdgpu_ttm_tt_unpopulate()
1404 		ttm_dma_unpopulate(&gtt->ttm, adev->dev);  in amdgpu_ttm_tt_unpopulate()
1410 	ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);  in amdgpu_ttm_tt_unpopulate()
1427 	struct amdgpu_ttm_tt *gtt;  in amdgpu_ttm_tt_set_userptr()  local
1436 	gtt = (void*)bo->ttm;  in amdgpu_ttm_tt_set_userptr()
1437 	gtt->userptr = addr;  in amdgpu_ttm_tt_set_userptr()
1438 	gtt->userflags = flags;  in amdgpu_ttm_tt_set_userptr()
1440 	if (gtt->usertask)  in amdgpu_ttm_tt_set_userptr()
1441 		put_task_struct(gtt->usertask);  in amdgpu_ttm_tt_set_userptr()
1442 	gtt->usertask = current->group_leader;  in amdgpu_ttm_tt_set_userptr()
1443 	get_task_struct(gtt->usertask);  in amdgpu_ttm_tt_set_userptr()
1453 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_usermm()  local
1455 	if (gtt == NULL)  in amdgpu_ttm_tt_get_usermm()
1458 	if (gtt->usertask == NULL)  in amdgpu_ttm_tt_get_usermm()
1461 	return gtt->usertask->mm;  in amdgpu_ttm_tt_get_usermm()
1472 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_affect_userptr()  local
1475 	if (gtt == NULL || !gtt->userptr)  in amdgpu_ttm_tt_affect_userptr()
1481 	size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;  in amdgpu_ttm_tt_affect_userptr()
1482 	if (gtt->userptr > end || gtt->userptr + size <= start)  in amdgpu_ttm_tt_affect_userptr()
1493 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_is_userptr()  local
1495 	if (gtt == NULL || !gtt->userptr)  in amdgpu_ttm_tt_is_userptr()
1506 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_is_readonly()  local
1508 	if (gtt == NULL)  in amdgpu_ttm_tt_is_readonly()
1511 	return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);  in amdgpu_ttm_tt_is_readonly()