808 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_user_pages()  local
809 	struct mm_struct *mm = gtt->usertask->mm;  in amdgpu_ttm_tt_get_user_pages()
817 	if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))  in amdgpu_ttm_tt_get_user_pages()
822 	if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {  in amdgpu_ttm_tt_get_user_pages()
827 		unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;  in amdgpu_ttm_tt_get_user_pages()
830 		vma = find_vma(mm, gtt->userptr);  in amdgpu_ttm_tt_get_user_pages()
840 		uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE;  in amdgpu_ttm_tt_get_user_pages()
845 		spin_lock(&gtt->guptasklock);  in amdgpu_ttm_tt_get_user_pages()
846 		list_add(&guptask.list, &gtt->guptasks);  in amdgpu_ttm_tt_get_user_pages()
847 		spin_unlock(&gtt->guptasklock);  in amdgpu_ttm_tt_get_user_pages()
852 			r = get_user_pages_remote(gtt->usertask,  in amdgpu_ttm_tt_get_user_pages()
856 		spin_lock(&gtt->guptasklock);  in amdgpu_ttm_tt_get_user_pages()
858 		spin_unlock(&gtt->guptasklock);  in amdgpu_ttm_tt_get_user_pages()
885 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_set_user_pages()  local
888 	gtt->last_set_pages = atomic_read(&gtt->mmu_invalidations);  in amdgpu_ttm_tt_set_user_pages()
904 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_mark_user_pages()  local
913 		if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))  in amdgpu_ttm_tt_mark_user_pages()
928 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_pin_userptr()  local
932 	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);  in amdgpu_ttm_tt_pin_userptr()
951 					 gtt->ttm.dma_address, ttm->num_pages);  in amdgpu_ttm_tt_pin_userptr()
966 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_unpin_userptr()  local
968 	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);  in amdgpu_ttm_tt_unpin_userptr()
991 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_gart_bind()  local
997 		r = amdgpu_gart_bind(adev, gtt->offset, page_idx,  in amdgpu_ttm_gart_bind()
998 				ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_gart_bind()
1007 				gtt->offset + (page_idx << PAGE_SHIFT),  in amdgpu_ttm_gart_bind()
1010 				&(gtt->ttm.dma_address[page_idx]), flags);  in amdgpu_ttm_gart_bind()
1012 		r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,  in amdgpu_ttm_gart_bind()
1013 				     ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_gart_bind()
1019 			  ttm->num_pages, gtt->offset);  in amdgpu_ttm_gart_bind()
1034 	struct amdgpu_ttm_tt *gtt = (void*)ttm;  in amdgpu_ttm_backend_bind()  local
1038 	if (gtt->userptr) {  in amdgpu_ttm_backend_bind()
1056 		gtt->offset = AMDGPU_BO_INVALID_OFFSET;  in amdgpu_ttm_backend_bind()
1064 	gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;  in amdgpu_ttm_backend_bind()
1065 	r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,  in amdgpu_ttm_backend_bind()
1066 		ttm->pages, gtt->ttm.dma_address, flags);  in amdgpu_ttm_backend_bind()
1070 			  ttm->num_pages, gtt->offset);  in amdgpu_ttm_backend_bind()
1081 	struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;  in amdgpu_ttm_alloc_gart()  local
1112 	gtt->offset = (u64)tmp.start << PAGE_SHIFT;  in amdgpu_ttm_alloc_gart()
1157 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_backend_unbind()  local
1161 	if (gtt->userptr)  in amdgpu_ttm_backend_unbind()
1164 	if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)  in amdgpu_ttm_backend_unbind()
1168 	r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);  in amdgpu_ttm_backend_unbind()
1171 			  gtt->ttm.ttm.num_pages, gtt->offset);  in amdgpu_ttm_backend_unbind()
1177 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_backend_destroy()  local
1179 	if (gtt->usertask)  in amdgpu_ttm_backend_destroy()
1180 		put_task_struct(gtt->usertask);  in amdgpu_ttm_backend_destroy()
1182 	ttm_dma_tt_fini(&gtt->ttm);  in amdgpu_ttm_backend_destroy()
1183 	kfree(gtt);  in amdgpu_ttm_backend_destroy()
1203 	struct amdgpu_ttm_tt *gtt;  in amdgpu_ttm_tt_create()  local
1207 	gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);  in amdgpu_ttm_tt_create()
1208 	if (gtt == NULL) {  in amdgpu_ttm_tt_create()
1211 	gtt->ttm.ttm.func = &amdgpu_backend_func;  in amdgpu_ttm_tt_create()
1214 	if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags)) {  in amdgpu_ttm_tt_create()
1215 		kfree(gtt);  in amdgpu_ttm_tt_create()
1218 	return &gtt->ttm.ttm;  in amdgpu_ttm_tt_create()
1231 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_populate()  local
1235 	if (gtt && gtt->userptr) {  in amdgpu_ttm_tt_populate()
1247 						 gtt->ttm.dma_address,  in amdgpu_ttm_tt_populate()
1255 		return ttm_dma_populate(&gtt->ttm, adev->dev, ctx);  in amdgpu_ttm_tt_populate()
1261 	return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);  in amdgpu_ttm_tt_populate()
1273 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_unpopulate()  local
1276 	if (gtt && gtt->userptr) {  in amdgpu_ttm_tt_unpopulate()
1290 		ttm_dma_unpopulate(&gtt->ttm, adev->dev);  in amdgpu_ttm_tt_unpopulate()
1296 	ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);  in amdgpu_ttm_tt_unpopulate()
1313 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_set_userptr()  local
1315 	if (gtt == NULL)  in amdgpu_ttm_tt_set_userptr()
1318 	gtt->userptr = addr;  in amdgpu_ttm_tt_set_userptr()
1319 	gtt->userflags = flags;  in amdgpu_ttm_tt_set_userptr()
1321 	if (gtt->usertask)  in amdgpu_ttm_tt_set_userptr()
1322 		put_task_struct(gtt->usertask);  in amdgpu_ttm_tt_set_userptr()
1323 	gtt->usertask = current->group_leader;  in amdgpu_ttm_tt_set_userptr()
1324 	get_task_struct(gtt->usertask);  in amdgpu_ttm_tt_set_userptr()
1326 	spin_lock_init(&gtt->guptasklock);  in amdgpu_ttm_tt_set_userptr()
1327 	INIT_LIST_HEAD(&gtt->guptasks);  in amdgpu_ttm_tt_set_userptr()
1328 	atomic_set(&gtt->mmu_invalidations, 0);  in amdgpu_ttm_tt_set_userptr()
1329 	gtt->last_set_pages = 0;  in amdgpu_ttm_tt_set_userptr()
1339 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_get_usermm()  local
1341 	if (gtt == NULL)  in amdgpu_ttm_tt_get_usermm()
1344 	if (gtt->usertask == NULL)  in amdgpu_ttm_tt_get_usermm()
1347 	return gtt->usertask->mm;  in amdgpu_ttm_tt_get_usermm()
1358 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_affect_userptr()  local
1362 	if (gtt == NULL || !gtt->userptr)  in amdgpu_ttm_tt_affect_userptr()
1368 	size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;  in amdgpu_ttm_tt_affect_userptr()
1369 	if (gtt->userptr > end || gtt->userptr + size <= start)  in amdgpu_ttm_tt_affect_userptr()
1375 	spin_lock(&gtt->guptasklock);  in amdgpu_ttm_tt_affect_userptr()
1376 	list_for_each_entry(entry, &gtt->guptasks, list) {  in amdgpu_ttm_tt_affect_userptr()
1378 			spin_unlock(&gtt->guptasklock);  in amdgpu_ttm_tt_affect_userptr()
1382 	spin_unlock(&gtt->guptasklock);  in amdgpu_ttm_tt_affect_userptr()
1384 	atomic_inc(&gtt->mmu_invalidations);  in amdgpu_ttm_tt_affect_userptr()
1395 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_userptr_invalidated()  local
1398 	*last_invalidated = atomic_read(&gtt->mmu_invalidations);  in amdgpu_ttm_tt_userptr_invalidated()
1408 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_userptr_needs_pages()  local
1410 	if (gtt == NULL || !gtt->userptr)  in amdgpu_ttm_tt_userptr_needs_pages()
1413 	return atomic_read(&gtt->mmu_invalidations) != gtt->last_set_pages;  in amdgpu_ttm_tt_userptr_needs_pages()
1421 	struct amdgpu_ttm_tt *gtt = (void *)ttm;  in amdgpu_ttm_tt_is_readonly()  local
1423 	if (gtt == NULL)  in amdgpu_ttm_tt_is_readonly()
1426 	return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);  in amdgpu_ttm_tt_is_readonly()
1959 	struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;  in amdgpu_map_buffer()  local
1998 	dma_address = &gtt->ttm.dma_address[offset >> PAGE_SHIFT];  in amdgpu_map_buffer()