37  * for the entire GPU, there are multiple VM page tables active
38  * at any given time.  The VM page tables can contain a mix
42  * Each VM has an ID associated with it and there is a page table
78  * radeon_vm_manager_init - init the vm manager
82  * Init the vm manager (cayman+).
100  * radeon_vm_manager_fini - tear down the vm manager
104  * Tear down the VM manager (cayman+).
120  * radeon_vm_get_bos - add the vm BOs to a validation list
122  * @vm: vm providing the BOs
129 					  struct radeon_vm *vm,  in radeon_vm_get_bos()  argument
135 	list = kvmalloc_array(vm->max_pde_used + 2,  in radeon_vm_get_bos()
140 	/* add the vm page table to the list */  in radeon_vm_get_bos()
141 	list[0].robj = vm->page_directory;  in radeon_vm_get_bos()
144 	list[0].tv.bo = &vm->page_directory->tbo;  in radeon_vm_get_bos()
149 	for (i = 0, idx = 1; i <= vm->max_pde_used; i++) {  in radeon_vm_get_bos()
150 		if (!vm->page_tables[i].bo)  in radeon_vm_get_bos()
153 		list[idx].robj = vm->page_tables[i].bo;  in radeon_vm_get_bos()
169  * @vm: vm to allocate id for
172  * Allocate an id for the vm (cayman+).
178 				       struct radeon_vm *vm, int ring)  in radeon_vm_grab_id()  argument
181 	struct radeon_vm_id *vm_id = &vm->ids[ring];  in radeon_vm_grab_id()
194 	/* skip over VMID 0, since it is the system VM */  in radeon_vm_grab_id()
225  * radeon_vm_flush - hardware flush the vm
228  * @vm: vm we want to flush
230  * @updates: last vm update that is waited for
232  * Flush the vm (cayman+).
237 		     struct radeon_vm *vm,  in radeon_vm_flush()  argument
240 	uint64_t pd_addr = radeon_bo_gpu_offset(vm->page_directory);  in radeon_vm_flush()
241 	struct radeon_vm_id *vm_id = &vm->ids[ring];  in radeon_vm_flush()
246 		trace_radeon_vm_flush(pd_addr, ring, vm->ids[ring].id);  in radeon_vm_flush()
257  * radeon_vm_fence - remember fence for vm
260  * @vm: vm we want to fence
263  * Fence the vm (cayman+).
269 		     struct radeon_vm *vm,  in radeon_vm_fence()  argument
272 	unsigned vm_id = vm->ids[fence->ring].id;  in radeon_vm_fence()
277 	radeon_fence_unref(&vm->ids[fence->ring].last_id_use);  in radeon_vm_fence()
278 	vm->ids[fence->ring].last_id_use = radeon_fence_ref(fence);  in radeon_vm_fence()
282  * radeon_vm_bo_find - find the bo_va for a specific vm & bo
284  * @vm: requested vm
287  * Find @bo inside the requested vm (cayman+).
288  * Search inside the @bos vm list for the requested vm
293 struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm,  in radeon_vm_bo_find()  argument
299 		if (bo_va->vm == vm)  in radeon_vm_bo_find()
307  * radeon_vm_bo_add - add a bo to a specific vm
310  * @vm: requested vm
313  * Add @bo into the requested vm (cayman+).
314  * Add @bo to the list of bos associated with the vm
320 				      struct radeon_vm *vm,  in radeon_vm_bo_add()  argument
329 	bo_va->vm = vm;  in radeon_vm_bo_add()
338 	mutex_lock(&vm->mutex);  in radeon_vm_bo_add()
340 	mutex_unlock(&vm->mutex);  in radeon_vm_bo_add()
433  * radeon_vm_bo_set_addr - set bos virtual address inside a vm
437  * @soffset: requested offset of the buffer in the VM address space
441  * Validate and set the offset requested within the vm address space.
452 	struct radeon_vm *vm = bo_va->vm;  in radeon_vm_bo_set_addr()  local
477 	mutex_lock(&vm->mutex);  in radeon_vm_bo_set_addr()
482 		it = interval_tree_iter_first(&vm->va, soffset, eoffset);  in radeon_vm_bo_set_addr()
490 			mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
501 			mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
507 		tmp->vm = vm;  in radeon_vm_bo_set_addr()
510 		interval_tree_remove(&bo_va->it, &vm->va);  in radeon_vm_bo_set_addr()
511 		spin_lock(&vm->status_lock);  in radeon_vm_bo_set_addr()
515 		list_add(&tmp->vm_status, &vm->freed);  in radeon_vm_bo_set_addr()
516 		spin_unlock(&vm->status_lock);  in radeon_vm_bo_set_addr()
520 		spin_lock(&vm->status_lock);  in radeon_vm_bo_set_addr()
523 		list_add(&bo_va->vm_status, &vm->cleared);  in radeon_vm_bo_set_addr()
524 		spin_unlock(&vm->status_lock);  in radeon_vm_bo_set_addr()
525 		interval_tree_insert(&bo_va->it, &vm->va);  in radeon_vm_bo_set_addr()
535 	if (eoffset > vm->max_pde_used)  in radeon_vm_bo_set_addr()
536 		vm->max_pde_used = eoffset;  in radeon_vm_bo_set_addr()
544 		if (vm->page_tables[pt_idx].bo)  in radeon_vm_bo_set_addr()
548 		mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
564 		mutex_lock(&vm->mutex);  in radeon_vm_bo_set_addr()
565 		if (vm->page_tables[pt_idx].bo) {  in radeon_vm_bo_set_addr()
567 			mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
569 			mutex_lock(&vm->mutex);  in radeon_vm_bo_set_addr()
573 		vm->page_tables[pt_idx].addr = 0;  in radeon_vm_bo_set_addr()
574 		vm->page_tables[pt_idx].bo = pt;  in radeon_vm_bo_set_addr()
577 	mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
631  * @vm: requested vm
642 				    struct radeon_vm *vm)  in radeon_vm_update_page_directory()  argument
644 	struct radeon_bo *pd = vm->page_directory;  in radeon_vm_update_page_directory()
656 	ndw += vm->max_pde_used * 6;  in radeon_vm_update_page_directory()
668 	for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {  in radeon_vm_update_page_directory()
669 		struct radeon_bo *bo = vm->page_tables[pt_idx].bo;  in radeon_vm_update_page_directory()
676 		if (vm->page_tables[pt_idx].addr == pt)  in radeon_vm_update_page_directory()
678 		vm->page_tables[pt_idx].addr = pt;  in radeon_vm_update_page_directory()
804  * @vm: requested vm
815 				 struct radeon_vm *vm,  in radeon_vm_update_ptes()  argument
828 		struct radeon_bo *pt = vm->page_tables[pt_idx].bo;  in radeon_vm_update_ptes()
877  * @vm: requested vm
886 static void radeon_vm_fence_pts(struct radeon_vm *vm,  in radeon_vm_fence_pts()  argument
896 		radeon_bo_fence(vm->page_tables[i].bo, fence, true);  in radeon_vm_fence_pts()
900  * radeon_vm_bo_update - map a bo into the vm page table
903  * @vm: requested vm
916 	struct radeon_vm *vm = bo_va->vm;  in radeon_vm_bo_update()  local
924 		dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n",  in radeon_vm_bo_update()
925 			bo_va->bo, vm);  in radeon_vm_bo_update()
929 	spin_lock(&vm->status_lock);  in radeon_vm_bo_update()
932 			spin_unlock(&vm->status_lock);  in radeon_vm_bo_update()
938 		list_add(&bo_va->vm_status, &vm->cleared);  in radeon_vm_bo_update()
940 	spin_unlock(&vm->status_lock);  in radeon_vm_bo_update()
1009 			radeon_sync_fence(&ib.sync, vm->ids[i].last_id_use);  in radeon_vm_bo_update()
1012 	r = radeon_vm_update_ptes(rdev, vm, &ib, bo_va->it.start,  in radeon_vm_bo_update()
1029 	radeon_vm_fence_pts(vm, bo_va->it.start, bo_va->it.last + 1, ib.fence);  in radeon_vm_bo_update()
1041  * @vm: requested vm
1049 			  struct radeon_vm *vm)  in radeon_vm_clear_freed()  argument
1054 	spin_lock(&vm->status_lock);  in radeon_vm_clear_freed()
1055 	while (!list_empty(&vm->freed)) {  in radeon_vm_clear_freed()
1056 		bo_va = list_first_entry(&vm->freed,  in radeon_vm_clear_freed()
1058 		spin_unlock(&vm->status_lock);  in radeon_vm_clear_freed()
1063 		spin_lock(&vm->status_lock);  in radeon_vm_clear_freed()
1070 	spin_unlock(&vm->status_lock);  in radeon_vm_clear_freed()
1079  * @vm: requested vm
1087 			     struct radeon_vm *vm)  in radeon_vm_clear_invalids()  argument
1092 	spin_lock(&vm->status_lock);  in radeon_vm_clear_invalids()
1093 	while (!list_empty(&vm->invalidated)) {  in radeon_vm_clear_invalids()
1094 		bo_va = list_first_entry(&vm->invalidated,  in radeon_vm_clear_invalids()
1096 		spin_unlock(&vm->status_lock);  in radeon_vm_clear_invalids()
1102 		spin_lock(&vm->status_lock);  in radeon_vm_clear_invalids()
1104 	spin_unlock(&vm->status_lock);  in radeon_vm_clear_invalids()
1110  * radeon_vm_bo_rmv - remove a bo to a specific vm
1115  * Remove @bo_va->bo from the requested vm (cayman+).
1122 	struct radeon_vm *vm = bo_va->vm;  in radeon_vm_bo_rmv()  local
1126 	mutex_lock(&vm->mutex);  in radeon_vm_bo_rmv()
1128 		interval_tree_remove(&bo_va->it, &vm->va);  in radeon_vm_bo_rmv()
1130 	spin_lock(&vm->status_lock);  in radeon_vm_bo_rmv()
1134 		list_add(&bo_va->vm_status, &vm->freed);  in radeon_vm_bo_rmv()
1139 	spin_unlock(&vm->status_lock);  in radeon_vm_bo_rmv()
1141 	mutex_unlock(&vm->mutex);  in radeon_vm_bo_rmv()
1148  * @vm: requested vm
1159 		spin_lock(&bo_va->vm->status_lock);  in radeon_vm_bo_invalidate()
1162 			list_add(&bo_va->vm_status, &bo_va->vm->invalidated);  in radeon_vm_bo_invalidate()
1163 		spin_unlock(&bo_va->vm->status_lock);  in radeon_vm_bo_invalidate()
1168  * radeon_vm_init - initialize a vm instance
1171  * @vm: requested vm
1173  * Init @vm fields (cayman+).
1175 int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)  in radeon_vm_init()  argument
1182 	vm->ib_bo_va = NULL;  in radeon_vm_init()
1184 		vm->ids[i].id = 0;  in radeon_vm_init()
1185 		vm->ids[i].flushed_updates = NULL;  in radeon_vm_init()
1186 		vm->ids[i].last_id_use = NULL;  in radeon_vm_init()
1188 	mutex_init(&vm->mutex);  in radeon_vm_init()
1189 	vm->va = RB_ROOT_CACHED;  in radeon_vm_init()
1190 	spin_lock_init(&vm->status_lock);  in radeon_vm_init()
1191 	INIT_LIST_HEAD(&vm->invalidated);  in radeon_vm_init()
1192 	INIT_LIST_HEAD(&vm->freed);  in radeon_vm_init()
1193 	INIT_LIST_HEAD(&vm->cleared);  in radeon_vm_init()
1200 	vm->page_tables = kzalloc(pts_size, GFP_KERNEL);  in radeon_vm_init()
1201 	if (vm->page_tables == NULL) {  in radeon_vm_init()
1208 			     NULL, &vm->page_directory);  in radeon_vm_init()
1212 	r = radeon_vm_clear_bo(rdev, vm->page_directory);  in radeon_vm_init()
1214 		radeon_bo_unref(&vm->page_directory);  in radeon_vm_init()
1215 		vm->page_directory = NULL;  in radeon_vm_init()
1223  * radeon_vm_fini - tear down a vm instance
1226  * @vm: requested vm
1228  * Tear down @vm (cayman+).
1229  * Unbind the VM and remove all bos from the vm bo list
1231 void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)  in radeon_vm_fini()  argument
1236 	if (!RB_EMPTY_ROOT(&vm->va.rb_root))  in radeon_vm_fini()
1237 		dev_err(rdev->dev, "still active bo inside vm\n");  in radeon_vm_fini()
1240 					     &vm->va.rb_root, it.rb) {  in radeon_vm_fini()
1241 		interval_tree_remove(&bo_va->it, &vm->va);  in radeon_vm_fini()
1250 	list_for_each_entry_safe(bo_va, tmp, &vm->freed, vm_status) {  in radeon_vm_fini()
1257 		radeon_bo_unref(&vm->page_tables[i].bo);  in radeon_vm_fini()
1258 	kfree(vm->page_tables);  in radeon_vm_fini()
1260 	radeon_bo_unref(&vm->page_directory);  in radeon_vm_fini()
1263 		radeon_fence_unref(&vm->ids[i].flushed_updates);  in radeon_vm_fini()
1264 		radeon_fence_unref(&vm->ids[i].last_id_use);  in radeon_vm_fini()
1267 	mutex_destroy(&vm->mutex);  in radeon_vm_fini()