Lines Matching full:vm
46 * for the entire GPU, there are multiple VM page tables active
47 * at any given time. The VM page tables can contain a mix
51 * Each VM has an ID associated with it and there is a page table
88 * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
92 static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm) in amdgpu_vm_eviction_lock() argument
94 mutex_lock(&vm->eviction_lock); in amdgpu_vm_eviction_lock()
95 vm->saved_flags = memalloc_nofs_save(); in amdgpu_vm_eviction_lock()
98 static inline int amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm) in amdgpu_vm_eviction_trylock() argument
100 if (mutex_trylock(&vm->eviction_lock)) { in amdgpu_vm_eviction_trylock()
101 vm->saved_flags = memalloc_nofs_save(); in amdgpu_vm_eviction_trylock()
107 static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm) in amdgpu_vm_eviction_unlock() argument
109 memalloc_nofs_restore(vm->saved_flags); in amdgpu_vm_eviction_unlock()
110 mutex_unlock(&vm->eviction_lock); in amdgpu_vm_eviction_unlock()
220 * State for PDs/PTs and per VM BOs which are not at the location they should
225 struct amdgpu_vm *vm = vm_bo->vm; in amdgpu_vm_bo_evicted() local
230 list_move(&vm_bo->vm_status, &vm->evicted); in amdgpu_vm_bo_evicted()
232 list_move_tail(&vm_bo->vm_status, &vm->evicted); in amdgpu_vm_bo_evicted()
239 * State for per VM BOs which are moved, but that change is not yet reflected
244 list_move(&vm_bo->vm_status, &vm_bo->vm->moved); in amdgpu_vm_bo_moved()
252 * State for PDs/PTs and per VM BOs which have gone through the state machine
257 list_move(&vm_bo->vm_status, &vm_bo->vm->idle); in amdgpu_vm_bo_idle()
271 spin_lock(&vm_bo->vm->invalidated_lock); in amdgpu_vm_bo_invalidated()
272 list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated); in amdgpu_vm_bo_invalidated()
273 spin_unlock(&vm_bo->vm->invalidated_lock); in amdgpu_vm_bo_invalidated()
287 list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); in amdgpu_vm_bo_relocated()
302 spin_lock(&vm_bo->vm->invalidated_lock); in amdgpu_vm_bo_done()
304 spin_unlock(&vm_bo->vm->invalidated_lock); in amdgpu_vm_bo_done()
308 * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
310 * @base: base structure for tracking BO usage in a VM
311 * @vm: vm to which bo is to be added
318 struct amdgpu_vm *vm, in amdgpu_vm_bo_base_init() argument
321 base->vm = vm; in amdgpu_vm_bo_base_init()
331 if (bo->tbo.base.resv != vm->root.base.bo->tbo.base.resv) in amdgpu_vm_bo_base_init()
334 vm->bulk_moveable = false; in amdgpu_vm_bo_base_init()
345 * we checked all the prerequisites, but it looks like this per vm bo in amdgpu_vm_bo_base_init()
347 * is validated on next vm use to avoid fault. in amdgpu_vm_bo_base_init()
384 * @vm: amdgpu_vm structure
391 struct amdgpu_vm *vm, uint64_t start, in amdgpu_vm_pt_start() argument
396 cursor->entry = &vm->root; in amdgpu_vm_pt_start()
510 * @vm: amdgpu_vm structure
517 struct amdgpu_vm *vm, in amdgpu_vm_pt_first_dfs() argument
524 amdgpu_vm_pt_start(adev, vm, 0, cursor); in amdgpu_vm_pt_first_dfs()
568 #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) \ argument
569 for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)), \
575 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
577 * @vm: vm providing the BOs
584 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, in amdgpu_vm_get_pd_bo() argument
589 entry->tv.bo = &vm->root.base.bo->tbo; in amdgpu_vm_get_pd_bo()
590 /* Two for VM updates, one for TTM and one for the CS job */ in amdgpu_vm_get_pd_bo()
619 struct amdgpu_vm *vm = bo_base->vm; in amdgpu_vm_del_from_lru_notify() local
621 if (abo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) in amdgpu_vm_del_from_lru_notify()
622 vm->bulk_moveable = false; in amdgpu_vm_del_from_lru_notify()
630 * @vm: vm providing the BOs
636 struct amdgpu_vm *vm) in amdgpu_vm_move_to_lru_tail() argument
640 if (vm->bulk_moveable) { in amdgpu_vm_move_to_lru_tail()
642 ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move); in amdgpu_vm_move_to_lru_tail()
647 memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move)); in amdgpu_vm_move_to_lru_tail()
650 list_for_each_entry(bo_base, &vm->idle, vm_status) { in amdgpu_vm_move_to_lru_tail()
656 ttm_bo_move_to_lru_tail(&bo->tbo, &vm->lru_bulk_move); in amdgpu_vm_move_to_lru_tail()
659 &vm->lru_bulk_move); in amdgpu_vm_move_to_lru_tail()
663 vm->bulk_moveable = true; in amdgpu_vm_move_to_lru_tail()
670 * @vm: vm providing the BOs
679 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, in amdgpu_vm_validate_pt_bos() argument
686 vm->bulk_moveable &= list_empty(&vm->evicted); in amdgpu_vm_validate_pt_bos()
688 list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) { in amdgpu_vm_validate_pt_bos()
698 vm->update_funcs->map_table(bo); in amdgpu_vm_validate_pt_bos()
703 amdgpu_vm_eviction_lock(vm); in amdgpu_vm_validate_pt_bos()
704 vm->evicting = false; in amdgpu_vm_validate_pt_bos()
705 amdgpu_vm_eviction_unlock(vm); in amdgpu_vm_validate_pt_bos()
711 * amdgpu_vm_ready - check VM is ready for updates
713 * @vm: VM to check
715 * Check if all VM PDs/PTs are ready for updates
720 bool amdgpu_vm_ready(struct amdgpu_vm *vm) in amdgpu_vm_ready() argument
722 return list_empty(&vm->evicted); in amdgpu_vm_ready()
729 * @vm: VM to clear BO from
739 struct amdgpu_vm *vm, in amdgpu_vm_clear_bo() argument
761 if (!vm->pte_support_ats) { in amdgpu_vm_clear_bo()
774 if ((pt - vm->root.entries) >= ats_entries) { in amdgpu_vm_clear_bo()
793 r = vm->update_funcs->map_table(bo); in amdgpu_vm_clear_bo()
799 params.vm = vm; in amdgpu_vm_clear_bo()
802 r = vm->update_funcs->prepare(¶ms, NULL, AMDGPU_SYNC_EXPLICIT); in amdgpu_vm_clear_bo()
817 r = vm->update_funcs->update(¶ms, bo, addr, 0, ats_entries, in amdgpu_vm_clear_bo()
840 r = vm->update_funcs->update(¶ms, bo, addr, 0, entries, in amdgpu_vm_clear_bo()
846 return vm->update_funcs->commit(¶ms, NULL); in amdgpu_vm_clear_bo()
853 * @vm: requesting vm
858 static void amdgpu_vm_bo_param(struct amdgpu_device *adev, struct amdgpu_vm *vm, in amdgpu_vm_bo_param() argument
870 if (vm->use_cpu_for_update) in amdgpu_vm_bo_param()
872 else if (!vm->root.base.bo || vm->root.base.bo->shadow) in amdgpu_vm_bo_param()
876 if (vm->root.base.bo) in amdgpu_vm_bo_param()
877 bp->resv = vm->root.base.bo->tbo.base.resv; in amdgpu_vm_bo_param()
884 * @vm: VM to allocate page tables for
895 struct amdgpu_vm *vm, in amdgpu_vm_alloc_pts() argument
918 amdgpu_vm_bo_param(adev, vm, cursor->level, immediate, &bp); in amdgpu_vm_alloc_pts()
928 amdgpu_vm_bo_base_init(&entry->base, vm, pt); in amdgpu_vm_alloc_pts()
930 r = amdgpu_vm_clear_bo(adev, vm, pt, immediate); in amdgpu_vm_alloc_pts()
963 * @vm: amdgpu vm structure
969 struct amdgpu_vm *vm, in amdgpu_vm_free_pts() argument
975 vm->bulk_moveable = false; in amdgpu_vm_free_pts()
977 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) in amdgpu_vm_free_pts()
985 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
1000 /* Compute has a VM bug for GFX version < 7. in amdgpu_vm_check_compute_bug()
1001 Compute has a VM bug for GFX 8 MEC firmware version < 673.*/ in amdgpu_vm_check_compute_bug()
1056 * amdgpu_vm_flush - hardware flush the vm
1062 * Emit a VM flush when it is necessary.
1085 bool update_spm_vmid_needed = (job->vm && (job->vm->reserved_vmid[vmhub] != NULL)); in amdgpu_vm_flush()
1175 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
1177 * @vm: requested vm
1180 * Find @bo inside the requested vm.
1181 * Search inside the @bos vm list for the requested vm
1189 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, in amdgpu_vm_bo_find() argument
1195 if (base->vm != vm) in amdgpu_vm_bo_find()
1234 * @vm: requested vm
1240 struct amdgpu_vm *vm, in amdgpu_vm_update_pde() argument
1254 return vm->update_funcs->update(params, bo, pde, pt, 1, 0, flags); in amdgpu_vm_update_pde()
1261 * @vm: related vm
1266 struct amdgpu_vm *vm) in amdgpu_vm_invalidate_pds() argument
1271 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry) in amdgpu_vm_invalidate_pds()
1280 * @vm: requested vm
1289 struct amdgpu_vm *vm, bool immediate) in amdgpu_vm_update_pdes() argument
1294 if (list_empty(&vm->relocated)) in amdgpu_vm_update_pdes()
1299 params.vm = vm; in amdgpu_vm_update_pdes()
1302 r = vm->update_funcs->prepare(¶ms, NULL, AMDGPU_SYNC_EXPLICIT); in amdgpu_vm_update_pdes()
1306 while (!list_empty(&vm->relocated)) { in amdgpu_vm_update_pdes()
1309 entry = list_first_entry(&vm->relocated, struct amdgpu_vm_pt, in amdgpu_vm_update_pdes()
1313 r = amdgpu_vm_update_pde(¶ms, vm, entry); in amdgpu_vm_update_pdes()
1318 r = vm->update_funcs->commit(¶ms, &vm->last_update); in amdgpu_vm_update_pdes()
1324 amdgpu_vm_invalidate_pds(adev, vm); in amdgpu_vm_update_pdes()
1352 params->vm->update_funcs->update(params, bo, pe, addr, count, incr, in amdgpu_vm_update_flags()
1445 amdgpu_vm_pt_start(adev, params->vm, start, &cursor); in amdgpu_vm_update_ptes()
1455 r = amdgpu_vm_alloc_pts(params->adev, params->vm, in amdgpu_vm_update_ptes()
1518 struct amdgpu_vm *vm = params->vm; in amdgpu_vm_update_ptes() local
1530 vm->task_info.pid, in amdgpu_vm_update_ptes()
1531 vm->immediate.fence_context); in amdgpu_vm_update_ptes()
1557 amdgpu_vm_free_pts(adev, params->vm, &cursor); in amdgpu_vm_update_ptes()
1571 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
1574 * @vm: requested vm
1591 struct amdgpu_vm *vm, bool immediate, in amdgpu_vm_bo_update_mapping() argument
1604 params.vm = vm; in amdgpu_vm_bo_update_mapping()
1609 /* Implicitly sync to command submissions in the same VM before in amdgpu_vm_bo_update_mapping()
1617 amdgpu_vm_eviction_lock(vm); in amdgpu_vm_bo_update_mapping()
1618 if (vm->evicting) { in amdgpu_vm_bo_update_mapping()
1623 if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) { in amdgpu_vm_bo_update_mapping()
1626 amdgpu_bo_fence(vm->root.base.bo, vm->last_unlocked, true); in amdgpu_vm_bo_update_mapping()
1627 swap(vm->last_unlocked, tmp); in amdgpu_vm_bo_update_mapping()
1631 r = vm->update_funcs->prepare(¶ms, resv, sync_mode); in amdgpu_vm_bo_update_mapping()
1639 r = vm->update_funcs->commit(¶ms, fence); in amdgpu_vm_bo_update_mapping()
1642 amdgpu_vm_eviction_unlock(vm); in amdgpu_vm_bo_update_mapping()
1652 * @vm: requested vm
1668 struct amdgpu_vm *vm, in amdgpu_vm_bo_split_mapping() argument
1742 r = amdgpu_vm_bo_update_mapping(adev, vm, false, false, resv, in amdgpu_vm_bo_split_mapping()
1761 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1764 * @bo_va: requested BO and VM object
1776 struct amdgpu_vm *vm = bo_va->base.vm; in amdgpu_vm_bo_update() local
1790 resv = vm->root.base.bo->tbo.base.resv; in amdgpu_vm_bo_update()
1824 vm->root.base.bo->tbo.base.resv)) in amdgpu_vm_bo_update()
1825 last_update = &vm->last_update; in amdgpu_vm_bo_update()
1838 r = amdgpu_vm_bo_split_mapping(adev, resv, pages_addr, vm, in amdgpu_vm_bo_update()
1849 if (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) { in amdgpu_vm_bo_update()
1960 * @vm: requested vm
1967 struct amdgpu_vm *vm, in amdgpu_vm_free_mapping() argument
1980 * @vm: requested vm
1982 * Register a cleanup callback to disable PRT support after VM dies.
1984 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) in amdgpu_vm_prt_fini() argument
1986 struct dma_resv *resv = vm->root.base.bo->tbo.base.resv; in amdgpu_vm_prt_fini()
2018 * @vm: requested vm
2030 struct amdgpu_vm *vm, in amdgpu_vm_clear_freed() argument
2033 struct dma_resv *resv = vm->root.base.bo->tbo.base.resv; in amdgpu_vm_clear_freed()
2039 while (!list_empty(&vm->freed)) { in amdgpu_vm_clear_freed()
2040 mapping = list_first_entry(&vm->freed, in amdgpu_vm_clear_freed()
2044 if (vm->pte_support_ats && in amdgpu_vm_clear_freed()
2048 r = amdgpu_vm_bo_update_mapping(adev, vm, false, false, resv, in amdgpu_vm_clear_freed()
2051 amdgpu_vm_free_mapping(adev, vm, mapping, f); in amdgpu_vm_clear_freed()
2073 * @vm: requested vm
2083 struct amdgpu_vm *vm) in amdgpu_vm_handle_moved() argument
2090 list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) { in amdgpu_vm_handle_moved()
2091 /* Per VM BOs never need to bo cleared in the page tables */ in amdgpu_vm_handle_moved()
2097 spin_lock(&vm->invalidated_lock); in amdgpu_vm_handle_moved()
2098 while (!list_empty(&vm->invalidated)) { in amdgpu_vm_handle_moved()
2099 bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, in amdgpu_vm_handle_moved()
2102 spin_unlock(&vm->invalidated_lock); in amdgpu_vm_handle_moved()
2117 spin_lock(&vm->invalidated_lock); in amdgpu_vm_handle_moved()
2119 spin_unlock(&vm->invalidated_lock); in amdgpu_vm_handle_moved()
2125 * amdgpu_vm_bo_add - add a bo to a specific vm
2128 * @vm: requested vm
2131 * Add @bo into the requested vm.
2132 * Add @bo to the list of bos associated with the vm
2140 struct amdgpu_vm *vm, in amdgpu_vm_bo_add() argument
2149 amdgpu_vm_bo_base_init(&bo_va->base, vm, bo); in amdgpu_vm_bo_add()
2181 struct amdgpu_vm *vm = bo_va->base.vm; in amdgpu_vm_bo_insert_map() local
2186 amdgpu_vm_it_insert(mapping, &vm->va); in amdgpu_vm_bo_insert_map()
2191 if (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv && in amdgpu_vm_bo_insert_map()
2193 list_move(&bo_va->base.vm_status, &vm->moved); in amdgpu_vm_bo_insert_map()
2199 * amdgpu_vm_bo_map - map bo inside a vm
2208 * Add a mapping of the BO at the specefied addr into the VM.
2222 struct amdgpu_vm *vm = bo_va->base.vm; in amdgpu_vm_bo_map() local
2240 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); in amdgpu_vm_bo_map()
2264 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
2273 * Add a mapping of the BO at the specefied addr into the VM. Replace existing
2308 r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size); in amdgpu_vm_bo_replace_map()
2328 * amdgpu_vm_bo_unmap - remove bo mapping from vm
2334 * Remove a mapping of the BO at the specefied addr from the VM.
2346 struct amdgpu_vm *vm = bo_va->base.vm; in amdgpu_vm_bo_unmap() local
2369 amdgpu_vm_it_remove(mapping, &vm->va); in amdgpu_vm_bo_unmap()
2374 list_add(&mapping->list, &vm->freed); in amdgpu_vm_bo_unmap()
2376 amdgpu_vm_free_mapping(adev, vm, mapping, in amdgpu_vm_bo_unmap()
2386 * @vm: VM structure to use
2396 struct amdgpu_vm *vm, in amdgpu_vm_bo_clear_mappings() argument
2421 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); in amdgpu_vm_bo_clear_mappings()
2452 amdgpu_vm_it_remove(tmp, &vm->va); in amdgpu_vm_bo_clear_mappings()
2461 list_add(&tmp->list, &vm->freed); in amdgpu_vm_bo_clear_mappings()
2467 amdgpu_vm_it_insert(before, &vm->va); in amdgpu_vm_bo_clear_mappings()
2476 amdgpu_vm_it_insert(after, &vm->va); in amdgpu_vm_bo_clear_mappings()
2489 * @vm: the requested VM
2498 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm, in amdgpu_vm_bo_lookup_mapping() argument
2501 return amdgpu_vm_it_iter_first(&vm->va, addr, addr); in amdgpu_vm_bo_lookup_mapping()
2507 * @vm: the requested vm
2512 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket) in amdgpu_vm_bo_trace_cs() argument
2519 for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping; in amdgpu_vm_bo_trace_cs()
2535 * amdgpu_vm_bo_rmv - remove a bo to a specific vm
2540 * Remove @bo_va->bo from the requested vm.
2549 struct amdgpu_vm *vm = bo_va->base.vm; in amdgpu_vm_bo_rmv() local
2553 if (bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) in amdgpu_vm_bo_rmv()
2554 vm->bulk_moveable = false; in amdgpu_vm_bo_rmv()
2566 spin_lock(&vm->invalidated_lock); in amdgpu_vm_bo_rmv()
2568 spin_unlock(&vm->invalidated_lock); in amdgpu_vm_bo_rmv()
2572 amdgpu_vm_it_remove(mapping, &vm->va); in amdgpu_vm_bo_rmv()
2575 list_add(&mapping->list, &vm->freed); in amdgpu_vm_bo_rmv()
2579 amdgpu_vm_it_remove(mapping, &vm->va); in amdgpu_vm_bo_rmv()
2580 amdgpu_vm_free_mapping(adev, vm, mapping, in amdgpu_vm_bo_rmv()
2593 * amdgpu_vm_evictable - check if we can evict a VM
2595 * @bo: A page table of the VM.
2597 * Check if it is possible to evict a VM.
2603 /* Page tables of a destroyed VM can go away immediately */ in amdgpu_vm_evictable()
2604 if (!bo_base || !bo_base->vm) in amdgpu_vm_evictable()
2607 /* Don't evict VM page tables while they are busy */ in amdgpu_vm_evictable()
2612 if (!amdgpu_vm_eviction_trylock(bo_base->vm)) in amdgpu_vm_evictable()
2615 /* Don't evict VM page tables while they are updated */ in amdgpu_vm_evictable()
2616 if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) { in amdgpu_vm_evictable()
2617 amdgpu_vm_eviction_unlock(bo_base->vm); in amdgpu_vm_evictable()
2621 bo_base->vm->evicting = true; in amdgpu_vm_evictable()
2622 amdgpu_vm_eviction_unlock(bo_base->vm); in amdgpu_vm_evictable()
2645 struct amdgpu_vm *vm = bo_base->vm; in amdgpu_vm_bo_invalidate() local
2647 if (evicted && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) { in amdgpu_vm_bo_invalidate()
2658 else if (bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) in amdgpu_vm_bo_invalidate()
2666 * amdgpu_vm_get_block_size - calculate VM page table size as power of two
2668 * @vm_size: VM size
2671 * VM page table as power of two
2687 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2690 * @min_vm_size: the minimum vm size in GB if it's set auto
2704 /* adjust vm size first */ in amdgpu_vm_adjust_size()
2708 dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n", in amdgpu_vm_adjust_size()
2716 /* Optimal VM size depends on the amount of physical in amdgpu_vm_adjust_size()
2723 * - On GFX8 and older, VM space can be segmented for in amdgpu_vm_adjust_size()
2729 * VM size with the given page table size. in amdgpu_vm_adjust_size()
2758 /* block size depends on vm size and hw setup*/ in amdgpu_vm_adjust_size()
2774 DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n", in amdgpu_vm_adjust_size()
2781 * amdgpu_vm_wait_idle - wait for the VM to become idle
2783 * @vm: VM object to wait for
2784 * @timeout: timeout to wait for VM to become idle
2786 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout) in amdgpu_vm_wait_idle() argument
2788 timeout = dma_resv_wait_timeout_rcu(vm->root.base.bo->tbo.base.resv, in amdgpu_vm_wait_idle()
2793 return dma_fence_wait_timeout(vm->last_unlocked, true, timeout); in amdgpu_vm_wait_idle()
2797 * amdgpu_vm_init - initialize a vm instance
2800 * @vm: requested vm
2804 * Init @vm fields.
2809 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, in amdgpu_vm_init() argument
2816 vm->va = RB_ROOT_CACHED; in amdgpu_vm_init()
2818 vm->reserved_vmid[i] = NULL; in amdgpu_vm_init()
2819 INIT_LIST_HEAD(&vm->evicted); in amdgpu_vm_init()
2820 INIT_LIST_HEAD(&vm->relocated); in amdgpu_vm_init()
2821 INIT_LIST_HEAD(&vm->moved); in amdgpu_vm_init()
2822 INIT_LIST_HEAD(&vm->idle); in amdgpu_vm_init()
2823 INIT_LIST_HEAD(&vm->invalidated); in amdgpu_vm_init()
2824 spin_lock_init(&vm->invalidated_lock); in amdgpu_vm_init()
2825 INIT_LIST_HEAD(&vm->freed); in amdgpu_vm_init()
2829 r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL, in amdgpu_vm_init()
2835 r = drm_sched_entity_init(&vm->delayed, DRM_SCHED_PRIORITY_NORMAL, in amdgpu_vm_init()
2841 vm->pte_support_ats = false; in amdgpu_vm_init()
2842 vm->is_compute_context = false; in amdgpu_vm_init()
2845 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & in amdgpu_vm_init()
2849 vm->pte_support_ats = true; in amdgpu_vm_init()
2851 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & in amdgpu_vm_init()
2854 DRM_DEBUG_DRIVER("VM update mode is %s\n", in amdgpu_vm_init()
2855 vm->use_cpu_for_update ? "CPU" : "SDMA"); in amdgpu_vm_init()
2856 WARN_ONCE((vm->use_cpu_for_update && in amdgpu_vm_init()
2858 "CPU update of VM recommended only for large BAR system\n"); in amdgpu_vm_init()
2860 if (vm->use_cpu_for_update) in amdgpu_vm_init()
2861 vm->update_funcs = &amdgpu_vm_cpu_funcs; in amdgpu_vm_init()
2863 vm->update_funcs = &amdgpu_vm_sdma_funcs; in amdgpu_vm_init()
2864 vm->last_update = NULL; in amdgpu_vm_init()
2865 vm->last_unlocked = dma_fence_get_stub(); in amdgpu_vm_init()
2867 mutex_init(&vm->eviction_lock); in amdgpu_vm_init()
2868 vm->evicting = false; in amdgpu_vm_init()
2870 amdgpu_vm_bo_param(adev, vm, adev->vm_manager.root_level, false, &bp); in amdgpu_vm_init()
2885 amdgpu_vm_bo_base_init(&vm->root.base, vm, root); in amdgpu_vm_init()
2887 r = amdgpu_vm_clear_bo(adev, vm, root, false); in amdgpu_vm_init()
2891 amdgpu_bo_unreserve(vm->root.base.bo); in amdgpu_vm_init()
2897 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1, in amdgpu_vm_init()
2903 vm->pasid = pasid; in amdgpu_vm_init()
2906 INIT_KFIFO(vm->faults); in amdgpu_vm_init()
2911 amdgpu_bo_unreserve(vm->root.base.bo); in amdgpu_vm_init()
2914 amdgpu_bo_unref(&vm->root.base.bo->shadow); in amdgpu_vm_init()
2915 amdgpu_bo_unref(&vm->root.base.bo); in amdgpu_vm_init()
2916 vm->root.base.bo = NULL; in amdgpu_vm_init()
2919 dma_fence_put(vm->last_unlocked); in amdgpu_vm_init()
2920 drm_sched_entity_destroy(&vm->delayed); in amdgpu_vm_init()
2923 drm_sched_entity_destroy(&vm->immediate); in amdgpu_vm_init()
2929 * amdgpu_vm_check_clean_reserved - check if a VM is clean
2932 * @vm: the VM to check
2936 * VM
2939 * 0 if this VM is clean
2942 struct amdgpu_vm *vm) in amdgpu_vm_check_clean_reserved() argument
2948 if (!(vm->root.entries)) in amdgpu_vm_check_clean_reserved()
2952 if (vm->root.entries[i].base.bo) in amdgpu_vm_check_clean_reserved()
2960 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
2963 * @vm: requested vm
2969 * Changes the following VM parameters:
2980 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, in amdgpu_vm_make_compute() argument
2986 r = amdgpu_bo_reserve(vm->root.base.bo, true); in amdgpu_vm_make_compute()
2991 r = amdgpu_vm_check_clean_reserved(adev, vm); in amdgpu_vm_make_compute()
2999 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1, in amdgpu_vm_make_compute()
3011 if (pte_support_ats != vm->pte_support_ats) { in amdgpu_vm_make_compute()
3012 vm->pte_support_ats = pte_support_ats; in amdgpu_vm_make_compute()
3013 r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo, false); in amdgpu_vm_make_compute()
3018 /* Update VM state */ in amdgpu_vm_make_compute()
3019 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & in amdgpu_vm_make_compute()
3021 DRM_DEBUG_DRIVER("VM update mode is %s\n", in amdgpu_vm_make_compute()
3022 vm->use_cpu_for_update ? "CPU" : "SDMA"); in amdgpu_vm_make_compute()
3023 WARN_ONCE((vm->use_cpu_for_update && in amdgpu_vm_make_compute()
3025 "CPU update of VM recommended only for large BAR system\n"); in amdgpu_vm_make_compute()
3027 if (vm->use_cpu_for_update) { in amdgpu_vm_make_compute()
3029 r = amdgpu_bo_sync_wait(vm->root.base.bo, in amdgpu_vm_make_compute()
3034 vm->update_funcs = &amdgpu_vm_cpu_funcs; in amdgpu_vm_make_compute()
3036 vm->update_funcs = &amdgpu_vm_sdma_funcs; in amdgpu_vm_make_compute()
3038 dma_fence_put(vm->last_update); in amdgpu_vm_make_compute()
3039 vm->last_update = NULL; in amdgpu_vm_make_compute()
3040 vm->is_compute_context = true; in amdgpu_vm_make_compute()
3042 if (vm->pasid) { in amdgpu_vm_make_compute()
3046 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); in amdgpu_vm_make_compute()
3052 amdgpu_pasid_free(vm->pasid); in amdgpu_vm_make_compute()
3053 vm->pasid = 0; in amdgpu_vm_make_compute()
3056 /* Free the shadow bo for compute VM */ in amdgpu_vm_make_compute()
3057 amdgpu_bo_unref(&vm->root.base.bo->shadow); in amdgpu_vm_make_compute()
3060 vm->pasid = pasid; in amdgpu_vm_make_compute()
3073 amdgpu_bo_unreserve(vm->root.base.bo); in amdgpu_vm_make_compute()
3078 * amdgpu_vm_release_compute - release a compute vm
3080 * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute
3083 * pasid from vm. Compute should stop use of vm after this call.
3085 void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm) in amdgpu_vm_release_compute() argument
3087 if (vm->pasid) { in amdgpu_vm_release_compute()
3091 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); in amdgpu_vm_release_compute()
3094 vm->pasid = 0; in amdgpu_vm_release_compute()
3095 vm->is_compute_context = false; in amdgpu_vm_release_compute()
3099 * amdgpu_vm_fini - tear down a vm instance
3102 * @vm: requested vm
3104 * Tear down @vm.
3105 * Unbind the VM and remove all bos from the vm bo list
3107 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) in amdgpu_vm_fini() argument
3114 amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm); in amdgpu_vm_fini()
3116 root = amdgpu_bo_ref(vm->root.base.bo); in amdgpu_vm_fini()
3118 if (vm->pasid) { in amdgpu_vm_fini()
3122 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); in amdgpu_vm_fini()
3124 vm->pasid = 0; in amdgpu_vm_fini()
3127 dma_fence_wait(vm->last_unlocked, false); in amdgpu_vm_fini()
3128 dma_fence_put(vm->last_unlocked); in amdgpu_vm_fini()
3130 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { in amdgpu_vm_fini()
3132 amdgpu_vm_prt_fini(adev, vm); in amdgpu_vm_fini()
3137 amdgpu_vm_free_mapping(adev, vm, mapping, NULL); in amdgpu_vm_fini()
3140 amdgpu_vm_free_pts(adev, vm, NULL); in amdgpu_vm_fini()
3143 WARN_ON(vm->root.base.bo); in amdgpu_vm_fini()
3145 drm_sched_entity_destroy(&vm->immediate); in amdgpu_vm_fini()
3146 drm_sched_entity_destroy(&vm->delayed); in amdgpu_vm_fini()
3148 if (!RB_EMPTY_ROOT(&vm->va.rb_root)) { in amdgpu_vm_fini()
3149 dev_err(adev->dev, "still active bo inside vm\n"); in amdgpu_vm_fini()
3152 &vm->va.rb_root, rb) { in amdgpu_vm_fini()
3160 dma_fence_put(vm->last_update); in amdgpu_vm_fini()
3162 amdgpu_vmid_free_reserved(adev, vm, i); in amdgpu_vm_fini()
3166 * amdgpu_vm_manager_init - init the VM manager
3170 * Initialize the VM manager structures
3187 * Compute VM tables will be updated by CPU in amdgpu_vm_manager_init()
3207 * amdgpu_vm_manager_fini - cleanup VM manager
3211 * Cleanup the VM manager and free resources.
3222 * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
3242 r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, in amdgpu_vm_ioctl()
3251 /* Wait vm idle to make sure the vmid set in SPM_VMID is in amdgpu_vm_ioctl()
3254 r = amdgpu_bo_reserve(fpriv->vm.root.base.bo, true); in amdgpu_vm_ioctl()
3258 r = amdgpu_vm_wait_idle(&fpriv->vm, timeout); in amdgpu_vm_ioctl()
3262 amdgpu_bo_unreserve(fpriv->vm.root.base.bo); in amdgpu_vm_ioctl()
3263 amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0); in amdgpu_vm_ioctl()
3276 * @pasid: PASID identifier for VM
3282 struct amdgpu_vm *vm; in amdgpu_vm_get_task_info() local
3287 vm = idr_find(&adev->vm_manager.pasid_idr, pasid); in amdgpu_vm_get_task_info()
3288 if (vm) in amdgpu_vm_get_task_info()
3289 *task_info = vm->task_info; in amdgpu_vm_get_task_info()
3297 * @vm: vm for which to set the info
3299 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm) in amdgpu_vm_set_task_info() argument
3301 if (vm->task_info.pid) in amdgpu_vm_set_task_info()
3304 vm->task_info.pid = current->pid; in amdgpu_vm_set_task_info()
3305 get_task_comm(vm->task_info.task_name, current); in amdgpu_vm_set_task_info()
3310 vm->task_info.tgid = current->group_leader->pid; in amdgpu_vm_set_task_info()
3311 get_task_comm(vm->task_info.process_name, current->group_leader); in amdgpu_vm_set_task_info()
3315 * amdgpu_vm_handle_fault - graceful handling of VM faults.
3317 * @pasid: PASID of the VM
3320 * Try to gracefully handle a VM fault. Return true if the fault was handled and
3328 struct amdgpu_vm *vm; in amdgpu_vm_handle_fault() local
3332 vm = idr_find(&adev->vm_manager.pasid_idr, pasid); in amdgpu_vm_handle_fault()
3333 if (vm) in amdgpu_vm_handle_fault()
3334 root = amdgpu_bo_ref(vm->root.base.bo); in amdgpu_vm_handle_fault()
3346 /* Double check that the VM still exists */ in amdgpu_vm_handle_fault()
3348 vm = idr_find(&adev->vm_manager.pasid_idr, pasid); in amdgpu_vm_handle_fault()
3349 if (vm && vm->root.base.bo != root) in amdgpu_vm_handle_fault()
3350 vm = NULL; in amdgpu_vm_handle_fault()
3352 if (!vm) in amdgpu_vm_handle_fault()
3359 if (vm->is_compute_context) { in amdgpu_vm_handle_fault()
3378 r = amdgpu_vm_bo_update_mapping(adev, vm, true, false, NULL, addr, in amdgpu_vm_handle_fault()
3383 r = amdgpu_vm_update_pdes(adev, vm, true); in amdgpu_vm_handle_fault()