81 		struct kgd_mem *mem)  in check_if_add_bo_to_vm()  argument
85 	list_for_each_entry(entry, &mem->bo_va_list, bo_list)  in check_if_add_bo_to_vm()
99 	uint64_t mem;  in amdgpu_amdkfd_gpuvm_init_mem_limits()  local
102 	mem = si.totalram - si.totalhigh;  in amdgpu_amdkfd_gpuvm_init_mem_limits()
103 	mem *= si.mem_unit;  in amdgpu_amdkfd_gpuvm_init_mem_limits()
106 	kfd_mem_limit.max_system_mem_limit = (mem >> 1) - (mem >> 3);  in amdgpu_amdkfd_gpuvm_init_mem_limits()
107 	kfd_mem_limit.max_userptr_mem_limit = mem - (mem >> 2);  in amdgpu_amdkfd_gpuvm_init_mem_limits()
443 static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem,  in add_bo_to_vm()  argument
450 	struct amdgpu_bo *bo = mem->bo;  in add_bo_to_vm()
451 	uint64_t va = mem->va;  in add_bo_to_vm()
452 	struct list_head *list_bo_va = &mem->bo_va_list;  in add_bo_to_vm()
453 	unsigned long bo_size = bo->tbo.mem.size;  in add_bo_to_vm()
481 							 mem->mapping_flags);  in add_bo_to_vm()
534 static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,  in add_kgd_mem_to_kfd_bo_list()  argument
538 	struct ttm_validate_buffer *entry = &mem->validate_list;  in add_kgd_mem_to_kfd_bo_list()
539 	struct amdgpu_bo *bo = mem->bo;  in add_kgd_mem_to_kfd_bo_list()
564 static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,  in init_user_pages()  argument
567 	struct amdkfd_process_info *process_info = mem->process_info;  in init_user_pages()
568 	struct amdgpu_bo *bo = mem->bo;  in init_user_pages()
590 	WARN(mem->user_pages, "Leaking user_pages array");  in init_user_pages()
592 	mem->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages,  in init_user_pages()
595 	if (!mem->user_pages) {  in init_user_pages()
601 	ret = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm, mem->user_pages);  in init_user_pages()
607 	amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, mem->user_pages);  in init_user_pages()
614 	amdgpu_bo_placement_from_domain(bo, mem->domain);  in init_user_pages()
622 		release_pages(mem->user_pages, bo->tbo.ttm->num_pages);  in init_user_pages()
624 	kvfree(mem->user_pages);  in init_user_pages()
625 	mem->user_pages = NULL;  in init_user_pages()
661 static int reserve_bo_and_vm(struct kgd_mem *mem,  in reserve_bo_and_vm()  argument
665 	struct amdgpu_bo *bo = mem->bo;  in reserve_bo_and_vm()
672 	ctx->sync = &mem->sync;  in reserve_bo_and_vm()
713 static int reserve_bo_and_cond_vms(struct kgd_mem *mem,  in reserve_bo_and_cond_vms()  argument
717 	struct amdgpu_bo *bo = mem->bo;  in reserve_bo_and_cond_vms()
725 	ctx->sync = &mem->sync;  in reserve_bo_and_cond_vms()
730 	list_for_each_entry(entry, &mem->bo_va_list, bo_list) {  in reserve_bo_and_cond_vms()
754 	list_for_each_entry(entry, &mem->bo_va_list, bo_list) {  in reserve_bo_and_cond_vms()
1129 		void *vm, struct kgd_mem **mem,  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()  argument
1166 	*mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1167 	if (!*mem)  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1169 	INIT_LIST_HEAD(&(*mem)->bo_va_list);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1170 	mutex_init(&(*mem)->lock);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1171 	(*mem)->aql_queue = !!(flags & ALLOC_MEM_FLAGS_AQL_QUEUE_MEM);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1177 	if ((*mem)->aql_queue)  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1196 	(*mem)->mapping_flags = mapping_flags;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1198 	amdgpu_sync_create(&(*mem)->sync);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1222 	bo->kfd_bo = *mem;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1223 	(*mem)->bo = bo;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1227 	(*mem)->va = va;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1228 	(*mem)->domain = domain;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1229 	(*mem)->mapped_to_gpu_memory = 0;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1230 	(*mem)->process_info = avm->process_info;  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1231 	add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1234 		ret = init_user_pages(*mem, current->mm, user_addr);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1237 			list_del(&(*mem)->validate_list.head);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1255 	mutex_destroy(&(*mem)->lock);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1256 	kfree(*mem);  in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1261 		struct kgd_dev *kgd, struct kgd_mem *mem)  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()  argument
1263 	struct amdkfd_process_info *process_info = mem->process_info;  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1264 	unsigned long bo_size = mem->bo->tbo.mem.size;  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1270 	mutex_lock(&mem->lock);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1272 	if (mem->mapped_to_gpu_memory > 0) {  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1274 				mem->va, bo_size);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1275 		mutex_unlock(&mem->lock);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1279 	mutex_unlock(&mem->lock);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1285 	amdgpu_mn_unregister(mem->bo);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1288 	bo_list_entry = &mem->validate_list;  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1294 	if (mem->user_pages) {  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1296 		if (mem->user_pages[0])  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1297 			release_pages(mem->user_pages,  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1298 					mem->bo->tbo.ttm->num_pages);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1299 		kvfree(mem->user_pages);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1302 	ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1310 	amdgpu_amdkfd_remove_eviction_fence(mem->bo,  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1313 	pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1314 		mem->va + bo_size * (1 + mem->aql_queue));  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1317 	list_for_each_entry_safe(entry, tmp, &mem->bo_va_list, bo_list)  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1324 	amdgpu_sync_free(&mem->sync);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1327 	amdgpu_bo_unref(&mem->bo);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1328 	mutex_destroy(&mem->lock);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1329 	kfree(mem);  in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1335 		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm)  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()  argument
1349 	bo = mem->bo;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1359 	mutex_lock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1367 		is_invalid_userptr = atomic_read(&mem->invalid);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1371 	mutex_lock(&mem->lock);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1373 	domain = mem->domain;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1374 	bo_size = bo->tbo.mem.size;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1377 			mem->va,  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1378 			mem->va + bo_size * (1 + mem->aql_queue),  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1381 	ret = reserve_bo_and_vm(mem, vm, &ctx);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1390 	if (bo->tbo.mem.mem_type == TTM_PL_SYSTEM)  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1393 	if (check_if_add_bo_to_vm(avm, mem)) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1394 		ret = add_bo_to_vm(adev, mem, avm, false,  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1398 		if (mem->aql_queue) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1399 			ret = add_bo_to_vm(adev, mem, avm,  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1410 	if (mem->mapped_to_gpu_memory == 0 &&  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1423 	list_for_each_entry(entry, &mem->bo_va_list, bo_list) {  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1443 			mem->mapped_to_gpu_memory++;  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1445 					mem->mapped_to_gpu_memory);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1466 	mutex_unlock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1467 	mutex_unlock(&mem->lock);  in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1472 		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm)  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()  argument
1477 	unsigned long bo_size = mem->bo->tbo.mem.size;  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1482 	mutex_lock(&mem->lock);  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1484 	ret = reserve_bo_and_cond_vms(mem, vm, BO_VM_MAPPED, &ctx);  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1498 		mem->va,  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1499 		mem->va + bo_size * (1 + mem->aql_queue),  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1502 	list_for_each_entry(entry, &mem->bo_va_list, bo_list) {  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1514 						mem->va);  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1518 			mem->mapped_to_gpu_memory--;  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1520 					mem->mapped_to_gpu_memory);  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1527 	if (mem->mapped_to_gpu_memory == 0 &&  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1528 	    !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && !mem->bo->pin_count)  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1529 		amdgpu_amdkfd_remove_eviction_fence(mem->bo,  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1536 	mutex_unlock(&mem->lock);  in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
1541 		struct kgd_dev *kgd, struct kgd_mem *mem, bool intr)  in amdgpu_amdkfd_gpuvm_sync_memory()  argument
1548 	mutex_lock(&mem->lock);  in amdgpu_amdkfd_gpuvm_sync_memory()
1549 	amdgpu_sync_clone(&mem->sync, &sync);  in amdgpu_amdkfd_gpuvm_sync_memory()
1550 	mutex_unlock(&mem->lock);  in amdgpu_amdkfd_gpuvm_sync_memory()
1558 		struct kgd_mem *mem, void **kptr, uint64_t *size)  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()  argument
1561 	struct amdgpu_bo *bo = mem->bo;  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1571 	mutex_lock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1592 		bo, mem->process_info->eviction_fence, NULL, NULL);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1593 	list_del_init(&mem->validate_list.head);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1600 	mutex_unlock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1608 	mutex_unlock(&mem->process_info->lock);  in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
1614 					      struct kfd_vm_fault_info *mem)  in amdgpu_amdkfd_gpuvm_get_vm_fault_info()  argument
1620 		*mem = *adev->gmc.vm_fault_info;  in amdgpu_amdkfd_gpuvm_get_vm_fault_info()
1638 int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem,  in amdgpu_amdkfd_evict_userptr()  argument
1641 	struct amdkfd_process_info *process_info = mem->process_info;  in amdgpu_amdkfd_evict_userptr()
1645 	invalid = atomic_inc_return(&mem->invalid);  in amdgpu_amdkfd_evict_userptr()
1668 	struct kgd_mem *mem, *tmp_mem;  in update_invalid_user_pages()  local
1676 	list_for_each_entry_safe(mem, tmp_mem,  in update_invalid_user_pages()
1679 		if (!atomic_read(&mem->invalid))  in update_invalid_user_pages()
1682 		bo = mem->bo;  in update_invalid_user_pages()
1695 		list_move_tail(&mem->validate_list.head,  in update_invalid_user_pages()
1703 	list_for_each_entry(mem, &process_info->userptr_inval_list,  in update_invalid_user_pages()
1705 		invalid = atomic_read(&mem->invalid);  in update_invalid_user_pages()
1712 		bo = mem->bo;  in update_invalid_user_pages()
1714 		if (!mem->user_pages) {  in update_invalid_user_pages()
1715 			mem->user_pages =  in update_invalid_user_pages()
1719 			if (!mem->user_pages) {  in update_invalid_user_pages()
1724 		} else if (mem->user_pages[0]) {  in update_invalid_user_pages()
1725 			release_pages(mem->user_pages, bo->tbo.ttm->num_pages);  in update_invalid_user_pages()
1730 						   mem->user_pages);  in update_invalid_user_pages()
1732 			mem->user_pages[0] = NULL;  in update_invalid_user_pages()
1745 		if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid)  in update_invalid_user_pages()
1766 	struct kgd_mem *mem, *tmp_mem;  in validate_invalid_user_pages()  local
1789 	list_for_each_entry(mem, &process_info->userptr_inval_list,  in validate_invalid_user_pages()
1791 		list_add_tail(&mem->resv_list.head, &resv_list);  in validate_invalid_user_pages()
1792 		mem->resv_list.bo = mem->validate_list.bo;  in validate_invalid_user_pages()
1793 		mem->resv_list.shared = mem->validate_list.shared;  in validate_invalid_user_pages()
1819 	list_for_each_entry_safe(mem, tmp_mem,  in validate_invalid_user_pages()
1824 		bo = mem->bo;  in validate_invalid_user_pages()
1827 		if (mem->user_pages[0]) {  in validate_invalid_user_pages()
1829 						     mem->user_pages);  in validate_invalid_user_pages()
1830 			amdgpu_bo_placement_from_domain(bo, mem->domain);  in validate_invalid_user_pages()
1843 		kvfree(mem->user_pages);  in validate_invalid_user_pages()
1844 		mem->user_pages = NULL;  in validate_invalid_user_pages()
1845 		list_move_tail(&mem->validate_list.head,  in validate_invalid_user_pages()
1854 		list_for_each_entry(bo_va_entry, &mem->bo_va_list, bo_list) {  in validate_invalid_user_pages()
1864 				atomic_inc(&mem->invalid);  in validate_invalid_user_pages()
1981 	struct kgd_mem *mem;  in amdgpu_amdkfd_gpuvm_restore_process_bos()  local
2007 	list_for_each_entry(mem, &process_info->kfd_bo_list,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2010 		list_add_tail(&mem->resv_list.head, &ctx.list);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2011 		mem->resv_list.bo = mem->validate_list.bo;  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2012 		mem->resv_list.shared = mem->validate_list.shared;  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2039 	list_for_each_entry(mem, &process_info->kfd_bo_list,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2042 		struct amdgpu_bo *bo = mem->bo;  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2043 		uint32_t domain = mem->domain;  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2052 		list_for_each_entry(bo_va_entry, &mem->bo_va_list,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2091 	list_for_each_entry(mem, &process_info->kfd_bo_list,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2093 		ttm_bo_wait(&mem->bo->tbo, false, false);  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2094 	list_for_each_entry(mem, &process_info->kfd_bo_list,  in amdgpu_amdkfd_gpuvm_restore_process_bos()
2096 		amdgpu_bo_fence(mem->bo,  in amdgpu_amdkfd_gpuvm_restore_process_bos()