58 	struct hl_vm_phys_pg_pack *phys_pg_pack;  in alloc_device_memory()  local
82 	phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);  in alloc_device_memory()
83 	if (!phys_pg_pack) {  in alloc_device_memory()
88 	phys_pg_pack->vm_type = VM_TYPE_PHYS_PACK;  in alloc_device_memory()
89 	phys_pg_pack->asid = ctx->asid;  in alloc_device_memory()
90 	phys_pg_pack->npages = num_pgs;  in alloc_device_memory()
91 	phys_pg_pack->page_size = page_size;  in alloc_device_memory()
92 	phys_pg_pack->total_size = total_size;  in alloc_device_memory()
93 	phys_pg_pack->flags = args->flags;  in alloc_device_memory()
94 	phys_pg_pack->contiguous = contiguous;  in alloc_device_memory()
96 	phys_pg_pack->pages = kvmalloc_array(num_pgs, sizeof(u64), GFP_KERNEL);  in alloc_device_memory()
97 	if (!phys_pg_pack->pages) {  in alloc_device_memory()
102 	if (phys_pg_pack->contiguous) {  in alloc_device_memory()
104 			phys_pg_pack->pages[i] = paddr + i * page_size;  in alloc_device_memory()
107 			phys_pg_pack->pages[i] = (u64) gen_pool_alloc(  in alloc_device_memory()
110 			if (!phys_pg_pack->pages[i]) {  in alloc_device_memory()
122 	handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0,  in alloc_device_memory()
135 	phys_pg_pack->handle = handle;  in alloc_device_memory()
137 	atomic64_add(phys_pg_pack->total_size, &ctx->dram_phys_mem);  in alloc_device_memory()
138 	atomic64_add(phys_pg_pack->total_size, &hdev->dram_used_mem);  in alloc_device_memory()
146 	if (!phys_pg_pack->contiguous)  in alloc_device_memory()
148 			gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[i],  in alloc_device_memory()
151 	kvfree(phys_pg_pack->pages);  in alloc_device_memory()
153 	kfree(phys_pg_pack);  in alloc_device_memory()
267 		struct hl_vm_phys_pg_pack *phys_pg_pack)  in free_phys_pg_pack()  argument
272 	if (!phys_pg_pack->created_from_userptr) {  in free_phys_pg_pack()
273 		if (phys_pg_pack->contiguous) {  in free_phys_pg_pack()
274 			gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[0],  in free_phys_pg_pack()
275 					phys_pg_pack->total_size);  in free_phys_pg_pack()
277 			for (i = 0; i < phys_pg_pack->npages ; i++)  in free_phys_pg_pack()
281 			for (i = 0 ; i < phys_pg_pack->npages ; i++) {  in free_phys_pg_pack()
283 						phys_pg_pack->pages[i],  in free_phys_pg_pack()
284 						phys_pg_pack->page_size);  in free_phys_pg_pack()
291 	kvfree(phys_pg_pack->pages);  in free_phys_pg_pack()
292 	kfree(phys_pg_pack);  in free_phys_pg_pack()
308 	struct hl_vm_phys_pg_pack *phys_pg_pack;  in free_device_memory()  local
311 	phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);  in free_device_memory()
312 	if (phys_pg_pack) {  in free_device_memory()
313 		if (atomic_read(&phys_pg_pack->mapping_cnt) > 0) {  in free_device_memory()
328 		atomic64_sub(phys_pg_pack->total_size, &ctx->dram_phys_mem);  in free_device_memory()
329 		atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem);  in free_device_memory()
331 		free_phys_pg_pack(hdev, phys_pg_pack);  in free_device_memory()
649 	struct hl_vm_phys_pg_pack *phys_pg_pack;  in init_phys_pg_pack_from_userptr()  local
657 	phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);  in init_phys_pg_pack_from_userptr()
658 	if (!phys_pg_pack)  in init_phys_pg_pack_from_userptr()
661 	phys_pg_pack->vm_type = userptr->vm_type;  in init_phys_pg_pack_from_userptr()
662 	phys_pg_pack->created_from_userptr = true;  in init_phys_pg_pack_from_userptr()
663 	phys_pg_pack->asid = ctx->asid;  in init_phys_pg_pack_from_userptr()
664 	atomic_set(&phys_pg_pack->mapping_cnt, 1);  in init_phys_pg_pack_from_userptr()
690 	phys_pg_pack->pages = kvmalloc_array(total_npages, sizeof(u64),  in init_phys_pg_pack_from_userptr()
692 	if (!phys_pg_pack->pages) {  in init_phys_pg_pack_from_userptr()
697 	phys_pg_pack->npages = total_npages;  in init_phys_pg_pack_from_userptr()
698 	phys_pg_pack->page_size = page_size;  in init_phys_pg_pack_from_userptr()
699 	phys_pg_pack->total_size = total_npages * page_size;  in init_phys_pg_pack_from_userptr()
708 			phys_pg_pack->offset = dma_addr & (page_size - 1);  in init_phys_pg_pack_from_userptr()
713 			phys_pg_pack->pages[j++] = dma_addr;  in init_phys_pg_pack_from_userptr()
723 	*pphys_pg_pack = phys_pg_pack;  in init_phys_pg_pack_from_userptr()
728 	kfree(phys_pg_pack);  in init_phys_pg_pack_from_userptr()
746 		struct hl_vm_phys_pg_pack *phys_pg_pack)  in map_phys_page_pack()  argument
750 	u32 page_size = phys_pg_pack->page_size;  in map_phys_page_pack()
753 	for (i = 0 ; i < phys_pg_pack->npages ; i++) {  in map_phys_page_pack()
754 		paddr = phys_pg_pack->pages[i];  in map_phys_page_pack()
760 				phys_pg_pack->handle, phys_pg_pack->npages,  in map_phys_page_pack()
777 					phys_pg_pack->handle, next_vaddr,  in map_phys_page_pack()
778 					phys_pg_pack->pages[i], page_size);  in map_phys_page_pack()
791 	struct hl_vm_phys_pg_pack *phys_pg_pack;  in get_paddr_from_handle()  local
796 	phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);  in get_paddr_from_handle()
797 	if (!phys_pg_pack) {  in get_paddr_from_handle()
803 	*paddr = phys_pg_pack->pages[0];  in get_paddr_from_handle()
829 	struct hl_vm_phys_pg_pack *phys_pg_pack;  in map_device_va()  local
849 				&phys_pg_pack);  in map_device_va()
863 		phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);  in map_device_va()
864 		if (!phys_pg_pack) {  in map_device_va()
872 		atomic_inc(&phys_pg_pack->mapping_cnt);  in map_device_va()
876 		vm_type = (enum vm_type_t *) phys_pg_pack;  in map_device_va()
885 	if (!is_userptr && !(phys_pg_pack->flags & HL_MEM_SHARED) &&  in map_device_va()
886 			phys_pg_pack->asid != ctx->asid) {  in map_device_va()
902 			phys_pg_pack->total_size, hint_addr, is_userptr);  in map_device_va()
912 	rc = map_phys_page_pack(ctx, ret_vaddr, phys_pg_pack);  in map_device_va()
924 	ret_vaddr += phys_pg_pack->offset;  in map_device_va()
936 		free_phys_pg_pack(hdev, phys_pg_pack);  in map_device_va()
944 			ret_vaddr + phys_pg_pack->total_size - 1))  in map_device_va()
953 	atomic_dec(&phys_pg_pack->mapping_cnt);  in map_device_va()
955 		free_phys_pg_pack(hdev, phys_pg_pack);  in map_device_va()
976 	struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;  in unmap_device_va()  local
1008 				&phys_pg_pack);  in unmap_device_va()
1017 		phys_pg_pack = hnode->ptr;  in unmap_device_va()
1026 	if (atomic_read(&phys_pg_pack->mapping_cnt) == 0) {  in unmap_device_va()
1032 	page_size = phys_pg_pack->page_size;  in unmap_device_va()
1039 	for (i = 0 ; i < phys_pg_pack->npages ; i++, next_vaddr += page_size) {  in unmap_device_va()
1058 			vaddr + phys_pg_pack->total_size - 1))  in unmap_device_va()
1062 	atomic_dec(&phys_pg_pack->mapping_cnt);  in unmap_device_va()
1066 		free_phys_pg_pack(hdev, phys_pg_pack);  in unmap_device_va()
1074 		free_phys_pg_pack(hdev, phys_pg_pack);  in unmap_device_va()