90 	struct hl_vm_phys_pg_pack *phys_pg_pack;  in alloc_device_memory()  local
126 	phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);  in alloc_device_memory()
127 	if (!phys_pg_pack) {  in alloc_device_memory()
132 	phys_pg_pack->vm_type = VM_TYPE_PHYS_PACK;  in alloc_device_memory()
133 	phys_pg_pack->asid = ctx->asid;  in alloc_device_memory()
134 	phys_pg_pack->npages = num_pgs;  in alloc_device_memory()
135 	phys_pg_pack->page_size = page_size;  in alloc_device_memory()
136 	phys_pg_pack->total_size = total_size;  in alloc_device_memory()
137 	phys_pg_pack->flags = args->flags;  in alloc_device_memory()
138 	phys_pg_pack->contiguous = contiguous;  in alloc_device_memory()
140 	phys_pg_pack->pages = kvmalloc_array(num_pgs, sizeof(u64), GFP_KERNEL);  in alloc_device_memory()
141 	if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) {  in alloc_device_memory()
146 	if (phys_pg_pack->contiguous) {  in alloc_device_memory()
148 			phys_pg_pack->pages[i] = paddr + i * page_size;  in alloc_device_memory()
152 				phys_pg_pack->pages[i] =  in alloc_device_memory()
157 				phys_pg_pack->pages[i] = gen_pool_alloc(vm->dram_pg_pool,  in alloc_device_memory()
160 			if (!phys_pg_pack->pages[i]) {  in alloc_device_memory()
172 	handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0,  in alloc_device_memory()
185 	phys_pg_pack->handle = handle;  in alloc_device_memory()
187 	atomic64_add(phys_pg_pack->total_size, &ctx->dram_phys_mem);  in alloc_device_memory()
188 	atomic64_add(phys_pg_pack->total_size, &hdev->dram_used_mem);  in alloc_device_memory()
196 	if (!phys_pg_pack->contiguous)  in alloc_device_memory()
198 			gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[i],  in alloc_device_memory()
201 	kvfree(phys_pg_pack->pages);  in alloc_device_memory()
203 	kfree(phys_pg_pack);  in alloc_device_memory()
313 				struct hl_vm_phys_pg_pack *phys_pg_pack)  in free_phys_pg_pack()  argument
318 	if (phys_pg_pack->created_from_userptr)  in free_phys_pg_pack()
321 	if (phys_pg_pack->contiguous) {  in free_phys_pg_pack()
322 		gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[0],  in free_phys_pg_pack()
323 			phys_pg_pack->total_size);  in free_phys_pg_pack()
325 		for (i = 0; i < phys_pg_pack->npages ; i++)  in free_phys_pg_pack()
329 		for (i = 0 ; i < phys_pg_pack->npages ; i++) {  in free_phys_pg_pack()
331 				phys_pg_pack->pages[i],  in free_phys_pg_pack()
332 				phys_pg_pack->page_size);  in free_phys_pg_pack()
339 	kvfree(phys_pg_pack->pages);  in free_phys_pg_pack()
340 	kfree(phys_pg_pack);  in free_phys_pg_pack()
357 	struct hl_vm_phys_pg_pack *phys_pg_pack;  in free_device_memory()  local
361 	phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);  in free_device_memory()
362 	if (!phys_pg_pack) {  in free_device_memory()
368 	if (atomic_read(&phys_pg_pack->mapping_cnt) > 0) {  in free_device_memory()
374 	if (phys_pg_pack->exporting_cnt) {  in free_device_memory()
386 	atomic64_sub(phys_pg_pack->total_size, &ctx->dram_phys_mem);  in free_device_memory()
387 	atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem);  in free_device_memory()
389 	free_phys_pg_pack(hdev, phys_pg_pack);  in free_device_memory()
848 	struct hl_vm_phys_pg_pack *phys_pg_pack;  in init_phys_pg_pack_from_userptr()  local
855 	phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL);  in init_phys_pg_pack_from_userptr()
856 	if (!phys_pg_pack)  in init_phys_pg_pack_from_userptr()
859 	phys_pg_pack->vm_type = userptr->vm_type;  in init_phys_pg_pack_from_userptr()
860 	phys_pg_pack->created_from_userptr = true;  in init_phys_pg_pack_from_userptr()
861 	phys_pg_pack->asid = ctx->asid;  in init_phys_pg_pack_from_userptr()
862 	atomic_set(&phys_pg_pack->mapping_cnt, 1);  in init_phys_pg_pack_from_userptr()
890 	phys_pg_pack->pages = kvmalloc_array(total_npages, sizeof(u64),  in init_phys_pg_pack_from_userptr()
892 	if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) {  in init_phys_pg_pack_from_userptr()
897 	phys_pg_pack->npages = total_npages;  in init_phys_pg_pack_from_userptr()
898 	phys_pg_pack->page_size = page_size;  in init_phys_pg_pack_from_userptr()
899 	phys_pg_pack->total_size = total_npages * page_size;  in init_phys_pg_pack_from_userptr()
908 			phys_pg_pack->offset = dma_addr & (page_size - 1);  in init_phys_pg_pack_from_userptr()
913 			phys_pg_pack->pages[j++] = dma_addr;  in init_phys_pg_pack_from_userptr()
923 	*pphys_pg_pack = phys_pg_pack;  in init_phys_pg_pack_from_userptr()
928 	kfree(phys_pg_pack);  in init_phys_pg_pack_from_userptr()
945 				struct hl_vm_phys_pg_pack *phys_pg_pack)  in map_phys_pg_pack()  argument
949 	u32 page_size = phys_pg_pack->page_size;  in map_phys_pg_pack()
953 	for (i = 0 ; i < phys_pg_pack->npages ; i++) {  in map_phys_pg_pack()
954 		paddr = phys_pg_pack->pages[i];  in map_phys_pg_pack()
957 				(i + 1) == phys_pg_pack->npages);  in map_phys_pg_pack()
961 				phys_pg_pack->handle, phys_pg_pack->npages,  in map_phys_pg_pack()
981 					phys_pg_pack->handle, next_vaddr,  in map_phys_pg_pack()
982 					phys_pg_pack->pages[i], page_size);  in map_phys_pg_pack()
1008 				struct hl_vm_phys_pg_pack *phys_pg_pack)  in unmap_phys_pg_pack()  argument
1016 	page_size = phys_pg_pack->page_size;  in unmap_phys_pg_pack()
1019 	for (i = 0 ; i < phys_pg_pack->npages ; i++, next_vaddr += page_size) {  in unmap_phys_pg_pack()
1021 				       (i + 1) == phys_pg_pack->npages))  in unmap_phys_pg_pack()
1043 	struct hl_vm_phys_pg_pack *phys_pg_pack;  in get_paddr_from_handle()  local
1048 	phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);  in get_paddr_from_handle()
1049 	if (!phys_pg_pack) {  in get_paddr_from_handle()
1055 	*paddr = phys_pg_pack->pages[0];  in get_paddr_from_handle()
1077 	struct hl_vm_phys_pg_pack *phys_pg_pack;  in map_device_va()  local
1110 				&phys_pg_pack, false);  in map_device_va()
1120 		handle = phys_pg_pack->handle;  in map_device_va()
1123 		if (phys_pg_pack->page_size == page_size) {  in map_device_va()
1147 		phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);  in map_device_va()
1148 		if (!phys_pg_pack) {  in map_device_va()
1156 		atomic_inc(&phys_pg_pack->mapping_cnt);  in map_device_va()
1160 		vm_type = (enum vm_type *) phys_pg_pack;  in map_device_va()
1174 	if (!is_userptr && !(phys_pg_pack->flags & HL_MEM_SHARED) &&  in map_device_va()
1175 			phys_pg_pack->asid != ctx->asid) {  in map_device_va()
1189 	if (hint_addr && phys_pg_pack->offset) {  in map_device_va()
1194 				hint_addr, phys_pg_pack->offset);  in map_device_va()
1200 			hint_addr, phys_pg_pack->offset);  in map_device_va()
1203 	ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size,  in map_device_va()
1215 	rc = map_phys_pg_pack(ctx, ret_vaddr, phys_pg_pack);  in map_device_va()
1223 				ctx->asid, ret_vaddr, phys_pg_pack->total_size);  in map_device_va()
1234 							phys_pg_pack->total_size);  in map_device_va()
1239 	ret_vaddr += phys_pg_pack->offset;  in map_device_va()
1251 		free_phys_pg_pack(hdev, phys_pg_pack);  in map_device_va()
1257 				ret_vaddr + phys_pg_pack->total_size - 1))  in map_device_va()
1266 	atomic_dec(&phys_pg_pack->mapping_cnt);  in map_device_va()
1268 		free_phys_pg_pack(hdev, phys_pg_pack);  in map_device_va()
1289 	struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;  in unmap_device_va()  local
1325 		rc = init_phys_pg_pack_from_userptr(ctx, userptr, &phys_pg_pack,  in unmap_device_va()
1334 		if (phys_pg_pack->page_size ==  in unmap_device_va()
1342 		phys_pg_pack = hnode->ptr;  in unmap_device_va()
1351 	if (atomic_read(&phys_pg_pack->mapping_cnt) == 0) {  in unmap_device_va()
1357 	if (!is_userptr && !is_power_of_2(phys_pg_pack->page_size))  in unmap_device_va()
1360 						phys_pg_pack->page_size) *  in unmap_device_va()
1361 							phys_pg_pack->page_size;  in unmap_device_va()
1363 		vaddr &= ~(((u64) phys_pg_pack->page_size) - 1);  in unmap_device_va()
1367 	unmap_phys_pg_pack(ctx, vaddr, phys_pg_pack);  in unmap_device_va()
1376 							phys_pg_pack->total_size);  in unmap_device_va()
1390 					vaddr + phys_pg_pack->total_size - 1);  in unmap_device_va()
1400 	atomic_dec(&phys_pg_pack->mapping_cnt);  in unmap_device_va()
1404 		free_phys_pg_pack(hdev, phys_pg_pack);  in unmap_device_va()
1412 		free_phys_pg_pack(hdev, phys_pg_pack);  in unmap_device_va()
1703 	struct hl_vm_phys_pg_pack *phys_pg_pack;  in hl_map_dmabuf()  local
1710 	phys_pg_pack = hl_dmabuf->phys_pg_pack;  in hl_map_dmabuf()
1717 	if (phys_pg_pack)  in hl_map_dmabuf()
1719 						phys_pg_pack->pages,  in hl_map_dmabuf()
1720 						phys_pg_pack->npages,  in hl_map_dmabuf()
1721 						phys_pg_pack->page_size,  in hl_map_dmabuf()
1770 	if (hl_dmabuf->phys_pg_pack) {  in hl_release_dmabuf()
1772 		hl_dmabuf->phys_pg_pack->exporting_cnt--;  in hl_release_dmabuf()
1921 	struct hl_vm_phys_pg_pack *phys_pg_pack;  in export_dmabuf_from_handle()  local
1938 	phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, (u32) handle);  in export_dmabuf_from_handle()
1939 	if (!phys_pg_pack) {  in export_dmabuf_from_handle()
1946 	phys_pg_pack->exporting_cnt++;  in export_dmabuf_from_handle()
1950 	if (phys_pg_pack->vm_type != VM_TYPE_PHYS_PACK) {  in export_dmabuf_from_handle()
1956 	for (i = 0 ; i < phys_pg_pack->npages ; i++) {  in export_dmabuf_from_handle()
1959 						(phys_pg_pack->pages[i] -  in export_dmabuf_from_handle()
1962 		if (bar_address + phys_pg_pack->page_size >  in export_dmabuf_from_handle()
1964 			bar_address + phys_pg_pack->page_size < bar_address) {  in export_dmabuf_from_handle()
1968 				phys_pg_pack->pages[i],  in export_dmabuf_from_handle()
1969 				phys_pg_pack->page_size);  in export_dmabuf_from_handle()
1982 	hl_dmabuf->phys_pg_pack = phys_pg_pack;  in export_dmabuf_from_handle()
1984 	rc = export_dmabuf_common(ctx, hl_dmabuf, phys_pg_pack->total_size,  in export_dmabuf_from_handle()
1996 	phys_pg_pack->exporting_cnt--;  in export_dmabuf_from_handle()