Lines Matching +full:vm +full:- +full:map
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
29 * two chunks - one to return as result and a remainder to stay in the list.
42 * alloc_device_memory - allocate device memory
49 * - Allocate the requested size rounded up to 2MB pages
50 * - Return unique handle
55 struct hl_device *hdev = ctx->hdev; in alloc_device_memory()
56 struct hl_vm *vm = &hdev->vm; in alloc_device_memory() local
64 page_size = hdev->asic_prop.dram_page_size; in alloc_device_memory()
66 num_pgs = (args->alloc.mem_size + (page_size - 1)) >> page_shift; in alloc_device_memory()
70 dev_err(hdev->dev, "Cannot allocate 0 bytes\n"); in alloc_device_memory()
71 return -EINVAL; in alloc_device_memory()
74 contiguous = args->flags & HL_MEM_CONTIGUOUS; in alloc_device_memory()
77 paddr = (u64) gen_pool_alloc(vm->dram_pg_pool, total_size); in alloc_device_memory()
79 dev_err(hdev->dev, in alloc_device_memory()
82 return -ENOMEM; in alloc_device_memory()
88 rc = -ENOMEM; in alloc_device_memory()
92 phys_pg_pack->vm_type = VM_TYPE_PHYS_PACK; in alloc_device_memory()
93 phys_pg_pack->asid = ctx->asid; in alloc_device_memory()
94 phys_pg_pack->npages = num_pgs; in alloc_device_memory()
95 phys_pg_pack->page_size = page_size; in alloc_device_memory()
96 phys_pg_pack->total_size = total_size; in alloc_device_memory()
97 phys_pg_pack->flags = args->flags; in alloc_device_memory()
98 phys_pg_pack->contiguous = contiguous; in alloc_device_memory()
100 phys_pg_pack->pages = kvmalloc_array(num_pgs, sizeof(u64), GFP_KERNEL); in alloc_device_memory()
101 if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) { in alloc_device_memory()
102 rc = -ENOMEM; in alloc_device_memory()
106 if (phys_pg_pack->contiguous) { in alloc_device_memory()
108 phys_pg_pack->pages[i] = paddr + i * page_size; in alloc_device_memory()
111 phys_pg_pack->pages[i] = (u64) gen_pool_alloc( in alloc_device_memory()
112 vm->dram_pg_pool, in alloc_device_memory()
114 if (!phys_pg_pack->pages[i]) { in alloc_device_memory()
115 dev_err(hdev->dev, in alloc_device_memory()
117 rc = -ENOMEM; in alloc_device_memory()
125 spin_lock(&vm->idr_lock); in alloc_device_memory()
126 handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0, in alloc_device_memory()
128 spin_unlock(&vm->idr_lock); in alloc_device_memory()
131 dev_err(hdev->dev, "Failed to get handle for page\n"); in alloc_device_memory()
132 rc = -EFAULT; in alloc_device_memory()
137 kref_get(&vm->dram_pg_pool_refcount); in alloc_device_memory()
139 phys_pg_pack->handle = handle; in alloc_device_memory()
141 atomic64_add(phys_pg_pack->total_size, &ctx->dram_phys_mem); in alloc_device_memory()
142 atomic64_add(phys_pg_pack->total_size, &hdev->dram_used_mem); in alloc_device_memory()
150 if (!phys_pg_pack->contiguous) in alloc_device_memory()
152 gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[i], in alloc_device_memory()
155 kvfree(phys_pg_pack->pages); in alloc_device_memory()
160 gen_pool_free(vm->dram_pg_pool, paddr, total_size); in alloc_device_memory()
166 * dma_map_host_va - DMA mapping of the given host virtual address.
173 * - Allocate userptr structure
174 * - Pin the given host memory using the userptr structure
175 * - Perform DMA mapping to have the DMA addresses of the pages
185 rc = -ENOMEM; in dma_map_host_va()
191 dev_err(hdev->dev, "Failed to pin host memory\n"); in dma_map_host_va()
195 rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl, in dma_map_host_va()
196 userptr->sgt->nents, DMA_BIDIRECTIONAL); in dma_map_host_va()
198 dev_err(hdev->dev, "failed to map sgt with DMA region\n"); in dma_map_host_va()
202 userptr->dma_mapped = true; in dma_map_host_va()
203 userptr->dir = DMA_BIDIRECTIONAL; in dma_map_host_va()
204 userptr->vm_type = VM_TYPE_USERPTR; in dma_map_host_va()
220 * dma_unmap_host_va - DMA unmapping of the given host virtual address.
225 * - Unpins the physical pages
226 * - Frees the userptr structure
236 * dram_pg_pool_do_release - free DRAM pages pool
241 * - Frees the idr structure of physical pages handles
242 * - Frees the generic pool of DRAM physical pages
246 struct hl_vm *vm = container_of(ref, struct hl_vm, in dram_pg_pool_do_release() local
253 idr_destroy(&vm->phys_pg_pack_handles); in dram_pg_pool_do_release()
254 gen_pool_destroy(vm->dram_pg_pool); in dram_pg_pool_do_release()
258 * free_phys_pg_pack - free physical page pack
263 * - For DRAM memory only, iterate over the pack and free each physical block
265 * - Free the hl_vm_phys_pg_pack structure
270 struct hl_vm *vm = &hdev->vm; in free_phys_pg_pack() local
273 if (!phys_pg_pack->created_from_userptr) { in free_phys_pg_pack()
274 if (phys_pg_pack->contiguous) { in free_phys_pg_pack()
275 gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[0], in free_phys_pg_pack()
276 phys_pg_pack->total_size); in free_phys_pg_pack()
278 for (i = 0; i < phys_pg_pack->npages ; i++) in free_phys_pg_pack()
279 kref_put(&vm->dram_pg_pool_refcount, in free_phys_pg_pack()
282 for (i = 0 ; i < phys_pg_pack->npages ; i++) { in free_phys_pg_pack()
283 gen_pool_free(vm->dram_pg_pool, in free_phys_pg_pack()
284 phys_pg_pack->pages[i], in free_phys_pg_pack()
285 phys_pg_pack->page_size); in free_phys_pg_pack()
286 kref_put(&vm->dram_pg_pool_refcount, in free_phys_pg_pack()
292 kvfree(phys_pg_pack->pages); in free_phys_pg_pack()
297 * free_device_memory - free device memory
303 * - Free the device memory related to the given handle
307 struct hl_device *hdev = ctx->hdev; in free_device_memory()
308 struct hl_vm *vm = &hdev->vm; in free_device_memory() local
311 spin_lock(&vm->idr_lock); in free_device_memory()
312 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle); in free_device_memory()
314 if (atomic_read(&phys_pg_pack->mapping_cnt) > 0) { in free_device_memory()
315 dev_err(hdev->dev, "handle %u is mapped, cannot free\n", in free_device_memory()
317 spin_unlock(&vm->idr_lock); in free_device_memory()
318 return -EINVAL; in free_device_memory()
326 idr_remove(&vm->phys_pg_pack_handles, handle); in free_device_memory()
327 spin_unlock(&vm->idr_lock); in free_device_memory()
329 atomic64_sub(phys_pg_pack->total_size, &ctx->dram_phys_mem); in free_device_memory()
330 atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem); in free_device_memory()
334 spin_unlock(&vm->idr_lock); in free_device_memory()
335 dev_err(hdev->dev, in free_device_memory()
338 return -EINVAL; in free_device_memory()
345 * clear_va_list_locked - free virtual addresses list
351 * - Iterate over the list and free each virtual addresses block
361 list_del(&va_block->node); in clear_va_list_locked()
367 * print_va_list_locked - print virtual addresses list
373 * - Iterate over the list and print each virtual addresses block
383 dev_dbg(hdev->dev, "print va list:\n"); in print_va_list_locked()
386 dev_dbg(hdev->dev, in print_va_list_locked()
388 va_block->start, va_block->end, va_block->size); in print_va_list_locked()
393 * merge_va_blocks_locked - merge a virtual block if possible
400 * - Merge the given blocks with the adjacent blocks if their virtual ranges
411 if (&prev->node != va_list && prev->end + 1 == va_block->start) { in merge_va_blocks_locked()
412 prev->end = va_block->end; in merge_va_blocks_locked()
413 prev->size = prev->end - prev->start; in merge_va_blocks_locked()
414 list_del(&va_block->node); in merge_va_blocks_locked()
420 if (&next->node != va_list && va_block->end + 1 == next->start) { in merge_va_blocks_locked()
421 next->start = va_block->start; in merge_va_blocks_locked()
422 next->size = next->end - next->start; in merge_va_blocks_locked()
423 list_del(&va_block->node); in merge_va_blocks_locked()
429 * add_va_block_locked - add a virtual block to the virtual addresses list
437 * - Add the given block to the virtual blocks list and merge with other
446 u64 size = end - start; in add_va_block_locked()
452 if (hl_mem_area_crosses_range(start, size, va_block->start, in add_va_block_locked()
453 va_block->end)) { in add_va_block_locked()
454 dev_err(hdev->dev, in add_va_block_locked()
456 va_block->start, va_block->end); in add_va_block_locked()
457 return -EINVAL; in add_va_block_locked()
460 if (va_block->end < start) in add_va_block_locked()
466 return -ENOMEM; in add_va_block_locked()
468 va_block->start = start; in add_va_block_locked()
469 va_block->end = end; in add_va_block_locked()
470 va_block->size = size; in add_va_block_locked()
473 list_add(&va_block->node, va_list); in add_va_block_locked()
475 list_add(&va_block->node, &res->node); in add_va_block_locked()
485 * add_va_block - wrapper for add_va_block_locked
493 * - Takes the list lock and calls add_va_block_locked
500 mutex_lock(&va_range->lock); in add_va_block()
501 rc = add_va_block_locked(hdev, &va_range->list, start, end); in add_va_block()
502 mutex_unlock(&va_range->lock); in add_va_block()
508 * get_va_block() - get a virtual block for the given size and alignment.
516 * - Iterate on the virtual block list to find a suitable virtual block for the
518 * - Reserve the requested block and update the list.
519 * - Return the start address of the virtual block.
529 align_mask = ~((u64)va_block_align - 1); in get_va_block()
532 if (hint_addr & (va_block_align - 1)) in get_va_block()
535 mutex_lock(&va_range->lock); in get_va_block()
537 print_va_list_locked(hdev, &va_range->list); in get_va_block()
539 list_for_each_entry(va_block, &va_range->list, node) { in get_va_block()
541 valid_start = va_block->start; in get_va_block()
543 if (valid_start & (va_block_align - 1)) { in get_va_block()
546 if (valid_start > va_block->end) in get_va_block()
550 valid_size = va_block->end - valid_start; in get_va_block()
560 ((hint_addr + size) <= va_block->end)) { in get_va_block()
569 dev_err(hdev->dev, "no available va block for size %llu\n", in get_va_block()
574 if (res_valid_start > new_va_block->start) { in get_va_block()
575 prev_start = new_va_block->start; in get_va_block()
576 prev_end = res_valid_start - 1; in get_va_block()
578 new_va_block->start = res_valid_start; in get_va_block()
579 new_va_block->size = res_valid_size; in get_va_block()
584 if (new_va_block->size > size) { in get_va_block()
585 new_va_block->start += size; in get_va_block()
586 new_va_block->size = new_va_block->end - new_va_block->start; in get_va_block()
588 list_del(&new_va_block->node); in get_va_block()
593 add_va_block_locked(hdev, &va_range->list, prev_start, in get_va_block()
596 print_va_list_locked(hdev, &va_range->list); in get_va_block()
598 mutex_unlock(&va_range->lock); in get_va_block()
604 * get_sg_info - get number of pages and the DMA address from SG list
617 return ((((*dma_addr) & (PAGE_SIZE - 1)) + sg_dma_len(sg)) + in get_sg_info()
618 (PAGE_SIZE - 1)) >> PAGE_SHIFT; in get_sg_info()
622 * init_phys_pg_pack_from_userptr - initialize physical page pack from host
629 * - Pin the physical pages related to the given virtual block
630 * - Create a physical page pack from the physical pages related to the given
642 huge_page_size = ctx->hdev->asic_prop.pmmu_huge.page_size; in init_phys_pg_pack_from_userptr()
649 return -ENOMEM; in init_phys_pg_pack_from_userptr()
651 phys_pg_pack->vm_type = userptr->vm_type; in init_phys_pg_pack_from_userptr()
652 phys_pg_pack->created_from_userptr = true; in init_phys_pg_pack_from_userptr()
653 phys_pg_pack->asid = ctx->asid; in init_phys_pg_pack_from_userptr()
654 atomic_set(&phys_pg_pack->mapping_cnt, 1); in init_phys_pg_pack_from_userptr()
663 for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) { in init_phys_pg_pack_from_userptr()
669 (dma_addr & (huge_page_size - 1))) in init_phys_pg_pack_from_userptr()
678 page_mask = ~(((u64) page_size) - 1); in init_phys_pg_pack_from_userptr()
680 phys_pg_pack->pages = kvmalloc_array(total_npages, sizeof(u64), in init_phys_pg_pack_from_userptr()
682 if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) { in init_phys_pg_pack_from_userptr()
683 rc = -ENOMEM; in init_phys_pg_pack_from_userptr()
687 phys_pg_pack->npages = total_npages; in init_phys_pg_pack_from_userptr()
688 phys_pg_pack->page_size = page_size; in init_phys_pg_pack_from_userptr()
689 phys_pg_pack->total_size = total_npages * page_size; in init_phys_pg_pack_from_userptr()
692 for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) { in init_phys_pg_pack_from_userptr()
698 phys_pg_pack->offset = dma_addr & (page_size - 1); in init_phys_pg_pack_from_userptr()
703 phys_pg_pack->pages[j++] = dma_addr; in init_phys_pg_pack_from_userptr()
707 npages -= pgs_in_huge_page; in init_phys_pg_pack_from_userptr()
709 npages--; in init_phys_pg_pack_from_userptr()
724 * map_phys_pg_pack - maps the physical page pack.
726 * @vaddr: start address of the virtual area to map from
727 * @phys_pg_pack: the pack of physical pages to map to
730 * - Maps each chunk of virtual memory to matching physical chunk
731 * - Stores number of successful mappings in the given argument
732 * - Returns 0 on success, error code otherwise
737 struct hl_device *hdev = ctx->hdev; in map_phys_pg_pack()
739 u32 page_size = phys_pg_pack->page_size; in map_phys_pg_pack()
742 for (i = 0 ; i < phys_pg_pack->npages ; i++) { in map_phys_pg_pack()
743 paddr = phys_pg_pack->pages[i]; in map_phys_pg_pack()
746 (i + 1) == phys_pg_pack->npages); in map_phys_pg_pack()
748 dev_err(hdev->dev, in map_phys_pg_pack()
749 "map failed for handle %u, npages: %llu, mapped: %llu", in map_phys_pg_pack()
750 phys_pg_pack->handle, phys_pg_pack->npages, in map_phys_pg_pack()
766 dev_warn_ratelimited(hdev->dev, in map_phys_pg_pack()
768 phys_pg_pack->handle, next_vaddr, in map_phys_pg_pack()
769 phys_pg_pack->pages[i], page_size); in map_phys_pg_pack()
778 * unmap_phys_pg_pack - unmaps the physical page pack
786 struct hl_device *hdev = ctx->hdev; in unmap_phys_pg_pack()
790 page_size = phys_pg_pack->page_size; in unmap_phys_pg_pack()
793 for (i = 0 ; i < phys_pg_pack->npages ; i++, next_vaddr += page_size) { in unmap_phys_pg_pack()
795 (i + 1) == phys_pg_pack->npages)) in unmap_phys_pg_pack()
796 dev_warn_ratelimited(hdev->dev, in unmap_phys_pg_pack()
803 if (hdev->pldm) in unmap_phys_pg_pack()
811 struct hl_device *hdev = ctx->hdev; in get_paddr_from_handle()
812 struct hl_vm *vm = &hdev->vm; in get_paddr_from_handle() local
816 handle = lower_32_bits(args->map_device.handle); in get_paddr_from_handle()
817 spin_lock(&vm->idr_lock); in get_paddr_from_handle()
818 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle); in get_paddr_from_handle()
820 spin_unlock(&vm->idr_lock); in get_paddr_from_handle()
821 dev_err(hdev->dev, "no match for handle %u\n", handle); in get_paddr_from_handle()
822 return -EINVAL; in get_paddr_from_handle()
825 *paddr = phys_pg_pack->pages[0]; in get_paddr_from_handle()
827 spin_unlock(&vm->idr_lock); in get_paddr_from_handle()
833 * map_device_va - map the given memory
840 * - If given a physical device memory handle, map to a device virtual block
842 * - If given a host virtual address and size, find the related physical pages,
843 * map a device virtual block to this pages and return the start address of
849 struct hl_device *hdev = ctx->hdev; in map_device_va()
850 struct hl_vm *vm = &hdev->vm; in map_device_va() local
859 bool is_userptr = args->flags & HL_MEM_USERPTR; in map_device_va()
865 u64 addr = args->map_host.host_virt_addr, in map_device_va()
866 size = args->map_host.mem_size; in map_device_va()
867 u32 page_size = hdev->asic_prop.pmmu.page_size, in map_device_va()
868 huge_page_size = hdev->asic_prop.pmmu_huge.page_size; in map_device_va()
872 dev_err(hdev->dev, "failed to get userptr from va\n"); in map_device_va()
879 dev_err(hdev->dev, in map_device_va()
886 hint_addr = args->map_host.hint_addr; in map_device_va()
887 handle = phys_pg_pack->handle; in map_device_va()
890 if (phys_pg_pack->page_size == page_size) { in map_device_va()
891 va_range = ctx->host_va_range; in map_device_va()
897 if (addr & (huge_page_size - 1)) in map_device_va()
906 va_range = ctx->host_huge_va_range; in map_device_va()
910 handle = lower_32_bits(args->map_device.handle); in map_device_va()
912 spin_lock(&vm->idr_lock); in map_device_va()
913 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle); in map_device_va()
915 spin_unlock(&vm->idr_lock); in map_device_va()
916 dev_err(hdev->dev, in map_device_va()
918 return -EINVAL; in map_device_va()
922 atomic_inc(&phys_pg_pack->mapping_cnt); in map_device_va()
924 spin_unlock(&vm->idr_lock); in map_device_va()
928 hint_addr = args->map_device.hint_addr; in map_device_va()
931 va_range = ctx->dram_va_range; in map_device_va()
932 va_block_align = hdev->asic_prop.dmmu.page_size; in map_device_va()
939 if (!is_userptr && !(phys_pg_pack->flags & HL_MEM_SHARED) && in map_device_va()
940 phys_pg_pack->asid != ctx->asid) { in map_device_va()
941 dev_err(hdev->dev, in map_device_va()
942 "Failed to map memory, handle %u is not shared\n", in map_device_va()
944 rc = -EPERM; in map_device_va()
950 rc = -ENOMEM; in map_device_va()
954 ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size, in map_device_va()
957 dev_err(hdev->dev, "no available va block for handle %u\n", in map_device_va()
959 rc = -ENOMEM; in map_device_va()
963 mutex_lock(&ctx->mmu_lock); in map_device_va()
967 mutex_unlock(&ctx->mmu_lock); in map_device_va()
968 dev_err(hdev->dev, "mapping page pack failed for handle %u\n", in map_device_va()
973 rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, false, *vm_type); in map_device_va()
975 mutex_unlock(&ctx->mmu_lock); in map_device_va()
978 dev_err(hdev->dev, in map_device_va()
984 ret_vaddr += phys_pg_pack->offset; in map_device_va()
986 hnode->ptr = vm_type; in map_device_va()
987 hnode->vaddr = ret_vaddr; in map_device_va()
989 mutex_lock(&ctx->mem_hash_lock); in map_device_va()
990 hash_add(ctx->mem_hash, &hnode->node, ret_vaddr); in map_device_va()
991 mutex_unlock(&ctx->mem_hash_lock); in map_device_va()
1002 ret_vaddr + phys_pg_pack->total_size - 1)) in map_device_va()
1003 dev_warn(hdev->dev, in map_device_va()
1011 atomic_dec(&phys_pg_pack->mapping_cnt); in map_device_va()
1022 * unmap_device_va - unmap the given device virtual address
1029 * - Unmap the physical pages related to the given virtual address
1030 * - return the device virtual block to the virtual block list
1034 struct hl_device *hdev = ctx->hdev; in unmap_device_va()
1044 mutex_lock(&ctx->mem_hash_lock); in unmap_device_va()
1045 hash_for_each_possible(ctx->mem_hash, hnode, node, (unsigned long)vaddr) in unmap_device_va()
1046 if (vaddr == hnode->vaddr) in unmap_device_va()
1050 mutex_unlock(&ctx->mem_hash_lock); in unmap_device_va()
1051 dev_err(hdev->dev, in unmap_device_va()
1054 return -EINVAL; in unmap_device_va()
1057 hash_del(&hnode->node); in unmap_device_va()
1058 mutex_unlock(&ctx->mem_hash_lock); in unmap_device_va()
1060 vm_type = hnode->ptr; in unmap_device_va()
1064 userptr = hnode->ptr; in unmap_device_va()
1068 dev_err(hdev->dev, in unmap_device_va()
1074 if (phys_pg_pack->page_size == in unmap_device_va()
1075 hdev->asic_prop.pmmu.page_size) in unmap_device_va()
1076 va_range = ctx->host_va_range; in unmap_device_va()
1078 va_range = ctx->host_huge_va_range; in unmap_device_va()
1081 va_range = ctx->dram_va_range; in unmap_device_va()
1082 phys_pg_pack = hnode->ptr; in unmap_device_va()
1084 dev_warn(hdev->dev, in unmap_device_va()
1085 "unmap failed, unknown vm desc for vaddr 0x%llx\n", in unmap_device_va()
1087 rc = -EFAULT; in unmap_device_va()
1091 if (atomic_read(&phys_pg_pack->mapping_cnt) == 0) { in unmap_device_va()
1092 dev_err(hdev->dev, "vaddr 0x%llx is not mapped\n", vaddr); in unmap_device_va()
1093 rc = -EINVAL; in unmap_device_va()
1097 vaddr &= ~(((u64) phys_pg_pack->page_size) - 1); in unmap_device_va()
1099 mutex_lock(&ctx->mmu_lock); in unmap_device_va()
1109 rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, true, in unmap_device_va()
1112 mutex_unlock(&ctx->mmu_lock); in unmap_device_va()
1124 dev_err(hdev->dev, in unmap_device_va()
1129 vaddr + phys_pg_pack->total_size - 1); in unmap_device_va()
1131 dev_warn(hdev->dev, in unmap_device_va()
1139 atomic_dec(&phys_pg_pack->mapping_cnt); in unmap_device_va()
1153 mutex_lock(&ctx->mem_hash_lock); in unmap_device_va()
1154 hash_add(ctx->mem_hash, &hnode->node, vaddr); in unmap_device_va()
1155 mutex_unlock(&ctx->mem_hash_lock); in unmap_device_va()
1162 struct hl_device *hdev = hpriv->hdev; in mem_ioctl_no_mmu()
1163 struct hl_ctx *ctx = hpriv->ctx; in mem_ioctl_no_mmu()
1168 switch (args->in.op) { in mem_ioctl_no_mmu()
1170 if (args->in.alloc.mem_size == 0) { in mem_ioctl_no_mmu()
1171 dev_err(hdev->dev, in mem_ioctl_no_mmu()
1173 rc = -EINVAL; in mem_ioctl_no_mmu()
1180 args->in.flags |= HL_MEM_CONTIGUOUS; in mem_ioctl_no_mmu()
1181 rc = alloc_device_memory(ctx, &args->in, &handle); in mem_ioctl_no_mmu()
1184 args->out.handle = (__u64) handle; in mem_ioctl_no_mmu()
1188 rc = free_device_memory(ctx, args->in.free.handle); in mem_ioctl_no_mmu()
1192 if (args->in.flags & HL_MEM_USERPTR) { in mem_ioctl_no_mmu()
1193 device_addr = args->in.map_host.host_virt_addr; in mem_ioctl_no_mmu()
1196 rc = get_paddr_from_handle(ctx, &args->in, in mem_ioctl_no_mmu()
1201 args->out.device_virt_addr = device_addr; in mem_ioctl_no_mmu()
1209 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n"); in mem_ioctl_no_mmu()
1210 rc = -ENOTTY; in mem_ioctl_no_mmu()
1221 struct hl_device *hdev = hpriv->hdev; in hl_mem_ioctl()
1222 struct hl_ctx *ctx = hpriv->ctx; in hl_mem_ioctl()
1228 dev_warn_ratelimited(hdev->dev, in hl_mem_ioctl()
1230 atomic_read(&hdev->in_reset) ? "in_reset" : "disabled"); in hl_mem_ioctl()
1231 return -EBUSY; in hl_mem_ioctl()
1234 if (!hdev->mmu_enable) in hl_mem_ioctl()
1237 switch (args->in.op) { in hl_mem_ioctl()
1239 if (!hdev->dram_supports_virtual_memory) { in hl_mem_ioctl()
1240 dev_err(hdev->dev, "DRAM alloc is not supported\n"); in hl_mem_ioctl()
1241 rc = -EINVAL; in hl_mem_ioctl()
1245 if (args->in.alloc.mem_size == 0) { in hl_mem_ioctl()
1246 dev_err(hdev->dev, in hl_mem_ioctl()
1248 rc = -EINVAL; in hl_mem_ioctl()
1251 rc = alloc_device_memory(ctx, &args->in, &handle); in hl_mem_ioctl()
1254 args->out.handle = (__u64) handle; in hl_mem_ioctl()
1258 rc = free_device_memory(ctx, args->in.free.handle); in hl_mem_ioctl()
1262 rc = map_device_va(ctx, &args->in, &device_addr); in hl_mem_ioctl()
1265 args->out.device_virt_addr = device_addr; in hl_mem_ioctl()
1269 rc = unmap_device_va(ctx, args->in.unmap.device_virt_addr, in hl_mem_ioctl()
1274 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n"); in hl_mem_ioctl()
1275 rc = -ENOTTY; in hl_mem_ioctl()
1290 dev_err(hdev->dev, "user pointer is invalid - 0x%llx\n", addr); in get_user_memory()
1291 return -EFAULT; in get_user_memory()
1294 userptr->vec = frame_vector_create(npages); in get_user_memory()
1295 if (!userptr->vec) { in get_user_memory()
1296 dev_err(hdev->dev, "Failed to create frame vector\n"); in get_user_memory()
1297 return -ENOMEM; in get_user_memory()
1301 userptr->vec); in get_user_memory()
1304 dev_err(hdev->dev, in get_user_memory()
1305 "Failed to map host memory, user ptr probably wrong\n"); in get_user_memory()
1308 rc = -EFAULT; in get_user_memory()
1312 if (frame_vector_to_pages(userptr->vec) < 0) { in get_user_memory()
1313 dev_err(hdev->dev, in get_user_memory()
1315 rc = -EFAULT; in get_user_memory()
1319 rc = sg_alloc_table_from_pages(userptr->sgt, in get_user_memory()
1320 frame_vector_pages(userptr->vec), in get_user_memory()
1323 dev_err(hdev->dev, "failed to create SG table from pages\n"); in get_user_memory()
1330 put_vaddr_frames(userptr->vec); in get_user_memory()
1332 frame_vector_destroy(userptr->vec); in get_user_memory()
1337 * hl_pin_host_memory - pins a chunk of host memory.
1344 * - Pins the physical pages
1345 * - Create an SG list from those pages
1355 dev_err(hdev->dev, "size to pin is invalid - %llu\n", size); in hl_pin_host_memory()
1356 return -EINVAL; in hl_pin_host_memory()
1365 dev_err(hdev->dev, in hl_pin_host_memory()
1368 return -EINVAL; in hl_pin_host_memory()
1375 userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_ATOMIC); in hl_pin_host_memory()
1376 if (!userptr->sgt) in hl_pin_host_memory()
1377 return -ENOMEM; in hl_pin_host_memory()
1382 npages = (end - start) >> PAGE_SHIFT; in hl_pin_host_memory()
1384 userptr->size = size; in hl_pin_host_memory()
1385 userptr->addr = addr; in hl_pin_host_memory()
1386 userptr->dma_mapped = false; in hl_pin_host_memory()
1387 INIT_LIST_HEAD(&userptr->job_node); in hl_pin_host_memory()
1392 dev_err(hdev->dev, in hl_pin_host_memory()
1403 kfree(userptr->sgt); in hl_pin_host_memory()
1408 * hl_unpin_host_memory - unpins a chunk of host memory.
1413 * - Unpins the physical pages related to the host memory
1414 * - Free the SG list
1422 if (userptr->dma_mapped) in hl_unpin_host_memory()
1423 hdev->asic_funcs->hl_dma_unmap_sg(hdev, userptr->sgt->sgl, in hl_unpin_host_memory()
1424 userptr->sgt->nents, in hl_unpin_host_memory()
1425 userptr->dir); in hl_unpin_host_memory()
1427 pages = frame_vector_pages(userptr->vec); in hl_unpin_host_memory()
1431 for (i = 0; i < frame_vector_count(userptr->vec); i++) in hl_unpin_host_memory()
1434 put_vaddr_frames(userptr->vec); in hl_unpin_host_memory()
1435 frame_vector_destroy(userptr->vec); in hl_unpin_host_memory()
1437 list_del(&userptr->job_node); in hl_unpin_host_memory()
1439 sg_free_table(userptr->sgt); in hl_unpin_host_memory()
1440 kfree(userptr->sgt); in hl_unpin_host_memory()
1444 * hl_userptr_delete_list - clear userptr list
1450 * - Iterates over the list and unpins the host memory and frees the userptr
1467 * hl_userptr_is_pinned - returns whether the given userptr is pinned
1474 * - Iterates over the list and checks if the given userptr is in it, means is
1482 if ((addr == (*userptr)->addr) && (size == (*userptr)->size)) in hl_userptr_is_pinned()
1490 * va_range_init - initialize virtual addresses range
1497 * - Initializes the virtual addresses list of the given range with the given
1505 INIT_LIST_HEAD(&va_range->list); in va_range_init()
1509 if (start & (PAGE_SIZE - 1)) { in va_range_init()
1514 if (end & (PAGE_SIZE - 1)) in va_range_init()
1518 dev_err(hdev->dev, "too small vm range for va list\n"); in va_range_init()
1519 return -EFAULT; in va_range_init()
1525 dev_err(hdev->dev, "Failed to init host va list\n"); in va_range_init()
1529 va_range->start_addr = start; in va_range_init()
1530 va_range->end_addr = end; in va_range_init()
1536 * va_range_fini() - clear a virtual addresses range
1541 * - Frees the virtual addresses block list and its lock
1546 mutex_lock(&va_range->lock); in va_range_fini()
1547 clear_va_list_locked(hdev, &va_range->list); in va_range_fini()
1548 mutex_unlock(&va_range->lock); in va_range_fini()
1550 mutex_destroy(&va_range->lock); in va_range_fini()
1555 * vm_ctx_init_with_ranges() - initialize virtual memory for context
1567 * - MMU for context
1568 * - Virtual address to area descriptor hashtable
1569 * - Virtual block list of available virtual memory
1579 struct hl_device *hdev = ctx->hdev; in vm_ctx_init_with_ranges()
1582 ctx->host_va_range = kzalloc(sizeof(*ctx->host_va_range), GFP_KERNEL); in vm_ctx_init_with_ranges()
1583 if (!ctx->host_va_range) in vm_ctx_init_with_ranges()
1584 return -ENOMEM; in vm_ctx_init_with_ranges()
1586 ctx->host_huge_va_range = kzalloc(sizeof(*ctx->host_huge_va_range), in vm_ctx_init_with_ranges()
1588 if (!ctx->host_huge_va_range) { in vm_ctx_init_with_ranges()
1589 rc = -ENOMEM; in vm_ctx_init_with_ranges()
1593 ctx->dram_va_range = kzalloc(sizeof(*ctx->dram_va_range), GFP_KERNEL); in vm_ctx_init_with_ranges()
1594 if (!ctx->dram_va_range) { in vm_ctx_init_with_ranges()
1595 rc = -ENOMEM; in vm_ctx_init_with_ranges()
1601 dev_err(hdev->dev, "failed to init context %d\n", ctx->asid); in vm_ctx_init_with_ranges()
1605 mutex_init(&ctx->mem_hash_lock); in vm_ctx_init_with_ranges()
1606 hash_init(ctx->mem_hash); in vm_ctx_init_with_ranges()
1608 mutex_init(&ctx->host_va_range->lock); in vm_ctx_init_with_ranges()
1610 rc = va_range_init(hdev, ctx->host_va_range, host_range_start, in vm_ctx_init_with_ranges()
1613 dev_err(hdev->dev, "failed to init host vm range\n"); in vm_ctx_init_with_ranges()
1617 if (hdev->pmmu_huge_range) { in vm_ctx_init_with_ranges()
1618 mutex_init(&ctx->host_huge_va_range->lock); in vm_ctx_init_with_ranges()
1620 rc = va_range_init(hdev, ctx->host_huge_va_range, in vm_ctx_init_with_ranges()
1624 dev_err(hdev->dev, in vm_ctx_init_with_ranges()
1625 "failed to init host huge vm range\n"); in vm_ctx_init_with_ranges()
1629 kfree(ctx->host_huge_va_range); in vm_ctx_init_with_ranges()
1630 ctx->host_huge_va_range = ctx->host_va_range; in vm_ctx_init_with_ranges()
1633 mutex_init(&ctx->dram_va_range->lock); in vm_ctx_init_with_ranges()
1635 rc = va_range_init(hdev, ctx->dram_va_range, dram_range_start, in vm_ctx_init_with_ranges()
1638 dev_err(hdev->dev, "failed to init dram vm range\n"); in vm_ctx_init_with_ranges()
1647 mutex_destroy(&ctx->dram_va_range->lock); in vm_ctx_init_with_ranges()
1649 if (hdev->pmmu_huge_range) { in vm_ctx_init_with_ranges()
1650 mutex_lock(&ctx->host_huge_va_range->lock); in vm_ctx_init_with_ranges()
1651 clear_va_list_locked(hdev, &ctx->host_huge_va_range->list); in vm_ctx_init_with_ranges()
1652 mutex_unlock(&ctx->host_huge_va_range->lock); in vm_ctx_init_with_ranges()
1655 if (hdev->pmmu_huge_range) in vm_ctx_init_with_ranges()
1656 mutex_destroy(&ctx->host_huge_va_range->lock); in vm_ctx_init_with_ranges()
1657 mutex_lock(&ctx->host_va_range->lock); in vm_ctx_init_with_ranges()
1658 clear_va_list_locked(hdev, &ctx->host_va_range->list); in vm_ctx_init_with_ranges()
1659 mutex_unlock(&ctx->host_va_range->lock); in vm_ctx_init_with_ranges()
1661 mutex_destroy(&ctx->host_va_range->lock); in vm_ctx_init_with_ranges()
1662 mutex_destroy(&ctx->mem_hash_lock); in vm_ctx_init_with_ranges()
1665 kfree(ctx->dram_va_range); in vm_ctx_init_with_ranges()
1667 kfree(ctx->host_huge_va_range); in vm_ctx_init_with_ranges()
1669 kfree(ctx->host_va_range); in vm_ctx_init_with_ranges()
1676 struct asic_fixed_properties *prop = &ctx->hdev->asic_prop; in hl_vm_ctx_init()
1680 atomic64_set(&ctx->dram_phys_mem, 0); in hl_vm_ctx_init()
1683 * - If MMU is enabled, init the ranges as usual. in hl_vm_ctx_init()
1684 * - If MMU is disabled, in case of host mapping, the returned address in hl_vm_ctx_init()
1689 if (ctx->hdev->mmu_enable) { in hl_vm_ctx_init()
1690 dram_range_start = prop->dmmu.start_addr; in hl_vm_ctx_init()
1691 dram_range_end = prop->dmmu.end_addr; in hl_vm_ctx_init()
1692 host_range_start = prop->pmmu.start_addr; in hl_vm_ctx_init()
1693 host_range_end = prop->pmmu.end_addr; in hl_vm_ctx_init()
1694 host_huge_range_start = prop->pmmu_huge.start_addr; in hl_vm_ctx_init()
1695 host_huge_range_end = prop->pmmu_huge.end_addr; in hl_vm_ctx_init()
1697 dram_range_start = prop->dram_user_base_address; in hl_vm_ctx_init()
1698 dram_range_end = prop->dram_end_address; in hl_vm_ctx_init()
1699 host_range_start = prop->dram_user_base_address; in hl_vm_ctx_init()
1700 host_range_end = prop->dram_end_address; in hl_vm_ctx_init()
1701 host_huge_range_start = prop->dram_user_base_address; in hl_vm_ctx_init()
1702 host_huge_range_end = prop->dram_end_address; in hl_vm_ctx_init()
1713 * hl_vm_ctx_fini - virtual memory teardown of context
1718 * - Virtual block list of available virtual memory
1719 * - Virtual address to area descriptor hashtable
1720 * - MMU for context
1723 * - Unmaps the existing hashtable nodes if the hashtable is not empty. The
1726 * - Frees any existing physical page list from the idr which relates to the
1728 * - This function checks the virtual block list for correctness. At this point
1734 struct hl_device *hdev = ctx->hdev; in hl_vm_ctx_fini()
1735 struct hl_vm *vm = &hdev->vm; in hl_vm_ctx_fini() local
1747 if (!hdev->hard_reset_pending && !hash_empty(ctx->mem_hash)) in hl_vm_ctx_fini()
1748 dev_notice(hdev->dev, in hl_vm_ctx_fini()
1751 hash_for_each_safe(ctx->mem_hash, i, tmp_node, hnode, node) { in hl_vm_ctx_fini()
1752 dev_dbg(hdev->dev, in hl_vm_ctx_fini()
1754 hnode->vaddr, ctx->asid); in hl_vm_ctx_fini()
1755 unmap_device_va(ctx, hnode->vaddr, true); in hl_vm_ctx_fini()
1759 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR); in hl_vm_ctx_fini()
1760 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_PHYS_PACK); in hl_vm_ctx_fini()
1762 spin_lock(&vm->idr_lock); in hl_vm_ctx_fini()
1763 idr_for_each_entry(&vm->phys_pg_pack_handles, phys_pg_list, i) in hl_vm_ctx_fini()
1764 if (phys_pg_list->asid == ctx->asid) { in hl_vm_ctx_fini()
1765 dev_dbg(hdev->dev, in hl_vm_ctx_fini()
1767 phys_pg_list, ctx->asid); in hl_vm_ctx_fini()
1768 atomic64_sub(phys_pg_list->total_size, in hl_vm_ctx_fini()
1769 &hdev->dram_used_mem); in hl_vm_ctx_fini()
1771 idr_remove(&vm->phys_pg_pack_handles, i); in hl_vm_ctx_fini()
1773 spin_unlock(&vm->idr_lock); in hl_vm_ctx_fini()
1775 va_range_fini(hdev, ctx->dram_va_range); in hl_vm_ctx_fini()
1776 if (hdev->pmmu_huge_range) in hl_vm_ctx_fini()
1777 va_range_fini(hdev, ctx->host_huge_va_range); in hl_vm_ctx_fini()
1778 va_range_fini(hdev, ctx->host_va_range); in hl_vm_ctx_fini()
1780 mutex_destroy(&ctx->mem_hash_lock); in hl_vm_ctx_fini()
1785 * hl_vm_init - initialize virtual memory module
1790 * - MMU module
1791 * - DRAM physical pages pool of 2MB
1792 * - Idr for device memory allocation handles
1796 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_vm_init()
1797 struct hl_vm *vm = &hdev->vm; in hl_vm_init() local
1800 vm->dram_pg_pool = gen_pool_create(__ffs(prop->dram_page_size), -1); in hl_vm_init()
1801 if (!vm->dram_pg_pool) { in hl_vm_init()
1802 dev_err(hdev->dev, "Failed to create dram page pool\n"); in hl_vm_init()
1803 return -ENOMEM; in hl_vm_init()
1806 kref_init(&vm->dram_pg_pool_refcount); in hl_vm_init()
1808 rc = gen_pool_add(vm->dram_pg_pool, prop->dram_user_base_address, in hl_vm_init()
1809 prop->dram_end_address - prop->dram_user_base_address, in hl_vm_init()
1810 -1); in hl_vm_init()
1813 dev_err(hdev->dev, in hl_vm_init()
1818 spin_lock_init(&vm->idr_lock); in hl_vm_init()
1819 idr_init(&vm->phys_pg_pack_handles); in hl_vm_init()
1821 atomic64_set(&hdev->dram_used_mem, 0); in hl_vm_init()
1823 vm->init_done = true; in hl_vm_init()
1828 gen_pool_destroy(vm->dram_pg_pool); in hl_vm_init()
1834 * hl_vm_fini - virtual memory module teardown
1839 * - Idr for device memory allocation handles
1840 * - DRAM physical pages pool of 2MB
1841 * - MMU module
1845 struct hl_vm *vm = &hdev->vm; in hl_vm_fini() local
1847 if (!vm->init_done) in hl_vm_fini()
1854 if (kref_put(&vm->dram_pg_pool_refcount, dram_pg_pool_do_release) != 1) in hl_vm_fini()
1855 dev_warn(hdev->dev, "dram_pg_pool was not destroyed on %s\n", in hl_vm_fini()
1858 vm->init_done = false; in hl_vm_fini()