Lines Matching +full:vm +full:- +full:map
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
17 /* use small pages for supporting non-pow2 (32M/40M/48M) DRAM phys page sizes */
31 * two chunks - one to return as result and a remainder to stay in the list.
44 * alloc_device_memory() - allocate device memory.
50 * - Allocate the requested size rounded up to 'dram_page_size' pages.
51 * - Return unique handle for later map/unmap/free.
56 struct hl_device *hdev = ctx->hdev; in alloc_device_memory()
57 struct hl_vm *vm = &hdev->vm; in alloc_device_memory() local
65 page_size = hdev->asic_prop.dram_page_size; in alloc_device_memory()
66 num_pgs = DIV_ROUND_UP_ULL(args->alloc.mem_size, page_size); in alloc_device_memory()
70 dev_err(hdev->dev, "Cannot allocate 0 bytes\n"); in alloc_device_memory()
71 return -EINVAL; in alloc_device_memory()
74 contiguous = args->flags & HL_MEM_CONTIGUOUS; in alloc_device_memory()
77 paddr = (u64) gen_pool_alloc(vm->dram_pg_pool, total_size); in alloc_device_memory()
79 dev_err(hdev->dev, in alloc_device_memory()
82 return -ENOMEM; in alloc_device_memory()
88 rc = -ENOMEM; in alloc_device_memory()
92 phys_pg_pack->vm_type = VM_TYPE_PHYS_PACK; in alloc_device_memory()
93 phys_pg_pack->asid = ctx->asid; in alloc_device_memory()
94 phys_pg_pack->npages = num_pgs; in alloc_device_memory()
95 phys_pg_pack->page_size = page_size; in alloc_device_memory()
96 phys_pg_pack->total_size = total_size; in alloc_device_memory()
97 phys_pg_pack->flags = args->flags; in alloc_device_memory()
98 phys_pg_pack->contiguous = contiguous; in alloc_device_memory()
100 phys_pg_pack->pages = kvmalloc_array(num_pgs, sizeof(u64), GFP_KERNEL); in alloc_device_memory()
101 if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) { in alloc_device_memory()
102 rc = -ENOMEM; in alloc_device_memory()
106 if (phys_pg_pack->contiguous) { in alloc_device_memory()
108 phys_pg_pack->pages[i] = paddr + i * page_size; in alloc_device_memory()
111 phys_pg_pack->pages[i] = (u64) gen_pool_alloc( in alloc_device_memory()
112 vm->dram_pg_pool, in alloc_device_memory()
114 if (!phys_pg_pack->pages[i]) { in alloc_device_memory()
115 dev_err(hdev->dev, in alloc_device_memory()
117 rc = -ENOMEM; in alloc_device_memory()
125 spin_lock(&vm->idr_lock); in alloc_device_memory()
126 handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0, in alloc_device_memory()
128 spin_unlock(&vm->idr_lock); in alloc_device_memory()
131 dev_err(hdev->dev, "Failed to get handle for page\n"); in alloc_device_memory()
132 rc = -EFAULT; in alloc_device_memory()
137 kref_get(&vm->dram_pg_pool_refcount); in alloc_device_memory()
139 phys_pg_pack->handle = handle; in alloc_device_memory()
141 atomic64_add(phys_pg_pack->total_size, &ctx->dram_phys_mem); in alloc_device_memory()
142 atomic64_add(phys_pg_pack->total_size, &hdev->dram_used_mem); in alloc_device_memory()
150 if (!phys_pg_pack->contiguous) in alloc_device_memory()
152 gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[i], in alloc_device_memory()
155 kvfree(phys_pg_pack->pages); in alloc_device_memory()
160 gen_pool_free(vm->dram_pg_pool, paddr, total_size); in alloc_device_memory()
166 * dma_map_host_va() - DMA mapping of the given host virtual address.
173 * - Allocate userptr structure.
174 * - Pin the given host memory using the userptr structure.
175 * - Perform DMA mapping to have the DMA addresses of the pages.
185 rc = -ENOMEM; in dma_map_host_va()
191 dev_err(hdev->dev, "Failed to pin host memory\n"); in dma_map_host_va()
195 rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl, in dma_map_host_va()
196 userptr->sgt->nents, DMA_BIDIRECTIONAL); in dma_map_host_va()
198 dev_err(hdev->dev, "failed to map sgt with DMA region\n"); in dma_map_host_va()
202 userptr->dma_mapped = true; in dma_map_host_va()
203 userptr->dir = DMA_BIDIRECTIONAL; in dma_map_host_va()
204 userptr->vm_type = VM_TYPE_USERPTR; in dma_map_host_va()
220 * dma_unmap_host_va() - DMA unmapping of the given host virtual address.
225 * - Unpins the physical pages.
226 * - Frees the userptr structure.
236 * dram_pg_pool_do_release() - free DRAM pages pool
240 * - Frees the idr structure of physical pages handles.
241 * - Frees the generic pool of DRAM physical pages.
245 struct hl_vm *vm = container_of(ref, struct hl_vm, in dram_pg_pool_do_release() local
252 idr_destroy(&vm->phys_pg_pack_handles); in dram_pg_pool_do_release()
253 gen_pool_destroy(vm->dram_pg_pool); in dram_pg_pool_do_release()
257 * free_phys_pg_pack() - free physical page pack.
262 * - For DRAM memory only
263 * - iterate over the pack, scrub and free each physical block structure by
268 * - Free the hl_vm_phys_pg_pack structure.
273 struct hl_vm *vm = &hdev->vm; in free_phys_pg_pack() local
277 if (phys_pg_pack->created_from_userptr) in free_phys_pg_pack()
280 if (phys_pg_pack->contiguous) { in free_phys_pg_pack()
281 if (hdev->memory_scrub && !hdev->disabled) { in free_phys_pg_pack()
282 rc = hdev->asic_funcs->scrub_device_mem(hdev, in free_phys_pg_pack()
283 phys_pg_pack->pages[0], in free_phys_pg_pack()
284 phys_pg_pack->total_size); in free_phys_pg_pack()
286 dev_err(hdev->dev, in free_phys_pg_pack()
290 gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[0], in free_phys_pg_pack()
291 phys_pg_pack->total_size); in free_phys_pg_pack()
293 for (i = 0; i < phys_pg_pack->npages ; i++) in free_phys_pg_pack()
294 kref_put(&vm->dram_pg_pool_refcount, in free_phys_pg_pack()
297 for (i = 0 ; i < phys_pg_pack->npages ; i++) { in free_phys_pg_pack()
298 if (hdev->memory_scrub && !hdev->disabled && rc == 0) { in free_phys_pg_pack()
299 rc = hdev->asic_funcs->scrub_device_mem( in free_phys_pg_pack()
301 phys_pg_pack->pages[i], in free_phys_pg_pack()
302 phys_pg_pack->page_size); in free_phys_pg_pack()
304 dev_err(hdev->dev, in free_phys_pg_pack()
307 gen_pool_free(vm->dram_pg_pool, in free_phys_pg_pack()
308 phys_pg_pack->pages[i], in free_phys_pg_pack()
309 phys_pg_pack->page_size); in free_phys_pg_pack()
310 kref_put(&vm->dram_pg_pool_refcount, in free_phys_pg_pack()
315 if (rc && !hdev->disabled) in free_phys_pg_pack()
319 kvfree(phys_pg_pack->pages); in free_phys_pg_pack()
326 * free_device_memory() - free device memory.
331 * - Free the device memory related to the given handle.
335 struct hl_device *hdev = ctx->hdev; in free_device_memory()
336 struct hl_vm *vm = &hdev->vm; in free_device_memory() local
338 u32 handle = args->free.handle; in free_device_memory()
340 spin_lock(&vm->idr_lock); in free_device_memory()
341 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle); in free_device_memory()
343 if (atomic_read(&phys_pg_pack->mapping_cnt) > 0) { in free_device_memory()
344 dev_err(hdev->dev, "handle %u is mapped, cannot free\n", in free_device_memory()
346 spin_unlock(&vm->idr_lock); in free_device_memory()
347 return -EINVAL; in free_device_memory()
355 idr_remove(&vm->phys_pg_pack_handles, handle); in free_device_memory()
356 spin_unlock(&vm->idr_lock); in free_device_memory()
358 atomic64_sub(phys_pg_pack->total_size, &ctx->dram_phys_mem); in free_device_memory()
359 atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem); in free_device_memory()
363 spin_unlock(&vm->idr_lock); in free_device_memory()
364 dev_err(hdev->dev, in free_device_memory()
367 return -EINVAL; in free_device_memory()
374 * clear_va_list_locked() - free virtual addresses list.
379 * - Iterate over the list and free each virtual addresses block.
389 list_del(&va_block->node); in clear_va_list_locked()
395 * print_va_list_locked() - print virtual addresses list.
400 * - Iterate over the list and print each virtual addresses block.
410 dev_dbg(hdev->dev, "print va list:\n"); in print_va_list_locked()
413 dev_dbg(hdev->dev, in print_va_list_locked()
415 va_block->start, va_block->end, va_block->size); in print_va_list_locked()
420 * merge_va_blocks_locked() - merge a virtual block if possible.
426 * - Merge the given blocks with the adjacent blocks if their virtual ranges
437 if (&prev->node != va_list && prev->end + 1 == va_block->start) { in merge_va_blocks_locked()
438 prev->end = va_block->end; in merge_va_blocks_locked()
439 prev->size = prev->end - prev->start; in merge_va_blocks_locked()
440 list_del(&va_block->node); in merge_va_blocks_locked()
446 if (&next->node != va_list && va_block->end + 1 == next->start) { in merge_va_blocks_locked()
447 next->start = va_block->start; in merge_va_blocks_locked()
448 next->size = next->end - next->start; in merge_va_blocks_locked()
449 list_del(&va_block->node); in merge_va_blocks_locked()
455 * add_va_block_locked() - add a virtual block to the virtual addresses list.
462 * - Add the given block to the virtual blocks list and merge with other blocks
471 u64 size = end - start; in add_va_block_locked()
477 if (hl_mem_area_crosses_range(start, size, va_block->start, in add_va_block_locked()
478 va_block->end)) { in add_va_block_locked()
479 dev_err(hdev->dev, in add_va_block_locked()
481 va_block->start, va_block->end); in add_va_block_locked()
482 return -EINVAL; in add_va_block_locked()
485 if (va_block->end < start) in add_va_block_locked()
491 return -ENOMEM; in add_va_block_locked()
493 va_block->start = start; in add_va_block_locked()
494 va_block->end = end; in add_va_block_locked()
495 va_block->size = size; in add_va_block_locked()
498 list_add(&va_block->node, va_list); in add_va_block_locked()
500 list_add(&va_block->node, &res->node); in add_va_block_locked()
510 * add_va_block() - wrapper for add_va_block_locked.
517 * - Takes the list lock and calls add_va_block_locked.
524 mutex_lock(&va_range->lock); in add_va_block()
525 rc = add_va_block_locked(hdev, &va_range->list, start, end); in add_va_block()
526 mutex_unlock(&va_range->lock); in add_va_block()
532 * is_hint_crossing_range() - check if hint address crossing specified reserved
542 prop->hints_dram_reserved_va_range.start_addr, in is_hint_crossing_range()
543 prop->hints_dram_reserved_va_range.end_addr); in is_hint_crossing_range()
547 prop->hints_host_reserved_va_range.start_addr, in is_hint_crossing_range()
548 prop->hints_host_reserved_va_range.end_addr); in is_hint_crossing_range()
552 prop->hints_host_hpage_reserved_va_range.start_addr, in is_hint_crossing_range()
553 prop->hints_host_hpage_reserved_va_range.end_addr); in is_hint_crossing_range()
559 * get_va_block() - get a virtual block for the given size and alignment.
570 * - Iterate on the virtual block list to find a suitable virtual block for the
572 * - Reserve the requested block and update the list.
573 * - Return the start address of the virtual block.
582 struct asic_fixed_properties *prop = &hdev->asic_prop; in get_va_block()
585 dram_hint_mask = prop->dram_hints_align_mask; in get_va_block()
587 bool is_align_pow_2 = is_power_of_2(va_range->page_size); in get_va_block()
592 align_mask = ~((u64)va_block_align - 1); in get_va_block()
595 * with non-power-of-2 range we work only with page granularity in get_va_block()
599 size = DIV_ROUND_UP_ULL(size, va_range->page_size) * in get_va_block()
600 va_range->page_size; in get_va_block()
605 if ((is_align_pow_2 && (hint_addr & (va_block_align - 1))) || in get_va_block()
607 do_div(tmp_hint_addr, va_range->page_size))) { in get_va_block()
611 dev_err(hdev->dev, in get_va_block()
612 "Hint address 0x%llx is not page aligned - cannot be respected\n", in get_va_block()
617 dev_dbg(hdev->dev, in get_va_block()
623 mutex_lock(&va_range->lock); in get_va_block()
625 print_va_list_locked(hdev, &va_range->list); in get_va_block()
627 list_for_each_entry(va_block, &va_range->list, node) { in get_va_block()
629 valid_start = va_block->start; in get_va_block()
631 if (is_align_pow_2 && (valid_start & (va_block_align - 1))) { in get_va_block()
634 if (valid_start > va_block->end) in get_va_block()
638 valid_size = va_block->end - valid_start; in get_va_block()
647 if (prop->hints_range_reservation && !hint_addr) in get_va_block()
660 (hint_addr + size) <= va_block->end) { in get_va_block()
669 dev_err(hdev->dev, "no available va block for size %llu\n", in get_va_block()
675 /* Hint address must be respected. If we are here - this means in get_va_block()
678 dev_err(hdev->dev, in get_va_block()
689 if (reserved_valid_start > new_va_block->start) { in get_va_block()
690 prev_start = new_va_block->start; in get_va_block()
691 prev_end = reserved_valid_start - 1; in get_va_block()
693 new_va_block->start = reserved_valid_start; in get_va_block()
694 new_va_block->size = reserved_valid_size; in get_va_block()
699 if (new_va_block->size > size) { in get_va_block()
700 new_va_block->start += size; in get_va_block()
701 new_va_block->size = new_va_block->end - new_va_block->start; in get_va_block()
703 list_del(&new_va_block->node); in get_va_block()
708 add_va_block_locked(hdev, &va_range->list, prev_start, in get_va_block()
711 print_va_list_locked(hdev, &va_range->list); in get_va_block()
713 mutex_unlock(&va_range->lock); in get_va_block()
719 * hl_reserve_va_block() - reserve a virtual block of a given size.
728 * - Iterate on the virtual block list to find a suitable virtual block for the
730 * - Reserve the requested block and update the list.
731 * - Return the start address of the virtual block.
736 return get_va_block(hdev, ctx->va_range[type], size, 0, in hl_reserve_va_block()
737 max(alignment, ctx->va_range[type]->page_size), in hl_reserve_va_block()
742 * hl_get_va_range_type() - get va_range type for the given address and size.
756 ctx->va_range[i]->start_addr, in hl_get_va_range_type()
757 ctx->va_range[i]->end_addr)) { in hl_get_va_range_type()
763 return -EINVAL; in hl_get_va_range_type()
767 * hl_unreserve_va_block() - wrapper for add_va_block to unreserve a va block.
774 * - Takes the list lock and calls add_va_block_locked.
784 dev_err(hdev->dev, in hl_unreserve_va_block()
790 rc = add_va_block(hdev, ctx->va_range[type], start_addr, in hl_unreserve_va_block()
791 start_addr + size - 1); in hl_unreserve_va_block()
793 dev_warn(hdev->dev, in hl_unreserve_va_block()
800 * init_phys_pg_pack_from_userptr() - initialize physical page pack from host
811 * - Pin the physical pages related to the given virtual block.
812 * - Create a physical page pack from the physical pages related to the given
821 huge_page_size = ctx->hdev->asic_prop.pmmu_huge.page_size; in init_phys_pg_pack_from_userptr()
832 return -ENOMEM; in init_phys_pg_pack_from_userptr()
834 phys_pg_pack->vm_type = userptr->vm_type; in init_phys_pg_pack_from_userptr()
835 phys_pg_pack->created_from_userptr = true; in init_phys_pg_pack_from_userptr()
836 phys_pg_pack->asid = ctx->asid; in init_phys_pg_pack_from_userptr()
837 atomic_set(&phys_pg_pack->mapping_cnt, 1); in init_phys_pg_pack_from_userptr()
848 for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) { in init_phys_pg_pack_from_userptr()
854 (dma_addr & (huge_page_size - 1))) in init_phys_pg_pack_from_userptr()
863 page_mask = ~(((u64) page_size) - 1); in init_phys_pg_pack_from_userptr()
865 phys_pg_pack->pages = kvmalloc_array(total_npages, sizeof(u64), in init_phys_pg_pack_from_userptr()
867 if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) { in init_phys_pg_pack_from_userptr()
868 rc = -ENOMEM; in init_phys_pg_pack_from_userptr()
872 phys_pg_pack->npages = total_npages; in init_phys_pg_pack_from_userptr()
873 phys_pg_pack->page_size = page_size; in init_phys_pg_pack_from_userptr()
874 phys_pg_pack->total_size = total_npages * page_size; in init_phys_pg_pack_from_userptr()
877 for_each_sg(userptr->sgt->sgl, sg, userptr->sgt->nents, i) { in init_phys_pg_pack_from_userptr()
883 phys_pg_pack->offset = dma_addr & (page_size - 1); in init_phys_pg_pack_from_userptr()
888 phys_pg_pack->pages[j++] = dma_addr; in init_phys_pg_pack_from_userptr()
892 npages -= pgs_in_huge_page; in init_phys_pg_pack_from_userptr()
894 npages--; in init_phys_pg_pack_from_userptr()
909 * map_phys_pg_pack() - maps the physical page pack..
911 * @vaddr: start address of the virtual area to map from.
912 * @phys_pg_pack: the pack of physical pages to map to.
915 * - Maps each chunk of virtual memory to matching physical chunk.
916 * - Stores number of successful mappings in the given argument.
917 * - Returns 0 on success, error code otherwise.
922 struct hl_device *hdev = ctx->hdev; in map_phys_pg_pack()
924 u32 page_size = phys_pg_pack->page_size; in map_phys_pg_pack()
928 for (i = 0 ; i < phys_pg_pack->npages ; i++) { in map_phys_pg_pack()
929 paddr = phys_pg_pack->pages[i]; in map_phys_pg_pack()
932 (i + 1) == phys_pg_pack->npages); in map_phys_pg_pack()
934 dev_err(hdev->dev, in map_phys_pg_pack()
935 "map failed for handle %u, npages: %llu, mapped: %llu", in map_phys_pg_pack()
936 phys_pg_pack->handle, phys_pg_pack->npages, in map_phys_pg_pack()
954 dev_warn_ratelimited(hdev->dev, in map_phys_pg_pack()
956 phys_pg_pack->handle, next_vaddr, in map_phys_pg_pack()
957 phys_pg_pack->pages[i], page_size); in map_phys_pg_pack()
969 if (hdev->pldm || (is_host_addr && (i & 0x7FFF) == 0)) in map_phys_pg_pack()
977 * unmap_phys_pg_pack() - unmaps the physical page pack.
985 struct hl_device *hdev = ctx->hdev; in unmap_phys_pg_pack()
991 page_size = phys_pg_pack->page_size; in unmap_phys_pg_pack()
994 for (i = 0 ; i < phys_pg_pack->npages ; i++, next_vaddr += page_size) { in unmap_phys_pg_pack()
996 (i + 1) == phys_pg_pack->npages)) in unmap_phys_pg_pack()
997 dev_warn_ratelimited(hdev->dev, in unmap_phys_pg_pack()
1008 if (hdev->pldm || (is_host_addr && (i & 0x7FFF) == 0)) in unmap_phys_pg_pack()
1016 struct hl_device *hdev = ctx->hdev; in get_paddr_from_handle()
1017 struct hl_vm *vm = &hdev->vm; in get_paddr_from_handle() local
1021 handle = lower_32_bits(args->map_device.handle); in get_paddr_from_handle()
1022 spin_lock(&vm->idr_lock); in get_paddr_from_handle()
1023 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle); in get_paddr_from_handle()
1025 spin_unlock(&vm->idr_lock); in get_paddr_from_handle()
1026 dev_err(hdev->dev, "no match for handle %u\n", handle); in get_paddr_from_handle()
1027 return -EINVAL; in get_paddr_from_handle()
1030 *paddr = phys_pg_pack->pages[0]; in get_paddr_from_handle()
1032 spin_unlock(&vm->idr_lock); in get_paddr_from_handle()
1038 * map_device_va() - map the given memory.
1044 * - If given a physical device memory handle, map to a device virtual block
1046 * - If given a host virtual address and size, find the related physical pages,
1047 * map a device virtual block to this pages and return the start address of
1053 struct hl_device *hdev = ctx->hdev; in map_device_va()
1054 struct hl_vm *vm = &hdev->vm; in map_device_va() local
1063 bool is_userptr = args->flags & HL_MEM_USERPTR; in map_device_va()
1070 u64 addr = args->map_host.host_virt_addr, in map_device_va()
1071 size = args->map_host.mem_size; in map_device_va()
1072 u32 page_size = hdev->asic_prop.pmmu.page_size, in map_device_va()
1073 huge_page_size = hdev->asic_prop.pmmu_huge.page_size; in map_device_va()
1077 dev_err(hdev->dev, "failed to get userptr from va\n"); in map_device_va()
1084 dev_err(hdev->dev, in map_device_va()
1091 hint_addr = args->map_host.hint_addr; in map_device_va()
1092 handle = phys_pg_pack->handle; in map_device_va()
1095 if (phys_pg_pack->page_size == page_size) { in map_device_va()
1096 va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST]; in map_device_va()
1102 if (addr & (huge_page_size - 1)) in map_device_va()
1111 va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]; in map_device_va()
1116 handle = lower_32_bits(args->map_device.handle); in map_device_va()
1118 spin_lock(&vm->idr_lock); in map_device_va()
1119 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle); in map_device_va()
1121 spin_unlock(&vm->idr_lock); in map_device_va()
1122 dev_err(hdev->dev, in map_device_va()
1124 return -EINVAL; in map_device_va()
1128 atomic_inc(&phys_pg_pack->mapping_cnt); in map_device_va()
1130 spin_unlock(&vm->idr_lock); in map_device_va()
1134 hint_addr = args->map_device.hint_addr; in map_device_va()
1137 va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM]; in map_device_va()
1139 va_block_align = hdev->asic_prop.dmmu.page_size; in map_device_va()
1146 if (!is_userptr && !(phys_pg_pack->flags & HL_MEM_SHARED) && in map_device_va()
1147 phys_pg_pack->asid != ctx->asid) { in map_device_va()
1148 dev_err(hdev->dev, in map_device_va()
1149 "Failed to map memory, handle %u is not shared\n", in map_device_va()
1151 rc = -EPERM; in map_device_va()
1157 rc = -ENOMEM; in map_device_va()
1161 if (hint_addr && phys_pg_pack->offset) { in map_device_va()
1162 if (args->flags & HL_MEM_FORCE_HINT) { in map_device_va()
1164 dev_err(hdev->dev, in map_device_va()
1166 hint_addr, phys_pg_pack->offset); in map_device_va()
1167 rc = -EINVAL; in map_device_va()
1170 dev_dbg(hdev->dev, in map_device_va()
1172 hint_addr, phys_pg_pack->offset); in map_device_va()
1175 ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size, in map_device_va()
1177 va_range_type, args->flags); in map_device_va()
1179 dev_err(hdev->dev, "no available va block for handle %u\n", in map_device_va()
1181 rc = -ENOMEM; in map_device_va()
1185 mutex_lock(&ctx->mmu_lock); in map_device_va()
1189 mutex_unlock(&ctx->mmu_lock); in map_device_va()
1190 dev_err(hdev->dev, "mapping page pack failed for handle %u\n", in map_device_va()
1195 rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, false, in map_device_va()
1196 *vm_type, ctx->asid, ret_vaddr, phys_pg_pack->total_size); in map_device_va()
1198 mutex_unlock(&ctx->mmu_lock); in map_device_va()
1201 dev_err(hdev->dev, in map_device_va()
1207 ret_vaddr += phys_pg_pack->offset; in map_device_va()
1209 hnode->ptr = vm_type; in map_device_va()
1210 hnode->vaddr = ret_vaddr; in map_device_va()
1212 mutex_lock(&ctx->mem_hash_lock); in map_device_va()
1213 hash_add(ctx->mem_hash, &hnode->node, ret_vaddr); in map_device_va()
1214 mutex_unlock(&ctx->mem_hash_lock); in map_device_va()
1225 ret_vaddr + phys_pg_pack->total_size - 1)) in map_device_va()
1226 dev_warn(hdev->dev, in map_device_va()
1234 atomic_dec(&phys_pg_pack->mapping_cnt); in map_device_va()
1245 * unmap_device_va() - unmap the given device virtual address.
1251 * - unmap the physical pages related to the given virtual address.
1252 * - return the device virtual block to the virtual block list.
1258 u64 vaddr = args->unmap.device_virt_addr; in unmap_device_va()
1261 struct hl_device *hdev = ctx->hdev; in unmap_device_va()
1268 prop = &hdev->asic_prop; in unmap_device_va()
1271 mutex_lock(&ctx->mem_hash_lock); in unmap_device_va()
1272 hash_for_each_possible(ctx->mem_hash, hnode, node, (unsigned long)vaddr) in unmap_device_va()
1273 if (vaddr == hnode->vaddr) in unmap_device_va()
1277 mutex_unlock(&ctx->mem_hash_lock); in unmap_device_va()
1278 dev_err(hdev->dev, in unmap_device_va()
1281 return -EINVAL; in unmap_device_va()
1284 hash_del(&hnode->node); in unmap_device_va()
1285 mutex_unlock(&ctx->mem_hash_lock); in unmap_device_va()
1287 vm_type = hnode->ptr; in unmap_device_va()
1291 userptr = hnode->ptr; in unmap_device_va()
1296 dev_err(hdev->dev, in unmap_device_va()
1302 if (phys_pg_pack->page_size == in unmap_device_va()
1303 hdev->asic_prop.pmmu.page_size) in unmap_device_va()
1304 va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST]; in unmap_device_va()
1306 va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]; in unmap_device_va()
1309 va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM]; in unmap_device_va()
1310 phys_pg_pack = hnode->ptr; in unmap_device_va()
1312 dev_warn(hdev->dev, in unmap_device_va()
1313 "unmap failed, unknown vm desc for vaddr 0x%llx\n", in unmap_device_va()
1315 rc = -EFAULT; in unmap_device_va()
1319 if (atomic_read(&phys_pg_pack->mapping_cnt) == 0) { in unmap_device_va()
1320 dev_err(hdev->dev, "vaddr 0x%llx is not mapped\n", vaddr); in unmap_device_va()
1321 rc = -EINVAL; in unmap_device_va()
1325 if (!is_userptr && !is_power_of_2(phys_pg_pack->page_size)) in unmap_device_va()
1326 vaddr = prop->dram_base_address + in unmap_device_va()
1327 DIV_ROUND_DOWN_ULL(vaddr - prop->dram_base_address, in unmap_device_va()
1328 phys_pg_pack->page_size) * in unmap_device_va()
1329 phys_pg_pack->page_size; in unmap_device_va()
1331 vaddr &= ~(((u64) phys_pg_pack->page_size) - 1); in unmap_device_va()
1333 mutex_lock(&ctx->mmu_lock); in unmap_device_va()
1343 rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, true, in unmap_device_va()
1344 *vm_type, ctx->asid, vaddr, in unmap_device_va()
1345 phys_pg_pack->total_size); in unmap_device_va()
1347 mutex_unlock(&ctx->mmu_lock); in unmap_device_va()
1359 dev_err(hdev->dev, in unmap_device_va()
1364 vaddr + phys_pg_pack->total_size - 1); in unmap_device_va()
1366 dev_warn(hdev->dev, in unmap_device_va()
1374 atomic_dec(&phys_pg_pack->mapping_cnt); in unmap_device_va()
1388 mutex_lock(&ctx->mem_hash_lock); in unmap_device_va()
1389 hash_add(ctx->mem_hash, &hnode->node, vaddr); in unmap_device_va()
1390 mutex_unlock(&ctx->mem_hash_lock); in unmap_device_va()
1401 rc = hdev->asic_funcs->get_hw_block_id(hdev, address, size, &block_id); in map_block()
1412 (struct hl_vm_hw_block_list_node *) vma->vm_private_data; in hw_block_vm_close()
1413 struct hl_ctx *ctx = lnode->ctx; in hw_block_vm_close()
1415 mutex_lock(&ctx->hw_block_list_lock); in hw_block_vm_close()
1416 list_del(&lnode->node); in hw_block_vm_close()
1417 mutex_unlock(&ctx->hw_block_list_lock); in hw_block_vm_close()
1420 vma->vm_private_data = NULL; in hw_block_vm_close()
1428 * hl_hw_block_mmap() - mmap a hw block to user.
1438 struct hl_device *hdev = hpriv->hdev; in hl_hw_block_mmap()
1439 struct hl_ctx *ctx = hpriv->ctx; in hl_hw_block_mmap()
1446 block_id = vma->vm_pgoff; in hl_hw_block_mmap()
1447 vma->vm_pgoff = 0; in hl_hw_block_mmap()
1450 block_size = vma->vm_end - vma->vm_start; in hl_hw_block_mmap()
1452 if (!access_ok((void __user *) (uintptr_t) vma->vm_start, block_size)) { in hl_hw_block_mmap()
1453 dev_err(hdev->dev, in hl_hw_block_mmap()
1454 "user pointer is invalid - 0x%lx\n", in hl_hw_block_mmap()
1455 vma->vm_start); in hl_hw_block_mmap()
1457 return -EINVAL; in hl_hw_block_mmap()
1462 return -ENOMEM; in hl_hw_block_mmap()
1464 vma->vm_ops = &hw_block_vm_ops; in hl_hw_block_mmap()
1465 vma->vm_private_data = lnode; in hl_hw_block_mmap()
1469 rc = hdev->asic_funcs->hw_block_mmap(hdev, vma, block_id, block_size); in hl_hw_block_mmap()
1476 lnode->ctx = ctx; in hl_hw_block_mmap()
1477 lnode->vaddr = vma->vm_start; in hl_hw_block_mmap()
1478 lnode->size = block_size; in hl_hw_block_mmap()
1479 lnode->id = block_id; in hl_hw_block_mmap()
1481 mutex_lock(&ctx->hw_block_list_lock); in hl_hw_block_mmap()
1482 list_add_tail(&lnode->node, &ctx->hw_block_mem_list); in hl_hw_block_mmap()
1483 mutex_unlock(&ctx->hw_block_list_lock); in hl_hw_block_mmap()
1485 vma->vm_pgoff = block_id; in hl_hw_block_mmap()
1492 struct hl_device *hdev = hpriv->hdev; in mem_ioctl_no_mmu()
1493 struct hl_ctx *ctx = hpriv->ctx; in mem_ioctl_no_mmu()
1498 switch (args->in.op) { in mem_ioctl_no_mmu()
1500 if (args->in.alloc.mem_size == 0) { in mem_ioctl_no_mmu()
1501 dev_err(hdev->dev, in mem_ioctl_no_mmu()
1503 rc = -EINVAL; in mem_ioctl_no_mmu()
1510 args->in.flags |= HL_MEM_CONTIGUOUS; in mem_ioctl_no_mmu()
1511 rc = alloc_device_memory(ctx, &args->in, &handle); in mem_ioctl_no_mmu()
1514 args->out.handle = (__u64) handle; in mem_ioctl_no_mmu()
1518 rc = free_device_memory(ctx, &args->in); in mem_ioctl_no_mmu()
1522 if (args->in.flags & HL_MEM_USERPTR) { in mem_ioctl_no_mmu()
1523 device_addr = args->in.map_host.host_virt_addr; in mem_ioctl_no_mmu()
1526 rc = get_paddr_from_handle(ctx, &args->in, in mem_ioctl_no_mmu()
1531 args->out.device_virt_addr = device_addr; in mem_ioctl_no_mmu()
1539 rc = map_block(hdev, args->in.map_block.block_addr, in mem_ioctl_no_mmu()
1541 args->out.block_handle = block_handle; in mem_ioctl_no_mmu()
1542 args->out.block_size = block_size; in mem_ioctl_no_mmu()
1546 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n"); in mem_ioctl_no_mmu()
1547 rc = -ENOTTY; in mem_ioctl_no_mmu()
1559 struct hl_device *hdev = hpriv->hdev; in hl_mem_ioctl()
1560 struct hl_ctx *ctx = hpriv->ctx; in hl_mem_ioctl()
1566 dev_warn_ratelimited(hdev->dev, in hl_mem_ioctl()
1568 hdev->status[status]); in hl_mem_ioctl()
1569 return -EBUSY; in hl_mem_ioctl()
1572 if (!hdev->mmu_enable) in hl_mem_ioctl()
1575 switch (args->in.op) { in hl_mem_ioctl()
1577 if (args->in.alloc.mem_size == 0) { in hl_mem_ioctl()
1578 dev_err(hdev->dev, in hl_mem_ioctl()
1580 rc = -EINVAL; in hl_mem_ioctl()
1592 if (!hdev->asic_prop.dram_supports_virtual_memory) { in hl_mem_ioctl()
1593 atomic64_add(args->in.alloc.mem_size, in hl_mem_ioctl()
1594 &ctx->dram_phys_mem); in hl_mem_ioctl()
1595 atomic64_add(args->in.alloc.mem_size, in hl_mem_ioctl()
1596 &hdev->dram_used_mem); in hl_mem_ioctl()
1598 dev_dbg(hdev->dev, "DRAM alloc is not supported\n"); in hl_mem_ioctl()
1602 args->out.handle = 0; in hl_mem_ioctl()
1606 rc = alloc_device_memory(ctx, &args->in, &handle); in hl_mem_ioctl()
1609 args->out.handle = (__u64) handle; in hl_mem_ioctl()
1621 if (!hdev->asic_prop.dram_supports_virtual_memory) { in hl_mem_ioctl()
1622 atomic64_sub(args->in.alloc.mem_size, in hl_mem_ioctl()
1623 &ctx->dram_phys_mem); in hl_mem_ioctl()
1624 atomic64_sub(args->in.alloc.mem_size, in hl_mem_ioctl()
1625 &hdev->dram_used_mem); in hl_mem_ioctl()
1627 dev_dbg(hdev->dev, "DRAM alloc is not supported\n"); in hl_mem_ioctl()
1633 rc = free_device_memory(ctx, &args->in); in hl_mem_ioctl()
1637 rc = map_device_va(ctx, &args->in, &device_addr); in hl_mem_ioctl()
1640 args->out.device_virt_addr = device_addr; in hl_mem_ioctl()
1644 rc = unmap_device_va(ctx, &args->in, false); in hl_mem_ioctl()
1648 rc = map_block(hdev, args->in.map_block.block_addr, in hl_mem_ioctl()
1650 args->out.block_handle = block_handle; in hl_mem_ioctl()
1651 args->out.block_size = block_size; in hl_mem_ioctl()
1655 dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n"); in hl_mem_ioctl()
1656 rc = -ENOTTY; in hl_mem_ioctl()
1671 dev_err(hdev->dev, "user pointer is invalid - 0x%llx\n", addr); in get_user_memory()
1672 return -EFAULT; in get_user_memory()
1675 userptr->pages = kvmalloc_array(npages, sizeof(*userptr->pages), in get_user_memory()
1677 if (!userptr->pages) in get_user_memory()
1678 return -ENOMEM; in get_user_memory()
1682 userptr->pages); in get_user_memory()
1685 dev_err(hdev->dev, in get_user_memory()
1691 rc = -EFAULT; in get_user_memory()
1694 userptr->npages = npages; in get_user_memory()
1696 rc = sg_alloc_table_from_pages(userptr->sgt, in get_user_memory()
1697 userptr->pages, in get_user_memory()
1700 dev_err(hdev->dev, "failed to create SG table from pages\n"); in get_user_memory()
1707 unpin_user_pages(userptr->pages, npages); in get_user_memory()
1709 kvfree(userptr->pages); in get_user_memory()
1714 * hl_pin_host_memory() - pins a chunk of host memory.
1721 * - Pins the physical pages.
1722 * - Create an SG list from those pages.
1732 dev_err(hdev->dev, "size to pin is invalid - %llu\n", size); in hl_pin_host_memory()
1733 return -EINVAL; in hl_pin_host_memory()
1742 dev_err(hdev->dev, in hl_pin_host_memory()
1745 return -EINVAL; in hl_pin_host_memory()
1748 userptr->pid = current->pid; in hl_pin_host_memory()
1749 userptr->sgt = kzalloc(sizeof(*userptr->sgt), GFP_KERNEL); in hl_pin_host_memory()
1750 if (!userptr->sgt) in hl_pin_host_memory()
1751 return -ENOMEM; in hl_pin_host_memory()
1756 npages = (end - start) >> PAGE_SHIFT; in hl_pin_host_memory()
1758 userptr->size = size; in hl_pin_host_memory()
1759 userptr->addr = addr; in hl_pin_host_memory()
1760 userptr->dma_mapped = false; in hl_pin_host_memory()
1761 INIT_LIST_HEAD(&userptr->job_node); in hl_pin_host_memory()
1766 dev_err(hdev->dev, in hl_pin_host_memory()
1777 kfree(userptr->sgt); in hl_pin_host_memory()
1782 * hl_unpin_host_memory - unpins a chunk of host memory.
1787 * - Unpins the physical pages related to the host memory
1788 * - Free the SG list
1794 if (userptr->dma_mapped) in hl_unpin_host_memory()
1795 hdev->asic_funcs->hl_dma_unmap_sg(hdev, userptr->sgt->sgl, in hl_unpin_host_memory()
1796 userptr->sgt->nents, in hl_unpin_host_memory()
1797 userptr->dir); in hl_unpin_host_memory()
1799 unpin_user_pages_dirty_lock(userptr->pages, userptr->npages, true); in hl_unpin_host_memory()
1800 kvfree(userptr->pages); in hl_unpin_host_memory()
1802 list_del(&userptr->job_node); in hl_unpin_host_memory()
1804 sg_free_table(userptr->sgt); in hl_unpin_host_memory()
1805 kfree(userptr->sgt); in hl_unpin_host_memory()
1809 * hl_userptr_delete_list() - clear userptr list.
1814 * - Iterates over the list and unpins the host memory and frees the userptr
1831 * hl_userptr_is_pinned() - returns whether the given userptr is pinned.
1837 * - Iterates over the list and checks if the given userptr is in it, means is
1845 if ((addr == (*userptr)->addr) && (size == (*userptr)->size)) in hl_userptr_is_pinned()
1853 * va_range_init() - initialize virtual addresses range.
1860 * - Initializes the virtual addresses list of the given range with the given
1868 INIT_LIST_HEAD(&va_range->list); in va_range_init()
1877 if (start & (PAGE_SIZE - 1)) { in va_range_init()
1882 if (end & (PAGE_SIZE - 1)) in va_range_init()
1887 dev_err(hdev->dev, "too small vm range for va list\n"); in va_range_init()
1888 return -EFAULT; in va_range_init()
1894 dev_err(hdev->dev, "Failed to init host va list\n"); in va_range_init()
1898 va_range->start_addr = start; in va_range_init()
1899 va_range->end_addr = end; in va_range_init()
1900 va_range->page_size = page_size; in va_range_init()
1906 * va_range_fini() - clear a virtual addresses range.
1911 * - Frees the virtual addresses block list and its lock.
1915 mutex_lock(&va_range->lock); in va_range_fini()
1916 clear_va_list_locked(hdev, &va_range->list); in va_range_fini()
1917 mutex_unlock(&va_range->lock); in va_range_fini()
1919 mutex_destroy(&va_range->lock); in va_range_fini()
1924 * vm_ctx_init_with_ranges() - initialize virtual memory for context.
1936 * - MMU for context.
1937 * - Virtual address to area descriptor hashtable.
1938 * - Virtual block list of available virtual memory.
1951 struct hl_device *hdev = ctx->hdev; in vm_ctx_init_with_ranges()
1955 ctx->va_range[i] = in vm_ctx_init_with_ranges()
1957 if (!ctx->va_range[i]) { in vm_ctx_init_with_ranges()
1958 rc = -ENOMEM; in vm_ctx_init_with_ranges()
1965 dev_err(hdev->dev, "failed to init context %d\n", ctx->asid); in vm_ctx_init_with_ranges()
1969 mutex_init(&ctx->mem_hash_lock); in vm_ctx_init_with_ranges()
1970 hash_init(ctx->mem_hash); in vm_ctx_init_with_ranges()
1972 mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock); in vm_ctx_init_with_ranges()
1974 rc = va_range_init(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST], in vm_ctx_init_with_ranges()
1977 dev_err(hdev->dev, "failed to init host vm range\n"); in vm_ctx_init_with_ranges()
1981 if (hdev->pmmu_huge_range) { in vm_ctx_init_with_ranges()
1982 mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock); in vm_ctx_init_with_ranges()
1985 ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE], in vm_ctx_init_with_ranges()
1989 dev_err(hdev->dev, in vm_ctx_init_with_ranges()
1990 "failed to init host huge vm range\n"); in vm_ctx_init_with_ranges()
1994 kfree(ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]); in vm_ctx_init_with_ranges()
1995 ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE] = in vm_ctx_init_with_ranges()
1996 ctx->va_range[HL_VA_RANGE_TYPE_HOST]; in vm_ctx_init_with_ranges()
1999 mutex_init(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock); in vm_ctx_init_with_ranges()
2001 rc = va_range_init(hdev, ctx->va_range[HL_VA_RANGE_TYPE_DRAM], in vm_ctx_init_with_ranges()
2004 dev_err(hdev->dev, "failed to init dram vm range\n"); in vm_ctx_init_with_ranges()
2013 mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_DRAM]->lock); in vm_ctx_init_with_ranges()
2015 if (hdev->pmmu_huge_range) { in vm_ctx_init_with_ranges()
2016 mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock); in vm_ctx_init_with_ranges()
2018 &ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->list); in vm_ctx_init_with_ranges()
2019 mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock); in vm_ctx_init_with_ranges()
2022 if (hdev->pmmu_huge_range) in vm_ctx_init_with_ranges()
2023 mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]->lock); in vm_ctx_init_with_ranges()
2024 mutex_lock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock); in vm_ctx_init_with_ranges()
2025 clear_va_list_locked(hdev, &ctx->va_range[HL_VA_RANGE_TYPE_HOST]->list); in vm_ctx_init_with_ranges()
2026 mutex_unlock(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock); in vm_ctx_init_with_ranges()
2028 mutex_destroy(&ctx->va_range[HL_VA_RANGE_TYPE_HOST]->lock); in vm_ctx_init_with_ranges()
2029 mutex_destroy(&ctx->mem_hash_lock); in vm_ctx_init_with_ranges()
2033 kfree(ctx->va_range[i]); in vm_ctx_init_with_ranges()
2040 struct asic_fixed_properties *prop = &ctx->hdev->asic_prop; in hl_vm_ctx_init()
2045 atomic64_set(&ctx->dram_phys_mem, 0); in hl_vm_ctx_init()
2048 * - If MMU is enabled, init the ranges as usual. in hl_vm_ctx_init()
2049 * - If MMU is disabled, in case of host mapping, the returned address in hl_vm_ctx_init()
2054 if (!ctx->hdev->mmu_enable) in hl_vm_ctx_init()
2057 dram_range_start = prop->dmmu.start_addr; in hl_vm_ctx_init()
2058 dram_range_end = prop->dmmu.end_addr; in hl_vm_ctx_init()
2059 dram_page_size = prop->dram_page_size ? in hl_vm_ctx_init()
2060 prop->dram_page_size : prop->dmmu.page_size; in hl_vm_ctx_init()
2061 host_range_start = prop->pmmu.start_addr; in hl_vm_ctx_init()
2062 host_range_end = prop->pmmu.end_addr; in hl_vm_ctx_init()
2063 host_page_size = prop->pmmu.page_size; in hl_vm_ctx_init()
2064 host_huge_range_start = prop->pmmu_huge.start_addr; in hl_vm_ctx_init()
2065 host_huge_range_end = prop->pmmu_huge.end_addr; in hl_vm_ctx_init()
2066 host_huge_page_size = prop->pmmu_huge.page_size; in hl_vm_ctx_init()
2075 * hl_vm_ctx_fini() - virtual memory teardown of context.
2079 * - Virtual block list of available virtual memory.
2080 * - Virtual address to area descriptor hashtable.
2081 * - MMU for context.
2084 * - Unmaps the existing hashtable nodes if the hashtable is not empty. The
2087 * - Frees any existing physical page list from the idr which relates to the
2089 * - This function checks the virtual block list for correctness. At this point
2095 struct hl_device *hdev = ctx->hdev; in hl_vm_ctx_fini()
2096 struct hl_vm *vm = &hdev->vm; in hl_vm_ctx_fini() local
2103 if (!hdev->mmu_enable) in hl_vm_ctx_fini()
2112 if (!hdev->hard_reset_pending && !hash_empty(ctx->mem_hash)) in hl_vm_ctx_fini()
2113 dev_dbg(hdev->dev, in hl_vm_ctx_fini()
2116 hash_for_each_safe(ctx->mem_hash, i, tmp_node, hnode, node) { in hl_vm_ctx_fini()
2117 dev_dbg(hdev->dev, in hl_vm_ctx_fini()
2119 hnode->vaddr, ctx->asid); in hl_vm_ctx_fini()
2120 args.unmap.device_virt_addr = hnode->vaddr; in hl_vm_ctx_fini()
2124 mutex_lock(&ctx->mmu_lock); in hl_vm_ctx_fini()
2127 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR); in hl_vm_ctx_fini()
2128 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_PHYS_PACK); in hl_vm_ctx_fini()
2130 mutex_unlock(&ctx->mmu_lock); in hl_vm_ctx_fini()
2132 spin_lock(&vm->idr_lock); in hl_vm_ctx_fini()
2133 idr_for_each_entry(&vm->phys_pg_pack_handles, phys_pg_list, i) in hl_vm_ctx_fini()
2134 if (phys_pg_list->asid == ctx->asid) { in hl_vm_ctx_fini()
2135 dev_dbg(hdev->dev, in hl_vm_ctx_fini()
2137 phys_pg_list, ctx->asid); in hl_vm_ctx_fini()
2138 atomic64_sub(phys_pg_list->total_size, in hl_vm_ctx_fini()
2139 &hdev->dram_used_mem); in hl_vm_ctx_fini()
2141 idr_remove(&vm->phys_pg_pack_handles, i); in hl_vm_ctx_fini()
2143 spin_unlock(&vm->idr_lock); in hl_vm_ctx_fini()
2145 va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_DRAM]); in hl_vm_ctx_fini()
2146 va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST]); in hl_vm_ctx_fini()
2148 if (hdev->pmmu_huge_range) in hl_vm_ctx_fini()
2149 va_range_fini(hdev, ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE]); in hl_vm_ctx_fini()
2151 mutex_destroy(&ctx->mem_hash_lock); in hl_vm_ctx_fini()
2158 if (ctx->asid != HL_KERNEL_ASID_ID && in hl_vm_ctx_fini()
2159 !hdev->asic_prop.dram_supports_virtual_memory) in hl_vm_ctx_fini()
2160 atomic64_set(&hdev->dram_used_mem, 0); in hl_vm_ctx_fini()
2164 * hl_vm_init() - initialize virtual memory module.
2168 * - MMU module.
2169 * - DRAM physical pages pool of 2MB.
2170 * - Idr for device memory allocation handles.
2174 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_vm_init()
2175 struct hl_vm *vm = &hdev->vm; in hl_vm_init() local
2178 if (is_power_of_2(prop->dram_page_size)) in hl_vm_init()
2179 vm->dram_pg_pool = in hl_vm_init()
2180 gen_pool_create(__ffs(prop->dram_page_size), -1); in hl_vm_init()
2182 vm->dram_pg_pool = in hl_vm_init()
2183 gen_pool_create(__ffs(DRAM_POOL_PAGE_SIZE), -1); in hl_vm_init()
2185 if (!vm->dram_pg_pool) { in hl_vm_init()
2186 dev_err(hdev->dev, "Failed to create dram page pool\n"); in hl_vm_init()
2187 return -ENOMEM; in hl_vm_init()
2190 kref_init(&vm->dram_pg_pool_refcount); in hl_vm_init()
2192 rc = gen_pool_add(vm->dram_pg_pool, prop->dram_user_base_address, in hl_vm_init()
2193 prop->dram_end_address - prop->dram_user_base_address, in hl_vm_init()
2194 -1); in hl_vm_init()
2197 dev_err(hdev->dev, in hl_vm_init()
2202 spin_lock_init(&vm->idr_lock); in hl_vm_init()
2203 idr_init(&vm->phys_pg_pack_handles); in hl_vm_init()
2205 atomic64_set(&hdev->dram_used_mem, 0); in hl_vm_init()
2207 vm->init_done = true; in hl_vm_init()
2212 gen_pool_destroy(vm->dram_pg_pool); in hl_vm_init()
2218 * hl_vm_fini() - virtual memory module teardown.
2222 * - Idr for device memory allocation handles.
2223 * - DRAM physical pages pool of 2MB.
2224 * - MMU module.
2228 struct hl_vm *vm = &hdev->vm; in hl_vm_fini() local
2230 if (!vm->init_done) in hl_vm_fini()
2237 if (kref_put(&vm->dram_pg_pool_refcount, dram_pg_pool_do_release) != 1) in hl_vm_fini()
2238 dev_warn(hdev->dev, "dram_pg_pool was not destroyed on %s\n", in hl_vm_fini()
2241 vm->init_done = false; in hl_vm_fini()
2245 * hl_hw_block_mem_init() - HW block memory initialization.
2253 mutex_init(&ctx->hw_block_list_lock); in hl_hw_block_mem_init()
2254 INIT_LIST_HEAD(&ctx->hw_block_mem_list); in hl_hw_block_mem_init()
2258 * hl_hw_block_mem_fini() - HW block memory teardown.
2268 if (!list_empty(&ctx->hw_block_mem_list)) in hl_hw_block_mem_fini()
2269 dev_crit(ctx->hdev->dev, "HW block mem list isn't empty\n"); in hl_hw_block_mem_fini()
2271 list_for_each_entry_safe(lnode, tmp, &ctx->hw_block_mem_list, node) { in hl_hw_block_mem_fini()
2272 list_del(&lnode->node); in hl_hw_block_mem_fini()
2276 mutex_destroy(&ctx->hw_block_list_lock); in hl_hw_block_mem_fini()