2061 	struct vm_struct *area;  in __get_vm_area_node()  local
2073 	area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);  in __get_vm_area_node()
2074 	if (unlikely(!area))  in __get_vm_area_node()
2082 		kfree(area);  in __get_vm_area_node()
2088 	setup_vmalloc_vm(area, va, flags, caller);  in __get_vm_area_node()
2090 	return area;  in __get_vm_area_node()
2181 static inline void set_area_direct_map(const struct vm_struct *area,  in set_area_direct_map()  argument
2186 	for (i = 0; i < area->nr_pages; i++)  in set_area_direct_map()
2187 		if (page_address(area->pages[i]))  in set_area_direct_map()
2188 			set_direct_map(area->pages[i]);  in set_area_direct_map()
2192 static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)  in vm_remove_mappings()  argument
2195 	int flush_reset = area->flags & VM_FLUSH_RESET_PERMS;  in vm_remove_mappings()
2199 	remove_vm_area(area->addr);  in vm_remove_mappings()
2219 	for (i = 0; i < area->nr_pages; i++) {  in vm_remove_mappings()
2220 		unsigned long addr = (unsigned long)page_address(area->pages[i]);  in vm_remove_mappings()
2233 	set_area_direct_map(area, set_direct_map_invalid_noflush);  in vm_remove_mappings()
2235 	set_area_direct_map(area, set_direct_map_default_noflush);  in vm_remove_mappings()
2240 	struct vm_struct *area;  in __vunmap()  local
2249 	area = find_vm_area(addr);  in __vunmap()
2250 	if (unlikely(!area)) {  in __vunmap()
2256 	debug_check_no_locks_freed(area->addr, get_vm_area_size(area));  in __vunmap()
2257 	debug_check_no_obj_freed(area->addr, get_vm_area_size(area));  in __vunmap()
2259 	kasan_poison_vmalloc(area->addr, area->size);  in __vunmap()
2261 	vm_remove_mappings(area, deallocate_pages);  in __vunmap()
2266 		for (i = 0; i < area->nr_pages; i++) {  in __vunmap()
2267 			struct page *page = area->pages[i];  in __vunmap()
2272 		atomic_long_sub(area->nr_pages, &nr_vmalloc_pages);  in __vunmap()
2274 		kvfree(area->pages);  in __vunmap()
2277 	kfree(area);  in __vunmap()
2389 	struct vm_struct *area;  in vmap()  local
2398 	area = get_vm_area_caller(size, flags, __builtin_return_address(0));  in vmap()
2399 	if (!area)  in vmap()
2402 	if (map_kernel_range((unsigned long)area->addr, size, pgprot_nx(prot),  in vmap()
2404 		vunmap(area->addr);  in vmap()
2409 		area->pages = pages;  in vmap()
2410 	return area->addr;  in vmap()
2443 	struct vm_struct *area;  in vmap_pfn()  local
2445 	area = get_vm_area_caller(count * PAGE_SIZE, VM_IOREMAP,  in vmap_pfn()
2447 	if (!area)  in vmap_pfn()
2449 	if (apply_to_page_range(&init_mm, (unsigned long)area->addr,  in vmap_pfn()
2451 		free_vm_area(area);  in vmap_pfn()
2454 	return area->addr;  in vmap_pfn()
2459 static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,  in __vmalloc_area_node()  argument
2463 	unsigned int nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;  in __vmalloc_area_node()
2474 					area->caller);  in __vmalloc_area_node()
2480 		remove_vm_area(area->addr);  in __vmalloc_area_node()
2481 		kfree(area);  in __vmalloc_area_node()
2485 	area->pages = pages;  in __vmalloc_area_node()
2486 	area->nr_pages = nr_pages;  in __vmalloc_area_node()
2488 	for (i = 0; i < area->nr_pages; i++) {  in __vmalloc_area_node()
2498 			area->nr_pages = i;  in __vmalloc_area_node()
2499 			atomic_long_add(area->nr_pages, &nr_vmalloc_pages);  in __vmalloc_area_node()
2502 		area->pages[i] = page;  in __vmalloc_area_node()
2506 	atomic_long_add(area->nr_pages, &nr_vmalloc_pages);  in __vmalloc_area_node()
2508 	if (map_kernel_range((unsigned long)area->addr, get_vm_area_size(area),  in __vmalloc_area_node()
2512 	return area->addr;  in __vmalloc_area_node()
2517 			  (area->nr_pages*PAGE_SIZE), area->size);  in __vmalloc_area_node()
2518 	__vfree(area->addr);  in __vmalloc_area_node()
2545 	struct vm_struct *area;  in __vmalloc_node_range()  local
2553 	area = __get_vm_area_node(real_size, align, VM_ALLOC | VM_UNINITIALIZED |  in __vmalloc_node_range()
2555 	if (!area)  in __vmalloc_node_range()
2558 	addr = __vmalloc_area_node(area, gfp_mask, prot, node);  in __vmalloc_node_range()
2567 	clear_vm_uninitialized_flag(area);  in __vmalloc_node_range()
2569 	kmemleak_vmalloc(area, size, gfp_mask);  in __vmalloc_node_range()
3016 	struct vm_struct *area;  in remap_vmalloc_range_partial()  local
3028 	area = find_vm_area(kaddr);  in remap_vmalloc_range_partial()
3029 	if (!area)  in remap_vmalloc_range_partial()
3032 	if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT)))  in remap_vmalloc_range_partial()
3036 	    end_index > get_vm_area_size(area))  in remap_vmalloc_range_partial()
3082 void free_vm_area(struct vm_struct *area)  in free_vm_area()  argument
3085 	ret = remove_vm_area(area->addr);  in free_vm_area()
3086 	BUG_ON(ret != area);  in free_vm_area()
3087 	kfree(area);  in free_vm_area()
3190 	int area, area2, last_area, term_area;  in pcpu_get_vm_areas()  local
3197 	for (last_area = 0, area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
3198 		start = offsets[area];  in pcpu_get_vm_areas()
3199 		end = start + sizes[area];  in pcpu_get_vm_areas()
3202 		BUG_ON(!IS_ALIGNED(offsets[area], align));  in pcpu_get_vm_areas()
3203 		BUG_ON(!IS_ALIGNED(sizes[area], align));  in pcpu_get_vm_areas()
3207 			last_area = area;  in pcpu_get_vm_areas()
3209 		for (area2 = area + 1; area2 < nr_vms; area2++) {  in pcpu_get_vm_areas()
3228 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
3229 		vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);  in pcpu_get_vm_areas()
3230 		vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);  in pcpu_get_vm_areas()
3231 		if (!vas[area] || !vms[area])  in pcpu_get_vm_areas()
3238 	area = term_area = last_area;  in pcpu_get_vm_areas()
3239 	start = offsets[area];  in pcpu_get_vm_areas()
3240 	end = start + sizes[area];  in pcpu_get_vm_areas()
3265 			term_area = area;  in pcpu_get_vm_areas()
3275 			term_area = area;  in pcpu_get_vm_areas()
3283 		area = (area + nr_vms - 1) % nr_vms;  in pcpu_get_vm_areas()
3284 		if (area == term_area)  in pcpu_get_vm_areas()
3287 		start = offsets[area];  in pcpu_get_vm_areas()
3288 		end = start + sizes[area];  in pcpu_get_vm_areas()
3293 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
3296 		start = base + offsets[area];  in pcpu_get_vm_areas()
3297 		size = sizes[area];  in pcpu_get_vm_areas()
3314 		va = vas[area];  in pcpu_get_vm_areas()
3322 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
3323 		if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area]))  in pcpu_get_vm_areas()
3326 		kasan_unpoison_vmalloc((void *)vas[area]->va_start,  in pcpu_get_vm_areas()
3327 				       sizes[area]);  in pcpu_get_vm_areas()
3332 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
3333 		insert_vmap_area(vas[area], &vmap_area_root, &vmap_area_list);  in pcpu_get_vm_areas()
3335 		setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC,  in pcpu_get_vm_areas()
3350 	while (area--) {  in pcpu_get_vm_areas()
3351 		orig_start = vas[area]->va_start;  in pcpu_get_vm_areas()
3352 		orig_end = vas[area]->va_end;  in pcpu_get_vm_areas()
3353 		va = merge_or_add_vmap_area(vas[area], &free_vmap_area_root,  in pcpu_get_vm_areas()
3358 		vas[area] = NULL;  in pcpu_get_vm_areas()
3368 		for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
3369 			if (vas[area])  in pcpu_get_vm_areas()
3372 			vas[area] = kmem_cache_zalloc(  in pcpu_get_vm_areas()
3374 			if (!vas[area])  in pcpu_get_vm_areas()
3382 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
3383 		if (vas[area])  in pcpu_get_vm_areas()
3384 			kmem_cache_free(vmap_area_cachep, vas[area]);  in pcpu_get_vm_areas()
3386 		kfree(vms[area]);  in pcpu_get_vm_areas()
3400 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
3401 		orig_start = vas[area]->va_start;  in pcpu_get_vm_areas()
3402 		orig_end = vas[area]->va_end;  in pcpu_get_vm_areas()
3403 		va = merge_or_add_vmap_area(vas[area], &free_vmap_area_root,  in pcpu_get_vm_areas()
3408 		vas[area] = NULL;  in pcpu_get_vm_areas()
3409 		kfree(vms[area]);  in pcpu_get_vm_areas()