362  * find a lowest match of free area.
682  * free area is inserted. If VA has been merged, it is
726 			/* Point to the new merged area. */  in merge_or_add_vmap_area()
1003  * Returns a start address of the newly allocated area, if success.
1081 	 * that we have it available when fit type of free area is  in alloc_vmap_area()
1283 		 * Finally insert or merge lazily-freed area. It is  in __purge_vmap_area_lazy()
1323  * Free a vmap area, caller ensuring that the area has been unmapped
1346  * Free and unmap a vmap area
1809  * vm_area_add_early - add vmap area early during boot
1812  * This function is used to add fixed kernel vm area to vmlist before
1835  * vm_area_register_early - register vmap area early during boot
1839  * This function is used to register kernel vm area before
1943  * map_kernel_range_noflush - map kernel VM area with the specified pages
1944  * @addr: start of the VM area to map
1945  * @size: size of the VM area to map
1949  * Map PFN_UP(@size) pages at @addr.  The VM area @addr and @size
1968  * unmap_kernel_range_noflush - unmap kernel VM area
1969  * @addr: start of the VM area to unmap
1970  * @size: size of the VM area to unmap
1972  * Unmap PFN_UP(@size) pages at @addr.  The VM area @addr and @size
1988  * unmap_kernel_range - unmap kernel VM area and flush cache and TLB
1989  * @addr: start of the VM area to unmap
1990  * @size: size of the VM area to unmap
2005 int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages)  in map_vm_area()  argument
2007 	unsigned long addr = (unsigned long)area->addr;  in map_vm_area()
2008 	unsigned long end = addr + get_vm_area_size(area);  in map_vm_area()
2045 	struct vm_struct *area;  in __get_vm_area_node()  local
2056 	area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);  in __get_vm_area_node()
2057 	if (unlikely(!area))  in __get_vm_area_node()
2065 		kfree(area);  in __get_vm_area_node()
2069 	setup_vmalloc_vm(area, va, flags, caller);  in __get_vm_area_node()
2071 	return area;  in __get_vm_area_node()
2091  * get_vm_area - reserve a contiguous kernel virtual area
2092  * @size:	 size of the area
2095  * Search an area of @size in the kernel virtual mapping area,
2096  * and reserved it for out purposes.  Returns the area descriptor
2099  * Return: the area descriptor on success or %NULL on failure.
2116  * find_vm_area - find a continuous kernel virtual area
2119  * Search for the kernel VM area starting at @addr, and return it.
2123  * Return: pointer to the found area or %NULL on faulure
2137  * remove_vm_area - find and remove a continuous kernel virtual area
2140  * Search for the kernel VM area starting at @addr, and remove it.
2141  * This function returns the found VM area, but using it is NOT safe
2144  * Return: pointer to the found area or %NULL on faulure
2170 static inline void set_area_direct_map(const struct vm_struct *area,  in set_area_direct_map()  argument
2175 	for (i = 0; i < area->nr_pages; i++)  in set_area_direct_map()
2176 		if (page_address(area->pages[i]))  in set_area_direct_map()
2177 			set_direct_map(area->pages[i]);  in set_area_direct_map()
2181 static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)  in vm_remove_mappings()  argument
2184 	int flush_reset = area->flags & VM_FLUSH_RESET_PERMS;  in vm_remove_mappings()
2188 	remove_vm_area(area->addr);  in vm_remove_mappings()
2195 	 * If not deallocating pages, just do the flush of the VM area and  in vm_remove_mappings()
2208 	for (i = 0; i < area->nr_pages; i++) {  in vm_remove_mappings()
2209 		unsigned long addr = (unsigned long)page_address(area->pages[i]);  in vm_remove_mappings()
2222 	set_area_direct_map(area, set_direct_map_invalid_noflush);  in vm_remove_mappings()
2224 	set_area_direct_map(area, set_direct_map_default_noflush);  in vm_remove_mappings()
2229 	struct vm_struct *area;  in __vunmap()  local
2238 	area = find_vm_area(addr);  in __vunmap()
2239 	if (unlikely(!area)) {  in __vunmap()
2240 		WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",  in __vunmap()
2245 	debug_check_no_locks_freed(area->addr, get_vm_area_size(area));  in __vunmap()
2246 	debug_check_no_obj_freed(area->addr, get_vm_area_size(area));  in __vunmap()
2248 	vm_remove_mappings(area, deallocate_pages);  in __vunmap()
2253 		for (i = 0; i < area->nr_pages; i++) {  in __vunmap()
2254 			struct page *page = area->pages[i];  in __vunmap()
2259 		atomic_long_sub(area->nr_pages, &nr_vmalloc_pages);  in __vunmap()
2261 		kvfree(area->pages);  in __vunmap()
2264 	kfree(area);  in __vunmap()
2312  * Free the virtually continuous memory area starting at @addr, as
2343  * Free the virtually contiguous memory area starting at @addr,
2367  * Return: the address of the area or %NULL on failure
2372 	struct vm_struct *area;  in vmap()  local
2381 	area = get_vm_area_caller(size, flags, __builtin_return_address(0));  in vmap()
2382 	if (!area)  in vmap()
2385 	if (map_vm_area(area, prot, pages)) {  in vmap()
2386 		vunmap(area->addr);  in vmap()
2390 	return area->addr;  in vmap()
2397 static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,  in __vmalloc_area_node()  argument
2408 	nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;  in __vmalloc_area_node()
2414 				PAGE_KERNEL, node, area->caller);  in __vmalloc_area_node()
2420 		remove_vm_area(area->addr);  in __vmalloc_area_node()
2421 		kfree(area);  in __vmalloc_area_node()
2425 	area->pages = pages;  in __vmalloc_area_node()
2426 	area->nr_pages = nr_pages;  in __vmalloc_area_node()
2428 	for (i = 0; i < area->nr_pages; i++) {  in __vmalloc_area_node()
2438 			area->nr_pages = i;  in __vmalloc_area_node()
2439 			atomic_long_add(area->nr_pages, &nr_vmalloc_pages);  in __vmalloc_area_node()
2442 		area->pages[i] = page;  in __vmalloc_area_node()
2446 	atomic_long_add(area->nr_pages, &nr_vmalloc_pages);  in __vmalloc_area_node()
2448 	if (map_vm_area(area, prot, pages))  in __vmalloc_area_node()
2450 	return area->addr;  in __vmalloc_area_node()
2455 			  (area->nr_pages*PAGE_SIZE), area->size);  in __vmalloc_area_node()
2456 	__vfree(area->addr);  in __vmalloc_area_node()
2464  * @start:		  vm area range start
2465  * @end:		  vm area range end
2468  * @vm_flags:		  additional vm area flags (e.g. %VM_NO_GUARD)
2476  * Return: the address of the area or %NULL on failure
2483 	struct vm_struct *area;  in __vmalloc_node_range()  local
2491 	area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED |  in __vmalloc_node_range()
2493 	if (!area)  in __vmalloc_node_range()
2496 	addr = __vmalloc_area_node(area, gfp_mask, prot, node);  in __vmalloc_node_range()
2505 	clear_vm_uninitialized_flag(area);  in __vmalloc_node_range()
2507 	kmemleak_vmalloc(area, size, gfp_mask);  in __vmalloc_node_range()
2619  * The resulting memory area is zeroed so it can be mapped to userspace
2726  * The resulting memory area is 32bit addressable and zeroed so it can be
2759 		 * To do safe access to this _mapped_ area, we need  in aligned_vread()
2798 		 * To do safe access to this _mapped_ area, we need  in aligned_vwrite()
2822  * vread() - read vmalloc area in a safe way.
2827  * This function checks that addr is a valid vmalloc'ed area, and
2828  * copy data from that area to a given buffer. If the given memory range
2830  * proper area of @buf. If there are memory holes, they'll be zero-filled.
2831  * IOREMAP area is treated as memory hole and no copy is done.
2834  * vm_struct area, returns 0. @buf should be kernel's buffer.
2837  * should know vmalloc() area is valid and can use memcpy().
2838  * This is for routines which have to access vmalloc area without
2843  * include any intersection with valid vmalloc area
2882 		else /* IOREMAP area is treated as memory hole */  in vread()
2901  * vwrite() - write vmalloc area in a safe way.
2906  * This function checks that addr is a valid vmalloc'ed area, and
2909  * proper area of @buf. If there are memory holes, no copy to hole.
2910  * IOREMAP area is treated as memory hole and no copy is done.
2913  * vm_struct area, returns 0. @buf should be kernel's buffer.
2916  * should know vmalloc() area is valid and can use memcpy().
2917  * This is for routines which have to access vmalloc area without
2922  * doesn't include any intersection with valid vmalloc area
2979  * @size:		size of map area
2983  * This function checks that @kaddr is a valid vmalloc'ed area,
2993 	struct vm_struct *area;  in remap_vmalloc_range_partial()  local
3000 	area = find_vm_area(kaddr);  in remap_vmalloc_range_partial()
3001 	if (!area)  in remap_vmalloc_range_partial()
3004 	if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT)))  in remap_vmalloc_range_partial()
3007 	if (kaddr + size > area->addr + get_vm_area_size(area))  in remap_vmalloc_range_partial()
3037  * This function checks that addr is a valid vmalloc'ed area, and
3056  * The purpose of this function is to make sure the vmalloc area
3077  * @size:	   size of the area
3087  * allocated for the VM area are returned.
3091 	struct vm_struct *area;  in alloc_vm_area()  local
3093 	area = get_vm_area_caller(size, VM_IOREMAP,  in alloc_vm_area()
3095 	if (area == NULL)  in alloc_vm_area()
3102 	if (apply_to_page_range(&init_mm, (unsigned long)area->addr,  in alloc_vm_area()
3104 		free_vm_area(area);  in alloc_vm_area()
3108 	return area;  in alloc_vm_area()
3112 void free_vm_area(struct vm_struct *area)  in free_vm_area()  argument
3115 	ret = remove_vm_area(area->addr);  in free_vm_area()
3116 	BUG_ON(ret != area);  in free_vm_area()
3117 	kfree(area);  in free_vm_area()
3131  * Returns: vmap_area if it is found. If there is no such area
3190  * @offsets: array containing offset of each area
3191  * @sizes: array containing size of each area
3208  * base address is pulled down to fit the area. Scanning is repeated till
3220 	int area, area2, last_area, term_area;  in pcpu_get_vm_areas()  local
3227 	for (last_area = 0, area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
3228 		start = offsets[area];  in pcpu_get_vm_areas()
3229 		end = start + sizes[area];  in pcpu_get_vm_areas()
3232 		BUG_ON(!IS_ALIGNED(offsets[area], align));  in pcpu_get_vm_areas()
3233 		BUG_ON(!IS_ALIGNED(sizes[area], align));  in pcpu_get_vm_areas()
3235 		/* detect the area with the highest address */  in pcpu_get_vm_areas()
3237 			last_area = area;  in pcpu_get_vm_areas()
3239 		for (area2 = area + 1; area2 < nr_vms; area2++) {  in pcpu_get_vm_areas()
3258 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
3259 		vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);  in pcpu_get_vm_areas()
3260 		vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);  in pcpu_get_vm_areas()
3261 		if (!vas[area] || !vms[area])  in pcpu_get_vm_areas()
3267 	/* start scanning - we scan from the top, begin with the last area */  in pcpu_get_vm_areas()
3268 	area = term_area = last_area;  in pcpu_get_vm_areas()
3269 	start = offsets[area];  in pcpu_get_vm_areas()
3270 	end = start + sizes[area];  in pcpu_get_vm_areas()
3295 			term_area = area;  in pcpu_get_vm_areas()
3305 			term_area = area;  in pcpu_get_vm_areas()
3310 		 * This area fits, move on to the previous one.  If  in pcpu_get_vm_areas()
3313 		area = (area + nr_vms - 1) % nr_vms;  in pcpu_get_vm_areas()
3314 		if (area == term_area)  in pcpu_get_vm_areas()
3317 		start = offsets[area];  in pcpu_get_vm_areas()
3318 		end = start + sizes[area];  in pcpu_get_vm_areas()
3323 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
3326 		start = base + offsets[area];  in pcpu_get_vm_areas()
3327 		size = sizes[area];  in pcpu_get_vm_areas()
3343 		/* Allocated area. */  in pcpu_get_vm_areas()
3344 		va = vas[area];  in pcpu_get_vm_areas()
3354 	for (area = 0; area < nr_vms; area++)  in pcpu_get_vm_areas()
3355 		setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC,  in pcpu_get_vm_areas()
3363 	while (area--) {  in pcpu_get_vm_areas()
3364 		__free_vmap_area(vas[area]);  in pcpu_get_vm_areas()
3365 		vas[area] = NULL;  in pcpu_get_vm_areas()
3375 		for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
3376 			if (vas[area])  in pcpu_get_vm_areas()
3379 			vas[area] = kmem_cache_zalloc(  in pcpu_get_vm_areas()
3381 			if (!vas[area])  in pcpu_get_vm_areas()
3389 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
3390 		if (vas[area])  in pcpu_get_vm_areas()
3391 			kmem_cache_free(vmap_area_cachep, vas[area]);  in pcpu_get_vm_areas()
3393 		kfree(vms[area]);  in pcpu_get_vm_areas()
3486 	 * of vmap area is being tear down or vm_map_ram allocation.  in s_show()