348 		struct vmap_area *va;  in __find_vmap_area()  local
350 		va = rb_entry(n, struct vmap_area, rb_node);  in __find_vmap_area()
351 		if (addr < va->va_start)  in __find_vmap_area()
353 		else if (addr >= va->va_end)  in __find_vmap_area()
356 			return va;  in __find_vmap_area()
362 static void __insert_vmap_area(struct vmap_area *va)  in __insert_vmap_area()  argument
373 		if (va->va_start < tmp_va->va_end)  in __insert_vmap_area()
375 		else if (va->va_end > tmp_va->va_start)  in __insert_vmap_area()
381 	rb_link_node(&va->rb_node, parent, p);  in __insert_vmap_area()
382 	rb_insert_color(&va->rb_node, &vmap_area_root);  in __insert_vmap_area()
385 	tmp = rb_prev(&va->rb_node);  in __insert_vmap_area()
389 		list_add_rcu(&va->list, &prev->list);  in __insert_vmap_area()
391 		list_add_rcu(&va->list, &vmap_area_list);  in __insert_vmap_area()
407 	struct vmap_area *va;  in alloc_vmap_area()  local
419 	va = kmalloc_node(sizeof(struct vmap_area),  in alloc_vmap_area()
421 	if (unlikely(!va))  in alloc_vmap_area()
428 	kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK);  in alloc_vmap_area()
504 	va->va_start = addr;  in alloc_vmap_area()
505 	va->va_end = addr + size;  in alloc_vmap_area()
506 	va->flags = 0;  in alloc_vmap_area()
507 	__insert_vmap_area(va);  in alloc_vmap_area()
508 	free_vmap_cache = &va->rb_node;  in alloc_vmap_area()
511 	BUG_ON(!IS_ALIGNED(va->va_start, align));  in alloc_vmap_area()
512 	BUG_ON(va->va_start < vstart);  in alloc_vmap_area()
513 	BUG_ON(va->va_end > vend);  in alloc_vmap_area()
515 	return va;  in alloc_vmap_area()
537 	kfree(va);  in alloc_vmap_area()
553 static void __free_vmap_area(struct vmap_area *va)  in __free_vmap_area()  argument
555 	BUG_ON(RB_EMPTY_NODE(&va->rb_node));  in __free_vmap_area()
558 		if (va->va_end < cached_vstart) {  in __free_vmap_area()
563 			if (va->va_start <= cache->va_start) {  in __free_vmap_area()
564 				free_vmap_cache = rb_prev(&va->rb_node);  in __free_vmap_area()
572 	rb_erase(&va->rb_node, &vmap_area_root);  in __free_vmap_area()
573 	RB_CLEAR_NODE(&va->rb_node);  in __free_vmap_area()
574 	list_del_rcu(&va->list);  in __free_vmap_area()
582 	if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END)  in __free_vmap_area()
583 		vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end);  in __free_vmap_area()
585 	kfree_rcu(va, rcu_head);  in __free_vmap_area()
591 static void free_vmap_area(struct vmap_area *va)  in free_vmap_area()  argument
594 	__free_vmap_area(va);  in free_vmap_area()
601 static void unmap_vmap_area(struct vmap_area *va)  in unmap_vmap_area()  argument
603 	vunmap_page_range(va->va_start, va->va_end);  in unmap_vmap_area()
658 	struct vmap_area *va;  in __purge_vmap_area_lazy()  local
665 	llist_for_each_entry(va, valist, purge_list) {  in __purge_vmap_area_lazy()
666 		if (va->va_start < start)  in __purge_vmap_area_lazy()
667 			start = va->va_start;  in __purge_vmap_area_lazy()
668 		if (va->va_end > end)  in __purge_vmap_area_lazy()
669 			end = va->va_end;  in __purge_vmap_area_lazy()
679 	llist_for_each_entry_safe(va, n_va, valist, purge_list) {  in __purge_vmap_area_lazy()
680 		int nr = (va->va_end - va->va_start) >> PAGE_SHIFT;  in __purge_vmap_area_lazy()
682 		__free_vmap_area(va);  in __purge_vmap_area_lazy()
718 static void free_vmap_area_noflush(struct vmap_area *va)  in free_vmap_area_noflush()  argument
722 	nr_lazy = atomic_add_return((va->va_end - va->va_start) >> PAGE_SHIFT,  in free_vmap_area_noflush()
726 	llist_add(&va->purge_list, &vmap_purge_list);  in free_vmap_area_noflush()
735 static void free_unmap_vmap_area(struct vmap_area *va)  in free_unmap_vmap_area()  argument
737 	flush_cache_vunmap(va->va_start, va->va_end);  in free_unmap_vmap_area()
738 	unmap_vmap_area(va);  in free_unmap_vmap_area()
740 		flush_tlb_kernel_range(va->va_start, va->va_end);  in free_unmap_vmap_area()
742 	free_vmap_area_noflush(va);  in free_unmap_vmap_area()
747 	struct vmap_area *va;  in find_vmap_area()  local
750 	va = __find_vmap_area(addr);  in find_vmap_area()
753 	return va;  in find_vmap_area()
795 	struct vmap_area *va;  member
849 	struct vmap_area *va;  in new_vmap_block()  local
861 	va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,  in new_vmap_block()
864 	if (IS_ERR(va)) {  in new_vmap_block()
866 		return ERR_CAST(va);  in new_vmap_block()
872 		free_vmap_area(va);  in new_vmap_block()
876 	vaddr = vmap_block_vaddr(va->va_start, 0);  in new_vmap_block()
878 	vb->va = va;  in new_vmap_block()
887 	vb_idx = addr_to_vb_idx(va->va_start);  in new_vmap_block()
908 	vb_idx = addr_to_vb_idx(vb->va->va_start);  in free_vmap_block()
914 	free_vmap_area_noflush(vb->va);  in free_vmap_block()
992 		vaddr = vmap_block_vaddr(vb->va->va_start, pages_off);  in vb_alloc()
1090 				unsigned long va_start = vb->va->va_start;  in vm_unmap_aliases()
1123 	struct vmap_area *va;  in vm_unmap_ram()  local
1137 	va = find_vmap_area(addr);  in vm_unmap_ram()
1138 	BUG_ON(!va);  in vm_unmap_ram()
1139 	debug_check_no_locks_freed((void *)va->va_start,  in vm_unmap_ram()
1140 				    (va->va_end - va->va_start));  in vm_unmap_ram()
1141 	free_unmap_vmap_area(va);  in vm_unmap_ram()
1172 		struct vmap_area *va;  in vm_map_ram()  local
1173 		va = alloc_vmap_area(size, PAGE_SIZE,  in vm_map_ram()
1175 		if (IS_ERR(va))  in vm_map_ram()
1178 		addr = va->va_start;  in vm_map_ram()
1243 	struct vmap_area *va;  in vmalloc_init()  local
1261 		va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);  in vmalloc_init()
1262 		va->flags = VM_VM_AREA;  in vmalloc_init()
1263 		va->va_start = (unsigned long)tmp->addr;  in vmalloc_init()
1264 		va->va_end = va->va_start + tmp->size;  in vmalloc_init()
1265 		va->vm = tmp;  in vmalloc_init()
1266 		__insert_vmap_area(va);  in vmalloc_init()
1349 static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,  in setup_vmalloc_vm()  argument
1354 	vm->addr = (void *)va->va_start;  in setup_vmalloc_vm()
1355 	vm->size = va->va_end - va->va_start;  in setup_vmalloc_vm()
1357 	va->vm = vm;  in setup_vmalloc_vm()
1358 	va->flags |= VM_VM_AREA;  in setup_vmalloc_vm()
1377 	struct vmap_area *va;  in __get_vm_area_node()  local
1396 	va = alloc_vmap_area(size, align, start, end, node, gfp_mask);  in __get_vm_area_node()
1397 	if (IS_ERR(va)) {  in __get_vm_area_node()
1402 	setup_vmalloc_vm(area, va, flags, caller);  in __get_vm_area_node()
1456 	struct vmap_area *va;  in find_vm_area()  local
1458 	va = find_vmap_area((unsigned long)addr);  in find_vm_area()
1459 	if (va && va->flags & VM_VM_AREA)  in find_vm_area()
1460 		return va->vm;  in find_vm_area()
1475 	struct vmap_area *va;  in remove_vm_area()  local
1479 	va = find_vmap_area((unsigned long)addr);  in remove_vm_area()
1480 	if (va && va->flags & VM_VM_AREA) {  in remove_vm_area()
1481 		struct vm_struct *vm = va->vm;  in remove_vm_area()
1484 		va->vm = NULL;  in remove_vm_area()
1485 		va->flags &= ~VM_VM_AREA;  in remove_vm_area()
1486 		va->flags |= VM_LAZY_FREE;  in remove_vm_area()
1490 		free_unmap_vmap_area(va);  in remove_vm_area()
2085 	struct vmap_area *va;  in vread()  local
2096 	list_for_each_entry(va, &vmap_area_list, list) {  in vread()
2100 		if (!(va->flags & VM_VM_AREA))  in vread()
2103 		vm = va->vm;  in vread()
2166 	struct vmap_area *va;  in vwrite()  local
2178 	list_for_each_entry(va, &vmap_area_list, list) {  in vwrite()
2182 		if (!(va->flags & VM_VM_AREA))  in vwrite()
2185 		vm = va->vm;  in vwrite()
2381 	struct vmap_area *va = NULL;  in pvm_find_next_prev()  local
2384 		va = rb_entry(n, struct vmap_area, rb_node);  in pvm_find_next_prev()
2385 		if (end < va->va_end)  in pvm_find_next_prev()
2387 		else if (end > va->va_end)  in pvm_find_next_prev()
2393 	if (!va)  in pvm_find_next_prev()
2396 	if (va->va_end > end) {  in pvm_find_next_prev()
2397 		*pnext = va;  in pvm_find_next_prev()
2400 		*pprev = va;  in pvm_find_next_prev()
2586 		struct vmap_area *va = vas[area];  in pcpu_get_vm_areas()  local
2588 		va->va_start = base + offsets[area];  in pcpu_get_vm_areas()
2589 		va->va_end = va->va_start + sizes[area];  in pcpu_get_vm_areas()
2590 		__insert_vmap_area(va);  in pcpu_get_vm_areas()
2678 	struct vmap_area *va;  in s_show()  local
2681 	va = list_entry(p, struct vmap_area, list);  in s_show()
2687 	if (!(va->flags & VM_VM_AREA)) {  in s_show()
2689 			(void *)va->va_start, (void *)va->va_end,  in s_show()
2690 			va->va_end - va->va_start,  in s_show()
2691 			va->flags & VM_LAZY_FREE ? "unpurged vm_area" : "vm_map_ram");  in s_show()
2696 	v = va->vm;  in s_show()