93 static void vmemmap_mark_sub_pmd_used(unsigned long start, unsigned long end)  in vmemmap_mark_sub_pmd_used()  argument
101 	memset((void *)start, 0, sizeof(struct page));  in vmemmap_mark_sub_pmd_used()
104 static void vmemmap_use_sub_pmd(unsigned long start, unsigned long end)  in vmemmap_use_sub_pmd()  argument
110 	if (unused_sub_pmd_start == start) {  in vmemmap_use_sub_pmd()
117 	vmemmap_mark_sub_pmd_used(start, end);  in vmemmap_use_sub_pmd()
120 static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end)  in vmemmap_use_new_sub_pmd()  argument
122 	unsigned long page = ALIGN_DOWN(start, PMD_SIZE);  in vmemmap_use_new_sub_pmd()
127 	vmemmap_mark_sub_pmd_used(start, end);  in vmemmap_use_new_sub_pmd()
130 	if (!IS_ALIGNED(start, PMD_SIZE))  in vmemmap_use_new_sub_pmd()
131 		memset((void *)page, PAGE_UNUSED, start - page);  in vmemmap_use_new_sub_pmd()
142 static bool vmemmap_unuse_sub_pmd(unsigned long start, unsigned long end)  in vmemmap_unuse_sub_pmd()  argument
144 	unsigned long page = ALIGN_DOWN(start, PMD_SIZE);  in vmemmap_unuse_sub_pmd()
147 	memset((void *)start, PAGE_UNUSED, end - start);  in vmemmap_unuse_sub_pmd()
193 static void try_free_pte_table(pmd_t *pmd, unsigned long start)  in try_free_pte_table()  argument
199 	pte = pte_offset_kernel(pmd, start);  in try_free_pte_table()
290 static void try_free_pmd_table(pud_t *pud, unsigned long start)  in try_free_pmd_table()  argument
292 	const unsigned long end = start + PUD_SIZE;  in try_free_pmd_table()
300 	if (start < KASAN_SHADOW_END && KASAN_SHADOW_START > end)  in try_free_pmd_table()
303 	pmd = pmd_offset(pud, start);  in try_free_pmd_table()
365 static void try_free_pud_table(p4d_t *p4d, unsigned long start)  in try_free_pud_table()  argument
367 	const unsigned long end = start + P4D_SIZE;  in try_free_pud_table()
375 	if (start < KASAN_SHADOW_END && KASAN_SHADOW_START > end)  in try_free_pud_table()
379 	pud = pud_offset(p4d, start);  in try_free_pud_table()
419 static void try_free_p4d_table(pgd_t *pgd, unsigned long start)  in try_free_p4d_table()  argument
421 	const unsigned long end = start + PGDIR_SIZE;  in try_free_p4d_table()
429 	if (start < KASAN_SHADOW_END && KASAN_SHADOW_START > end)  in try_free_p4d_table()
433 	p4d = p4d_offset(pgd, start);  in try_free_p4d_table()
442 static int modify_pagetable(unsigned long start, unsigned long end, bool add,  in modify_pagetable()  argument
450 	if (WARN_ON_ONCE(!PAGE_ALIGNED(start | end)))  in modify_pagetable()
452 	for (addr = start; addr < end; addr = next) {  in modify_pagetable()
474 		flush_tlb_kernel_range(start, end);  in modify_pagetable()
478 static int add_pagetable(unsigned long start, unsigned long end, bool direct)  in add_pagetable()  argument
480 	return modify_pagetable(start, end, true, direct);  in add_pagetable()
483 static int remove_pagetable(unsigned long start, unsigned long end, bool direct)  in remove_pagetable()  argument
485 	return modify_pagetable(start, end, false, direct);  in remove_pagetable()
491 static int vmem_add_range(unsigned long start, unsigned long size)  in vmem_add_range()  argument
493 	return add_pagetable(start, start + size, true);  in vmem_add_range()
499 static void vmem_remove_range(unsigned long start, unsigned long size)  in vmem_remove_range()  argument
501 	remove_pagetable(start, start + size, true);  in vmem_remove_range()
507 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,  in vmemmap_populate()  argument
514 	ret = add_pagetable(start, end, false);  in vmemmap_populate()
516 		remove_pagetable(start, end, false);  in vmemmap_populate()
521 void vmemmap_free(unsigned long start, unsigned long end,  in vmemmap_free()  argument
525 	remove_pagetable(start, end, false);  in vmemmap_free()
529 void vmem_remove_mapping(unsigned long start, unsigned long size)  in vmem_remove_mapping()  argument
532 	vmem_remove_range(start, size);  in vmem_remove_mapping()
540 	mhp_range.start = 0;  in arch_get_mappable_range()
545 int vmem_add_mapping(unsigned long start, unsigned long size)  in vmem_add_mapping()  argument
550 	if (start < range.start ||  in vmem_add_mapping()
551 	    start + size > range.end + 1 ||  in vmem_add_mapping()
552 	    start + size < start)  in vmem_add_mapping()
556 	ret = vmem_add_range(start, size);  in vmem_add_mapping()
558 		vmem_remove_range(start, size);  in vmem_add_mapping()