155 			p4d_t *p4d;  in sync_global_pgds_l4()  local
159 			p4d = p4d_offset(pgd, addr);  in sync_global_pgds_l4()
164 			if (!p4d_none(*p4d_ref) && !p4d_none(*p4d))  in sync_global_pgds_l4()
165 				BUG_ON(p4d_page_vaddr(*p4d)  in sync_global_pgds_l4()
168 			if (p4d_none(*p4d))  in sync_global_pgds_l4()
169 				set_p4d(p4d, *p4d_ref);  in sync_global_pgds_l4()
215 		p4d_t *p4d = (p4d_t *)spp_getpage();  in fill_p4d()  local
216 		pgd_populate(&init_mm, pgd, p4d);  in fill_p4d()
217 		if (p4d != p4d_offset(pgd, 0))  in fill_p4d()
219 			       p4d, p4d_offset(pgd, 0));  in fill_p4d()
224 static pud_t *fill_pud(p4d_t *p4d, unsigned long vaddr)  in fill_pud()  argument
226 	if (p4d_none(*p4d)) {  in fill_pud()
228 		p4d_populate(&init_mm, p4d, pud);  in fill_pud()
229 		if (pud != pud_offset(p4d, 0))  in fill_pud()
231 			       pud, pud_offset(p4d, 0));  in fill_pud()
233 	return pud_offset(p4d, vaddr);  in fill_pud()
275 	p4d_t *p4d = p4d_page + p4d_index(vaddr);  in set_pte_vaddr_p4d()  local
276 	pud_t *pud = fill_pud(p4d, vaddr);  in set_pte_vaddr_p4d()
309 	p4d_t *p4d;  in populate_extra_pmd()  local
313 	p4d = fill_p4d(pgd, vaddr);  in populate_extra_pmd()
314 	pud = fill_pud(p4d, vaddr);  in populate_extra_pmd()
333 	p4d_t *p4d;  in __init_extra_mapping()  local
344 			p4d = (p4d_t *) spp_getpage();  in __init_extra_mapping()
345 			set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE |  in __init_extra_mapping()
348 		p4d = p4d_offset(pgd, (unsigned long)__va(phys));  in __init_extra_mapping()
349 		if (p4d_none(*p4d)) {  in __init_extra_mapping()
351 			set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE |  in __init_extra_mapping()
354 		pud = pud_offset(p4d, (unsigned long)__va(phys));  in __init_extra_mapping()
650 		p4d_t *p4d;  in phys_p4d_init()  local
654 		p4d = p4d_page + p4d_index(vaddr);  in phys_p4d_init()
663 				set_p4d(p4d, __p4d(0));  in phys_p4d_init()
667 		if (!p4d_none(*p4d)) {  in phys_p4d_init()
668 			pud = pud_offset(p4d, 0);  in phys_p4d_init()
681 		p4d_populate(&init_mm, p4d, pud);  in phys_p4d_init()
709 		p4d_t *p4d;  in kernel_physical_mapping_init()  local
714 			p4d = (p4d_t *)pgd_page_vaddr(*pgd);  in kernel_physical_mapping_init()
715 			paddr_last = phys_p4d_init(p4d, __pa(vaddr),  in kernel_physical_mapping_init()
721 		p4d = alloc_low_page();  in kernel_physical_mapping_init()
722 		paddr_last = phys_p4d_init(p4d, __pa(vaddr), __pa(vaddr_end),  in kernel_physical_mapping_init()
727 			pgd_populate(&init_mm, pgd, p4d);  in kernel_physical_mapping_init()
729 			p4d_populate(&init_mm, p4d_offset(pgd, vaddr), (pud_t *) p4d);  in kernel_physical_mapping_init()
879 static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d)  in free_pud_table()  argument
891 	free_pagetable(p4d_page(*p4d), 0);  in free_pud_table()
893 	p4d_clear(p4d);  in free_pud_table()
1085 	p4d_t *p4d;  in remove_p4d_table()  local
1087 	p4d = p4d_start + p4d_index(addr);  in remove_p4d_table()
1088 	for (; addr < end; addr = next, p4d++) {  in remove_p4d_table()
1091 		if (!p4d_present(*p4d))  in remove_p4d_table()
1094 		BUILD_BUG_ON(p4d_large(*p4d));  in remove_p4d_table()
1096 		pud_base = pud_offset(p4d, 0);  in remove_p4d_table()
1104 			free_pud_table(pud_base, p4d);  in remove_p4d_table()
1119 	p4d_t *p4d;  in remove_pagetable()  local
1128 		p4d = p4d_offset(pgd, 0);  in remove_pagetable()
1129 		remove_p4d_table(p4d, addr, next, altmap, direct);  in remove_pagetable()
1296 	p4d_t *p4d;  in kern_addr_valid()  local
1308 	p4d = p4d_offset(pgd, addr);  in kern_addr_valid()
1309 	if (p4d_none(*p4d))  in kern_addr_valid()
1312 	pud = pud_offset(p4d, addr);  in kern_addr_valid()
1406 	p4d_t *p4d;  in vmemmap_populate_hugepages()  local
1417 		p4d = vmemmap_p4d_populate(pgd, addr, node);  in vmemmap_populate_hugepages()
1418 		if (!p4d)  in vmemmap_populate_hugepages()
1421 		pud = vmemmap_pud_populate(p4d, addr, node);  in vmemmap_populate_hugepages()
1491 	p4d_t *p4d;  in register_page_bootmem_memmap()  local
1507 		p4d = p4d_offset(pgd, addr);  in register_page_bootmem_memmap()
1508 		if (p4d_none(*p4d)) {  in register_page_bootmem_memmap()
1512 		get_page_bootmem(section_nr, p4d_page(*p4d), MIX_SECTION_INFO);  in register_page_bootmem_memmap()
1514 		pud = pud_offset(p4d, addr);  in register_page_bootmem_memmap()