151 static void clear_stage2_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr)  in clear_stage2_pgd_entry()  argument
153 	pud_t *pud_table __maybe_unused = stage2_pud_offset(pgd, 0UL);  in clear_stage2_pgd_entry()
154 	stage2_pgd_clear(pgd);  in clear_stage2_pgd_entry()
157 	put_page(virt_to_page(pgd));  in clear_stage2_pgd_entry()
288 static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd,  in unmap_stage2_puds()  argument
294 	start_pud = pud = stage2_pud_offset(pgd, addr);  in unmap_stage2_puds()
312 		clear_stage2_pgd_entry(kvm, pgd, start_addr);  in unmap_stage2_puds()
328 	pgd_t *pgd;  in unmap_stage2_range()  local
335 	pgd = kvm->arch.pgd + stage2_pgd_index(addr);  in unmap_stage2_range()
342 		if (!READ_ONCE(kvm->arch.pgd))  in unmap_stage2_range()
345 		if (!stage2_pgd_none(*pgd))  in unmap_stage2_range()
346 			unmap_stage2_puds(kvm, pgd, addr, next);  in unmap_stage2_range()
353 	} while (pgd++, addr = next, addr != end);  in unmap_stage2_range()
386 static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,  in stage2_flush_puds()  argument
392 	pud = stage2_pud_offset(pgd, addr);  in stage2_flush_puds()
410 	pgd_t *pgd;  in stage2_flush_memslot()  local
412 	pgd = kvm->arch.pgd + stage2_pgd_index(addr);  in stage2_flush_memslot()
415 		stage2_flush_puds(kvm, pgd, addr, next);  in stage2_flush_memslot()
416 	} while (pgd++, addr = next, addr != end);  in stage2_flush_memslot()
443 static void clear_hyp_pgd_entry(pgd_t *pgd)  in clear_hyp_pgd_entry()  argument
445 	pud_t *pud_table __maybe_unused = pud_offset(pgd, 0UL);  in clear_hyp_pgd_entry()
446 	pgd_clear(pgd);  in clear_hyp_pgd_entry()
448 	put_page(virt_to_page(pgd));  in clear_hyp_pgd_entry()
502 static void unmap_hyp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)  in unmap_hyp_puds()  argument
507 	start_pud = pud = pud_offset(pgd, addr);  in unmap_hyp_puds()
516 		clear_hyp_pgd_entry(pgd);  in unmap_hyp_puds()
527 	pgd_t *pgd;  in __unmap_hyp_range()  local
535 	pgd = pgdp + kvm_pgd_index(addr, ptrs_per_pgd);  in __unmap_hyp_range()
538 		if (!pgd_none(*pgd))  in __unmap_hyp_range()
539 			unmap_hyp_puds(pgd, addr, next);  in __unmap_hyp_range()
540 	} while (pgd++, addr = next, addr != end);  in __unmap_hyp_range()
649 static int create_hyp_pud_mappings(pgd_t *pgd, unsigned long start,  in create_hyp_pud_mappings()  argument
660 		pud = pud_offset(pgd, addr);  in create_hyp_pud_mappings()
686 	pgd_t *pgd;  in __create_hyp_mappings()  local
695 		pgd = pgdp + kvm_pgd_index(addr, ptrs_per_pgd);  in __create_hyp_mappings()
697 		if (pgd_none(*pgd)) {  in __create_hyp_mappings()
704 			kvm_pgd_populate(pgd, pud);  in __create_hyp_mappings()
705 			get_page(virt_to_page(pgd));  in __create_hyp_mappings()
709 		err = create_hyp_pud_mappings(pgd, addr, next, pfn, prot);  in __create_hyp_mappings()
771 	pgd_t *pgd = hyp_pgd;  in __create_hyp_private_mapping()  local
804 		pgd = boot_hyp_pgd;  in __create_hyp_private_mapping()
806 	ret = __create_hyp_mappings(pgd, __kvm_idmap_ptrs_per_pgd(),  in __create_hyp_private_mapping()
892 	pgd_t *pgd;  in kvm_alloc_stage2_pgd()  local
894 	if (kvm->arch.pgd != NULL) {  in kvm_alloc_stage2_pgd()
900 	pgd = alloc_pages_exact(S2_PGD_SIZE, GFP_KERNEL | __GFP_ZERO);  in kvm_alloc_stage2_pgd()
901 	if (!pgd)  in kvm_alloc_stage2_pgd()
904 	kvm->arch.pgd = pgd;  in kvm_alloc_stage2_pgd()
985 	void *pgd = NULL;  in kvm_free_stage2_pgd()  local
988 	if (kvm->arch.pgd) {  in kvm_free_stage2_pgd()
990 		pgd = READ_ONCE(kvm->arch.pgd);  in kvm_free_stage2_pgd()
991 		kvm->arch.pgd = NULL;  in kvm_free_stage2_pgd()
996 	if (pgd)  in kvm_free_stage2_pgd()
997 		free_pages_exact(pgd, S2_PGD_SIZE);  in kvm_free_stage2_pgd()
1003 	pgd_t *pgd;  in stage2_get_pud()  local
1006 	pgd = kvm->arch.pgd + stage2_pgd_index(addr);  in stage2_get_pud()
1007 	if (WARN_ON(stage2_pgd_none(*pgd))) {  in stage2_get_pud()
1011 		stage2_pgd_populate(pgd, pud);  in stage2_get_pud()
1012 		get_page(virt_to_page(pgd));  in stage2_get_pud()
1015 	return stage2_pud_offset(pgd, addr);  in stage2_get_pud()
1331 static void  stage2_wp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)  in stage2_wp_puds()  argument
1336 	pud = stage2_pud_offset(pgd, addr);  in stage2_wp_puds()
1355 	pgd_t *pgd;  in stage2_wp_range()  local
1358 	pgd = kvm->arch.pgd + stage2_pgd_index(addr);  in stage2_wp_range()
1370 		if (!READ_ONCE(kvm->arch.pgd))  in stage2_wp_range()
1373 		if (stage2_pgd_present(*pgd))  in stage2_wp_range()
1374 			stage2_wp_puds(pgd, addr, next);  in stage2_wp_range()
1375 	} while (pgd++, addr = next, addr != end);  in stage2_wp_range()
1823 	if (!kvm->arch.pgd)  in kvm_unmap_hva_range()
1854 	if (!kvm->arch.pgd)  in kvm_set_spte_hva()
1910 	if (!kvm->arch.pgd)  in kvm_age_hva()
1918 	if (!kvm->arch.pgd)  in kvm_test_age_hva()
1942 static int kvm_map_idmap_text(pgd_t *pgd)  in kvm_map_idmap_text()  argument
1947 	err = 	__create_hyp_mappings(pgd, __kvm_idmap_ptrs_per_pgd(),  in kvm_map_idmap_text()