155 static void clear_stage2_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr)  in clear_stage2_pgd_entry()  argument
157 	pud_t *pud_table __maybe_unused = stage2_pud_offset(kvm, pgd, 0UL);  in clear_stage2_pgd_entry()
158 	stage2_pgd_clear(kvm, pgd);  in clear_stage2_pgd_entry()
161 	put_page(virt_to_page(pgd));  in clear_stage2_pgd_entry()
292 static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd,  in unmap_stage2_puds()  argument
298 	start_pud = pud = stage2_pud_offset(kvm, pgd, addr);  in unmap_stage2_puds()
316 		clear_stage2_pgd_entry(kvm, pgd, start_addr);  in unmap_stage2_puds()
332 	pgd_t *pgd;  in unmap_stage2_range()  local
339 	pgd = kvm->arch.pgd + stage2_pgd_index(kvm, addr);  in unmap_stage2_range()
346 		if (!READ_ONCE(kvm->arch.pgd))  in unmap_stage2_range()
349 		if (!stage2_pgd_none(kvm, *pgd))  in unmap_stage2_range()
350 			unmap_stage2_puds(kvm, pgd, addr, next);  in unmap_stage2_range()
357 	} while (pgd++, addr = next, addr != end);  in unmap_stage2_range()
390 static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,  in stage2_flush_puds()  argument
396 	pud = stage2_pud_offset(kvm, pgd, addr);  in stage2_flush_puds()
414 	pgd_t *pgd;  in stage2_flush_memslot()  local
416 	pgd = kvm->arch.pgd + stage2_pgd_index(kvm, addr);  in stage2_flush_memslot()
419 		if (!stage2_pgd_none(kvm, *pgd))  in stage2_flush_memslot()
420 			stage2_flush_puds(kvm, pgd, addr, next);  in stage2_flush_memslot()
421 	} while (pgd++, addr = next, addr != end);  in stage2_flush_memslot()
448 static void clear_hyp_pgd_entry(pgd_t *pgd)  in clear_hyp_pgd_entry()  argument
450 	pud_t *pud_table __maybe_unused = pud_offset(pgd, 0UL);  in clear_hyp_pgd_entry()
451 	pgd_clear(pgd);  in clear_hyp_pgd_entry()
453 	put_page(virt_to_page(pgd));  in clear_hyp_pgd_entry()
507 static void unmap_hyp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)  in unmap_hyp_puds()  argument
512 	start_pud = pud = pud_offset(pgd, addr);  in unmap_hyp_puds()
521 		clear_hyp_pgd_entry(pgd);  in unmap_hyp_puds()
532 	pgd_t *pgd;  in __unmap_hyp_range()  local
540 	pgd = pgdp + kvm_pgd_index(addr, ptrs_per_pgd);  in __unmap_hyp_range()
543 		if (!pgd_none(*pgd))  in __unmap_hyp_range()
544 			unmap_hyp_puds(pgd, addr, next);  in __unmap_hyp_range()
545 	} while (pgd++, addr = next, addr != end);  in __unmap_hyp_range()
654 static int create_hyp_pud_mappings(pgd_t *pgd, unsigned long start,  in create_hyp_pud_mappings()  argument
665 		pud = pud_offset(pgd, addr);  in create_hyp_pud_mappings()
691 	pgd_t *pgd;  in __create_hyp_mappings()  local
700 		pgd = pgdp + kvm_pgd_index(addr, ptrs_per_pgd);  in __create_hyp_mappings()
702 		if (pgd_none(*pgd)) {  in __create_hyp_mappings()
709 			kvm_pgd_populate(pgd, pud);  in __create_hyp_mappings()
710 			get_page(virt_to_page(pgd));  in __create_hyp_mappings()
714 		err = create_hyp_pud_mappings(pgd, addr, next, pfn, prot);  in __create_hyp_mappings()
776 	pgd_t *pgd = hyp_pgd;  in __create_hyp_private_mapping()  local
809 		pgd = boot_hyp_pgd;  in __create_hyp_private_mapping()
811 	ret = __create_hyp_mappings(pgd, __kvm_idmap_ptrs_per_pgd(),  in __create_hyp_private_mapping()
897 	pgd_t *pgd;  in kvm_alloc_stage2_pgd()  local
899 	if (kvm->arch.pgd != NULL) {  in kvm_alloc_stage2_pgd()
905 	pgd = alloc_pages_exact(stage2_pgd_size(kvm), GFP_KERNEL | __GFP_ZERO);  in kvm_alloc_stage2_pgd()
906 	if (!pgd)  in kvm_alloc_stage2_pgd()
909 	pgd_phys = virt_to_phys(pgd);  in kvm_alloc_stage2_pgd()
913 	kvm->arch.pgd = pgd;  in kvm_alloc_stage2_pgd()
995 	void *pgd = NULL;  in kvm_free_stage2_pgd()  local
998 	if (kvm->arch.pgd) {  in kvm_free_stage2_pgd()
1000 		pgd = READ_ONCE(kvm->arch.pgd);  in kvm_free_stage2_pgd()
1001 		kvm->arch.pgd = NULL;  in kvm_free_stage2_pgd()
1007 	if (pgd)  in kvm_free_stage2_pgd()
1008 		free_pages_exact(pgd, stage2_pgd_size(kvm));  in kvm_free_stage2_pgd()
1014 	pgd_t *pgd;  in stage2_get_pud()  local
1017 	pgd = kvm->arch.pgd + stage2_pgd_index(kvm, addr);  in stage2_get_pud()
1018 	if (stage2_pgd_none(kvm, *pgd)) {  in stage2_get_pud()
1022 		stage2_pgd_populate(kvm, pgd, pud);  in stage2_get_pud()
1023 		get_page(virt_to_page(pgd));  in stage2_get_pud()
1026 	return stage2_pud_offset(kvm, pgd, addr);  in stage2_get_pud()
1471 static void  stage2_wp_puds(struct kvm *kvm, pgd_t *pgd,  in stage2_wp_puds()  argument
1477 	pud = stage2_pud_offset(kvm, pgd, addr);  in stage2_wp_puds()
1499 	pgd_t *pgd;  in stage2_wp_range()  local
1502 	pgd = kvm->arch.pgd + stage2_pgd_index(kvm, addr);  in stage2_wp_range()
1514 		if (!READ_ONCE(kvm->arch.pgd))  in stage2_wp_range()
1517 		if (stage2_pgd_present(kvm, *pgd))  in stage2_wp_range()
1518 			stage2_wp_puds(kvm, pgd, addr, next);  in stage2_wp_range()
1519 	} while (pgd++, addr = next, addr != end);  in stage2_wp_range()
2041 	if (!kvm->arch.pgd)  in kvm_unmap_hva_range()
2072 	if (!kvm->arch.pgd)  in kvm_set_spte_hva()
2126 	if (!kvm->arch.pgd)  in kvm_age_hva()
2134 	if (!kvm->arch.pgd)  in kvm_test_age_hva()
2158 static int kvm_map_idmap_text(pgd_t *pgd)  in kvm_map_idmap_text()  argument
2163 	err = 	__create_hyp_mappings(pgd, __kvm_idmap_ptrs_per_pgd(),  in kvm_map_idmap_text()