30 				   pte_t *ptep, int nodat)  in ptep_ipte_local()  argument
43 		__ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL);  in ptep_ipte_local()
45 		__ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL);  in ptep_ipte_local()
50 				    pte_t *ptep, int nodat)  in ptep_ipte_global()  argument
63 		__ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL);  in ptep_ipte_global()
65 		__ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);  in ptep_ipte_global()
70 				      unsigned long addr, pte_t *ptep,  in ptep_flush_direct()  argument
75 	old = *ptep;  in ptep_flush_direct()
81 		ptep_ipte_local(mm, addr, ptep, nodat);  in ptep_flush_direct()
83 		ptep_ipte_global(mm, addr, ptep, nodat);  in ptep_flush_direct()
89 				    unsigned long addr, pte_t *ptep,  in ptep_flush_lazy()  argument
94 	old = *ptep;  in ptep_flush_lazy()
100 		pte_val(*ptep) |= _PAGE_INVALID;  in ptep_flush_lazy()
103 		ptep_ipte_global(mm, addr, ptep, nodat);  in ptep_flush_lazy()
108 static inline pgste_t pgste_get_lock(pte_t *ptep)  in pgste_get_lock()  argument
121 		: "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])  in pgste_get_lock()
122 		: "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory");  in pgste_get_lock()
127 static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)  in pgste_set_unlock()  argument
133 		: "=Q" (ptep[PTRS_PER_PTE])  in pgste_set_unlock()
134 		: "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE])  in pgste_set_unlock()
139 static inline pgste_t pgste_get(pte_t *ptep)  in pgste_get()  argument
143 	pgste = *(unsigned long *)(ptep + PTRS_PER_PTE);  in pgste_get()
148 static inline void pgste_set(pte_t *ptep, pgste_t pgste)  in pgste_set()  argument
151 	*(pgste_t *)(ptep + PTRS_PER_PTE) = pgste;  in pgste_set()
176 static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,  in pgste_set_key()  argument
185 	VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));  in pgste_set_key()
198 static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)  in pgste_set_pte()  argument
217 	*ptep = entry;  in pgste_set_pte()
223 				       pte_t *ptep, pgste_t pgste)  in pgste_pte_notify()  argument
231 		ptep_notify(mm, addr, ptep, bits);  in pgste_pte_notify()
238 				      unsigned long addr, pte_t *ptep)  in ptep_xchg_start()  argument
243 		pgste = pgste_get_lock(ptep);  in ptep_xchg_start()
244 		pgste = pgste_pte_notify(mm, addr, ptep, pgste);  in ptep_xchg_start()
250 				    unsigned long addr, pte_t *ptep,  in ptep_xchg_commit()  argument
255 			pgste_set_key(ptep, pgste, new, mm);  in ptep_xchg_commit()
262 		pgste = pgste_set_pte(ptep, pgste, new);  in ptep_xchg_commit()
263 		pgste_set_unlock(ptep, pgste);  in ptep_xchg_commit()
265 		*ptep = new;  in ptep_xchg_commit()
271 		       pte_t *ptep, pte_t new)  in ptep_xchg_direct()  argument
278 	pgste = ptep_xchg_start(mm, addr, ptep);  in ptep_xchg_direct()
280 	old = ptep_flush_direct(mm, addr, ptep, nodat);  in ptep_xchg_direct()
281 	old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);  in ptep_xchg_direct()
288 		     pte_t *ptep, pte_t new)  in ptep_xchg_lazy()  argument
295 	pgste = ptep_xchg_start(mm, addr, ptep);  in ptep_xchg_lazy()
297 	old = ptep_flush_lazy(mm, addr, ptep, nodat);  in ptep_xchg_lazy()
298 	old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);  in ptep_xchg_lazy()
305 			     pte_t *ptep)  in ptep_modify_prot_start()  argument
312 	pgste = ptep_xchg_start(mm, addr, ptep);  in ptep_modify_prot_start()
314 	old = ptep_flush_lazy(mm, addr, ptep, nodat);  in ptep_modify_prot_start()
317 		pgste_set(ptep, pgste);  in ptep_modify_prot_start()
324 			     pte_t *ptep, pte_t pte)  in ptep_modify_prot_commit()  argument
331 		pgste = pgste_get(ptep);  in ptep_modify_prot_commit()
332 		pgste_set_key(ptep, pgste, pte, mm);  in ptep_modify_prot_commit()
333 		pgste = pgste_set_pte(ptep, pgste, pte);  in ptep_modify_prot_commit()
334 		pgste_set_unlock(ptep, pgste);  in ptep_modify_prot_commit()
336 		*ptep = pte;  in ptep_modify_prot_commit()
534 	pte_t *ptep;  in pgtable_trans_huge_withdraw()  local
547 	ptep = (pte_t *) pgtable;  in pgtable_trans_huge_withdraw()
548 	pte_val(*ptep) = _PAGE_INVALID;  in pgtable_trans_huge_withdraw()
549 	ptep++;  in pgtable_trans_huge_withdraw()
550 	pte_val(*ptep) = _PAGE_INVALID;  in pgtable_trans_huge_withdraw()
557 		     pte_t *ptep, pte_t entry)  in ptep_set_pte_at()  argument
563 	pgste = pgste_get_lock(ptep);  in ptep_set_pte_at()
565 	pgste_set_key(ptep, pgste, entry, mm);  in ptep_set_pte_at()
566 	pgste = pgste_set_pte(ptep, pgste, entry);  in ptep_set_pte_at()
567 	pgste_set_unlock(ptep, pgste);  in ptep_set_pte_at()
571 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep)  in ptep_set_notify()  argument
576 	pgste = pgste_get_lock(ptep);  in ptep_set_notify()
578 	pgste_set_unlock(ptep, pgste);  in ptep_set_notify()
594 		    pte_t *ptep, int prot, unsigned long bit)  in ptep_force_prot()  argument
600 	pgste = pgste_get_lock(ptep);  in ptep_force_prot()
601 	entry = *ptep;  in ptep_force_prot()
607 		pgste_set_unlock(ptep, pgste);  in ptep_force_prot()
613 		ptep_flush_direct(mm, addr, ptep, nodat);  in ptep_force_prot()
618 		ptep_flush_direct(mm, addr, ptep, nodat);  in ptep_force_prot()
623 	pgste = pgste_set_pte(ptep, pgste, entry);  in ptep_force_prot()
624 	pgste_set_unlock(ptep, pgste);  in ptep_force_prot()
655 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep)  in ptep_unshadow_pte()  argument
660 	pgste = pgste_get_lock(ptep);  in ptep_unshadow_pte()
663 	ptep_flush_direct(mm, saddr, ptep, nodat);  in ptep_unshadow_pte()
665 	pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID));  in ptep_unshadow_pte()
666 	pgste_set_unlock(ptep, pgste);  in ptep_unshadow_pte()
682 		     pte_t *ptep, int reset)  in ptep_zap_unused()  argument
690 	pgste = pgste_get_lock(ptep);  in ptep_zap_unused()
692 	pte = *ptep;  in ptep_zap_unused()
697 		pte_clear(mm, addr, ptep);  in ptep_zap_unused()
701 	pgste_set_unlock(ptep, pgste);  in ptep_zap_unused()
705 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)  in ptep_zap_key()  argument
712 	pgste = pgste_get_lock(ptep);  in ptep_zap_key()
715 	ptev = pte_val(*ptep);  in ptep_zap_key()
718 	pgste_set_unlock(ptep, pgste);  in ptep_zap_key()
726 		       pte_t *ptep)  in ptep_test_and_clear_uc()  argument
733 	pgste = pgste_get_lock(ptep);  in ptep_test_and_clear_uc()
736 	pte = *ptep;  in ptep_test_and_clear_uc()
738 		pgste = pgste_pte_notify(mm, addr, ptep, pgste);  in ptep_test_and_clear_uc()
740 		ptep_ipte_global(mm, addr, ptep, nodat);  in ptep_test_and_clear_uc()
745 		*ptep = pte;  in ptep_test_and_clear_uc()
747 	pgste_set_unlock(ptep, pgste);  in ptep_test_and_clear_uc()
759 	pte_t *ptep;  in set_guest_storage_key()  local
784 	ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);  in set_guest_storage_key()
785 	if (unlikely(!ptep))  in set_guest_storage_key()
788 	new = old = pgste_get_lock(ptep);  in set_guest_storage_key()
794 	if (!(pte_val(*ptep) & _PAGE_INVALID)) {  in set_guest_storage_key()
797 		paddr = pte_val(*ptep) & PAGE_MASK;  in set_guest_storage_key()
811 	pgste_set_unlock(ptep, new);  in set_guest_storage_key()
812 	pte_unmap_unlock(ptep, ptl);  in set_guest_storage_key()
861 	pte_t *ptep;  in reset_guest_reference_bit()  local
883 	ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);  in reset_guest_reference_bit()
884 	if (unlikely(!ptep))  in reset_guest_reference_bit()
887 	new = old = pgste_get_lock(ptep);  in reset_guest_reference_bit()
891 	if (!(pte_val(*ptep) & _PAGE_INVALID)) {  in reset_guest_reference_bit()
892 		paddr = pte_val(*ptep) & PAGE_MASK;  in reset_guest_reference_bit()
903 	pgste_set_unlock(ptep, new);  in reset_guest_reference_bit()
904 	pte_unmap_unlock(ptep, ptl);  in reset_guest_reference_bit()
916 	pte_t *ptep;  in get_guest_storage_key()  local
939 	ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);  in get_guest_storage_key()
940 	if (unlikely(!ptep))  in get_guest_storage_key()
943 	pgste = pgste_get_lock(ptep);  in get_guest_storage_key()
945 	paddr = pte_val(*ptep) & PAGE_MASK;  in get_guest_storage_key()
946 	if (!(pte_val(*ptep) & _PAGE_INVALID))  in get_guest_storage_key()
950 	pgste_set_unlock(ptep, pgste);  in get_guest_storage_key()
951 	pte_unmap_unlock(ptep, ptl);  in get_guest_storage_key()
974 	pte_t *ptep;  in pgste_perform_essa()  local
980 	ptep = get_locked_pte(mm, hva, &ptl);  in pgste_perform_essa()
981 	if (unlikely(!ptep))  in pgste_perform_essa()
983 	pgste = pgste_get_lock(ptep);  in pgste_perform_essa()
986 		*oldpte = pte_val(*ptep);  in pgste_perform_essa()
1000 		if (pte_val(*ptep) & _PAGE_INVALID)  in pgste_perform_essa()
1006 		if (pte_val(*ptep) & _PAGE_INVALID)  in pgste_perform_essa()
1011 		if (!(pte_val(*ptep) & _PAGE_INVALID)) {  in pgste_perform_essa()
1035 		if (!(pte_val(*ptep) & _PAGE_INVALID)) {  in pgste_perform_essa()
1053 	pgste_set_unlock(ptep, pgste);  in pgste_perform_essa()
1054 	pte_unmap_unlock(ptep, ptl);  in pgste_perform_essa()
1074 	pte_t *ptep;  in set_pgste_bits()  local
1076 	ptep = get_locked_pte(mm, hva, &ptl);  in set_pgste_bits()
1077 	if (unlikely(!ptep))  in set_pgste_bits()
1079 	new = pgste_get_lock(ptep);  in set_pgste_bits()
1084 	pgste_set_unlock(ptep, new);  in set_pgste_bits()
1085 	pte_unmap_unlock(ptep, ptl);  in set_pgste_bits()
1101 	pte_t *ptep;  in get_pgste()  local
1103 	ptep = get_locked_pte(mm, hva, &ptl);  in get_pgste()
1104 	if (unlikely(!ptep))  in get_pgste()
1106 	*pgstep = pgste_val(pgste_get(ptep));  in get_pgste()
1107 	pte_unmap_unlock(ptep, ptl);  in get_pgste()