18 #error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED
41  * This defines the generic helper for accessing PMD page
46 #define pmd_pgtable(pmd) pmd_page(pmd)  argument
88 static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)  in pte_offset_kernel()  argument
90 	return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);  in pte_offset_kernel()
143  * In many cases it is known that a virtual address is mapped at PMD or PTE
145  * pointer to the PMD entry in user or kernel page table or translate a virtual
161 	pmd_t *pmd = pmd_off_k(vaddr);  in virt_to_kpte()  local
163 	return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr);  in virt_to_kpte()
219 	pmd_t pmd = *pmdp;  in pmdp_test_and_clear_young()  local
221 	if (!pmd_young(pmd))  in pmdp_test_and_clear_young()
224 		set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd));  in pmdp_test_and_clear_young()
340 	pmd_t pmd = *pmdp;  in pmdp_huge_get_and_clear()  local
342 	return pmd;  in pmdp_huge_get_and_clear()
551 		unsigned long address, pmd_t *pmdp, pmd_t pmd)  in generic_pmdp_establish()  argument
554 	set_pmd_at(vma->vm_mm, address, pmdp, pmd);  in generic_pmdp_establish()
590 #define pmd_access_permitted(pmd, write) \  argument
591 	(pmd_present(pmd) && (!(write) || pmd_write(pmd)))
649 #define set_pmd_safe(pmdp, pmd) \  argument
651 	WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \
652 	set_pmd(pmdp, pmd); \
839 static inline int pmd_none_or_clear_bad(pmd_t *pmd)  in pmd_none_or_clear_bad()  argument
841 	if (pmd_none(*pmd))  in pmd_none_or_clear_bad()
843 	if (unlikely(pmd_bad(*pmd))) {  in pmd_none_or_clear_bad()
844 		pmd_clear_bad(pmd);  in pmd_none_or_clear_bad()
999 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)  in pmd_swp_mksoft_dirty()  argument
1001 	return pmd;  in pmd_swp_mksoft_dirty()
1004 static inline int pmd_swp_soft_dirty(pmd_t pmd)  in pmd_swp_soft_dirty()  argument
1009 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)  in pmd_swp_clear_soft_dirty()  argument
1011 	return pmd;  in pmd_swp_clear_soft_dirty()
1020 static inline int pmd_soft_dirty(pmd_t pmd)  in pmd_soft_dirty()  argument
1030 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)  in pmd_mksoft_dirty()  argument
1032 	return pmd;  in pmd_mksoft_dirty()
1040 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)  in pmd_clear_soft_dirty()  argument
1042 	return pmd;  in pmd_clear_soft_dirty()
1060 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)  in pmd_swp_mksoft_dirty()  argument
1062 	return pmd;  in pmd_swp_mksoft_dirty()
1065 static inline int pmd_swp_soft_dirty(pmd_t pmd)  in pmd_swp_soft_dirty()  argument
1070 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)  in pmd_swp_clear_soft_dirty()  argument
1072 	return pmd;  in pmd_swp_clear_soft_dirty()
1179 static inline int pmd_trans_huge(pmd_t pmd)  in pmd_trans_huge()  argument
1184 static inline int pmd_write(pmd_t pmd)  in pmd_write()  argument
1201 static inline int pmd_devmap(pmd_t pmd)  in pmd_devmap()  argument
1253 	 * Depend on compiler for an atomic pmd read. NOTE: this is  in pmd_read_atomic()
1267  * transhuge page faults. MADV_DONTNEED can convert a transhuge pmd
1268  * into a null pmd and the transhuge page fault can convert a null pmd
1269  * into an hugepmd or into a regular pmd (if the hugepage allocation
1270  * fails). While holding the mmap_lock in read mode the pmd becomes
1272  * transhuge pmd. When those races occurs and this function makes a
1274  * undefined so behaving like if the pmd was none is safe (because it
1279  * care of reading the pmd atomically to avoid SMP race conditions
1283  * fault can populate the pmd from under us).
1285 static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd)  in pmd_none_or_trans_huge_or_clear_bad()  argument
1287 	pmd_t pmdval = pmd_read_atomic(pmd);  in pmd_none_or_trans_huge_or_clear_bad()
1295 	 * mapped in the pmd). The below checks will only care about  in pmd_none_or_trans_huge_or_clear_bad()
1296 	 * the low part of the pmd with 32bit PAE x86 anyway, with the  in pmd_none_or_trans_huge_or_clear_bad()
1298 	 * the low part of the pmd is found null, the high part will  in pmd_none_or_trans_huge_or_clear_bad()
1306 	 * !pmd_present() checks for pmd migration entries  in pmd_none_or_trans_huge_or_clear_bad()
1316 	 * pmd_none() is preserved for future condition checks on pmd migration  in pmd_none_or_trans_huge_or_clear_bad()
1324 		pmd_clear_bad(pmd);  in pmd_none_or_trans_huge_or_clear_bad()
1334  * places that already verified the pmd is not none and they want to
1336  * need this). If THP is not enabled, the pmd can't go away under the
1339  * split_huge_pmd returns (because it may have run when the pmd become
1342 static inline int pmd_trans_unstable(pmd_t *pmd)  in pmd_trans_unstable()  argument
1345 	return pmd_none_or_trans_huge_or_clear_bad(pmd);  in pmd_trans_unstable()
1357 static inline int pmd_devmap_trans_unstable(pmd_t *pmd)  in pmd_devmap_trans_unstable()  argument
1359 	return pmd_devmap(*pmd) || pmd_trans_unstable(pmd);  in pmd_devmap_trans_unstable()
1376 static inline int pmd_protnone(pmd_t pmd)  in pmd_protnone()  argument
1401 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot);
1403 int pmd_clear_huge(pmd_t *pmd);
1406 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr);
1416 static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)  in pmd_set_huge()  argument
1428 static inline int pmd_clear_huge(pmd_t *pmd)  in pmd_clear_huge()  argument
1440 static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)  in pmd_free_pte_page()  argument
1552  * On some architectures it depends on the mm if the p4d/pud or pmd