463 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)  in maybe_pmd_mkwrite()  argument
466 		pmd = pmd_mkwrite(pmd);  in maybe_pmd_mkwrite()
467 	return pmd;  in maybe_pmd_mkwrite()
597 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in __do_huge_pmd_anonymous_page()
598 	if (unlikely(!pmd_none(*vmf->pmd))) {  in __do_huge_pmd_anonymous_page()
623 		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);  in __do_huge_pmd_anonymous_page()
624 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in __do_huge_pmd_anonymous_page()
680 		struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,  in set_huge_zero_page()  argument
684 	if (!pmd_none(*pmd))  in set_huge_zero_page()
689 		pgtable_trans_huge_deposit(mm, pmd, pgtable);  in set_huge_zero_page()
690 	set_pmd_at(mm, haddr, pmd, entry);  in set_huge_zero_page()
723 		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_anonymous_page()
725 		if (pmd_none(*vmf->pmd)) {  in do_huge_pmd_anonymous_page()
737 						   haddr, vmf->pmd, zero_page);  in do_huge_pmd_anonymous_page()
757 		pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write,  in insert_pfn_pmd()  argument
764 	ptl = pmd_lock(mm, pmd);  in insert_pfn_pmd()
765 	if (!pmd_none(*pmd)) {  in insert_pfn_pmd()
767 			if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) {  in insert_pfn_pmd()
768 				WARN_ON_ONCE(!is_huge_zero_pmd(*pmd));  in insert_pfn_pmd()
771 			entry = pmd_mkyoung(*pmd);  in insert_pfn_pmd()
773 			if (pmdp_set_access_flags(vma, addr, pmd, entry, 1))  in insert_pfn_pmd()
774 				update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
789 		pgtable_trans_huge_deposit(mm, pmd, pgtable);  in insert_pfn_pmd()
794 	set_pmd_at(mm, addr, pmd, entry);  in insert_pfn_pmd()
795 	update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
845 	insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);  in vmf_insert_pfn_pmd_prot()
936 		pmd_t *pmd, int flags)  in touch_pmd()  argument
940 	_pmd = pmd_mkyoung(*pmd);  in touch_pmd()
944 				pmd, _pmd, flags & FOLL_WRITE))  in touch_pmd()
945 		update_mmu_cache_pmd(vma, addr, pmd);  in touch_pmd()
949 		pmd_t *pmd, int flags, struct dev_pagemap **pgmap)  in follow_devmap_pmd()  argument
951 	unsigned long pfn = pmd_pfn(*pmd);  in follow_devmap_pmd()
955 	assert_spin_locked(pmd_lockptr(mm, pmd));  in follow_devmap_pmd()
968 	if (flags & FOLL_WRITE && !pmd_write(*pmd))  in follow_devmap_pmd()
971 	if (pmd_present(*pmd) && pmd_devmap(*pmd))  in follow_devmap_pmd()
977 		touch_pmd(vma, addr, pmd, flags);  in follow_devmap_pmd()
1003 	pmd_t pmd;  in copy_huge_pmd()  local
1020 	pmd = *src_pmd;  in copy_huge_pmd()
1028 		pmd = pmd_clear_uffd_wp(pmd);  in copy_huge_pmd()
1031 	if (unlikely(is_swap_pmd(pmd))) {  in copy_huge_pmd()
1032 		swp_entry_t entry = pmd_to_swp_entry(pmd);  in copy_huge_pmd()
1034 		VM_BUG_ON(!is_pmd_migration_entry(pmd));  in copy_huge_pmd()
1037 			pmd = swp_entry_to_pmd(entry);  in copy_huge_pmd()
1039 				pmd = pmd_swp_mksoft_dirty(pmd);  in copy_huge_pmd()
1040 			set_pmd_at(src_mm, addr, src_pmd, pmd);  in copy_huge_pmd()
1045 		set_pmd_at(dst_mm, addr, dst_pmd, pmd);  in copy_huge_pmd()
1051 	if (unlikely(!pmd_trans_huge(pmd))) {  in copy_huge_pmd()
1060 	if (is_huge_zero_pmd(pmd)) {  in copy_huge_pmd()
1074 	src_page = pmd_page(pmd);  in copy_huge_pmd()
1101 	pmd = pmd_mkold(pmd_wrprotect(pmd));  in copy_huge_pmd()
1102 	set_pmd_at(dst_mm, addr, dst_pmd, pmd);  in copy_huge_pmd()
1246 	vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);  in huge_pmd_set_accessed()
1247 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in huge_pmd_set_accessed()
1254 	if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write))  in huge_pmd_set_accessed()
1255 		update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd);  in huge_pmd_set_accessed()
1267 	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page()
1275 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1289 		if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1306 		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1))  in do_huge_pmd_wp_page()
1307 			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1316 	__split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);  in do_huge_pmd_wp_page()
1324 static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)  in can_follow_write_pmd()  argument
1326 	return pmd_write(pmd) ||  in can_follow_write_pmd()
1327 	       ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd));  in can_follow_write_pmd()
1332 				   pmd_t *pmd,  in follow_trans_huge_pmd()  argument
1338 	assert_spin_locked(pmd_lockptr(mm, pmd));  in follow_trans_huge_pmd()
1340 	if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags))  in follow_trans_huge_pmd()
1344 	if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))  in follow_trans_huge_pmd()
1348 	if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))  in follow_trans_huge_pmd()
1351 	page = pmd_page(*pmd);  in follow_trans_huge_pmd()
1358 		touch_pmd(vma, addr, pmd, flags);  in follow_trans_huge_pmd()
1401 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)  in do_huge_pmd_numa_page()  argument
1414 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_numa_page()
1415 	if (unlikely(!pmd_same(pmd, *vmf->pmd)))  in do_huge_pmd_numa_page()
1423 	if (unlikely(pmd_trans_migrating(*vmf->pmd))) {  in do_huge_pmd_numa_page()
1424 		page = pmd_page(*vmf->pmd);  in do_huge_pmd_numa_page()
1432 	page = pmd_page(pmd);  in do_huge_pmd_numa_page()
1443 	if (!pmd_savedwrite(pmd))  in do_huge_pmd_numa_page()
1478 	if (unlikely(!pmd_same(pmd, *vmf->pmd))) {  in do_huge_pmd_numa_page()
1525 				vmf->pmd, pmd, vmf->address, page, target_nid);  in do_huge_pmd_numa_page()
1535 	was_writable = pmd_savedwrite(pmd);  in do_huge_pmd_numa_page()
1536 	pmd = pmd_modify(pmd, vma->vm_page_prot);  in do_huge_pmd_numa_page()
1537 	pmd = pmd_mkyoung(pmd);  in do_huge_pmd_numa_page()
1539 		pmd = pmd_mkwrite(pmd);  in do_huge_pmd_numa_page()
1540 	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);  in do_huge_pmd_numa_page()
1541 	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_numa_page()
1562 		pmd_t *pmd, unsigned long addr, unsigned long next)  in madvise_free_huge_pmd()  argument
1572 	ptl = pmd_trans_huge_lock(pmd, vma);  in madvise_free_huge_pmd()
1576 	orig_pmd = *pmd;  in madvise_free_huge_pmd()
1615 		pmdp_invalidate(vma, addr, pmd);  in madvise_free_huge_pmd()
1619 		set_pmd_at(mm, addr, pmd, orig_pmd);  in madvise_free_huge_pmd()
1620 		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);  in madvise_free_huge_pmd()
1631 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)  in zap_deposited_table()  argument
1635 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in zap_deposited_table()
1641 		 pmd_t *pmd, unsigned long addr)  in zap_huge_pmd()  argument
1648 	ptl = __pmd_trans_huge_lock(pmd, vma);  in zap_huge_pmd()
1657 	orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd,  in zap_huge_pmd()
1659 	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);  in zap_huge_pmd()
1662 			zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1667 		zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1690 			zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1694 				zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1720 static pmd_t move_soft_dirty_pmd(pmd_t pmd)  in move_soft_dirty_pmd()  argument
1723 	if (unlikely(is_pmd_migration_entry(pmd)))  in move_soft_dirty_pmd()
1724 		pmd = pmd_swp_mksoft_dirty(pmd);  in move_soft_dirty_pmd()
1725 	else if (pmd_present(pmd))  in move_soft_dirty_pmd()
1726 		pmd = pmd_mksoft_dirty(pmd);  in move_soft_dirty_pmd()
1728 	return pmd;  in move_soft_dirty_pmd()
1735 	pmd_t pmd;  in move_huge_pmd()  local
1757 		pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);  in move_huge_pmd()
1758 		if (pmd_present(pmd))  in move_huge_pmd()
1767 		pmd = move_soft_dirty_pmd(pmd);  in move_huge_pmd()
1768 		set_pmd_at(mm, new_addr, new_pmd, pmd);  in move_huge_pmd()
1785 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,  in change_huge_pmd()  argument
1797 	ptl = __pmd_trans_huge_lock(pmd, vma);  in change_huge_pmd()
1801 	preserve_write = prot_numa && pmd_write(*pmd);  in change_huge_pmd()
1805 	if (is_swap_pmd(*pmd)) {  in change_huge_pmd()
1806 		swp_entry_t entry = pmd_to_swp_entry(*pmd);  in change_huge_pmd()
1808 		VM_BUG_ON(!is_pmd_migration_entry(*pmd));  in change_huge_pmd()
1817 			if (pmd_swp_soft_dirty(*pmd))  in change_huge_pmd()
1819 			set_pmd_at(mm, addr, pmd, newpmd);  in change_huge_pmd()
1830 	if (prot_numa && is_huge_zero_pmd(*pmd))  in change_huge_pmd()
1833 	if (prot_numa && pmd_protnone(*pmd))  in change_huge_pmd()
1857 	entry = pmdp_invalidate(vma, addr, pmd);  in change_huge_pmd()
1874 	set_pmd_at(mm, addr, pmd, entry);  in change_huge_pmd()
1887 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)  in __pmd_trans_huge_lock()  argument
1890 	ptl = pmd_lock(vma->vm_mm, pmd);  in __pmd_trans_huge_lock()
1891 	if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) ||  in __pmd_trans_huge_lock()
1892 			pmd_devmap(*pmd)))  in __pmd_trans_huge_lock()
1981 		unsigned long haddr, pmd_t *pmd)  in __split_huge_zero_page_pmd()  argument
1996 	pmdp_huge_clear_flush(vma, haddr, pmd);  in __split_huge_zero_page_pmd()
1998 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in __split_huge_zero_page_pmd()
2011 	pmd_populate(mm, pmd, pgtable);  in __split_huge_zero_page_pmd()
2014 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,  in __split_huge_pmd_locked()  argument
2028 	VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)  in __split_huge_pmd_locked()
2029 				&& !pmd_devmap(*pmd));  in __split_huge_pmd_locked()
2034 		_pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);  in __split_huge_pmd_locked()
2040 			zap_deposited_table(mm, pmd);  in __split_huge_pmd_locked()
2052 	} else if (pmd_trans_huge(*pmd) && is_huge_zero_pmd(*pmd)) {  in __split_huge_pmd_locked()
2062 		return __split_huge_zero_page_pmd(vma, haddr, pmd);  in __split_huge_pmd_locked()
2085 	old_pmd = pmdp_invalidate(vma, haddr, pmd);  in __split_huge_pmd_locked()
2113 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in __split_huge_pmd_locked()
2176 	pmd_populate(mm, pmd, pgtable);  in __split_huge_pmd_locked()
2186 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,  in __split_huge_pmd()  argument
2198 	ptl = pmd_lock(vma->vm_mm, pmd);  in __split_huge_pmd()
2208 		if (page != pmd_page(*pmd))  in __split_huge_pmd()
2213 	if (pmd_trans_huge(*pmd)) {  in __split_huge_pmd()
2215 			page = pmd_page(*pmd);  in __split_huge_pmd()
2218 				_pmd = *pmd;  in __split_huge_pmd()
2222 				if (unlikely(!pmd_same(*pmd, _pmd))) {  in __split_huge_pmd()
2233 	} else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd)))  in __split_huge_pmd()
2235 	__split_huge_pmd_locked(vma, pmd, range.start, freeze);  in __split_huge_pmd()
2262 	pmd_t *pmd;  in split_huge_pmd_address()  local
2276 	pmd = pmd_offset(pud, address);  in split_huge_pmd_address()
2278 	__split_huge_pmd(vma, pmd, address, freeze, page);  in split_huge_pmd_address()
2942 	if (!(pvmw->pmd && !pvmw->pte))  in set_pmd_migration_entry()
2946 	pmdval = pmdp_invalidate(vma, address, pvmw->pmd);  in set_pmd_migration_entry()
2953 	set_pmd_at(mm, address, pvmw->pmd, pmdswp);  in set_pmd_migration_entry()
2967 	if (!(pvmw->pmd && !pvmw->pte))  in remove_migration_pmd()
2970 	entry = pmd_to_swp_entry(*pvmw->pmd);  in remove_migration_pmd()
2973 	if (pmd_swp_soft_dirty(*pvmw->pmd))  in remove_migration_pmd()
2983 	set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);  in remove_migration_pmd()
2986 	update_mmu_cache_pmd(vma, address, pvmw->pmd);  in remove_migration_pmd()