492 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)  in maybe_pmd_mkwrite()  argument
495 		pmd = pmd_mkwrite(pmd);  in maybe_pmd_mkwrite()
496 	return pmd;  in maybe_pmd_mkwrite()
604 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in __do_huge_pmd_anonymous_page()
605 	if (unlikely(!pmd_none(*vmf->pmd))) {  in __do_huge_pmd_anonymous_page()
632 		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);  in __do_huge_pmd_anonymous_page()
633 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in __do_huge_pmd_anonymous_page()
690 		struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,  in set_huge_zero_page()  argument
694 	if (!pmd_none(*pmd))  in set_huge_zero_page()
699 		pgtable_trans_huge_deposit(mm, pmd, pgtable);  in set_huge_zero_page()
700 	set_pmd_at(mm, haddr, pmd, entry);  in set_huge_zero_page()
734 		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_anonymous_page()
737 		if (pmd_none(*vmf->pmd)) {  in do_huge_pmd_anonymous_page()
747 						   haddr, vmf->pmd, zero_page);  in do_huge_pmd_anonymous_page()
768 		pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write,  in insert_pfn_pmd()  argument
775 	ptl = pmd_lock(mm, pmd);  in insert_pfn_pmd()
776 	if (!pmd_none(*pmd)) {  in insert_pfn_pmd()
778 			if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) {  in insert_pfn_pmd()
779 				WARN_ON_ONCE(!is_huge_zero_pmd(*pmd));  in insert_pfn_pmd()
782 			entry = pmd_mkyoung(*pmd);  in insert_pfn_pmd()
784 			if (pmdp_set_access_flags(vma, addr, pmd, entry, 1))  in insert_pfn_pmd()
785 				update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
800 		pgtable_trans_huge_deposit(mm, pmd, pgtable);  in insert_pfn_pmd()
805 	set_pmd_at(mm, addr, pmd, entry);  in insert_pfn_pmd()
806 	update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
843 	insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);  in vmf_insert_pfn_pmd()
921 		pmd_t *pmd, int flags)  in touch_pmd()  argument
925 	_pmd = pmd_mkyoung(*pmd);  in touch_pmd()
929 				pmd, _pmd, flags & FOLL_WRITE))  in touch_pmd()
930 		update_mmu_cache_pmd(vma, addr, pmd);  in touch_pmd()
934 		pmd_t *pmd, int flags, struct dev_pagemap **pgmap)  in follow_devmap_pmd()  argument
936 	unsigned long pfn = pmd_pfn(*pmd);  in follow_devmap_pmd()
940 	assert_spin_locked(pmd_lockptr(mm, pmd));  in follow_devmap_pmd()
948 	if (flags & FOLL_WRITE && !pmd_write(*pmd))  in follow_devmap_pmd()
951 	if (pmd_present(*pmd) && pmd_devmap(*pmd))  in follow_devmap_pmd()
957 		touch_pmd(vma, addr, pmd, flags);  in follow_devmap_pmd()
982 	pmd_t pmd;  in copy_huge_pmd()  local
999 	pmd = *src_pmd;  in copy_huge_pmd()
1002 	if (unlikely(is_swap_pmd(pmd))) {  in copy_huge_pmd()
1003 		swp_entry_t entry = pmd_to_swp_entry(pmd);  in copy_huge_pmd()
1005 		VM_BUG_ON(!is_pmd_migration_entry(pmd));  in copy_huge_pmd()
1008 			pmd = swp_entry_to_pmd(entry);  in copy_huge_pmd()
1010 				pmd = pmd_swp_mksoft_dirty(pmd);  in copy_huge_pmd()
1011 			set_pmd_at(src_mm, addr, src_pmd, pmd);  in copy_huge_pmd()
1016 		set_pmd_at(dst_mm, addr, dst_pmd, pmd);  in copy_huge_pmd()
1022 	if (unlikely(!pmd_trans_huge(pmd))) {  in copy_huge_pmd()
1031 	if (is_huge_zero_pmd(pmd)) {  in copy_huge_pmd()
1045 	src_page = pmd_page(pmd);  in copy_huge_pmd()
1054 	pmd = pmd_mkold(pmd_wrprotect(pmd));  in copy_huge_pmd()
1055 	set_pmd_at(dst_mm, addr, dst_pmd, pmd);  in copy_huge_pmd()
1181 	vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);  in huge_pmd_set_accessed()
1182 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in huge_pmd_set_accessed()
1189 	if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write))  in huge_pmd_set_accessed()
1190 		update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd);  in huge_pmd_set_accessed()
1249 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page_fallback()
1250 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in do_huge_pmd_wp_page_fallback()
1262 	pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);  in do_huge_pmd_wp_page_fallback()
1264 	pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page_fallback()
1284 	pmd_populate(vma->vm_mm, vmf->pmd, pgtable);  in do_huge_pmd_wp_page_fallback()
1323 	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page()
1328 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))  in do_huge_pmd_wp_page()
1342 		if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1353 		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry,  1))  in do_huge_pmd_wp_page()
1354 			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1374 			split_huge_pmd(vma, vmf->pmd, vmf->address);  in do_huge_pmd_wp_page()
1379 				split_huge_pmd(vma, vmf->pmd, vmf->address);  in do_huge_pmd_wp_page()
1391 		split_huge_pmd(vma, vmf->pmd, vmf->address);  in do_huge_pmd_wp_page()
1416 	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {  in do_huge_pmd_wp_page()
1425 		pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);  in do_huge_pmd_wp_page()
1429 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in do_huge_pmd_wp_page()
1430 		update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1458 static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)  in can_follow_write_pmd()  argument
1460 	return pmd_write(pmd) ||  in can_follow_write_pmd()
1461 	       ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd));  in can_follow_write_pmd()
1466 				   pmd_t *pmd,  in follow_trans_huge_pmd()  argument
1472 	assert_spin_locked(pmd_lockptr(mm, pmd));  in follow_trans_huge_pmd()
1474 	if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags))  in follow_trans_huge_pmd()
1478 	if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))  in follow_trans_huge_pmd()
1482 	if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))  in follow_trans_huge_pmd()
1485 	page = pmd_page(*pmd);  in follow_trans_huge_pmd()
1488 		touch_pmd(vma, addr, pmd, flags);  in follow_trans_huge_pmd()
1533 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)  in do_huge_pmd_numa_page()  argument
1546 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_numa_page()
1547 	if (unlikely(!pmd_same(pmd, *vmf->pmd)))  in do_huge_pmd_numa_page()
1555 	if (unlikely(pmd_trans_migrating(*vmf->pmd))) {  in do_huge_pmd_numa_page()
1556 		page = pmd_page(*vmf->pmd);  in do_huge_pmd_numa_page()
1564 	page = pmd_page(pmd);  in do_huge_pmd_numa_page()
1575 	if (!pmd_savedwrite(pmd))  in do_huge_pmd_numa_page()
1610 	if (unlikely(!pmd_same(pmd, *vmf->pmd))) {  in do_huge_pmd_numa_page()
1657 				vmf->pmd, pmd, vmf->address, page, target_nid);  in do_huge_pmd_numa_page()
1667 	was_writable = pmd_savedwrite(pmd);  in do_huge_pmd_numa_page()
1668 	pmd = pmd_modify(pmd, vma->vm_page_prot);  in do_huge_pmd_numa_page()
1669 	pmd = pmd_mkyoung(pmd);  in do_huge_pmd_numa_page()
1671 		pmd = pmd_mkwrite(pmd);  in do_huge_pmd_numa_page()
1672 	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);  in do_huge_pmd_numa_page()
1673 	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_numa_page()
1694 		pmd_t *pmd, unsigned long addr, unsigned long next)  in madvise_free_huge_pmd()  argument
1704 	ptl = pmd_trans_huge_lock(pmd, vma);  in madvise_free_huge_pmd()
1708 	orig_pmd = *pmd;  in madvise_free_huge_pmd()
1747 		pmdp_invalidate(vma, addr, pmd);  in madvise_free_huge_pmd()
1751 		set_pmd_at(mm, addr, pmd, orig_pmd);  in madvise_free_huge_pmd()
1752 		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);  in madvise_free_huge_pmd()
1763 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)  in zap_deposited_table()  argument
1767 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in zap_deposited_table()
1773 		 pmd_t *pmd, unsigned long addr)  in zap_huge_pmd()  argument
1780 	ptl = __pmd_trans_huge_lock(pmd, vma);  in zap_huge_pmd()
1789 	orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,  in zap_huge_pmd()
1791 	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);  in zap_huge_pmd()
1794 			zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1799 		zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1822 			zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1826 				zap_deposited_table(tlb->mm, pmd);  in zap_huge_pmd()
1852 static pmd_t move_soft_dirty_pmd(pmd_t pmd)  in move_soft_dirty_pmd()  argument
1855 	if (unlikely(is_pmd_migration_entry(pmd)))  in move_soft_dirty_pmd()
1856 		pmd = pmd_swp_mksoft_dirty(pmd);  in move_soft_dirty_pmd()
1857 	else if (pmd_present(pmd))  in move_soft_dirty_pmd()
1858 		pmd = pmd_mksoft_dirty(pmd);  in move_soft_dirty_pmd()
1860 	return pmd;  in move_soft_dirty_pmd()
1868 	pmd_t pmd;  in move_huge_pmd()  local
1895 		pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);  in move_huge_pmd()
1896 		if (pmd_present(pmd))  in move_huge_pmd()
1905 		pmd = move_soft_dirty_pmd(pmd);  in move_huge_pmd()
1906 		set_pmd_at(mm, new_addr, new_pmd, pmd);  in move_huge_pmd()
1923 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,  in change_huge_pmd()  argument
1932 	ptl = __pmd_trans_huge_lock(pmd, vma);  in change_huge_pmd()
1936 	preserve_write = prot_numa && pmd_write(*pmd);  in change_huge_pmd()
1940 	if (is_swap_pmd(*pmd)) {  in change_huge_pmd()
1941 		swp_entry_t entry = pmd_to_swp_entry(*pmd);  in change_huge_pmd()
1943 		VM_BUG_ON(!is_pmd_migration_entry(*pmd));  in change_huge_pmd()
1952 			if (pmd_swp_soft_dirty(*pmd))  in change_huge_pmd()
1954 			set_pmd_at(mm, addr, pmd, newpmd);  in change_huge_pmd()
1965 	if (prot_numa && is_huge_zero_pmd(*pmd))  in change_huge_pmd()
1968 	if (prot_numa && pmd_protnone(*pmd))  in change_huge_pmd()
1992 	entry = pmdp_invalidate(vma, addr, pmd);  in change_huge_pmd()
1998 	set_pmd_at(mm, addr, pmd, entry);  in change_huge_pmd()
2011 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)  in __pmd_trans_huge_lock()  argument
2014 	ptl = pmd_lock(vma->vm_mm, pmd);  in __pmd_trans_huge_lock()
2015 	if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) ||  in __pmd_trans_huge_lock()
2016 			pmd_devmap(*pmd)))  in __pmd_trans_huge_lock()
2105 		unsigned long haddr, pmd_t *pmd)  in __split_huge_zero_page_pmd()  argument
2120 	pmdp_huge_clear_flush(vma, haddr, pmd);  in __split_huge_zero_page_pmd()
2122 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in __split_huge_zero_page_pmd()
2135 	pmd_populate(mm, pmd, pgtable);  in __split_huge_zero_page_pmd()
2138 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,  in __split_huge_pmd_locked()  argument
2152 	VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)  in __split_huge_pmd_locked()
2153 				&& !pmd_devmap(*pmd));  in __split_huge_pmd_locked()
2158 		_pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);  in __split_huge_pmd_locked()
2164 			zap_deposited_table(mm, pmd);  in __split_huge_pmd_locked()
2176 	} else if (is_huge_zero_pmd(*pmd)) {  in __split_huge_pmd_locked()
2186 		return __split_huge_zero_page_pmd(vma, haddr, pmd);  in __split_huge_pmd_locked()
2209 	old_pmd = pmdp_invalidate(vma, haddr, pmd);  in __split_huge_pmd_locked()
2235 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);  in __split_huge_pmd_locked()
2288 	pmd_populate(mm, pmd, pgtable);  in __split_huge_pmd_locked()
2298 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,  in __split_huge_pmd()  argument
2308 	ptl = pmd_lock(vma->vm_mm, pmd);  in __split_huge_pmd()
2315 	if (page && page != pmd_page(*pmd))  in __split_huge_pmd()
2318 	if (pmd_trans_huge(*pmd)) {  in __split_huge_pmd()
2319 		page = pmd_page(*pmd);  in __split_huge_pmd()
2322 	} else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd)))  in __split_huge_pmd()
2324 	__split_huge_pmd_locked(vma, pmd, range.start, freeze);  in __split_huge_pmd()
2349 	pmd_t *pmd;  in split_huge_pmd_address()  local
2363 	pmd = pmd_offset(pud, address);  in split_huge_pmd_address()
2365 	__split_huge_pmd(vma, pmd, address, freeze, page);  in split_huge_pmd_address()
3029 	if (!(pvmw->pmd && !pvmw->pte))  in set_pmd_migration_entry()
3033 	pmdval = *pvmw->pmd;  in set_pmd_migration_entry()
3034 	pmdp_invalidate(vma, address, pvmw->pmd);  in set_pmd_migration_entry()
3041 	set_pmd_at(mm, address, pvmw->pmd, pmdswp);  in set_pmd_migration_entry()
3055 	if (!(pvmw->pmd && !pvmw->pte))  in remove_migration_pmd()
3058 	entry = pmd_to_swp_entry(*pvmw->pmd);  in remove_migration_pmd()
3061 	if (pmd_swp_soft_dirty(*pvmw->pmd))  in remove_migration_pmd()
3071 	set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);  in remove_migration_pmd()
3074 	update_mmu_cache_pmd(vma, address, pvmw->pmd);  in remove_migration_pmd()