74 bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags,  in hugepage_vma_check()  argument
77 	if (!vma->vm_mm)		/* vdso */  in hugepage_vma_check()
86 	    test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))  in hugepage_vma_check()
95 	if (vma_is_dax(vma))  in hugepage_vma_check()
113 	    !transhuge_vma_suitable(vma, (vma->vm_end - HPAGE_PMD_SIZE)))  in hugepage_vma_check()
121 	if (!in_pf && shmem_file(vma->vm_file))  in hugepage_vma_check()
122 		return shmem_is_huge(file_inode(vma->vm_file), vma->vm_pgoff,  in hugepage_vma_check()
123 				     !enforce_sysfs, vma->vm_mm, vm_flags);  in hugepage_vma_check()
132 	if (!in_pf && file_thp_enabled(vma))  in hugepage_vma_check()
135 	if (!vma_is_anonymous(vma))  in hugepage_vma_check()
138 	if (vma_is_temporary_stack(vma))  in hugepage_vma_check()
148 	if (!vma->anon_vma)  in hugepage_vma_check()
551 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)  in maybe_pmd_mkwrite()  argument
553 	if (likely(vma->vm_flags & VM_WRITE))  in maybe_pmd_mkwrite()
554 		pmd = pmd_mkwrite(pmd, vma);  in maybe_pmd_mkwrite()
649 	struct vm_area_struct *vma = vmf->vma;  in __do_huge_pmd_anonymous_page()  local
657 	if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) {  in __do_huge_pmd_anonymous_page()
665 	pgtable = pte_alloc_one(vma->vm_mm);  in __do_huge_pmd_anonymous_page()
679 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in __do_huge_pmd_anonymous_page()
685 		ret = check_stable_address_space(vma->vm_mm);  in __do_huge_pmd_anonymous_page()
690 		if (userfaultfd_missing(vma)) {  in __do_huge_pmd_anonymous_page()
693 			pte_free(vma->vm_mm, pgtable);  in __do_huge_pmd_anonymous_page()
699 		entry = mk_huge_pmd(page, vma->vm_page_prot);  in __do_huge_pmd_anonymous_page()
700 		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);  in __do_huge_pmd_anonymous_page()
701 		folio_add_new_anon_rmap(folio, vma, haddr);  in __do_huge_pmd_anonymous_page()
702 		folio_add_lru_vma(folio, vma);  in __do_huge_pmd_anonymous_page()
703 		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);  in __do_huge_pmd_anonymous_page()
704 		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);  in __do_huge_pmd_anonymous_page()
705 		update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in __do_huge_pmd_anonymous_page()
706 		add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);  in __do_huge_pmd_anonymous_page()
707 		mm_inc_nr_ptes(vma->vm_mm);  in __do_huge_pmd_anonymous_page()
710 		count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC);  in __do_huge_pmd_anonymous_page()
718 		pte_free(vma->vm_mm, pgtable);  in __do_huge_pmd_anonymous_page()
733 gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma)  in vma_thp_gfp_mask()  argument
735 	const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE);  in vma_thp_gfp_mask()
761 		struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,  in set_huge_zero_page()  argument
767 	entry = mk_pmd(zero_page, vma->vm_page_prot);  in set_huge_zero_page()
776 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_anonymous_page()  local
781 	if (!transhuge_vma_suitable(vma, haddr))  in do_huge_pmd_anonymous_page()
783 	if (unlikely(anon_vma_prepare(vma)))  in do_huge_pmd_anonymous_page()
785 	khugepaged_enter_vma(vma, vma->vm_flags);  in do_huge_pmd_anonymous_page()
788 			!mm_forbids_zeropage(vma->vm_mm) &&  in do_huge_pmd_anonymous_page()
793 		pgtable = pte_alloc_one(vma->vm_mm);  in do_huge_pmd_anonymous_page()
796 		zero_page = mm_get_huge_zero_page(vma->vm_mm);  in do_huge_pmd_anonymous_page()
798 			pte_free(vma->vm_mm, pgtable);  in do_huge_pmd_anonymous_page()
802 		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_anonymous_page()
805 			ret = check_stable_address_space(vma->vm_mm);  in do_huge_pmd_anonymous_page()
808 				pte_free(vma->vm_mm, pgtable);  in do_huge_pmd_anonymous_page()
809 			} else if (userfaultfd_missing(vma)) {  in do_huge_pmd_anonymous_page()
811 				pte_free(vma->vm_mm, pgtable);  in do_huge_pmd_anonymous_page()
815 				set_huge_zero_page(pgtable, vma->vm_mm, vma,  in do_huge_pmd_anonymous_page()
817 				update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_anonymous_page()
822 			pte_free(vma->vm_mm, pgtable);  in do_huge_pmd_anonymous_page()
826 	gfp = vma_thp_gfp_mask(vma);  in do_huge_pmd_anonymous_page()
827 	folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, vma, haddr, true);  in do_huge_pmd_anonymous_page()
835 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,  in insert_pfn_pmd()  argument
839 	struct mm_struct *mm = vma->vm_mm;  in insert_pfn_pmd()
851 			entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);  in insert_pfn_pmd()
852 			if (pmdp_set_access_flags(vma, addr, pmd, entry, 1))  in insert_pfn_pmd()
853 				update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
864 		entry = maybe_pmd_mkwrite(entry, vma);  in insert_pfn_pmd()
874 	update_mmu_cache_pmd(vma, addr, pmd);  in insert_pfn_pmd()
895 	struct vm_area_struct *vma = vmf->vma;  in vmf_insert_pfn_pmd()  local
896 	pgprot_t pgprot = vma->vm_page_prot;  in vmf_insert_pfn_pmd()
904 	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&  in vmf_insert_pfn_pmd()
906 	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==  in vmf_insert_pfn_pmd()
908 	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));  in vmf_insert_pfn_pmd()
910 	if (addr < vma->vm_start || addr >= vma->vm_end)  in vmf_insert_pfn_pmd()
914 		pgtable = pte_alloc_one(vma->vm_mm);  in vmf_insert_pfn_pmd()
919 	track_pfn_insert(vma, &pgprot, pfn);  in vmf_insert_pfn_pmd()
921 	insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);  in vmf_insert_pfn_pmd()
927 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma)  in maybe_pud_mkwrite()  argument
929 	if (likely(vma->vm_flags & VM_WRITE))  in maybe_pud_mkwrite()
934 static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,  in insert_pfn_pud()  argument
937 	struct mm_struct *mm = vma->vm_mm;  in insert_pfn_pud()
938 	pgprot_t prot = vma->vm_page_prot;  in insert_pfn_pud()
950 			entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma);  in insert_pfn_pud()
951 			if (pudp_set_access_flags(vma, addr, pud, entry, 1))  in insert_pfn_pud()
952 				update_mmu_cache_pud(vma, addr, pud);  in insert_pfn_pud()
962 		entry = maybe_pud_mkwrite(entry, vma);  in insert_pfn_pud()
965 	update_mmu_cache_pud(vma, addr, pud);  in insert_pfn_pud()
984 	struct vm_area_struct *vma = vmf->vma;  in vmf_insert_pfn_pud()  local
985 	pgprot_t pgprot = vma->vm_page_prot;  in vmf_insert_pfn_pud()
992 	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&  in vmf_insert_pfn_pud()
994 	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==  in vmf_insert_pfn_pud()
996 	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));  in vmf_insert_pfn_pud()
998 	if (addr < vma->vm_start || addr >= vma->vm_end)  in vmf_insert_pfn_pud()
1001 	track_pfn_insert(vma, &pgprot, pfn);  in vmf_insert_pfn_pud()
1003 	insert_pfn_pud(vma, addr, vmf->pud, pfn, write);  in vmf_insert_pfn_pud()
1009 static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,  in touch_pmd()  argument
1017 	if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,  in touch_pmd()
1019 		update_mmu_cache_pmd(vma, addr, pmd);  in touch_pmd()
1022 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,  in follow_devmap_pmd()  argument
1026 	struct mm_struct *mm = vma->vm_mm;  in follow_devmap_pmd()
1041 		touch_pmd(vma, addr, pmd, flags & FOLL_WRITE);  in follow_devmap_pmd()
1164 static void touch_pud(struct vm_area_struct *vma, unsigned long addr,  in touch_pud()  argument
1172 	if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK,  in touch_pud()
1174 		update_mmu_cache_pud(vma, addr, pud);  in touch_pud()
1177 struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,  in follow_devmap_pud()  argument
1181 	struct mm_struct *mm = vma->vm_mm;  in follow_devmap_pud()
1196 		touch_pud(vma, addr, pud, flags & FOLL_WRITE);  in follow_devmap_pud()
1222 		  struct vm_area_struct *vma)  in copy_huge_pud()  argument
1265 	vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud);  in huge_pud_set_accessed()
1269 	touch_pud(vmf->vma, vmf->address, vmf->pud, write);  in huge_pud_set_accessed()
1279 	vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);  in huge_pmd_set_accessed()
1283 	touch_pmd(vmf->vma, vmf->address, vmf->pmd, write);  in huge_pmd_set_accessed()
1292 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_wp_page()  local
1298 	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);  in do_huge_pmd_wp_page()
1299 	VM_BUG_ON_VMA(!vma->anon_vma, vma);  in do_huge_pmd_wp_page()
1352 		page_move_anon_rmap(page, vma);  in do_huge_pmd_wp_page()
1360 		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);  in do_huge_pmd_wp_page()
1361 		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1))  in do_huge_pmd_wp_page()
1362 			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_wp_page()
1371 	__split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);  in do_huge_pmd_wp_page()
1375 static inline bool can_change_pmd_writable(struct vm_area_struct *vma,  in can_change_pmd_writable()  argument
1380 	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))  in can_change_pmd_writable()
1388 	if (vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd))  in can_change_pmd_writable()
1392 	if (userfaultfd_huge_pmd_wp(vma, pmd))  in can_change_pmd_writable()
1395 	if (!(vma->vm_flags & VM_SHARED)) {  in can_change_pmd_writable()
1397 		page = vm_normal_page_pmd(vma, addr, pmd);  in can_change_pmd_writable()
1407 					struct vm_area_struct *vma,  in can_follow_write_pmd()  argument
1419 	if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED))  in can_follow_write_pmd()
1423 	if (!(vma->vm_flags & VM_MAYWRITE))  in can_follow_write_pmd()
1427 	if (vma->vm_flags & VM_WRITE)  in can_follow_write_pmd()
1438 	if (vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd))  in can_follow_write_pmd()
1440 	return !userfaultfd_huge_pmd_wp(vma, pmd);  in can_follow_write_pmd()
1443 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,  in follow_trans_huge_pmd()  argument
1448 	struct mm_struct *mm = vma->vm_mm;  in follow_trans_huge_pmd()
1458 	    !can_follow_write_pmd(*pmd, page, vma, flags))  in follow_trans_huge_pmd()
1465 	if (pmd_protnone(*pmd) && !gup_can_follow_protnone(vma, flags))  in follow_trans_huge_pmd()
1468 	if (!pmd_write(*pmd) && gup_must_unshare(vma, flags, page))  in follow_trans_huge_pmd()
1479 		touch_pmd(vma, addr, pmd, flags & FOLL_WRITE);  in follow_trans_huge_pmd()
1490 	struct vm_area_struct *vma = vmf->vma;  in do_huge_pmd_numa_page()  local
1500 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_numa_page()
1506 	pmd = pmd_modify(oldpmd, vma->vm_page_prot);  in do_huge_pmd_numa_page()
1513 	if (!writable && vma_wants_manual_pte_write_upgrade(vma) &&  in do_huge_pmd_numa_page()
1514 	    can_change_pmd_writable(vma, vmf->address, pmd))  in do_huge_pmd_numa_page()
1517 	page = vm_normal_page_pmd(vma, haddr, pmd);  in do_huge_pmd_numa_page()
1532 	target_nid = numa_migrate_prep(page, vma, haddr, page_nid,  in do_huge_pmd_numa_page()
1543 	migrated = migrate_misplaced_page(page, vma, target_nid);  in do_huge_pmd_numa_page()
1549 		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);  in do_huge_pmd_numa_page()
1566 	pmd = pmd_modify(oldpmd, vma->vm_page_prot);  in do_huge_pmd_numa_page()
1569 		pmd = pmd_mkwrite(pmd, vma);  in do_huge_pmd_numa_page()
1570 	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);  in do_huge_pmd_numa_page()
1571 	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);  in do_huge_pmd_numa_page()
1580 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,  in madvise_free_huge_pmd()  argument
1591 	ptl = pmd_trans_huge_lock(pmd, vma);  in madvise_free_huge_pmd()
1634 		pmdp_invalidate(vma, addr, pmd);  in madvise_free_huge_pmd()
1659 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,  in zap_huge_pmd()  argument
1667 	ptl = __pmd_trans_huge_lock(pmd, vma);  in zap_huge_pmd()
1676 	orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd,  in zap_huge_pmd()
1678 	arch_check_zapped_pmd(vma, orig_pmd);  in zap_huge_pmd()
1680 	if (vma_is_special_huge(vma)) {  in zap_huge_pmd()
1693 			page_remove_rmap(page, vma, true);  in zap_huge_pmd()
1725 					 struct vm_area_struct *vma)  in pmd_move_must_withdraw()  argument
1733 	return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma);  in pmd_move_must_withdraw()
1748 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,  in move_huge_pmd()  argument
1753 	struct mm_struct *mm = vma->vm_mm;  in move_huge_pmd()
1770 	old_ptl = __pmd_trans_huge_lock(old_pmd, vma);  in move_huge_pmd()
1780 		if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) {  in move_huge_pmd()
1788 			flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE);  in move_huge_pmd()
1804 int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,  in change_huge_pmd()  argument
1808 	struct mm_struct *mm = vma->vm_mm;  in change_huge_pmd()
1821 	ptl = __pmd_trans_huge_lock(pmd, vma);  in change_huge_pmd()
1907 	oldpmd = pmdp_invalidate_ad(vma, addr, pmd);  in change_huge_pmd()
1922 	    can_change_pmd_writable(vma, addr, entry))  in change_huge_pmd()
1923 		entry = pmd_mkwrite(entry, vma);  in change_huge_pmd()
1941 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)  in __pmd_trans_huge_lock()  argument
1944 	ptl = pmd_lock(vma->vm_mm, pmd);  in __pmd_trans_huge_lock()
1958 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma)  in __pud_trans_huge_lock()  argument
1962 	ptl = pud_lock(vma->vm_mm, pud);  in __pud_trans_huge_lock()
1970 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,  in zap_huge_pud()  argument
1975 	ptl = __pud_trans_huge_lock(pud, vma);  in zap_huge_pud()
1979 	pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm);  in zap_huge_pud()
1981 	if (vma_is_special_huge(vma)) {  in zap_huge_pud()
1991 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud,  in __split_huge_pud_locked()  argument
1995 	VM_BUG_ON_VMA(vma->vm_start > haddr, vma);  in __split_huge_pud_locked()
1996 	VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma);  in __split_huge_pud_locked()
2001 	pudp_huge_clear_flush(vma, haddr, pud);  in __split_huge_pud_locked()
2004 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,  in __split_huge_pud()  argument
2010 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,  in __split_huge_pud()
2014 	ptl = pud_lock(vma->vm_mm, pud);  in __split_huge_pud()
2017 	__split_huge_pud_locked(vma, pud, range.start);  in __split_huge_pud()
2025 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,  in __split_huge_zero_page_pmd()  argument
2028 	struct mm_struct *mm = vma->vm_mm;  in __split_huge_zero_page_pmd()
2043 	old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd);  in __split_huge_zero_page_pmd()
2053 		entry = pfn_pte(my_zero_pfn(addr), vma->vm_page_prot);  in __split_huge_zero_page_pmd()
2066 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,  in __split_huge_pmd_locked()  argument
2069 	struct mm_struct *mm = vma->vm_mm;  in __split_huge_pmd_locked()
2080 	VM_BUG_ON_VMA(vma->vm_start > haddr, vma);  in __split_huge_pmd_locked()
2081 	VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);  in __split_huge_pmd_locked()
2087 	if (!vma_is_anonymous(vma)) {  in __split_huge_pmd_locked()
2088 		old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd);  in __split_huge_pmd_locked()
2095 		if (vma_is_special_huge(vma))  in __split_huge_pmd_locked()
2108 			page_remove_rmap(page, vma, true);  in __split_huge_pmd_locked()
2125 		return __split_huge_zero_page_pmd(vma, haddr, pmd);  in __split_huge_pmd_locked()
2148 	old_pmd = pmdp_invalidate(vma, haddr, pmd);  in __split_huge_pmd_locked()
2235 			entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot));  in __split_huge_pmd_locked()
2237 				entry = pte_mkwrite(entry, vma);  in __split_huge_pmd_locked()
2249 			page_add_anon_rmap(page + i, vma, addr, RMAP_NONE);  in __split_huge_pmd_locked()
2258 		page_remove_rmap(page, vma, true);  in __split_huge_pmd_locked()
2266 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,  in __split_huge_pmd()  argument
2272 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,  in __split_huge_pmd()
2276 	ptl = pmd_lock(vma->vm_mm, pmd);  in __split_huge_pmd()
2293 		__split_huge_pmd_locked(vma, pmd, range.start, freeze);  in __split_huge_pmd()
2301 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,  in split_huge_pmd_address()  argument
2304 	pmd_t *pmd = mm_find_pmd(vma->vm_mm, address);  in split_huge_pmd_address()
2309 	__split_huge_pmd(vma, pmd, address, freeze, folio);  in split_huge_pmd_address()
2312 static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address)  in split_huge_pmd_if_needed()  argument
2319 	    range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE),  in split_huge_pmd_if_needed()
2321 		split_huge_pmd_address(vma, address, false, NULL);  in split_huge_pmd_if_needed()
2324 void vma_adjust_trans_huge(struct vm_area_struct *vma,  in vma_adjust_trans_huge()  argument
2330 	split_huge_pmd_if_needed(vma, start);  in vma_adjust_trans_huge()
2333 	split_huge_pmd_if_needed(vma, end);  in vma_adjust_trans_huge()
2340 		struct vm_area_struct *next = find_vma(vma->vm_mm, vma->vm_end);  in vma_adjust_trans_huge()
2964 static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma)  in vma_not_suitable_for_thp_split()  argument
2966 	return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) ||  in vma_not_suitable_for_thp_split()
2967 		    is_vm_hugetlb_page(vma);  in vma_not_suitable_for_thp_split()
3011 		struct vm_area_struct *vma = vma_lookup(mm, addr);  in split_huge_pages_pid()  local
3015 		if (!vma)  in split_huge_pages_pid()
3019 		if (vma_not_suitable_for_thp_split(vma)) {  in split_huge_pages_pid()
3020 			addr = vma->vm_end;  in split_huge_pages_pid()
3025 		page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP);  in split_huge_pages_pid()
3205 	struct vm_area_struct *vma = pvmw->vma;  in set_pmd_migration_entry()  local
3206 	struct mm_struct *mm = vma->vm_mm;  in set_pmd_migration_entry()
3216 	flush_cache_range(vma, address, address + HPAGE_PMD_SIZE);  in set_pmd_migration_entry()
3217 	pmdval = pmdp_invalidate(vma, address, pvmw->pmd);  in set_pmd_migration_entry()
3244 	page_remove_rmap(page, vma, true);  in set_pmd_migration_entry()
3253 	struct vm_area_struct *vma = pvmw->vma;  in remove_migration_pmd()  local
3254 	struct mm_struct *mm = vma->vm_mm;  in remove_migration_pmd()
3265 	pmde = mk_huge_pmd(new, READ_ONCE(vma->vm_page_prot));  in remove_migration_pmd()
3269 		pmde = pmd_mkwrite(pmde, vma);  in remove_migration_pmd()
3284 		page_add_anon_rmap(new, vma, haddr, rmap_flags);  in remove_migration_pmd()
3286 		page_add_file_rmap(new, vma, true);  in remove_migration_pmd()
3292 	update_mmu_cache_pmd(vma, address, pvmw->pmd);  in remove_migration_pmd()