23 struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm,  in find_dst_vma()  argument
33 	dst_vma = find_vma(dst_mm, dst_start);  in find_dst_vma()
58 int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,  in mfill_atomic_install_pte()  argument
95 	dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);  in mfill_atomic_install_pte()
130 	inc_mm_counter(dst_mm, mm_counter(page));  in mfill_atomic_install_pte()
132 	set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);  in mfill_atomic_install_pte()
142 static int mcopy_atomic_pte(struct mm_struct *dst_mm,  in mcopy_atomic_pte()  argument
205 	if (mem_cgroup_charge(page_folio(page), dst_mm, GFP_KERNEL))  in mcopy_atomic_pte()
208 	ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,  in mcopy_atomic_pte()
219 static int mfill_zeropage_pte(struct mm_struct *dst_mm,  in mfill_zeropage_pte()  argument
232 	dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);  in mfill_zeropage_pte()
245 	set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);  in mfill_zeropage_pte()
255 static int mcontinue_atomic_pte(struct mm_struct *dst_mm,  in mcontinue_atomic_pte()  argument
284 	ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,  in mcontinue_atomic_pte()
325 static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,  in __mcopy_atomic_hugetlb()  argument
351 		mmap_read_unlock(dst_mm);  in __mcopy_atomic_hugetlb()
375 		dst_vma = find_dst_vma(dst_mm, dst_start, len);  in __mcopy_atomic_hugetlb()
411 		dst_pte = huge_pte_alloc(dst_mm, dst_vma, dst_addr, vma_hpagesize);  in __mcopy_atomic_hugetlb()
426 		err = hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma,  in __mcopy_atomic_hugetlb()
436 			mmap_read_unlock(dst_mm);  in __mcopy_atomic_hugetlb()
447 			mmap_read_lock(dst_mm);  in __mcopy_atomic_hugetlb()
467 	mmap_read_unlock(dst_mm);  in __mcopy_atomic_hugetlb()
478 extern ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,
487 static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm,  in mfill_atomic_pte()  argument
499 		return mcontinue_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr,  in mfill_atomic_pte()
515 			err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma,  in mfill_atomic_pte()
519 			err = mfill_zeropage_pte(dst_mm, dst_pmd,  in mfill_atomic_pte()
522 		err = shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,  in mfill_atomic_pte()
531 static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm,  in __mcopy_atomic()  argument
562 	mmap_read_lock(dst_mm);  in __mcopy_atomic()
578 	dst_vma = find_dst_vma(dst_mm, dst_start, len);  in __mcopy_atomic()
603 		return  __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start,  in __mcopy_atomic()
627 		dst_pmd = mm_alloc_pmd(dst_mm, dst_addr);  in __mcopy_atomic()
643 		    unlikely(__pte_alloc(dst_mm, dst_pmd))) {  in __mcopy_atomic()
656 		err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr,  in __mcopy_atomic()
663 			mmap_read_unlock(dst_mm);  in __mcopy_atomic()
693 	mmap_read_unlock(dst_mm);  in __mcopy_atomic()
703 ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,  in mcopy_atomic()  argument
707 	return __mcopy_atomic(dst_mm, dst_start, src_start, len,  in mcopy_atomic()
711 ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start,  in mfill_zeropage()  argument
714 	return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_ZEROPAGE,  in mfill_zeropage()
718 ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long start,  in mcopy_continue()  argument
721 	return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_CONTINUE,  in mcopy_continue()
725 void uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *dst_vma,  in uffd_wp_range()  argument
736 	tlb_gather_mmu(&tlb, dst_mm);  in uffd_wp_range()
742 int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start,  in mwriteprotect_range()  argument
759 	mmap_read_lock(dst_mm);  in mwriteprotect_range()
771 	dst_vma = find_dst_vma(dst_mm, start, len);  in mwriteprotect_range()
787 	uffd_wp_range(dst_mm, dst_vma, start, len, enable_wp);  in mwriteprotect_range()
791 	mmap_read_unlock(dst_mm);  in mwriteprotect_range()