22 struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm,  in find_dst_vma()  argument
32 	dst_vma = find_vma(dst_mm, dst_start);  in find_dst_vma()
51 static int mcopy_atomic_pte(struct mm_struct *dst_mm,  in mcopy_atomic_pte()  argument
99 	if (mem_cgroup_charge(page, dst_mm, GFP_KERNEL))  in mcopy_atomic_pte()
110 	dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);  in mcopy_atomic_pte()
124 	inc_mm_counter(dst_mm, MM_ANONPAGES);  in mcopy_atomic_pte()
128 	set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);  in mcopy_atomic_pte()
144 static int mfill_zeropage_pte(struct mm_struct *dst_mm,  in mfill_zeropage_pte()  argument
157 	dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);  in mfill_zeropage_pte()
170 	set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);  in mfill_zeropage_pte()
205 static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,  in __mcopy_atomic_hugetlb()  argument
231 		mmap_read_unlock(dst_mm);  in __mcopy_atomic_hugetlb()
255 		dst_vma = find_dst_vma(dst_mm, dst_start, len);  in __mcopy_atomic_hugetlb()
293 		dst_pte = huge_pte_alloc(dst_mm, dst_addr, vma_hpagesize);  in __mcopy_atomic_hugetlb()
308 		err = hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma,  in __mcopy_atomic_hugetlb()
318 			mmap_read_unlock(dst_mm);  in __mcopy_atomic_hugetlb()
329 			mmap_read_lock(dst_mm);  in __mcopy_atomic_hugetlb()
349 	mmap_read_unlock(dst_mm);  in __mcopy_atomic_hugetlb()
406 extern ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,
414 static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm,  in mfill_atomic_pte()  argument
437 			err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma,  in mfill_atomic_pte()
441 			err = mfill_zeropage_pte(dst_mm, dst_pmd,  in mfill_atomic_pte()
446 			err = shmem_mcopy_atomic_pte(dst_mm, dst_pmd,  in mfill_atomic_pte()
450 			err = shmem_mfill_zeropage_pte(dst_mm, dst_pmd,  in mfill_atomic_pte()
457 static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm,  in __mcopy_atomic()  argument
488 	mmap_read_lock(dst_mm);  in __mcopy_atomic()
504 	dst_vma = find_dst_vma(dst_mm, dst_start, len);  in __mcopy_atomic()
529 		return  __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start,  in __mcopy_atomic()
550 		dst_pmd = mm_alloc_pmd(dst_mm, dst_addr);  in __mcopy_atomic()
566 		    unlikely(__pte_alloc(dst_mm, dst_pmd))) {  in __mcopy_atomic()
579 		err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr,  in __mcopy_atomic()
586 			mmap_read_unlock(dst_mm);  in __mcopy_atomic()
615 	mmap_read_unlock(dst_mm);  in __mcopy_atomic()
625 ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,  in mcopy_atomic()  argument
629 	return __mcopy_atomic(dst_mm, dst_start, src_start, len, false,  in mcopy_atomic()
633 ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start,  in mfill_zeropage()  argument
636 	return __mcopy_atomic(dst_mm, start, 0, len, true, mmap_changing, 0);  in mfill_zeropage()
639 int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start,  in mwriteprotect_range()  argument
655 	mmap_read_lock(dst_mm);  in mwriteprotect_range()
667 	dst_vma = find_dst_vma(dst_mm, start, len);  in mwriteprotect_range()
689 	mmap_read_unlock(dst_mm);  in mwriteprotect_range()