Lines Matching refs:dst_mm
23 static int mcopy_atomic_pte(struct mm_struct *dst_mm, in mcopy_atomic_pte() argument
69 if (mem_cgroup_try_charge(page, dst_mm, GFP_KERNEL, &memcg, false)) in mcopy_atomic_pte()
77 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in mcopy_atomic_pte()
81 inc_mm_counter(dst_mm, MM_ANONPAGES); in mcopy_atomic_pte()
86 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); in mcopy_atomic_pte()
103 static int mfill_zeropage_pte(struct mm_struct *dst_mm, in mfill_zeropage_pte() argument
115 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in mfill_zeropage_pte()
118 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); in mfill_zeropage_pte()
153 static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, in __mcopy_atomic_hugetlb() argument
180 up_read(&dst_mm->mmap_sem); in __mcopy_atomic_hugetlb()
204 dst_vma = find_vma(dst_mm, dst_start); in __mcopy_atomic_hugetlb()
251 hash = hugetlb_fault_mutex_hash(h, dst_mm, dst_vma, mapping, in __mcopy_atomic_hugetlb()
256 dst_pte = huge_pte_alloc(dst_mm, dst_addr, huge_page_size(h)); in __mcopy_atomic_hugetlb()
269 err = hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma, in __mcopy_atomic_hugetlb()
278 up_read(&dst_mm->mmap_sem); in __mcopy_atomic_hugetlb()
288 down_read(&dst_mm->mmap_sem); in __mcopy_atomic_hugetlb()
308 up_read(&dst_mm->mmap_sem); in __mcopy_atomic_hugetlb()
365 extern ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,
373 static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, in mfill_atomic_pte() argument
385 err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma, in mfill_atomic_pte()
388 err = mfill_zeropage_pte(dst_mm, dst_pmd, in mfill_atomic_pte()
392 err = shmem_mcopy_atomic_pte(dst_mm, dst_pmd, in mfill_atomic_pte()
396 err = shmem_mfill_zeropage_pte(dst_mm, dst_pmd, in mfill_atomic_pte()
403 static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm, in __mcopy_atomic() argument
432 down_read(&dst_mm->mmap_sem); in __mcopy_atomic()
448 dst_vma = find_vma(dst_mm, dst_start); in __mcopy_atomic()
480 return __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start, in __mcopy_atomic()
500 dst_pmd = mm_alloc_pmd(dst_mm, dst_addr); in __mcopy_atomic()
516 unlikely(__pte_alloc(dst_mm, dst_pmd, dst_addr))) { in __mcopy_atomic()
529 err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr, in __mcopy_atomic()
536 up_read(&dst_mm->mmap_sem); in __mcopy_atomic()
565 up_read(&dst_mm->mmap_sem); in __mcopy_atomic()
575 ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, in mcopy_atomic() argument
579 return __mcopy_atomic(dst_mm, dst_start, src_start, len, false, in mcopy_atomic()
583 ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start, in mfill_zeropage() argument
586 return __mcopy_atomic(dst_mm, start, 0, len, true, mmap_changing); in mfill_zeropage()