24 				unsigned long end, struct mm_walk *walk)  in walk_pte_range_inner()  argument
26 	const struct mm_walk_ops *ops = walk->ops;  in walk_pte_range_inner()
30 		err = ops->pte_entry(pte, addr, addr + PAGE_SIZE, walk);  in walk_pte_range_inner()
42 			  struct mm_walk *walk)  in walk_pte_range()  argument
48 	if (walk->no_vma) {  in walk_pte_range()
50 		err = walk_pte_range_inner(pte, addr, end, walk);  in walk_pte_range()
53 		pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);  in walk_pte_range()
54 		err = walk_pte_range_inner(pte, addr, end, walk);  in walk_pte_range()
62 			  struct mm_walk *walk)  in walk_pmd_range()  argument
66 	const struct mm_walk_ops *ops = walk->ops;  in walk_pmd_range()
74 		if (pmd_none(*pmd) || (!walk->vma && !walk->no_vma)) {  in walk_pmd_range()
76 				err = ops->pte_hole(addr, next, depth, walk);  in walk_pmd_range()
82 		walk->action = ACTION_SUBTREE;  in walk_pmd_range()
89 			err = ops->pmd_entry(pmd, addr, next, walk);  in walk_pmd_range()
93 		if (walk->action == ACTION_AGAIN)  in walk_pmd_range()
100 		if ((!walk->vma && (pmd_leaf(*pmd) || !pmd_present(*pmd))) ||  in walk_pmd_range()
101 		    walk->action == ACTION_CONTINUE ||  in walk_pmd_range()
105 		if (walk->vma) {  in walk_pmd_range()
106 			split_huge_pmd(walk->vma, pmd, addr);  in walk_pmd_range()
111 		err = walk_pte_range(pmd, addr, next, walk);  in walk_pmd_range()
120 			  struct mm_walk *walk)  in walk_pud_range()  argument
124 	const struct mm_walk_ops *ops = walk->ops;  in walk_pud_range()
132 		if (pud_none(*pud) || (!walk->vma && !walk->no_vma)) {  in walk_pud_range()
134 				err = ops->pte_hole(addr, next, depth, walk);  in walk_pud_range()
140 		walk->action = ACTION_SUBTREE;  in walk_pud_range()
143 			err = ops->pud_entry(pud, addr, next, walk);  in walk_pud_range()
147 		if (walk->action == ACTION_AGAIN)  in walk_pud_range()
150 		if ((!walk->vma && (pud_leaf(*pud) || !pud_present(*pud))) ||  in walk_pud_range()
151 		    walk->action == ACTION_CONTINUE ||  in walk_pud_range()
155 		if (walk->vma)  in walk_pud_range()
156 			split_huge_pud(walk->vma, pud, addr);  in walk_pud_range()
160 		err = walk_pmd_range(pud, addr, next, walk);  in walk_pud_range()
169 			  struct mm_walk *walk)  in walk_p4d_range()  argument
173 	const struct mm_walk_ops *ops = walk->ops;  in walk_p4d_range()
182 				err = ops->pte_hole(addr, next, depth, walk);  in walk_p4d_range()
188 			err = ops->p4d_entry(p4d, addr, next, walk);  in walk_p4d_range()
193 			err = walk_pud_range(p4d, addr, next, walk);  in walk_p4d_range()
202 			  struct mm_walk *walk)  in walk_pgd_range()  argument
206 	const struct mm_walk_ops *ops = walk->ops;  in walk_pgd_range()
209 	if (walk->pgd)  in walk_pgd_range()
210 		pgd = walk->pgd + pgd_index(addr);  in walk_pgd_range()
212 		pgd = pgd_offset(walk->mm, addr);  in walk_pgd_range()
217 				err = ops->pte_hole(addr, next, 0, walk);  in walk_pgd_range()
223 			err = ops->pgd_entry(pgd, addr, next, walk);  in walk_pgd_range()
229 			err = walk_p4d_range(pgd, addr, next, walk);  in walk_pgd_range()
246 			      struct mm_walk *walk)  in walk_hugetlb_range()  argument
248 	struct vm_area_struct *vma = walk->vma;  in walk_hugetlb_range()
254 	const struct mm_walk_ops *ops = walk->ops;  in walk_hugetlb_range()
259 		pte = huge_pte_offset(walk->mm, addr & hmask, sz);  in walk_hugetlb_range()
262 			err = ops->hugetlb_entry(pte, hmask, addr, next, walk);  in walk_hugetlb_range()
264 			err = ops->pte_hole(addr, next, -1, walk);  in walk_hugetlb_range()
275 			      struct mm_walk *walk)  in walk_hugetlb_range()  argument
289 			struct mm_walk *walk)  in walk_page_test()  argument
291 	struct vm_area_struct *vma = walk->vma;  in walk_page_test()
292 	const struct mm_walk_ops *ops = walk->ops;  in walk_page_test()
295 		return ops->test_walk(start, end, walk);  in walk_page_test()
308 			err = ops->pte_hole(start, end, -1, walk);  in walk_page_test()
315 			struct mm_walk *walk)  in __walk_page_range()  argument
318 	struct vm_area_struct *vma = walk->vma;  in __walk_page_range()
319 	const struct mm_walk_ops *ops = walk->ops;  in __walk_page_range()
322 		err = ops->pre_vma(start, end, walk);  in __walk_page_range()
329 			err = walk_hugetlb_range(start, end, walk);  in __walk_page_range()
331 		err = walk_pgd_range(start, end, walk);  in __walk_page_range()
334 		ops->post_vma(walk);  in __walk_page_range()
386 	struct mm_walk walk = {  in walk_page_range()  local
395 	if (!walk.mm)  in walk_page_range()
398 	mmap_assert_locked(walk.mm);  in walk_page_range()
400 	vma = find_vma(walk.mm, start);  in walk_page_range()
403 			walk.vma = NULL;  in walk_page_range()
406 			walk.vma = NULL;  in walk_page_range()
409 			walk.vma = vma;  in walk_page_range()
413 			err = walk_page_test(start, next, &walk);  in walk_page_range()
426 		if (walk.vma || walk.ops->pte_hole)  in walk_page_range()
427 			err = __walk_page_range(start, next, &walk);  in walk_page_range()
445 	struct mm_walk walk = {  in walk_page_range_novma()  local
453 	if (start >= end || !walk.mm)  in walk_page_range_novma()
456 	mmap_assert_locked(walk.mm);  in walk_page_range_novma()
458 	return __walk_page_range(start, end, &walk);  in walk_page_range_novma()
464 	struct mm_walk walk = {  in walk_page_vma()  local
472 	if (!walk.mm)  in walk_page_vma()
475 	mmap_assert_locked(walk.mm);  in walk_page_vma()
477 	err = walk_page_test(vma->vm_start, vma->vm_end, &walk);  in walk_page_vma()
482 	return __walk_page_range(vma->vm_start, vma->vm_end, &walk);  in walk_page_vma()
519 	struct mm_walk walk = {  in walk_page_mapping()  local
544 		walk.vma = vma;  in walk_page_mapping()
545 		walk.mm = vma->vm_mm;  in walk_page_mapping()
547 		err = walk_page_test(vma->vm_start, vma->vm_end, &walk);  in walk_page_mapping()
554 		err = __walk_page_range(start_addr, end_addr, &walk);  in walk_page_mapping()