131 	struct vm_area_struct *vma;  in m_start()  local
158 	vma = find_vma(mm, last_addr);  in m_start()
159 	if (vma)  in m_start()
160 		return vma;  in m_start()
168 	struct vm_area_struct *next, *vma = v;  in m_next()  local
170 	if (vma == priv->tail_vma)  in m_next()
172 	else if (vma->vm_next)  in m_next()
173 		next = vma->vm_next;  in m_next()
239 static int is_stack(struct vm_area_struct *vma)  in is_stack()  argument
246 	return vma->vm_start <= vma->vm_mm->start_stack &&  in is_stack()
247 		vma->vm_end >= vma->vm_mm->start_stack;  in is_stack()
271 show_map_vma(struct seq_file *m, struct vm_area_struct *vma)  in show_map_vma()  argument
273 	struct mm_struct *mm = vma->vm_mm;  in show_map_vma()
274 	struct file *file = vma->vm_file;  in show_map_vma()
275 	vm_flags_t flags = vma->vm_flags;  in show_map_vma()
283 		struct inode *inode = file_inode(vma->vm_file);  in show_map_vma()
286 		pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;  in show_map_vma()
289 	start = vma->vm_start;  in show_map_vma()
290 	end = vma->vm_end;  in show_map_vma()
303 	if (vma->vm_ops && vma->vm_ops->name) {  in show_map_vma()
304 		name = vma->vm_ops->name(vma);  in show_map_vma()
309 	name = arch_vma_name(vma);  in show_map_vma()
316 		if (vma->vm_start <= mm->brk &&  in show_map_vma()
317 		    vma->vm_end >= mm->start_brk) {  in show_map_vma()
322 		if (is_stack(vma))  in show_map_vma()
483 			walk->vma->vm_file->f_mapping, addr, end);  in smaps_pte_hole()
495 	struct vm_area_struct *vma = walk->vma;  in smaps_pte_entry()  local
496 	bool locked = !!(vma->vm_flags & VM_LOCKED);  in smaps_pte_entry()
500 		page = vm_normal_page(vma, addr, *pte);  in smaps_pte_entry()
521 		page = xa_load(&vma->vm_file->f_mapping->i_pages,  in smaps_pte_entry()
522 						linear_page_index(vma, addr));  in smaps_pte_entry()
539 	struct vm_area_struct *vma = walk->vma;  in smaps_pmd_entry()  local
540 	bool locked = !!(vma->vm_flags & VM_LOCKED);  in smaps_pmd_entry()
545 		page = follow_trans_huge_pmd(vma, addr, pmd, FOLL_DUMP);  in smaps_pmd_entry()
574 	struct vm_area_struct *vma = walk->vma;  in smaps_pte_range()  local
578 	ptl = pmd_trans_huge_lock(pmd, vma);  in smaps_pte_range()
592 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);  in smaps_pte_range()
601 static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)  in show_smap_vma_flags()  argument
671 		if (vma->vm_flags & (1UL << i)) {  in show_smap_vma_flags()
686 	struct vm_area_struct *vma = walk->vma;  in smaps_hugetlb_range()  local
690 		page = vm_normal_page(vma, addr, *pte);  in smaps_hugetlb_range()
701 			mss->shared_hugetlb += huge_page_size(hstate_vma(vma));  in smaps_hugetlb_range()
703 			mss->private_hugetlb += huge_page_size(hstate_vma(vma));  in smaps_hugetlb_range()
728 static void smap_gather_stats(struct vm_area_struct *vma,  in smap_gather_stats()  argument
734 	if (start >= vma->vm_end)  in smap_gather_stats()
740 	if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {  in smap_gather_stats()
751 		unsigned long shmem_swapped = shmem_swap_usage(vma);  in smap_gather_stats()
753 		if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||  in smap_gather_stats()
754 					!(vma->vm_flags & VM_WRITE))) {  in smap_gather_stats()
764 		walk_page_vma(vma, ops, mss);  in smap_gather_stats()
766 		walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss);  in smap_gather_stats()
813 	struct vm_area_struct *vma = v;  in show_smap()  local
818 	smap_gather_stats(vma, &mss, 0);  in show_smap()
820 	show_map_vma(m, vma);  in show_smap()
822 	SEQ_PUT_DEC("Size:           ", vma->vm_end - vma->vm_start);  in show_smap()
823 	SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma));  in show_smap()
824 	SEQ_PUT_DEC(" kB\nMMUPageSize:    ", vma_mmu_pagesize(vma));  in show_smap()
830 		   transparent_hugepage_active(vma));  in show_smap()
833 		seq_printf(m, "ProtectionKey:  %8u\n", vma_pkey(vma));  in show_smap()
834 	show_smap_vma_flags(m, vma);  in show_smap()
844 	struct vm_area_struct *vma;  in show_smaps_rollup()  local
866 	for (vma = priv->mm->mmap; vma;) {  in show_smaps_rollup()
867 		smap_gather_stats(vma, &mss, 0);  in show_smaps_rollup()
868 		last_vma_end = vma->vm_end;  in show_smaps_rollup()
918 			vma = find_vma(mm, last_vma_end - 1);  in show_smaps_rollup()
920 			if (!vma)  in show_smaps_rollup()
924 			if (vma->vm_start >= last_vma_end)  in show_smaps_rollup()
928 			if (vma->vm_end > last_vma_end)  in show_smaps_rollup()
929 				smap_gather_stats(vma, &mss, last_vma_end);  in show_smaps_rollup()
932 		vma = vma->vm_next;  in show_smaps_rollup()
1037 static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte)  in pte_is_pinned()  argument
1043 	if (!is_cow_mapping(vma->vm_flags))  in pte_is_pinned()
1045 	if (likely(!test_bit(MMF_HAS_PINNED, &vma->vm_mm->flags)))  in pte_is_pinned()
1047 	page = vm_normal_page(vma, addr, pte);  in pte_is_pinned()
1053 static inline void clear_soft_dirty(struct vm_area_struct *vma,  in clear_soft_dirty()  argument
1067 		if (pte_is_pinned(vma, addr, ptent))  in clear_soft_dirty()
1069 		old_pte = ptep_modify_prot_start(vma, addr, pte);  in clear_soft_dirty()
1072 		ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent);  in clear_soft_dirty()
1075 		set_pte_at(vma->vm_mm, addr, pte, ptent);  in clear_soft_dirty()
1079 static inline void clear_soft_dirty(struct vm_area_struct *vma,  in clear_soft_dirty()  argument
1086 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,  in clear_soft_dirty_pmd()  argument
1093 		old = pmdp_invalidate(vma, addr, pmdp);  in clear_soft_dirty_pmd()
1102 		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);  in clear_soft_dirty_pmd()
1105 		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);  in clear_soft_dirty_pmd()
1109 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,  in clear_soft_dirty_pmd()  argument
1119 	struct vm_area_struct *vma = walk->vma;  in clear_refs_pte_range()  local
1124 	ptl = pmd_trans_huge_lock(pmd, vma);  in clear_refs_pte_range()
1127 			clear_soft_dirty_pmd(vma, addr, pmd);  in clear_refs_pte_range()
1137 		pmdp_test_and_clear_young(vma, addr, pmd);  in clear_refs_pte_range()
1148 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);  in clear_refs_pte_range()
1153 			clear_soft_dirty(vma, addr, pte);  in clear_refs_pte_range()
1160 		page = vm_normal_page(vma, addr, ptent);  in clear_refs_pte_range()
1165 		ptep_test_and_clear_young(vma, addr, pte);  in clear_refs_pte_range()
1178 	struct vm_area_struct *vma = walk->vma;  in clear_refs_test_walk()  local
1180 	if (vma->vm_flags & VM_PFNMAP)  in clear_refs_test_walk()
1189 	if (cp->type == CLEAR_REFS_ANON && vma->vm_file)  in clear_refs_test_walk()
1191 	if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)  in clear_refs_test_walk()
1207 	struct vm_area_struct *vma;  in clear_refs_write()  local
1248 			for (vma = mm->mmap; vma; vma = vma->vm_next) {  in clear_refs_write()
1249 				if (!(vma->vm_flags & VM_SOFTDIRTY))  in clear_refs_write()
1251 				vma->vm_flags &= ~VM_SOFTDIRTY;  in clear_refs_write()
1252 				vma_set_page_prot(vma);  in clear_refs_write()
1329 		struct vm_area_struct *vma = find_vma(walk->mm, addr);  in pagemap_pte_hole()  local
1334 		if (vma)  in pagemap_pte_hole()
1335 			hole_end = min(end, vma->vm_start);  in pagemap_pte_hole()
1345 		if (!vma)  in pagemap_pte_hole()
1349 		if (vma->vm_flags & VM_SOFTDIRTY)  in pagemap_pte_hole()
1351 		for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {  in pagemap_pte_hole()
1362 		struct vm_area_struct *vma, unsigned long addr, pte_t pte)  in pte_to_pagemap_entry()  argument
1371 		page = vm_normal_page(vma, addr, pte);  in pte_to_pagemap_entry()
1395 	if (vma->vm_flags & VM_SOFTDIRTY)  in pte_to_pagemap_entry()
1404 	struct vm_area_struct *vma = walk->vma;  in pagemap_pmd_range()  local
1411 	ptl = pmd_trans_huge_lock(pmdp, vma);  in pagemap_pmd_range()
1417 		if (vma->vm_flags & VM_SOFTDIRTY)  in pagemap_pmd_range()
1485 		pme = pte_to_pagemap_entry(pm, vma, addr, *pte);  in pagemap_pmd_range()
1504 	struct vm_area_struct *vma = walk->vma;  in pagemap_hugetlb_range()  local
1509 	if (vma->vm_flags & VM_SOFTDIRTY)  in pagemap_hugetlb_range()
1740 static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,  in can_gather_numa_stats()  argument
1749 	page = vm_normal_page(vma, addr, pte);  in can_gather_numa_stats()
1765 					      struct vm_area_struct *vma,  in can_gather_numa_stats_pmd()  argument
1774 	page = vm_normal_page_pmd(vma, addr, pmd);  in can_gather_numa_stats_pmd()
1793 	struct vm_area_struct *vma = walk->vma;  in gather_pte_stats()  local
1799 	ptl = pmd_trans_huge_lock(pmd, vma);  in gather_pte_stats()
1803 		page = can_gather_numa_stats_pmd(*pmd, vma, addr);  in gather_pte_stats()
1816 		struct page *page = can_gather_numa_stats(*pte, vma, addr);  in gather_pte_stats()
1866 	struct vm_area_struct *vma = v;  in show_numa_map()  local
1868 	struct file *file = vma->vm_file;  in show_numa_map()
1869 	struct mm_struct *mm = vma->vm_mm;  in show_numa_map()
1880 	pol = __get_vma_policy(vma, vma->vm_start);  in show_numa_map()
1888 	seq_printf(m, "%08lx %s", vma->vm_start, buffer);  in show_numa_map()
1893 	} else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {  in show_numa_map()
1895 	} else if (is_stack(vma)) {  in show_numa_map()
1899 	if (is_vm_hugetlb_page(vma))  in show_numa_map()
1903 	walk_page_vma(vma, &show_numa_ops, md);  in show_numa_map()
1923 	if (md->active < md->pages && !is_vm_hugetlb_page(vma))  in show_numa_map()
1933 	seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);  in show_numa_map()