Lines Matching refs:pvmw
810 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in folio_referenced_one()
813 while (page_vma_mapped_walk(&pvmw)) { in folio_referenced_one()
814 address = pvmw.address; in folio_referenced_one()
817 (!folio_test_large(folio) || !pvmw.pte)) { in folio_referenced_one()
819 mlock_vma_folio(folio, vma, !pvmw.pte); in folio_referenced_one()
820 page_vma_mapped_walk_done(&pvmw); in folio_referenced_one()
825 if (pvmw.pte) { in folio_referenced_one()
826 if (lru_gen_enabled() && pte_young(*pvmw.pte) && in folio_referenced_one()
828 lru_gen_look_around(&pvmw); in folio_referenced_one()
833 pvmw.pte)) { in folio_referenced_one()
847 pvmw.pmd)) in folio_referenced_one()
942 static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw) in page_vma_mkclean_one() argument
945 struct vm_area_struct *vma = pvmw->vma; in page_vma_mkclean_one()
947 unsigned long address = pvmw->address; in page_vma_mkclean_one()
955 vma_address_end(pvmw)); in page_vma_mkclean_one()
958 while (page_vma_mapped_walk(pvmw)) { in page_vma_mkclean_one()
961 address = pvmw->address; in page_vma_mkclean_one()
962 if (pvmw->pte) { in page_vma_mkclean_one()
964 pte_t *pte = pvmw->pte; in page_vma_mkclean_one()
977 pmd_t *pmd = pvmw->pmd; in page_vma_mkclean_one()
1015 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC); in page_mkclean_one()
1018 *cleaned += page_vma_mkclean_one(&pvmw); in page_mkclean_one()
1071 struct page_vma_mapped_walk pvmw = { in pfn_mkclean_range() local
1082 pvmw.address = vma_pgoff_address(pgoff, nr_pages, vma); in pfn_mkclean_range()
1083 VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma); in pfn_mkclean_range()
1085 return page_vma_mkclean_one(&pvmw); in pfn_mkclean_range()
1474 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in try_to_unmap_one()
1488 pvmw.flags = PVMW_SYNC; in try_to_unmap_one()
1501 range.end = vma_address_end(&pvmw); in try_to_unmap_one()
1514 while (page_vma_mapped_walk(&pvmw)) { in try_to_unmap_one()
1516 VM_BUG_ON_FOLIO(!pvmw.pte, folio); in try_to_unmap_one()
1525 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1531 pte_pfn(*pvmw.pte) - folio_pfn(folio)); in try_to_unmap_one()
1532 address = pvmw.address; in try_to_unmap_one()
1566 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1570 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { in try_to_unmap_one()
1586 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1591 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); in try_to_unmap_one()
1593 flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); in try_to_unmap_one()
1604 pteval = ptep_get_and_clear(mm, address, pvmw.pte); in try_to_unmap_one()
1608 pteval = ptep_clear_flush(vma, address, pvmw.pte); in try_to_unmap_one()
1617 pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval); in try_to_unmap_one()
1630 set_huge_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1633 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1665 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1706 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1709 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1714 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1716 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1721 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1723 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1731 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1733 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1762 set_pte_at(mm, address, pvmw.pte, swp_pte); in try_to_unmap_one()
1845 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in try_to_migrate_one()
1859 pvmw.flags = PVMW_SYNC; in try_to_migrate_one()
1876 range.end = vma_address_end(&pvmw); in try_to_migrate_one()
1889 while (page_vma_mapped_walk(&pvmw)) { in try_to_migrate_one()
1892 if (!pvmw.pte) { in try_to_migrate_one()
1894 pmd_pfn(*pvmw.pmd) - folio_pfn(folio)); in try_to_migrate_one()
1898 if (set_pmd_migration_entry(&pvmw, subpage)) { in try_to_migrate_one()
1900 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
1908 VM_BUG_ON_FOLIO(!pvmw.pte, folio); in try_to_migrate_one()
1925 pte_pfn(*pvmw.pte) - folio_pfn(folio)); in try_to_migrate_one()
1927 address = pvmw.address; in try_to_migrate_one()
1956 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
1960 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { in try_to_migrate_one()
1977 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
1983 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); in try_to_migrate_one()
1985 flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); in try_to_migrate_one()
1987 pteval = ptep_clear_flush(vma, address, pvmw.pte); in try_to_migrate_one()
2027 set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte); in try_to_migrate_one()
2028 trace_set_migration_pte(pvmw.address, pte_val(swp_pte), in try_to_migrate_one()
2038 set_huge_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2041 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2065 set_huge_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2067 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2069 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
2079 set_huge_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2081 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2083 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
2111 set_huge_pte_at(mm, address, pvmw.pte, swp_pte); in try_to_migrate_one()
2113 set_pte_at(mm, address, pvmw.pte, swp_pte); in try_to_migrate_one()
2198 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in page_make_device_exclusive_one()
2213 while (page_vma_mapped_walk(&pvmw)) { in page_make_device_exclusive_one()
2215 VM_BUG_ON_FOLIO(!pvmw.pte, folio); in page_make_device_exclusive_one()
2217 if (!pte_present(*pvmw.pte)) { in page_make_device_exclusive_one()
2219 page_vma_mapped_walk_done(&pvmw); in page_make_device_exclusive_one()
2224 pte_pfn(*pvmw.pte) - folio_pfn(folio)); in page_make_device_exclusive_one()
2225 address = pvmw.address; in page_make_device_exclusive_one()
2228 flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); in page_make_device_exclusive_one()
2229 pteval = ptep_clear_flush(vma, address, pvmw.pte); in page_make_device_exclusive_one()
2260 set_pte_at(mm, address, pvmw.pte, swp_pte); in page_make_device_exclusive_one()