Lines Matching refs:pvmw
808 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in folio_referenced_one()
811 while (page_vma_mapped_walk(&pvmw)) { in folio_referenced_one()
812 address = pvmw.address; in folio_referenced_one()
815 (!folio_test_large(folio) || !pvmw.pte)) { in folio_referenced_one()
817 mlock_vma_folio(folio, vma, !pvmw.pte); in folio_referenced_one()
818 page_vma_mapped_walk_done(&pvmw); in folio_referenced_one()
823 if (pvmw.pte) { in folio_referenced_one()
825 pte_young(ptep_get(pvmw.pte))) { in folio_referenced_one()
826 lru_gen_look_around(&pvmw); in folio_referenced_one()
831 pvmw.pte)) in folio_referenced_one()
835 pvmw.pmd)) in folio_referenced_one()
935 static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw) in page_vma_mkclean_one() argument
938 struct vm_area_struct *vma = pvmw->vma; in page_vma_mkclean_one()
940 unsigned long address = pvmw->address; in page_vma_mkclean_one()
947 vma->vm_mm, address, vma_address_end(pvmw)); in page_vma_mkclean_one()
950 while (page_vma_mapped_walk(pvmw)) { in page_vma_mkclean_one()
953 address = pvmw->address; in page_vma_mkclean_one()
954 if (pvmw->pte) { in page_vma_mkclean_one()
955 pte_t *pte = pvmw->pte; in page_vma_mkclean_one()
969 pmd_t *pmd = pvmw->pmd; in page_vma_mkclean_one()
1000 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC); in page_mkclean_one()
1003 *cleaned += page_vma_mkclean_one(&pvmw); in page_mkclean_one()
1056 struct page_vma_mapped_walk pvmw = { in pfn_mkclean_range() local
1067 pvmw.address = vma_pgoff_address(pgoff, nr_pages, vma); in pfn_mkclean_range()
1068 VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma); in pfn_mkclean_range()
1070 return page_vma_mkclean_one(&pvmw); in pfn_mkclean_range()
1476 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in try_to_unmap_one()
1492 pvmw.flags = PVMW_SYNC; in try_to_unmap_one()
1505 range.end = vma_address_end(&pvmw); in try_to_unmap_one()
1521 while (page_vma_mapped_walk(&pvmw)) { in try_to_unmap_one()
1523 VM_BUG_ON_FOLIO(!pvmw.pte, folio); in try_to_unmap_one()
1532 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1537 pfn = pte_pfn(ptep_get(pvmw.pte)); in try_to_unmap_one()
1539 address = pvmw.address; in try_to_unmap_one()
1573 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1577 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { in try_to_unmap_one()
1591 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1596 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); in try_to_unmap_one()
1609 pteval = ptep_get_and_clear(mm, address, pvmw.pte); in try_to_unmap_one()
1613 pteval = ptep_clear_flush(vma, address, pvmw.pte); in try_to_unmap_one()
1622 pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval); in try_to_unmap_one()
1635 set_huge_pte_at(mm, address, pvmw.pte, pteval, in try_to_unmap_one()
1639 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1665 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1703 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1706 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1711 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1713 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1718 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1720 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1728 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_unmap_one()
1730 page_vma_mapped_walk_done(&pvmw); in try_to_unmap_one()
1748 set_pte_at(mm, address, pvmw.pte, swp_pte); in try_to_unmap_one()
1821 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in try_to_migrate_one()
1837 pvmw.flags = PVMW_SYNC; in try_to_migrate_one()
1854 range.end = vma_address_end(&pvmw); in try_to_migrate_one()
1870 while (page_vma_mapped_walk(&pvmw)) { in try_to_migrate_one()
1873 if (!pvmw.pte) { in try_to_migrate_one()
1875 pmd_pfn(*pvmw.pmd) - folio_pfn(folio)); in try_to_migrate_one()
1879 if (set_pmd_migration_entry(&pvmw, subpage)) { in try_to_migrate_one()
1881 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
1889 VM_BUG_ON_FOLIO(!pvmw.pte, folio); in try_to_migrate_one()
1891 pfn = pte_pfn(ptep_get(pvmw.pte)); in try_to_migrate_one()
1909 address = pvmw.address; in try_to_migrate_one()
1938 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
1942 if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { in try_to_migrate_one()
1957 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
1963 pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); in try_to_migrate_one()
1976 pteval = ptep_get_and_clear(mm, address, pvmw.pte); in try_to_migrate_one()
1980 pteval = ptep_clear_flush(vma, address, pvmw.pte); in try_to_migrate_one()
2021 set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte); in try_to_migrate_one()
2022 trace_set_migration_pte(pvmw.address, pte_val(swp_pte), in try_to_migrate_one()
2032 set_huge_pte_at(mm, address, pvmw.pte, pteval, in try_to_migrate_one()
2036 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2057 set_huge_pte_at(mm, address, pvmw.pte, in try_to_migrate_one()
2060 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2062 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
2072 set_huge_pte_at(mm, address, pvmw.pte, in try_to_migrate_one()
2075 set_pte_at(mm, address, pvmw.pte, pteval); in try_to_migrate_one()
2077 page_vma_mapped_walk_done(&pvmw); in try_to_migrate_one()
2105 set_huge_pte_at(mm, address, pvmw.pte, swp_pte, in try_to_migrate_one()
2108 set_pte_at(mm, address, pvmw.pte, swp_pte); in try_to_migrate_one()
2186 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); in page_make_device_exclusive_one()
2202 while (page_vma_mapped_walk(&pvmw)) { in page_make_device_exclusive_one()
2204 VM_BUG_ON_FOLIO(!pvmw.pte, folio); in page_make_device_exclusive_one()
2206 ptent = ptep_get(pvmw.pte); in page_make_device_exclusive_one()
2209 page_vma_mapped_walk_done(&pvmw); in page_make_device_exclusive_one()
2215 address = pvmw.address; in page_make_device_exclusive_one()
2219 pteval = ptep_clear_flush(vma, address, pvmw.pte); in page_make_device_exclusive_one()
2250 set_pte_at(mm, address, pvmw.pte, swp_pte); in page_make_device_exclusive_one()