Lines Matching +full:tlb +full:- +full:split
1 // SPDX-License-Identifier: GPL-2.0-only
38 #include <asm/tlb.h>
69 return transhuge_vma_enabled(vma, vma->vm_flags) && vma->vm_file && in file_thp_enabled()
70 !inode_is_open_for_write(vma->vm_file->f_inode) && in file_thp_enabled()
71 (vma->vm_flags & VM_EXEC); in file_thp_enabled()
77 unsigned long addr = (vma->vm_end & HPAGE_PMD_MASK) - HPAGE_PMD_SIZE; in transparent_hugepage_active()
130 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) in mm_get_huge_zero_page()
136 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) in mm_get_huge_zero_page()
144 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) in mm_put_huge_zero_page()
208 ret = -EINVAL; in enabled_store()
240 return -EINVAL; in single_hugepage_flag_store()
303 return -EINVAL; in defrag_store()
355 return -ENOMEM; in hugepage_init_sysfs()
407 return -EINVAL; in hugepage_init()
415 * we use page->mapping and page->index in second tail page in hugepage_init()
437 * where the extra memory used could hurt more than TLB overhead in hugepage_init()
440 if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) { in hugepage_init()
496 if (likely(vma->vm_flags & VM_WRITE)) in maybe_pmd_mkwrite()
508 return &memcg->deferred_split_queue; in get_deferred_split_queue()
510 return &pgdat->deferred_split_queue; in get_deferred_split_queue()
517 return &pgdat->deferred_split_queue; in get_deferred_split_queue()
524 * we use page->mapping and page->indexlru in second tail page in prep_transhuge_page()
551 if (off_end <= off_align || (off_end - off_align) < size) in __thp_get_unmapped_area()
558 ret = current->mm->get_unmapped_area(filp, addr, len_pad, in __thp_get_unmapped_area()
575 ret += (off - ret) & (size - 1); in __thp_get_unmapped_area()
585 if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD)) in thp_get_unmapped_area()
592 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); in thp_get_unmapped_area()
599 struct vm_area_struct *vma = vmf->vma; in __do_huge_pmd_anonymous_page()
601 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in __do_huge_pmd_anonymous_page()
606 if (mem_cgroup_charge(page, vma->vm_mm, gfp)) { in __do_huge_pmd_anonymous_page()
614 pgtable = pte_alloc_one(vma->vm_mm); in __do_huge_pmd_anonymous_page()
620 clear_huge_page(page, vmf->address, HPAGE_PMD_NR); in __do_huge_pmd_anonymous_page()
628 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in __do_huge_pmd_anonymous_page()
629 if (unlikely(!pmd_none(*vmf->pmd))) { in __do_huge_pmd_anonymous_page()
634 ret = check_stable_address_space(vma->vm_mm); in __do_huge_pmd_anonymous_page()
640 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
642 pte_free(vma->vm_mm, pgtable); in __do_huge_pmd_anonymous_page()
648 entry = mk_huge_pmd(page, vma->vm_page_prot); in __do_huge_pmd_anonymous_page()
652 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); in __do_huge_pmd_anonymous_page()
653 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); in __do_huge_pmd_anonymous_page()
654 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in __do_huge_pmd_anonymous_page()
655 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); in __do_huge_pmd_anonymous_page()
656 mm_inc_nr_ptes(vma->vm_mm); in __do_huge_pmd_anonymous_page()
657 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
659 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); in __do_huge_pmd_anonymous_page()
664 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
667 pte_free(vma->vm_mm, pgtable); in __do_huge_pmd_anonymous_page()
684 const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE); in vma_thp_gfp_mask()
716 entry = mk_pmd(zero_page, vma->vm_page_prot); in set_huge_zero_page()
726 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_anonymous_page()
729 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_anonymous_page()
735 if (unlikely(khugepaged_enter(vma, vma->vm_flags))) in do_huge_pmd_anonymous_page()
737 if (!(vmf->flags & FAULT_FLAG_WRITE) && in do_huge_pmd_anonymous_page()
738 !mm_forbids_zeropage(vma->vm_mm) && in do_huge_pmd_anonymous_page()
743 pgtable = pte_alloc_one(vma->vm_mm); in do_huge_pmd_anonymous_page()
746 zero_page = mm_get_huge_zero_page(vma->vm_mm); in do_huge_pmd_anonymous_page()
748 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
752 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_anonymous_page()
754 if (pmd_none(*vmf->pmd)) { in do_huge_pmd_anonymous_page()
755 ret = check_stable_address_space(vma->vm_mm); in do_huge_pmd_anonymous_page()
757 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
758 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
760 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
761 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
765 set_huge_zero_page(pgtable, vma->vm_mm, vma, in do_huge_pmd_anonymous_page()
766 haddr, vmf->pmd, zero_page); in do_huge_pmd_anonymous_page()
767 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_anonymous_page()
768 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
771 spin_unlock(vmf->ptl); in do_huge_pmd_anonymous_page()
772 pte_free(vma->vm_mm, pgtable); in do_huge_pmd_anonymous_page()
790 struct mm_struct *mm = vma->vm_mm; in insert_pfn_pmd()
834 * vmf_insert_pfn_pmd_prot - insert a pmd size pfn
842 * @pgprot != @vmf->vma->vm_page_prot.
849 unsigned long addr = vmf->address & PMD_MASK; in vmf_insert_pfn_pmd_prot()
850 struct vm_area_struct *vma = vmf->vma; in vmf_insert_pfn_pmd_prot()
858 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && in vmf_insert_pfn_pmd_prot()
860 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == in vmf_insert_pfn_pmd_prot()
862 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); in vmf_insert_pfn_pmd_prot()
864 if (addr < vma->vm_start || addr >= vma->vm_end) in vmf_insert_pfn_pmd_prot()
868 pgtable = pte_alloc_one(vma->vm_mm); in vmf_insert_pfn_pmd_prot()
875 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable); in vmf_insert_pfn_pmd_prot()
883 if (likely(vma->vm_flags & VM_WRITE)) in maybe_pud_mkwrite()
891 struct mm_struct *mm = vma->vm_mm; in insert_pfn_pud()
925 * vmf_insert_pfn_pud_prot - insert a pud size pfn
933 * @pgprot != @vmf->vma->vm_page_prot.
940 unsigned long addr = vmf->address & PUD_MASK; in vmf_insert_pfn_pud_prot()
941 struct vm_area_struct *vma = vmf->vma; in vmf_insert_pfn_pud_prot()
948 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && in vmf_insert_pfn_pud_prot()
950 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == in vmf_insert_pfn_pud_prot()
952 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); in vmf_insert_pfn_pud_prot()
954 if (addr < vma->vm_start || addr >= vma->vm_end) in vmf_insert_pfn_pud_prot()
959 insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write); in vmf_insert_pfn_pud_prot()
982 struct mm_struct *mm = vma->vm_mm; in follow_devmap_pmd()
988 * When we COW a devmap PMD entry, we split it into PTEs, so we should in follow_devmap_pmd()
1014 return ERR_PTR(-EEXIST); in follow_devmap_pmd()
1019 return ERR_PTR(-EFAULT); in follow_devmap_pmd()
1022 page = ERR_PTR(-ENOMEM); in follow_devmap_pmd()
1035 int ret = -ENOMEM; in copy_huge_pmd()
1037 /* Skip if can be re-fill on fault */ in copy_huge_pmd()
1049 ret = -EAGAIN; in copy_huge_pmd()
1084 * under splitting since we don't split the page itself, only pmd to in copy_huge_pmd()
1101 * If this page is a potentially pinned page, split and retry the fault in copy_huge_pmd()
1105 * random page during the coming copy-on-write. in copy_huge_pmd()
1112 return -EAGAIN; in copy_huge_pmd()
1153 struct mm_struct *mm = vma->vm_mm; in follow_devmap_pud()
1181 return ERR_PTR(-EEXIST); in follow_devmap_pud()
1186 return ERR_PTR(-EFAULT); in follow_devmap_pud()
1189 page = ERR_PTR(-ENOMEM); in follow_devmap_pud()
1206 ret = -EAGAIN; in copy_huge_pud()
1213 * under splitting since we don't split the page itself, only pud to in copy_huge_pud()
1225 return -EAGAIN; in copy_huge_pud()
1243 bool write = vmf->flags & FAULT_FLAG_WRITE; in huge_pud_set_accessed()
1245 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); in huge_pud_set_accessed()
1246 if (unlikely(!pud_same(*vmf->pud, orig_pud))) in huge_pud_set_accessed()
1252 haddr = vmf->address & HPAGE_PUD_MASK; in huge_pud_set_accessed()
1253 if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write)) in huge_pud_set_accessed()
1254 update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud); in huge_pud_set_accessed()
1257 spin_unlock(vmf->ptl); in huge_pud_set_accessed()
1265 bool write = vmf->flags & FAULT_FLAG_WRITE; in huge_pmd_set_accessed()
1266 pmd_t orig_pmd = vmf->orig_pmd; in huge_pmd_set_accessed()
1268 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); in huge_pmd_set_accessed()
1269 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) in huge_pmd_set_accessed()
1275 haddr = vmf->address & HPAGE_PMD_MASK; in huge_pmd_set_accessed()
1276 if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write)) in huge_pmd_set_accessed()
1277 update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd); in huge_pmd_set_accessed()
1280 spin_unlock(vmf->ptl); in huge_pmd_set_accessed()
1285 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_wp_page()
1287 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_wp_page()
1288 pmd_t orig_pmd = vmf->orig_pmd; in do_huge_pmd_wp_page()
1290 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); in do_huge_pmd_wp_page()
1291 VM_BUG_ON_VMA(!vma->anon_vma, vma); in do_huge_pmd_wp_page()
1296 spin_lock(vmf->ptl); in do_huge_pmd_wp_page()
1298 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1299 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1309 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1311 spin_lock(vmf->ptl); in do_huge_pmd_wp_page()
1312 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { in do_huge_pmd_wp_page()
1313 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1329 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) in do_huge_pmd_wp_page()
1330 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_wp_page()
1332 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1337 spin_unlock(vmf->ptl); in do_huge_pmd_wp_page()
1339 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL); in do_huge_pmd_wp_page()
1358 struct mm_struct *mm = vma->vm_mm; in follow_trans_huge_pmd()
1368 return ERR_PTR(-EFAULT); in follow_trans_huge_pmd()
1378 return ERR_PTR(-ENOMEM); in follow_trans_huge_pmd()
1383 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { in follow_trans_huge_pmd()
1385 * We don't mlock() pte-mapped THPs. This way we can avoid in follow_trans_huge_pmd()
1386 * leaking mlocked pages into non-VM_LOCKED VMAs. in follow_trans_huge_pmd()
1391 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for in follow_trans_huge_pmd()
1395 * mlocking read-only mapping shared over fork(). We skip in follow_trans_huge_pmd()
1407 if (PageDoubleMap(page) || !page->mapping) in follow_trans_huge_pmd()
1411 if (page->mapping && !PageDoubleMap(page)) in follow_trans_huge_pmd()
1426 struct vm_area_struct *vma = vmf->vma; in do_huge_pmd_numa_page()
1427 pmd_t oldpmd = vmf->orig_pmd; in do_huge_pmd_numa_page()
1430 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in do_huge_pmd_numa_page()
1432 int target_nid, last_cpupid = -1; in do_huge_pmd_numa_page()
1437 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_numa_page()
1438 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) { in do_huge_pmd_numa_page()
1439 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1443 pmd = pmd_modify(oldpmd, vma->vm_page_prot); in do_huge_pmd_numa_page()
1462 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1470 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in do_huge_pmd_numa_page()
1471 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) { in do_huge_pmd_numa_page()
1472 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1487 pmd = pmd_modify(oldpmd, vma->vm_page_prot); in do_huge_pmd_numa_page()
1491 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); in do_huge_pmd_numa_page()
1492 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); in do_huge_pmd_numa_page()
1493 spin_unlock(vmf->ptl); in do_huge_pmd_numa_page()
1501 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, in madvise_free_huge_pmd() argument
1507 struct mm_struct *mm = tlb->mm; in madvise_free_huge_pmd()
1510 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in madvise_free_huge_pmd()
1538 * If user want to discard part-pages of THP, split it so MADV_FREE in madvise_free_huge_pmd()
1541 if (next - addr != HPAGE_PMD_SIZE) { in madvise_free_huge_pmd()
1560 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in madvise_free_huge_pmd()
1580 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, in zap_huge_pmd() argument
1586 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in zap_huge_pmd()
1598 tlb->fullmm); in zap_huge_pmd()
1599 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in zap_huge_pmd()
1602 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1605 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1627 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1628 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); in zap_huge_pmd()
1631 zap_deposited_table(tlb->mm, pmd); in zap_huge_pmd()
1632 add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR); in zap_huge_pmd()
1637 tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE); in zap_huge_pmd()
1648 * With split pmd lock we also need to move preallocated in pmd_move_must_withdraw()
1673 struct mm_struct *mm = vma->vm_mm; in move_huge_pmd()
1718 * - 0 if PMD could not be locked
1719 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary
1721 * - HPAGE_PMD_NR if protections changed and TLB flush necessary
1726 struct mm_struct *mm = vma->vm_mm; in change_huge_pmd()
1770 * Avoid trapping faults against the zero page. The read-only in change_huge_pmd()
1771 * data is likely to be read-cached on the local CPU and in change_huge_pmd()
1793 * // pmd is re-established in change_huge_pmd()
1834 ptl = pmd_lock(vma->vm_mm, pmd); in __pmd_trans_huge_lock()
1852 ptl = pud_lock(vma->vm_mm, pud); in __pud_trans_huge_lock()
1860 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, in zap_huge_pud() argument
1874 pudp_huge_get_and_clear_full(tlb->mm, addr, pud, tlb->fullmm); in zap_huge_pud()
1875 tlb_remove_pud_tlb_entry(tlb, pud, addr); in zap_huge_pud()
1890 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); in __split_huge_pud_locked()
1891 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); in __split_huge_pud_locked()
1905 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, in __split_huge_pud()
1909 ptl = pud_lock(vma->vm_mm, pud); in __split_huge_pud()
1917 * No need to double call mmu_notifier->invalidate_range() callback as in __split_huge_pud()
1927 struct mm_struct *mm = vma->vm_mm; in __split_huge_zero_page_pmd()
1947 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot); in __split_huge_zero_page_pmd()
1961 struct mm_struct *mm = vma->vm_mm; in __split_huge_pmd_locked()
1970 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); in __split_huge_pmd_locked()
1971 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); in __split_huge_pmd_locked()
2001 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR); in __split_huge_pmd_locked()
2020 * whole access to the hugepage during the split (which happens in in __split_huge_pmd_locked()
2021 * place). If we overwrite the pmd with the not-huge version pointing in __split_huge_pmd_locked()
2023 * free), userland could trigger a small page size TLB miss on the in __split_huge_pmd_locked()
2024 * small sized TLB while the hugepage TLB entry is still established in in __split_huge_pmd_locked()
2025 * the huge TLB. Some CPU doesn't like that. in __split_huge_pmd_locked()
2029 * loaded in the two TLB is identical (which should be the case here). in __split_huge_pmd_locked()
2030 * But it is generally safer to never allow small and huge TLB entries in __split_huge_pmd_locked()
2034 * must remain set at all times on the pmd until the split is complete in __split_huge_pmd_locked()
2035 * for this pmd), then we flush the SMP TLB and finally we write the in __split_huge_pmd_locked()
2036 * non-huge version of the pmd entry with pmd_populate. in __split_huge_pmd_locked()
2060 page_ref_add(page, HPAGE_PMD_NR - 1); in __split_huge_pmd_locked()
2090 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot)); in __split_huge_pmd_locked()
2112 * false-negative page_mapped(). in __split_huge_pmd_locked()
2121 if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { in __split_huge_pmd_locked()
2124 -HPAGE_PMD_NR); in __split_huge_pmd_locked()
2153 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, in __split_huge_pmd()
2157 ptl = pmd_lock(vma->vm_mm, pmd); in __split_huge_pmd()
2209 * No need to double call mmu_notifier->invalidate_range() callback. in __split_huge_pmd()
2216 * 3) Split a huge pmd into pte pointing to the same page. No need in __split_huge_pmd()
2217 * to invalidate secondary tlb entry they are all still valid. in __split_huge_pmd()
2219 * to call mmu_notifier->invalidate_range() in __split_huge_pmd()
2232 pgd = pgd_offset(vma->vm_mm, address); in split_huge_pmd_address()
2253 * contain an hugepage: check if we need to split an huge pmd. in split_huge_pmd_if_needed()
2266 /* Check if we need to split start first. */ in vma_adjust_trans_huge()
2269 /* Check if we need to split end next. */ in vma_adjust_trans_huge()
2273 * If we're also updating the vma->vm_next->vm_start, in vma_adjust_trans_huge()
2274 * check if we need to split it. in vma_adjust_trans_huge()
2277 struct vm_area_struct *next = vma->vm_next; in vma_adjust_trans_huge()
2278 unsigned long nstart = next->vm_start; in vma_adjust_trans_huge()
2325 lockdep_assert_held(&lruvec->lru_lock); in lru_add_page_tail()
2331 list_add_tail(&tail->lru, list); in lru_add_page_tail()
2336 list_add_tail(&tail->lru, &head->lru); in lru_add_page_tail()
2345 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail); in __split_huge_page_tail()
2353 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; in __split_huge_page_tail()
2354 page_tail->flags |= (head->flags & in __split_huge_page_tail()
2369 /* ->mapping in first tail page is compound_mapcount */ in __split_huge_page_tail()
2370 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, in __split_huge_page_tail()
2372 page_tail->mapping = head->mapping; in __split_huge_page_tail()
2373 page_tail->index = head->index + tail; in __split_huge_page_tail()
2375 /* Page flags must be visible before we make the page non-compound. */ in __split_huge_page_tail()
2399 * pages to show after the currently processed elements - e.g. in __split_huge_page_tail()
2423 xa_lock(&swap_cache->i_pages); in __split_huge_page()
2431 for (i = nr - 1; i >= 1; i--) { in __split_huge_page()
2437 if (shmem_mapping(head->mapping)) in __split_huge_page()
2438 shmem_uncharge(head->mapping->host, 1); in __split_huge_page()
2441 __xa_store(&head->mapping->i_pages, head[i].index, in __split_huge_page()
2444 __xa_store(&swap_cache->i_pages, offset + i, in __split_huge_page()
2460 xa_unlock(&swap_cache->i_pages); in __split_huge_page()
2467 xa_unlock(&head->mapping->i_pages); in __split_huge_page()
2490 * of the tail pages after the split is complete. in __split_huge_page()
2503 return atomic_read(&page->_mapcount) + 1; in total_mapcount()
2514 return ret - compound * nr; in total_mapcount()
2516 ret -= nr; in total_mapcount()
2523 * accuracy is primarily needed to know if copy-on-write faults can
2524 * reuse the page and change the mapping to read-write instead of
2535 * anon_vma of the transparent hugepage can become the vma->anon_vma
2540 * page_trans_huge_mapcount() in the copy-on-write faults where we
2552 mapcount = atomic_read(&page->_mapcount) + 1; in page_trans_huge_mapcount()
2567 ret -= 1; in page_trans_huge_mapcount()
2568 _total_mapcount -= thp_nr_pages(page); in page_trans_huge_mapcount()
2578 /* Racy check whether the huge page can be split */
2590 return total_mapcount(page) == page_count(page) - extra_pins - 1; in can_split_huge_page()
2595 * subpage of huge page to split. Split doesn't change the position of @page.
2597 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
2608 * Returns 0 if the hugepage is split successfully.
2609 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
2626 return -EBUSY; in split_huge_page_to_list()
2634 * is taken to serialise against parallel split or collapse in split_huge_page_to_list()
2639 ret = -EBUSY; in split_huge_page_to_list()
2642 end = -1; in split_huge_page_to_list()
2646 mapping = head->mapping; in split_huge_page_to_list()
2650 ret = -EBUSY; in split_huge_page_to_list()
2659 * but on 32-bit, i_size_read() takes an irq-unsafe seqlock, in split_huge_page_to_list()
2664 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); in split_huge_page_to_list()
2666 end = shmem_fallocend(mapping->host, end); in split_huge_page_to_list()
2670 * Racy check if we can split the page, before unmap_page() will in split_huge_page_to_list()
2671 * split PMDs in split_huge_page_to_list()
2674 ret = -EBUSY; in split_huge_page_to_list()
2683 XA_STATE(xas, &mapping->i_pages, page_index(head)); in split_huge_page_to_list()
2689 xa_lock(&mapping->i_pages); in split_huge_page_to_list()
2694 /* Prevent deferred_split_scan() touching ->_refcount */ in split_huge_page_to_list()
2695 spin_lock(&ds_queue->split_queue_lock); in split_huge_page_to_list()
2698 ds_queue->split_queue_len--; in split_huge_page_to_list()
2701 spin_unlock(&ds_queue->split_queue_lock); in split_huge_page_to_list()
2707 -nr); in split_huge_page_to_list()
2710 -nr); in split_huge_page_to_list()
2718 spin_unlock(&ds_queue->split_queue_lock); in split_huge_page_to_list()
2721 xa_unlock(&mapping->i_pages); in split_huge_page_to_list()
2724 ret = -EBUSY; in split_huge_page_to_list()
2744 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in free_transhuge_page()
2746 ds_queue->split_queue_len--; in free_transhuge_page()
2749 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in free_transhuge_page()
2765 * this may cause a race condition to corrupt deferred split queue. in deferred_split_huge_page()
2776 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in deferred_split_huge_page()
2779 list_add_tail(page_deferred_list(page), &ds_queue->split_queue); in deferred_split_huge_page()
2780 ds_queue->split_queue_len++; in deferred_split_huge_page()
2787 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in deferred_split_huge_page()
2793 struct pglist_data *pgdata = NODE_DATA(sc->nid); in deferred_split_count()
2794 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; in deferred_split_count()
2797 if (sc->memcg) in deferred_split_count()
2798 ds_queue = &sc->memcg->deferred_split_queue; in deferred_split_count()
2800 return READ_ONCE(ds_queue->split_queue_len); in deferred_split_count()
2806 struct pglist_data *pgdata = NODE_DATA(sc->nid); in deferred_split_scan()
2807 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; in deferred_split_scan()
2811 int split = 0; in deferred_split_scan() local
2814 if (sc->memcg) in deferred_split_scan()
2815 ds_queue = &sc->memcg->deferred_split_queue; in deferred_split_scan()
2818 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2820 list_for_each_safe(pos, next, &ds_queue->split_queue) { in deferred_split_scan()
2828 ds_queue->split_queue_len--; in deferred_split_scan()
2830 if (!--sc->nr_to_scan) in deferred_split_scan()
2833 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2841 split++; in deferred_split_scan()
2847 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2848 list_splice_tail(&list, &ds_queue->split_queue); in deferred_split_scan()
2849 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); in deferred_split_scan()
2852 * Stop shrinker if we didn't split any page, but the queue is empty. in deferred_split_scan()
2855 if (!split && list_empty(&ds_queue->split_queue)) in deferred_split_scan()
2857 return split; in deferred_split_scan()
2874 unsigned long total = 0, split = 0; in split_huge_pages_all() local
2876 pr_debug("Split all THPs\n"); in split_huge_pages_all()
2879 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { in split_huge_pages_all()
2896 split++; in split_huge_pages_all()
2904 pr_debug("%lu of %lu THP split\n", split, total); in split_huge_pages_all()
2909 return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) || in vma_not_suitable_for_thp_split()
2919 unsigned long total = 0, split = 0; in split_huge_pages_pid() local
2930 ret = -ESRCH; in split_huge_pages_pid()
2941 ret = -EINVAL; in split_huge_pages_pid()
2945 pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx]\n", in split_huge_pages_pid()
2951 * table filled with PTE-mapped THPs, each of which is distinct. in split_huge_pages_pid()
2958 if (!vma || addr < vma->vm_start) in split_huge_pages_pid()
2963 addr = vma->vm_end; in split_huge_pages_pid()
2987 split++; in split_huge_pages_pid()
2997 pr_debug("%lu of %lu THP split\n", split, total); in split_huge_pages_pid()
3009 int ret = -EINVAL; in split_huge_pages_in_file()
3012 unsigned long total = 0, split = 0; in split_huge_pages_in_file() local
3022 pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx]\n", in split_huge_pages_in_file()
3025 mapping = candidate->f_mapping; in split_huge_pages_in_file()
3045 split++; in split_huge_pages_in_file()
3056 pr_debug("%lu of %lu file-backed THP split\n", split, total); in split_huge_pages_in_file()
3078 ret = -EFAULT; in split_huge_pages_write()
3084 input_buf[MAX_INPUT_BUF_SZ - 1] = '\0'; in split_huge_pages_write()
3097 ret = -EINVAL; in split_huge_pages_write()
3103 ret = -EINVAL; in split_huge_pages_write()
3119 ret = -EINVAL; in split_huge_pages_write()
3151 struct vm_area_struct *vma = pvmw->vma; in set_pmd_migration_entry()
3152 struct mm_struct *mm = vma->vm_mm; in set_pmd_migration_entry()
3153 unsigned long address = pvmw->address; in set_pmd_migration_entry()
3158 if (!(pvmw->pmd && !pvmw->pte)) in set_pmd_migration_entry()
3162 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); in set_pmd_migration_entry()
3172 set_pmd_at(mm, address, pvmw->pmd, pmdswp); in set_pmd_migration_entry()
3179 struct vm_area_struct *vma = pvmw->vma; in remove_migration_pmd()
3180 struct mm_struct *mm = vma->vm_mm; in remove_migration_pmd()
3181 unsigned long address = pvmw->address; in remove_migration_pmd()
3186 if (!(pvmw->pmd && !pvmw->pte)) in remove_migration_pmd()
3189 entry = pmd_to_swp_entry(*pvmw->pmd); in remove_migration_pmd()
3191 pmde = pmd_mkold(mk_huge_pmd(new, vma->vm_page_prot)); in remove_migration_pmd()
3192 if (pmd_swp_soft_dirty(*pvmw->pmd)) in remove_migration_pmd()
3196 if (pmd_swp_uffd_wp(*pvmw->pmd)) in remove_migration_pmd()
3204 set_pmd_at(mm, mmun_start, pvmw->pmd, pmde); in remove_migration_pmd()
3205 if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new)) in remove_migration_pmd()
3207 update_mmu_cache_pmd(vma, address, pvmw->pmd); in remove_migration_pmd()