Lines Matching full:page
24 #include <linux/page-flags.h>
95 #include <asm/page.h>
123 * a zero page mapping on a read fault.
126 * related to the physical page in case of virtualization.
139 /* This function must be updated when the size of struct page grows above 80
146 static inline void __mm_zero_struct_page(struct page *page) in __mm_zero_struct_page() argument
148 unsigned long *_pp = (void *)page; in __mm_zero_struct_page()
150 /* Check that struct page is either 56, 64, 72, or 80 bytes */ in __mm_zero_struct_page()
151 BUILD_BUG_ON(sizeof(struct page) & 7); in __mm_zero_struct_page()
152 BUILD_BUG_ON(sizeof(struct page) < 56); in __mm_zero_struct_page()
153 BUILD_BUG_ON(sizeof(struct page) > 80); in __mm_zero_struct_page()
155 switch (sizeof(struct page)) { in __mm_zero_struct_page()
176 #define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page)))
215 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n)) argument
218 #define nth_page(page,n) ((page) + (n)) argument
219 #define folio_page_idx(folio, p) ((p) - &(folio)->page)
222 /* to align the pointer to the (next) page boundary */
225 /* to align the pointer to the (prev) page boundary */
231 #define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
279 #define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
294 #define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
295 #define VM_SYNC 0x00800000 /* Synchronous page faults */
306 #define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
349 # define VM_ARM64_BTI VM_ARCH_1 /* BTI guarded page, a.k.a. GP bit */
429 * low four bits) to a page protection mask..
434 * arch-specific page fault handlers.
450 * Return: true if the page fault allows retry and this is the first
476 * MM layer fills up gfp_mask for page allocations but fault handler might
485 pgoff_t pgoff; /* Logical page offset based on vma */
503 struct page *cow_page; /* Page handler may use for COW fault */
504 struct page *page; /* ->fault handlers should return a member
505 * page here, unless VM_FAULT_NOPAGE
511 * the 'address'. NULL if the page
514 spinlock_t *ptl; /* Page table lock.
515 * Protects pte page table if 'pte'
518 pgtable_t prealloc_pte; /* Pre-allocated pte page table.
519 * vm_ops->map_pages() sets up a page
522 * page table to avoid allocation from
527 /* page entry size for vm->huge_fault() */
537 * to the functions called when a no-page or a wp-page exception occurs.
563 /* notification that a previously read-only page is about to become
607 * page for @addr. This is useful if the default behavior
608 * (using pte_page()) would not find the correct page.
610 struct page *(*find_special_page)(struct vm_area_struct *vma,
714 static inline unsigned int compound_order(struct page *page) in compound_order() argument
716 if (!PageHead(page)) in compound_order()
718 return page[1].compound_order; in compound_order()
740 * Methods to modify the page usage count.
742 * What counts for a page usage:
743 * - cache mapping (page->mapping)
744 * - private data (page->private)
745 * - page mapped in a task's page tables, each mapping
748 * Also, many kernel routines increase the page count before a critical
749 * routine so they can be sure the page doesn't go away from under them.
753 * Drop a ref, return true if the refcount fell to zero (the page has no users)
755 static inline int put_page_testzero(struct page *page) in put_page_testzero() argument
757 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); in put_page_testzero()
758 return page_ref_dec_and_test(page); in put_page_testzero()
763 return put_page_testzero(&folio->page); in folio_put_testzero()
767 * Try to grab a ref unless the page has a refcount of zero, return false if
772 static inline bool get_page_unless_zero(struct page *page) in get_page_unless_zero() argument
774 return page_ref_add_unless(page, 1, 0); in get_page_unless_zero()
789 struct page *vmalloc_to_page(const void *addr);
830 * Mapcount of compound page as a whole, does not include mapped sub-pages.
834 static inline int compound_mapcount(struct page *page) in compound_mapcount() argument
836 return folio_entire_mapcount(page_folio(page)); in compound_mapcount()
840 * The atomic page->_mapcount, starts from -1: so that transitions
844 static inline void page_mapcount_reset(struct page *page) in page_mapcount_reset() argument
846 atomic_set(&(page)->_mapcount, -1); in page_mapcount_reset()
849 int __page_mapcount(struct page *page);
852 * Mapcount of 0-order page; when compound sub-page, includes
857 * They use this place in struct page differently.
859 static inline int page_mapcount(struct page *page) in page_mapcount() argument
861 if (unlikely(PageCompound(page))) in page_mapcount()
862 return __page_mapcount(page); in page_mapcount()
863 return atomic_read(&page->_mapcount) + 1; in page_mapcount()
869 static inline int total_mapcount(struct page *page) in total_mapcount() argument
871 return folio_mapcount(page_folio(page)); in total_mapcount()
875 static inline int total_mapcount(struct page *page) in total_mapcount() argument
877 return page_mapcount(page); in total_mapcount()
881 static inline struct page *virt_to_head_page(const void *x) in virt_to_head_page()
883 struct page *page = virt_to_page(x); in virt_to_head_page() local
885 return compound_head(page); in virt_to_head_page()
890 struct page *page = virt_to_page(x); in virt_to_folio() local
892 return page_folio(page); in virt_to_folio()
899 void split_page(struct page *page, unsigned int order);
907 * These are _only_ valid on the head of a compound page.
909 typedef void compound_page_dtor(struct page *);
925 static inline void set_compound_page_dtor(struct page *page, in set_compound_page_dtor() argument
928 VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page); in set_compound_page_dtor()
929 page[1].compound_dtor = compound_dtor; in set_compound_page_dtor()
934 static inline int head_compound_pincount(struct page *head) in head_compound_pincount()
939 static inline void set_compound_order(struct page *page, unsigned int order) in set_compound_order() argument
941 page[1].compound_order = order; in set_compound_order()
943 page[1].compound_nr = 1U << order; in set_compound_order()
947 /* Returns the number of pages in this potentially compound page. */
948 static inline unsigned long compound_nr(struct page *page) in compound_nr() argument
950 if (!PageHead(page)) in compound_nr()
953 return page[1].compound_nr; in compound_nr()
955 return 1UL << compound_order(page); in compound_nr()
959 /* Returns the number of bytes in this potentially compound page. */
960 static inline unsigned long page_size(struct page *page) in page_size() argument
962 return PAGE_SIZE << compound_order(page); in page_size()
965 /* Returns the number of bits needed for the number of bytes in a page */
966 static inline unsigned int page_shift(struct page *page) in page_shift() argument
968 return PAGE_SHIFT + compound_order(page); in page_shift()
972 * thp_order - Order of a transparent huge page.
973 * @page: Head page of a transparent huge page.
975 static inline unsigned int thp_order(struct page *page) in thp_order() argument
977 VM_BUG_ON_PGFLAGS(PageTail(page), page); in thp_order()
978 return compound_order(page); in thp_order()
982 * thp_nr_pages - The number of regular pages in this huge page.
983 * @page: The head page of a huge page.
985 static inline int thp_nr_pages(struct page *page) in thp_nr_pages() argument
987 VM_BUG_ON_PGFLAGS(PageTail(page), page); in thp_nr_pages()
988 return compound_nr(page); in thp_nr_pages()
992 * thp_size - Size of a transparent huge page.
993 * @page: Head page of a transparent huge page.
995 * Return: Number of bytes in this page.
997 static inline unsigned long thp_size(struct page *page) in thp_size() argument
999 return PAGE_SIZE << thp_order(page); in thp_size()
1002 void free_compound_page(struct page *page);
1018 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page);
1019 void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr);
1026 * Multiple processes may "see" the same page. E.g. for untouched
1027 * mappings of /dev/null, all processes see the same page full of
1031 * For the non-reserved pages, page_count(page) denotes a reference count.
1032 * page_count() == 0 means the page is free. page->lru is then used for
1034 * page_count() > 0 means the page has been allocated.
1038 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
1042 * A page may be used by anyone else who does a __get_free_page().
1044 * be used through the normal accessor functions. The top bits of page->flags
1045 * and page->virtual store page management information, but all other fields
1047 * page is the responsibility of the one who allocated it, and those who have
1054 * A pagecache page contains an opaque `private' member, which belongs to the
1055 * page's address_space. Usually, this is the address of a circular list of
1056 * the page's disk buffers. PG_private must be set to tell the VM to call
1059 * A page may belong to an inode's memory mapping. In this case, page->mapping
1060 * is the pointer to the inode, and page->index is the file offset of the page,
1065 * case PG_swapcache is set, and page->private is an offset into the swapcache.
1068 * reference to the page. Setting PG_private should also increment the
1069 * refcount. The each user mapping also has a reference to the page.
1088 bool __put_devmap_managed_page_refs(struct page *page, int refs);
1089 static inline bool put_devmap_managed_page_refs(struct page *page, int refs) in put_devmap_managed_page_refs() argument
1093 if (!is_zone_device_page(page)) in put_devmap_managed_page_refs()
1095 return __put_devmap_managed_page_refs(page, refs); in put_devmap_managed_page_refs()
1098 static inline bool put_devmap_managed_page_refs(struct page *page, int refs) in put_devmap_managed_page_refs() argument
1104 static inline bool put_devmap_managed_page(struct page *page) in put_devmap_managed_page() argument
1106 return put_devmap_managed_page_refs(page, 1); in put_devmap_managed_page()
1127 static inline void get_page(struct page *page) in get_page() argument
1129 folio_get(page_folio(page)); in get_page()
1132 bool __must_check try_grab_page(struct page *page, unsigned int flags);
1134 static inline __must_check bool try_get_page(struct page *page) in try_get_page() argument
1136 page = compound_head(page); in try_get_page()
1137 if (WARN_ON_ONCE(page_ref_count(page) <= 0)) in try_get_page()
1139 page_ref_inc(page); in try_get_page()
1148 * released back to the page allocator and may be used by another
1168 * released back to the page allocator and may be used by another
1182 void release_pages(struct page **pages, int nr);
1198 release_pages((struct page **)folios, nr); in folios_put()
1201 static inline void put_page(struct page *page) in put_page() argument
1203 struct folio *folio = page_folio(page); in put_page()
1209 if (put_devmap_managed_page(&folio->page)) in put_page()
1216 * the page's refcount so that two separate items are tracked: the original page
1218 * made against the page. ("gup-pinned" is another term for the latter).
1226 * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference
1228 * simpler, due to the fact that adding an even power of two to the page
1237 * applications that don't have huge page reference counts, this won't be an
1242 * other calls that race to set up page table entries.
1246 void unpin_user_page(struct page *page);
1247 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
1249 void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,
1251 void unpin_user_pages(struct page **pages, unsigned long npages);
1266 * node id available in page flags.
1270 static inline int page_zone_id(struct page *page) in page_zone_id() argument
1272 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK; in page_zone_id()
1276 extern int page_to_nid(const struct page *page);
1278 static inline int page_to_nid(const struct page *page) in page_to_nid() argument
1280 struct page *p = (struct page *)page; in page_to_nid()
1288 return page_to_nid(&folio->page); in folio_nid()
1292 /* page access time bits needs to hold at least 4 seconds */
1341 static inline int page_cpupid_xchg_last(struct page *page, int cpupid) in page_cpupid_xchg_last() argument
1343 return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK); in page_cpupid_xchg_last()
1346 static inline int page_cpupid_last(struct page *page) in page_cpupid_last() argument
1348 return page->_last_cpupid; in page_cpupid_last()
1350 static inline void page_cpupid_reset_last(struct page *page) in page_cpupid_reset_last() argument
1352 page->_last_cpupid = -1 & LAST_CPUPID_MASK; in page_cpupid_reset_last()
1355 static inline int page_cpupid_last(struct page *page) in page_cpupid_last() argument
1357 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK; in page_cpupid_last()
1360 extern int page_cpupid_xchg_last(struct page *page, int cpupid);
1362 static inline void page_cpupid_reset_last(struct page *page) in page_cpupid_reset_last() argument
1364 page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT; in page_cpupid_reset_last()
1368 static inline int xchg_page_access_time(struct page *page, int time) in xchg_page_access_time() argument
1372 last_time = page_cpupid_xchg_last(page, time >> PAGE_ACCESS_TIME_BUCKETS); in xchg_page_access_time()
1376 static inline int page_cpupid_xchg_last(struct page *page, int cpupid) in page_cpupid_xchg_last() argument
1378 return page_to_nid(page); /* XXX */ in page_cpupid_xchg_last()
1381 static inline int xchg_page_access_time(struct page *page, int time) in xchg_page_access_time() argument
1386 static inline int page_cpupid_last(struct page *page) in page_cpupid_last() argument
1388 return page_to_nid(page); /* XXX */ in page_cpupid_last()
1416 static inline void page_cpupid_reset_last(struct page *page) in page_cpupid_reset_last() argument
1429 * KASAN per-page tags are stored xor'ed with 0xff. This allows to avoid
1434 static inline u8 page_kasan_tag(const struct page *page) in page_kasan_tag() argument
1439 tag = (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK; in page_kasan_tag()
1446 static inline void page_kasan_tag_set(struct page *page, u8 tag) in page_kasan_tag_set() argument
1454 old_flags = READ_ONCE(page->flags); in page_kasan_tag_set()
1459 } while (unlikely(!try_cmpxchg(&page->flags, &old_flags, flags))); in page_kasan_tag_set()
1462 static inline void page_kasan_tag_reset(struct page *page) in page_kasan_tag_reset() argument
1465 page_kasan_tag_set(page, 0xff); in page_kasan_tag_reset()
1470 static inline u8 page_kasan_tag(const struct page *page) in page_kasan_tag() argument
1475 static inline void page_kasan_tag_set(struct page *page, u8 tag) { } in page_kasan_tag_set() argument
1476 static inline void page_kasan_tag_reset(struct page *page) { } in page_kasan_tag_reset() argument
1480 static inline struct zone *page_zone(const struct page *page) in page_zone() argument
1482 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; in page_zone()
1485 static inline pg_data_t *page_pgdat(const struct page *page) in page_pgdat() argument
1487 return NODE_DATA(page_to_nid(page)); in page_pgdat()
1492 return page_zone(&folio->page); in folio_zone()
1497 return page_pgdat(&folio->page); in folio_pgdat()
1501 static inline void set_page_section(struct page *page, unsigned long section) in set_page_section() argument
1503 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); in set_page_section()
1504 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; in set_page_section()
1507 static inline unsigned long page_to_section(const struct page *page) in page_to_section() argument
1509 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK; in page_to_section()
1514 * folio_pfn - Return the Page Frame Number of a folio.
1518 * Page Frame Numbers.
1520 * Return: The Page Frame Number of the first page in the folio.
1524 return page_to_pfn(&folio->page); in folio_pfn()
1559 * Return: True, if it is likely that the page has been "dma-pinned".
1560 * False, if the page is definitely not dma-pinned.
1579 static inline bool page_maybe_dma_pinned(struct page *page) in page_maybe_dma_pinned() argument
1581 return folio_maybe_dma_pinned(page_folio(page)); in page_maybe_dma_pinned()
1586 * should break the cow immediately for an anon page on the src mm.
1591 struct page *page) in page_needs_cow_for_dma() argument
1598 return page_maybe_dma_pinned(page); in page_needs_cow_for_dma()
1603 static inline bool is_longterm_pinnable_page(struct page *page) in is_longterm_pinnable_page() argument
1606 int mt = get_pageblock_migratetype(page); in is_longterm_pinnable_page()
1611 /* The zero page may always be pinned */ in is_longterm_pinnable_page()
1612 if (is_zero_pfn(page_to_pfn(page))) in is_longterm_pinnable_page()
1616 if (is_device_coherent_page(page)) in is_longterm_pinnable_page()
1620 return !is_zone_movable_page(page); in is_longterm_pinnable_page()
1623 static inline bool is_longterm_pinnable_page(struct page *page) in is_longterm_pinnable_page() argument
1631 return is_longterm_pinnable_page(&folio->page); in folio_is_longterm_pinnable()
1634 static inline void set_page_zone(struct page *page, enum zone_type zone) in set_page_zone() argument
1636 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT); in set_page_zone()
1637 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT; in set_page_zone()
1640 static inline void set_page_node(struct page *page, unsigned long node) in set_page_node() argument
1642 page->flags &= ~(NODES_MASK << NODES_PGSHIFT); in set_page_node()
1643 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT; in set_page_node()
1646 static inline void set_page_links(struct page *page, enum zone_type zone, in set_page_links() argument
1649 set_page_zone(page, zone); in set_page_links()
1650 set_page_node(page, node); in set_page_links()
1652 set_page_section(page, pfn_to_section_nr(pfn)); in set_page_links()
1681 * other. This is the folio equivalent to writing ``page++``.
1723 static inline int arch_make_page_accessible(struct page *page) in arch_make_page_accessible() argument
1750 static __always_inline void *lowmem_page_address(const struct page *page) in lowmem_page_address() argument
1752 return page_to_virt(page); in lowmem_page_address()
1760 static inline void *page_address(const struct page *page) in page_address() argument
1762 return page->virtual; in page_address()
1764 static inline void set_page_address(struct page *page, void *address) in set_page_address() argument
1766 page->virtual = address; in set_page_address()
1772 void *page_address(const struct page *page);
1773 void set_page_address(struct page *page, void *virtual);
1778 #define page_address(page) lowmem_page_address(page) argument
1779 #define set_page_address(page, address) do { } while(0) argument
1785 return page_address(&folio->page); in folio_address()
1788 extern void *page_rmapping(struct page *page);
1789 extern pgoff_t __page_file_index(struct page *page);
1792 * Return the pagecache index of the passed page. Regular pagecache pages
1795 static inline pgoff_t page_index(struct page *page) in page_index() argument
1797 if (unlikely(PageSwapCache(page))) in page_index()
1798 return __page_file_index(page); in page_index()
1799 return page->index; in page_index()
1802 bool page_mapped(struct page *page);
1806 * Return true only if the page has been allocated with
1810 static inline bool page_is_pfmemalloc(const struct page *page) in page_is_pfmemalloc() argument
1813 * lru.next has bit 1 set if the page is allocated from the in page_is_pfmemalloc()
1817 return (uintptr_t)page->lru.next & BIT(1); in page_is_pfmemalloc()
1821 * Only to be called by the page allocator on a freshly allocated
1822 * page.
1824 static inline void set_page_pfmemalloc(struct page *page) in set_page_pfmemalloc() argument
1826 page->lru.next = (void *)BIT(1); in set_page_pfmemalloc()
1829 static inline void clear_page_pfmemalloc(struct page *page) in clear_page_pfmemalloc() argument
1831 page->lru.next = NULL; in clear_page_pfmemalloc()
1840 #define offset_in_thp(page, p) ((unsigned long)(p) & (thp_size(page) - 1)) argument
1867 * drop the page in the mm, either by truncation or unmapping of the vma. By
1882 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1884 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
1916 int generic_error_remove_page(struct address_space *mapping, struct page *page);
1966 unsigned int gup_flags, struct page **pages,
1970 unsigned int gup_flags, struct page **pages,
1973 unsigned int gup_flags, struct page **pages,
1976 unsigned int gup_flags, struct page **pages,
1979 struct page **pages, unsigned int gup_flags);
1981 struct page **pages, unsigned int gup_flags);
1984 unsigned int gup_flags, struct page **pages);
1986 unsigned int gup_flags, struct page **pages);
1994 struct page **pages);
1995 struct page *get_dump_page(unsigned long addr);
1998 bool set_page_dirty(struct page *page);
1999 int set_page_dirty_lock(struct page *page);
2040 unsigned int gup_flags, struct page **pages);
2042 unsigned int gup_flags, struct page **pages);
2045 unsigned int gup_flags, struct page **pagep) in get_user_page_fast_only()
2090 /* Optimized variant when page is already known not to be PageAnon */
2091 static inline int mm_counter_file(struct page *page) in mm_counter_file() argument
2093 if (PageSwapBacked(page)) in mm_counter_file()
2098 static inline int mm_counter(struct page *page) in mm_counter() argument
2100 if (PageAnon(page)) in mm_counter()
2102 return mm_counter_file(page); in mm_counter()
2315 extern bool ptlock_alloc(struct page *page);
2316 extern void ptlock_free(struct page *page);
2318 static inline spinlock_t *ptlock_ptr(struct page *page) in ptlock_ptr() argument
2320 return page->ptl; in ptlock_ptr()
2327 static inline bool ptlock_alloc(struct page *page) in ptlock_alloc() argument
2332 static inline void ptlock_free(struct page *page) in ptlock_free() argument
2336 static inline spinlock_t *ptlock_ptr(struct page *page) in ptlock_ptr() argument
2338 return &page->ptl; in ptlock_ptr()
2347 static inline bool ptlock_init(struct page *page) in ptlock_init() argument
2350 * prep_new_page() initialize page->private (and therefore page->ptl) in ptlock_init()
2353 * It can happen if arch try to use slab for page table allocation: in ptlock_init()
2354 * slab code uses page->slab_cache, which share storage with page->ptl. in ptlock_init()
2356 VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page); in ptlock_init()
2357 if (!ptlock_alloc(page)) in ptlock_init()
2359 spin_lock_init(ptlock_ptr(page)); in ptlock_init()
2372 static inline bool ptlock_init(struct page *page) { return true; } in ptlock_init() argument
2373 static inline void ptlock_free(struct page *page) {} in ptlock_free() argument
2382 static inline bool pgtable_pte_page_ctor(struct page *page) in pgtable_pte_page_ctor() argument
2384 if (!ptlock_init(page)) in pgtable_pte_page_ctor()
2386 __SetPageTable(page); in pgtable_pte_page_ctor()
2387 inc_lruvec_page_state(page, NR_PAGETABLE); in pgtable_pte_page_ctor()
2391 static inline void pgtable_pte_page_dtor(struct page *page) in pgtable_pte_page_dtor() argument
2393 ptlock_free(page); in pgtable_pte_page_dtor()
2394 __ClearPageTable(page); in pgtable_pte_page_dtor()
2395 dec_lruvec_page_state(page, NR_PAGETABLE); in pgtable_pte_page_dtor()
2427 static struct page *pmd_to_page(pmd_t *pmd) in pmd_to_page()
2438 static inline bool pmd_ptlock_init(struct page *page) in pmd_ptlock_init() argument
2441 page->pmd_huge_pte = NULL; in pmd_ptlock_init()
2443 return ptlock_init(page); in pmd_ptlock_init()
2446 static inline void pmd_ptlock_free(struct page *page) in pmd_ptlock_free() argument
2449 VM_BUG_ON_PAGE(page->pmd_huge_pte, page); in pmd_ptlock_free()
2451 ptlock_free(page); in pmd_ptlock_free()
2463 static inline bool pmd_ptlock_init(struct page *page) { return true; } in pmd_ptlock_init() argument
2464 static inline void pmd_ptlock_free(struct page *page) {} in pmd_ptlock_free() argument
2477 static inline bool pgtable_pmd_page_ctor(struct page *page) in pgtable_pmd_page_ctor() argument
2479 if (!pmd_ptlock_init(page)) in pgtable_pmd_page_ctor()
2481 __SetPageTable(page); in pgtable_pmd_page_ctor()
2482 inc_lruvec_page_state(page, NR_PAGETABLE); in pgtable_pmd_page_ctor()
2486 static inline void pgtable_pmd_page_dtor(struct page *page) in pgtable_pmd_page_dtor() argument
2488 pmd_ptlock_free(page); in pgtable_pmd_page_dtor()
2489 __ClearPageTable(page); in pgtable_pmd_page_dtor()
2490 dec_lruvec_page_state(page, NR_PAGETABLE); in pgtable_pmd_page_dtor()
2524 extern void adjust_managed_page_count(struct page *page, long count);
2529 /* Free the reserved page into the buddy system, so it gets managed. */
2530 static inline void free_reserved_page(struct page *page) in free_reserved_page() argument
2532 ClearPageReserved(page); in free_reserved_page()
2533 init_page_count(page); in free_reserved_page()
2534 __free_page(page); in free_reserved_page()
2535 adjust_managed_page_count(page, 1); in free_reserved_page()
2537 #define free_highmem_page(page) free_reserved_page(page) argument
2539 static inline void mark_page_reserved(struct page *page) in mark_page_reserved() argument
2541 SetPageReserved(page); in mark_page_reserved()
2542 adjust_managed_page_count(page, -1); in mark_page_reserved()
2575 * An architecture is expected to register range of page frames backed by
2739 unsigned long flags, struct page **pages);
2914 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
2916 struct page **pages, unsigned long *num);
2917 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2919 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
2934 unsigned long addr, struct page *page) in vmf_insert_page() argument
2936 int err = vm_insert_page(vma, addr, page); in vmf_insert_page()
2962 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
2966 #define FOLL_TOUCH 0x02 /* mark page accessed */
2967 #define FOLL_GET 0x04 /* do get_page on page */
2973 #define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
2974 #define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
2987 * FOLL_LONGTERM indicates that the page will be held for an indefinite time
2995 * pin, delaying writeback, bounce buffer page writeback, etc. As FS DAX was
3007 * that region. And so, CMA attempts to migrate the page before pinning, when
3010 * FOLL_PIN indicates that a special kind of tracking (not just page->_refcount,
3012 * anything that gets a page reference and then touches page data (for example,
3051 * Indicates for which pages that are write-protected in the page table,
3053 * GUP pin will remain consistent with the pages mapped into the page tables
3063 * Must be called with the (sub)page that's actually referenced via the
3064 * page table entry, which might not necessarily be the head page for a
3067 static inline bool gup_must_unshare(unsigned int flags, struct page *page) in gup_must_unshare() argument
3070 * FOLL_WRITE is implicitly handled correctly as the page table entry in gup_must_unshare()
3077 * Note: PageAnon(page) is stable until the page is actually getting in gup_must_unshare()
3080 if (!PageAnon(page)) in gup_must_unshare()
3091 return !PageAnonExclusive(page); in gup_must_unshare()
3095 * Indicates whether GUP can follow a PROT_NONE mapped page, or whether
3118 extern void __kernel_poison_pages(struct page *page, int numpages);
3119 extern void __kernel_unpoison_pages(struct page *page, int numpages);
3134 static inline void kernel_poison_pages(struct page *page, int numpages) in kernel_poison_pages() argument
3137 __kernel_poison_pages(page, numpages); in kernel_poison_pages()
3139 static inline void kernel_unpoison_pages(struct page *page, int numpages) in kernel_unpoison_pages() argument
3142 __kernel_unpoison_pages(page, numpages); in kernel_unpoison_pages()
3147 static inline void __kernel_poison_pages(struct page *page, int nunmpages) { } in __kernel_poison_pages() argument
3148 static inline void kernel_poison_pages(struct page *page, int numpages) { } in kernel_poison_pages() argument
3149 static inline void kernel_unpoison_pages(struct page *page, int numpages) { } in kernel_unpoison_pages() argument
3194 extern void __kernel_map_pages(struct page *page, int numpages, int enable);
3196 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) in debug_pagealloc_map_pages() argument
3199 __kernel_map_pages(page, numpages, 1); in debug_pagealloc_map_pages()
3202 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) in debug_pagealloc_unmap_pages() argument
3205 __kernel_map_pages(page, numpages, 0); in debug_pagealloc_unmap_pages()
3208 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {} in debug_pagealloc_map_pages() argument
3209 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {} in debug_pagealloc_unmap_pages() argument
3254 struct page * __populate_section_memmap(unsigned long pfn,
3262 struct vmem_altmap *altmap, struct page *reuse);
3277 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
3297 extern void shake_page(struct page *p);
3357 extern void clear_huge_page(struct page *page,
3360 extern void copy_user_huge_page(struct page *dst, struct page *src,
3364 extern long copy_huge_page_from_user(struct page *dst_page,
3370 * vma_is_special_huge - Are transhuge page-table entries considered special?
3373 * Whether transhuge page-table entries are considered "special" following
3376 * Return: true if transhuge page-table entries should be considered special,
3401 static inline bool page_is_guard(struct page *page) in page_is_guard() argument
3406 return PageGuard(page); in page_is_guard()
3411 static inline bool page_is_guard(struct page *page) { return false; } in page_is_guard() argument
3420 extern int memcmp_pages(struct page *page1, struct page *page2);
3422 static inline int pages_identical(struct page *page1, struct page *page2) in pages_identical()