146 static inline void __mm_zero_struct_page(struct page *page)  in __mm_zero_struct_page()  argument
148 	unsigned long *_pp = (void *)page;  in __mm_zero_struct_page()
151 	BUILD_BUG_ON(sizeof(struct page) & 7);  in __mm_zero_struct_page()
152 	BUILD_BUG_ON(sizeof(struct page) < 56);  in __mm_zero_struct_page()
153 	BUILD_BUG_ON(sizeof(struct page) > 80);  in __mm_zero_struct_page()
155 	switch (sizeof(struct page)) {  in __mm_zero_struct_page()
176 #define mm_zero_struct_page(pp)  ((void)memset((pp), 0, sizeof(struct page)))
215 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))  argument
218 #define nth_page(page,n) ((page) + (n))  argument
219 #define folio_page_idx(folio, p)	((p) - &(folio)->page)
231 #define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
503 	struct page *cow_page;		/* Page handler may use for COW fault */
504 	struct page *page;		/* ->fault handlers should return a  member
610 	struct page *(*find_special_page)(struct vm_area_struct *vma,
714 static inline unsigned int compound_order(struct page *page)  in compound_order()  argument
716 	if (!PageHead(page))  in compound_order()
718 	return page[1].compound_order;  in compound_order()
755 static inline int put_page_testzero(struct page *page)  in put_page_testzero()  argument
757 	VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);  in put_page_testzero()
758 	return page_ref_dec_and_test(page);  in put_page_testzero()
763 	return put_page_testzero(&folio->page);  in folio_put_testzero()
772 static inline bool get_page_unless_zero(struct page *page)  in get_page_unless_zero()  argument
774 	return page_ref_add_unless(page, 1, 0);  in get_page_unless_zero()
789 struct page *vmalloc_to_page(const void *addr);
834 static inline int compound_mapcount(struct page *page)  in compound_mapcount()  argument
836 	return folio_entire_mapcount(page_folio(page));  in compound_mapcount()
844 static inline void page_mapcount_reset(struct page *page)  in page_mapcount_reset()  argument
846 	atomic_set(&(page)->_mapcount, -1);  in page_mapcount_reset()
849 int __page_mapcount(struct page *page);
859 static inline int page_mapcount(struct page *page)  in page_mapcount()  argument
861 	if (unlikely(PageCompound(page)))  in page_mapcount()
862 		return __page_mapcount(page);  in page_mapcount()
863 	return atomic_read(&page->_mapcount) + 1;  in page_mapcount()
869 static inline int total_mapcount(struct page *page)  in total_mapcount()  argument
871 	return folio_mapcount(page_folio(page));  in total_mapcount()
875 static inline int total_mapcount(struct page *page)  in total_mapcount()  argument
877 	return page_mapcount(page);  in total_mapcount()
881 static inline struct page *virt_to_head_page(const void *x)  in virt_to_head_page()
883 	struct page *page = virt_to_page(x);  in virt_to_head_page()  local
885 	return compound_head(page);  in virt_to_head_page()
890 	struct page *page = virt_to_page(x);  in virt_to_folio()  local
892 	return page_folio(page);  in virt_to_folio()
899 void split_page(struct page *page, unsigned int order);
909 typedef void compound_page_dtor(struct page *);
925 static inline void set_compound_page_dtor(struct page *page,  in set_compound_page_dtor()  argument
928 	VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page);  in set_compound_page_dtor()
929 	page[1].compound_dtor = compound_dtor;  in set_compound_page_dtor()
934 static inline int head_compound_pincount(struct page *head)  in head_compound_pincount()
939 static inline void set_compound_order(struct page *page, unsigned int order)  in set_compound_order()  argument
941 	page[1].compound_order = order;  in set_compound_order()
943 	page[1].compound_nr = 1U << order;  in set_compound_order()
948 static inline unsigned long compound_nr(struct page *page)  in compound_nr()  argument
950 	if (!PageHead(page))  in compound_nr()
953 	return page[1].compound_nr;  in compound_nr()
955 	return 1UL << compound_order(page);  in compound_nr()
960 static inline unsigned long page_size(struct page *page)  in page_size()  argument
962 	return PAGE_SIZE << compound_order(page);  in page_size()
966 static inline unsigned int page_shift(struct page *page)  in page_shift()  argument
968 	return PAGE_SHIFT + compound_order(page);  in page_shift()
975 static inline unsigned int thp_order(struct page *page)  in thp_order()  argument
977 	VM_BUG_ON_PGFLAGS(PageTail(page), page);  in thp_order()
978 	return compound_order(page);  in thp_order()
985 static inline int thp_nr_pages(struct page *page)  in thp_nr_pages()  argument
987 	VM_BUG_ON_PGFLAGS(PageTail(page), page);  in thp_nr_pages()
988 	return compound_nr(page);  in thp_nr_pages()
997 static inline unsigned long thp_size(struct page *page)  in thp_size()  argument
999 	return PAGE_SIZE << thp_order(page);  in thp_size()
1002 void free_compound_page(struct page *page);
1018 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page);
1019 void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr);
1088 bool __put_devmap_managed_page_refs(struct page *page, int refs);
1089 static inline bool put_devmap_managed_page_refs(struct page *page, int refs)  in put_devmap_managed_page_refs()  argument
1093 	if (!is_zone_device_page(page))  in put_devmap_managed_page_refs()
1095 	return __put_devmap_managed_page_refs(page, refs);  in put_devmap_managed_page_refs()
1098 static inline bool put_devmap_managed_page_refs(struct page *page, int refs)  in put_devmap_managed_page_refs()  argument
1104 static inline bool put_devmap_managed_page(struct page *page)  in put_devmap_managed_page()  argument
1106 	return put_devmap_managed_page_refs(page, 1);  in put_devmap_managed_page()
1127 static inline void get_page(struct page *page)  in get_page()  argument
1129 	folio_get(page_folio(page));  in get_page()
1132 bool __must_check try_grab_page(struct page *page, unsigned int flags);
1134 static inline __must_check bool try_get_page(struct page *page)  in try_get_page()  argument
1136 	page = compound_head(page);  in try_get_page()
1137 	if (WARN_ON_ONCE(page_ref_count(page) <= 0))  in try_get_page()
1139 	page_ref_inc(page);  in try_get_page()
1182 void release_pages(struct page **pages, int nr);
1198 	release_pages((struct page **)folios, nr);  in folios_put()
1201 static inline void put_page(struct page *page)  in put_page()  argument
1203 	struct folio *folio = page_folio(page);  in put_page()
1209 	if (put_devmap_managed_page(&folio->page))  in put_page()
1246 void unpin_user_page(struct page *page);
1247 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
1249 void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,
1251 void unpin_user_pages(struct page **pages, unsigned long npages);
1270 static inline int page_zone_id(struct page *page)  in page_zone_id()  argument
1272 	return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;  in page_zone_id()
1276 extern int page_to_nid(const struct page *page);
1278 static inline int page_to_nid(const struct page *page)  in page_to_nid()  argument
1280 	struct page *p = (struct page *)page;  in page_to_nid()
1288 	return page_to_nid(&folio->page);  in folio_nid()
1341 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)  in page_cpupid_xchg_last()  argument
1343 	return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);  in page_cpupid_xchg_last()
1346 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
1348 	return page->_last_cpupid;  in page_cpupid_last()
1350 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
1352 	page->_last_cpupid = -1 & LAST_CPUPID_MASK;  in page_cpupid_reset_last()
1355 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
1357 	return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;  in page_cpupid_last()
1360 extern int page_cpupid_xchg_last(struct page *page, int cpupid);
1362 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
1364 	page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;  in page_cpupid_reset_last()
1368 static inline int xchg_page_access_time(struct page *page, int time)  in xchg_page_access_time()  argument
1372 	last_time = page_cpupid_xchg_last(page, time >> PAGE_ACCESS_TIME_BUCKETS);  in xchg_page_access_time()
1376 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)  in page_cpupid_xchg_last()  argument
1378 	return page_to_nid(page); /* XXX */  in page_cpupid_xchg_last()
1381 static inline int xchg_page_access_time(struct page *page, int time)  in xchg_page_access_time()  argument
1386 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
1388 	return page_to_nid(page); /* XXX */  in page_cpupid_last()
1416 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
1434 static inline u8 page_kasan_tag(const struct page *page)  in page_kasan_tag()  argument
1439 		tag = (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK;  in page_kasan_tag()
1446 static inline void page_kasan_tag_set(struct page *page, u8 tag)  in page_kasan_tag_set()  argument
1454 	old_flags = READ_ONCE(page->flags);  in page_kasan_tag_set()
1459 	} while (unlikely(!try_cmpxchg(&page->flags, &old_flags, flags)));  in page_kasan_tag_set()
1462 static inline void page_kasan_tag_reset(struct page *page)  in page_kasan_tag_reset()  argument
1465 		page_kasan_tag_set(page, 0xff);  in page_kasan_tag_reset()
1470 static inline u8 page_kasan_tag(const struct page *page)  in page_kasan_tag()  argument
1475 static inline void page_kasan_tag_set(struct page *page, u8 tag) { }  in page_kasan_tag_set()  argument
1476 static inline void page_kasan_tag_reset(struct page *page) { }  in page_kasan_tag_reset()  argument
1480 static inline struct zone *page_zone(const struct page *page)  in page_zone()  argument
1482 	return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];  in page_zone()
1485 static inline pg_data_t *page_pgdat(const struct page *page)  in page_pgdat()  argument
1487 	return NODE_DATA(page_to_nid(page));  in page_pgdat()
1492 	return page_zone(&folio->page);  in folio_zone()
1497 	return page_pgdat(&folio->page);  in folio_pgdat()
1501 static inline void set_page_section(struct page *page, unsigned long section)  in set_page_section()  argument
1503 	page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);  in set_page_section()
1504 	page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;  in set_page_section()
1507 static inline unsigned long page_to_section(const struct page *page)  in page_to_section()  argument
1509 	return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;  in page_to_section()
1524 	return page_to_pfn(&folio->page);  in folio_pfn()
1579 static inline bool page_maybe_dma_pinned(struct page *page)  in page_maybe_dma_pinned()  argument
1581 	return folio_maybe_dma_pinned(page_folio(page));  in page_maybe_dma_pinned()
1591 					  struct page *page)  in page_needs_cow_for_dma()  argument
1598 	return page_maybe_dma_pinned(page);  in page_needs_cow_for_dma()
1603 static inline bool is_longterm_pinnable_page(struct page *page)  in is_longterm_pinnable_page()  argument
1606 	int mt = get_pageblock_migratetype(page);  in is_longterm_pinnable_page()
1612 	if (is_zero_pfn(page_to_pfn(page)))  in is_longterm_pinnable_page()
1616 	if (is_device_coherent_page(page))  in is_longterm_pinnable_page()
1620 	return !is_zone_movable_page(page);  in is_longterm_pinnable_page()
1623 static inline bool is_longterm_pinnable_page(struct page *page)  in is_longterm_pinnable_page()  argument
1631 	return is_longterm_pinnable_page(&folio->page);  in folio_is_longterm_pinnable()
1634 static inline void set_page_zone(struct page *page, enum zone_type zone)  in set_page_zone()  argument
1636 	page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);  in set_page_zone()
1637 	page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;  in set_page_zone()
1640 static inline void set_page_node(struct page *page, unsigned long node)  in set_page_node()  argument
1642 	page->flags &= ~(NODES_MASK << NODES_PGSHIFT);  in set_page_node()
1643 	page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;  in set_page_node()
1646 static inline void set_page_links(struct page *page, enum zone_type zone,  in set_page_links()  argument
1649 	set_page_zone(page, zone);  in set_page_links()
1650 	set_page_node(page, node);  in set_page_links()
1652 	set_page_section(page, pfn_to_section_nr(pfn));  in set_page_links()
1723 static inline int arch_make_page_accessible(struct page *page)  in arch_make_page_accessible()  argument
1750 static __always_inline void *lowmem_page_address(const struct page *page)  in lowmem_page_address()  argument
1752 	return page_to_virt(page);  in lowmem_page_address()
1760 static inline void *page_address(const struct page *page)  in page_address()  argument
1762 	return page->virtual;  in page_address()
1764 static inline void set_page_address(struct page *page, void *address)  in set_page_address()  argument
1766 	page->virtual = address;  in set_page_address()
1772 void *page_address(const struct page *page);
1773 void set_page_address(struct page *page, void *virtual);
1778 #define page_address(page) lowmem_page_address(page)  argument
1779 #define set_page_address(page, address)  do { } while(0)  argument
1785 	return page_address(&folio->page);  in folio_address()
1788 extern void *page_rmapping(struct page *page);
1789 extern pgoff_t __page_file_index(struct page *page);
1795 static inline pgoff_t page_index(struct page *page)  in page_index()  argument
1797 	if (unlikely(PageSwapCache(page)))  in page_index()
1798 		return __page_file_index(page);  in page_index()
1799 	return page->index;  in page_index()
1802 bool page_mapped(struct page *page);
1810 static inline bool page_is_pfmemalloc(const struct page *page)  in page_is_pfmemalloc()  argument
1817 	return (uintptr_t)page->lru.next & BIT(1);  in page_is_pfmemalloc()
1824 static inline void set_page_pfmemalloc(struct page *page)  in set_page_pfmemalloc()  argument
1826 	page->lru.next = (void *)BIT(1);  in set_page_pfmemalloc()
1829 static inline void clear_page_pfmemalloc(struct page *page)  in clear_page_pfmemalloc()  argument
1831 	page->lru.next = NULL;  in clear_page_pfmemalloc()
1840 #define offset_in_thp(page, p)	((unsigned long)(p) & (thp_size(page) - 1))  argument
1882 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1884 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
1916 int generic_error_remove_page(struct address_space *mapping, struct page *page);
1966 			    unsigned int gup_flags, struct page **pages,
1970 			   unsigned int gup_flags, struct page **pages,
1973 			    unsigned int gup_flags, struct page **pages,
1976 		    unsigned int gup_flags, struct page **pages,
1979 		    struct page **pages, unsigned int gup_flags);
1981 		    struct page **pages, unsigned int gup_flags);
1984 			unsigned int gup_flags, struct page **pages);
1986 			unsigned int gup_flags, struct page **pages);
1994 			struct page **pages);
1995 struct page *get_dump_page(unsigned long addr);
1998 bool set_page_dirty(struct page *page);
1999 int set_page_dirty_lock(struct page *page);
2040 			     unsigned int gup_flags, struct page **pages);
2042 			     unsigned int gup_flags, struct page **pages);
2045 			unsigned int gup_flags, struct page **pagep)  in get_user_page_fast_only()
2091 static inline int mm_counter_file(struct page *page)  in mm_counter_file()  argument
2093 	if (PageSwapBacked(page))  in mm_counter_file()
2098 static inline int mm_counter(struct page *page)  in mm_counter()  argument
2100 	if (PageAnon(page))  in mm_counter()
2102 	return mm_counter_file(page);  in mm_counter()
2315 extern bool ptlock_alloc(struct page *page);
2316 extern void ptlock_free(struct page *page);
2318 static inline spinlock_t *ptlock_ptr(struct page *page)  in ptlock_ptr()  argument
2320 	return page->ptl;  in ptlock_ptr()
2327 static inline bool ptlock_alloc(struct page *page)  in ptlock_alloc()  argument
2332 static inline void ptlock_free(struct page *page)  in ptlock_free()  argument
2336 static inline spinlock_t *ptlock_ptr(struct page *page)  in ptlock_ptr()  argument
2338 	return &page->ptl;  in ptlock_ptr()
2347 static inline bool ptlock_init(struct page *page)  in ptlock_init()  argument
2356 	VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);  in ptlock_init()
2357 	if (!ptlock_alloc(page))  in ptlock_init()
2359 	spin_lock_init(ptlock_ptr(page));  in ptlock_init()
2372 static inline bool ptlock_init(struct page *page) { return true; }  in ptlock_init()  argument
2373 static inline void ptlock_free(struct page *page) {}  in ptlock_free()  argument
2382 static inline bool pgtable_pte_page_ctor(struct page *page)  in pgtable_pte_page_ctor()  argument
2384 	if (!ptlock_init(page))  in pgtable_pte_page_ctor()
2386 	__SetPageTable(page);  in pgtable_pte_page_ctor()
2387 	inc_lruvec_page_state(page, NR_PAGETABLE);  in pgtable_pte_page_ctor()
2391 static inline void pgtable_pte_page_dtor(struct page *page)  in pgtable_pte_page_dtor()  argument
2393 	ptlock_free(page);  in pgtable_pte_page_dtor()
2394 	__ClearPageTable(page);  in pgtable_pte_page_dtor()
2395 	dec_lruvec_page_state(page, NR_PAGETABLE);  in pgtable_pte_page_dtor()
2427 static struct page *pmd_to_page(pmd_t *pmd)  in pmd_to_page()
2438 static inline bool pmd_ptlock_init(struct page *page)  in pmd_ptlock_init()  argument
2441 	page->pmd_huge_pte = NULL;  in pmd_ptlock_init()
2443 	return ptlock_init(page);  in pmd_ptlock_init()
2446 static inline void pmd_ptlock_free(struct page *page)  in pmd_ptlock_free()  argument
2449 	VM_BUG_ON_PAGE(page->pmd_huge_pte, page);  in pmd_ptlock_free()
2451 	ptlock_free(page);  in pmd_ptlock_free()
2463 static inline bool pmd_ptlock_init(struct page *page) { return true; }  in pmd_ptlock_init()  argument
2464 static inline void pmd_ptlock_free(struct page *page) {}  in pmd_ptlock_free()  argument
2477 static inline bool pgtable_pmd_page_ctor(struct page *page)  in pgtable_pmd_page_ctor()  argument
2479 	if (!pmd_ptlock_init(page))  in pgtable_pmd_page_ctor()
2481 	__SetPageTable(page);  in pgtable_pmd_page_ctor()
2482 	inc_lruvec_page_state(page, NR_PAGETABLE);  in pgtable_pmd_page_ctor()
2486 static inline void pgtable_pmd_page_dtor(struct page *page)  in pgtable_pmd_page_dtor()  argument
2488 	pmd_ptlock_free(page);  in pgtable_pmd_page_dtor()
2489 	__ClearPageTable(page);  in pgtable_pmd_page_dtor()
2490 	dec_lruvec_page_state(page, NR_PAGETABLE);  in pgtable_pmd_page_dtor()
2524 extern void adjust_managed_page_count(struct page *page, long count);
2530 static inline void free_reserved_page(struct page *page)  in free_reserved_page()  argument
2532 	ClearPageReserved(page);  in free_reserved_page()
2533 	init_page_count(page);  in free_reserved_page()
2534 	__free_page(page);  in free_reserved_page()
2535 	adjust_managed_page_count(page, 1);  in free_reserved_page()
2537 #define free_highmem_page(page) free_reserved_page(page)  argument
2539 static inline void mark_page_reserved(struct page *page)  in mark_page_reserved()  argument
2541 	SetPageReserved(page);  in mark_page_reserved()
2542 	adjust_managed_page_count(page, -1);  in mark_page_reserved()
2739 				   unsigned long flags, struct page **pages);
2914 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
2916 			struct page **pages, unsigned long *num);
2917 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2919 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
2934 				unsigned long addr, struct page *page)  in vmf_insert_page()  argument
2936 	int err = vm_insert_page(vma, addr, page);  in vmf_insert_page()
2962 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
3067 static inline bool gup_must_unshare(unsigned int flags, struct page *page)  in gup_must_unshare()  argument
3080 	if (!PageAnon(page))  in gup_must_unshare()
3091 	return !PageAnonExclusive(page);  in gup_must_unshare()
3118 extern void __kernel_poison_pages(struct page *page, int numpages);
3119 extern void __kernel_unpoison_pages(struct page *page, int numpages);
3134 static inline void kernel_poison_pages(struct page *page, int numpages)  in kernel_poison_pages()  argument
3137 		__kernel_poison_pages(page, numpages);  in kernel_poison_pages()
3139 static inline void kernel_unpoison_pages(struct page *page, int numpages)  in kernel_unpoison_pages()  argument
3142 		__kernel_unpoison_pages(page, numpages);  in kernel_unpoison_pages()
3147 static inline void __kernel_poison_pages(struct page *page, int nunmpages) { }  in __kernel_poison_pages()  argument
3148 static inline void kernel_poison_pages(struct page *page, int numpages) { }  in kernel_poison_pages()  argument
3149 static inline void kernel_unpoison_pages(struct page *page, int numpages) { }  in kernel_unpoison_pages()  argument
3194 extern void __kernel_map_pages(struct page *page, int numpages, int enable);
3196 static inline void debug_pagealloc_map_pages(struct page *page, int numpages)  in debug_pagealloc_map_pages()  argument
3199 		__kernel_map_pages(page, numpages, 1);  in debug_pagealloc_map_pages()
3202 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages)  in debug_pagealloc_unmap_pages()  argument
3205 		__kernel_map_pages(page, numpages, 0);  in debug_pagealloc_unmap_pages()
3208 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {}  in debug_pagealloc_map_pages()  argument
3209 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {}  in debug_pagealloc_unmap_pages()  argument
3254 struct page * __populate_section_memmap(unsigned long pfn,
3262 			    struct vmem_altmap *altmap, struct page *reuse);
3277 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
3297 extern void shake_page(struct page *p);
3357 extern void clear_huge_page(struct page *page,
3360 extern void copy_user_huge_page(struct page *dst, struct page *src,
3364 extern long copy_huge_page_from_user(struct page *dst_page,
3401 static inline bool page_is_guard(struct page *page)  in page_is_guard()  argument
3406 	return PageGuard(page);  in page_is_guard()
3411 static inline bool page_is_guard(struct page *page) { return false; }  in page_is_guard()  argument
3420 extern int memcmp_pages(struct page *page1, struct page *page2);
3422 static inline int pages_identical(struct page *page1, struct page *page2)  in pages_identical()