138 static inline void __mm_zero_struct_page(struct page *page)  in __mm_zero_struct_page()  argument
140 	unsigned long *_pp = (void *)page;  in __mm_zero_struct_page()
143 	BUILD_BUG_ON(sizeof(struct page) & 7);  in __mm_zero_struct_page()
144 	BUILD_BUG_ON(sizeof(struct page) < 56);  in __mm_zero_struct_page()
145 	BUILD_BUG_ON(sizeof(struct page) > 96);  in __mm_zero_struct_page()
147 	switch (sizeof(struct page)) {  in __mm_zero_struct_page()
174 #define mm_zero_struct_page(pp)  ((void)memset((pp), 0, sizeof(struct page)))
213 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))  argument
216 #define nth_page(page,n) ((page) + (n))  argument
217 #define folio_page_idx(folio, p)	((p) - &(folio)->page)
229 #define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
530 	struct page *cow_page;		/* Page handler may use for COW fault */
531 	struct page *page;		/* ->fault handlers should return a  member
629 	struct page *(*find_special_page)(struct vm_area_struct *vma,
1050 static inline unsigned int compound_order(struct page *page)  in compound_order()  argument
1052 	struct folio *folio = (struct folio *)page;  in compound_order()
1093 static inline int put_page_testzero(struct page *page)  in put_page_testzero()  argument
1095 	VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);  in put_page_testzero()
1096 	return page_ref_dec_and_test(page);  in put_page_testzero()
1101 	return put_page_testzero(&folio->page);  in folio_put_testzero()
1110 static inline bool get_page_unless_zero(struct page *page)  in get_page_unless_zero()  argument
1112 	return page_ref_add_unless(page, 1, 0);  in get_page_unless_zero()
1115 static inline struct folio *folio_get_nontail_page(struct page *page)  in folio_get_nontail_page()  argument
1117 	if (unlikely(!get_page_unless_zero(page)))  in folio_get_nontail_page()
1119 	return (struct folio *)page;  in folio_get_nontail_page()
1134 struct page *vmalloc_to_page(const void *addr);
1174 static inline void page_mapcount_reset(struct page *page)  in page_mapcount_reset()  argument
1176 	atomic_set(&(page)->_mapcount, -1);  in page_mapcount_reset()
1191 static inline int page_mapcount(struct page *page)  in page_mapcount()  argument
1193 	int mapcount = atomic_read(&page->_mapcount) + 1;  in page_mapcount()
1195 	if (unlikely(PageCompound(page)))  in page_mapcount()
1196 		mapcount += folio_entire_mapcount(page_folio(page));  in page_mapcount()
1221 static inline int total_mapcount(struct page *page)  in total_mapcount()  argument
1223 	if (likely(!PageCompound(page)))  in total_mapcount()
1224 		return atomic_read(&page->_mapcount) + 1;  in total_mapcount()
1225 	return folio_total_mapcount(page_folio(page));  in total_mapcount()
1256 static inline bool page_mapped(struct page *page)  in page_mapped()  argument
1258 	if (likely(!PageCompound(page)))  in page_mapped()
1259 		return atomic_read(&page->_mapcount) >= 0;  in page_mapped()
1260 	return folio_large_is_mapped(page_folio(page));  in page_mapped()
1263 static inline struct page *virt_to_head_page(const void *x)  in virt_to_head_page()
1265 	struct page *page = virt_to_page(x);  in virt_to_head_page()  local
1267 	return compound_head(page);  in virt_to_head_page()
1272 	struct page *page = virt_to_page(x);  in virt_to_folio()  local
1274 	return page_folio(page);  in virt_to_folio()
1281 void split_page(struct page *page, unsigned int order);
1289 static inline unsigned long page_size(struct page *page)  in page_size()  argument
1291 	return PAGE_SIZE << compound_order(page);  in page_size()
1295 static inline unsigned int page_shift(struct page *page)  in page_shift()  argument
1297 	return PAGE_SHIFT + compound_order(page);  in page_shift()
1304 static inline unsigned int thp_order(struct page *page)  in thp_order()  argument
1306 	VM_BUG_ON_PGFLAGS(PageTail(page), page);  in thp_order()
1307 	return compound_order(page);  in thp_order()
1316 static inline unsigned long thp_size(struct page *page)  in thp_size()  argument
1318 	return PAGE_SIZE << thp_order(page);  in thp_size()
1335 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page);
1337 		struct page *page, unsigned int nr, unsigned long addr);
1406 bool __put_devmap_managed_page_refs(struct page *page, int refs);
1407 static inline bool put_devmap_managed_page_refs(struct page *page, int refs)  in put_devmap_managed_page_refs()  argument
1411 	if (!is_zone_device_page(page))  in put_devmap_managed_page_refs()
1413 	return __put_devmap_managed_page_refs(page, refs);  in put_devmap_managed_page_refs()
1416 static inline bool put_devmap_managed_page_refs(struct page *page, int refs)  in put_devmap_managed_page_refs()  argument
1422 static inline bool put_devmap_managed_page(struct page *page)  in put_devmap_managed_page()  argument
1424 	return put_devmap_managed_page_refs(page, 1);  in put_devmap_managed_page()
1445 static inline void get_page(struct page *page)  in get_page()  argument
1447 	folio_get(page_folio(page));  in get_page()
1450 static inline __must_check bool try_get_page(struct page *page)  in try_get_page()  argument
1452 	page = compound_head(page);  in try_get_page()
1453 	if (WARN_ON_ONCE(page_ref_count(page) <= 0))  in try_get_page()
1455 	page_ref_inc(page);  in try_get_page()
1510 	struct page **pages;
1534 static inline void put_page(struct page *page)  in put_page()  argument
1536 	struct folio *folio = page_folio(page);  in put_page()
1542 	if (put_devmap_managed_page(&folio->page))  in put_page()
1579 void unpin_user_page(struct page *page);
1580 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
1582 void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,
1584 void unpin_user_pages(struct page **pages, unsigned long npages);
1618 static inline int page_zone_id(struct page *page)  in page_zone_id()  argument
1620 	return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;  in page_zone_id()
1624 extern int page_to_nid(const struct page *page);
1626 static inline int page_to_nid(const struct page *page)  in page_to_nid()  argument
1628 	struct page *p = (struct page *)page;  in page_to_nid()
1636 	return page_to_nid(&folio->page);  in folio_nid()
1689 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)  in page_cpupid_xchg_last()  argument
1691 	return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);  in page_cpupid_xchg_last()
1694 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
1696 	return page->_last_cpupid;  in page_cpupid_last()
1698 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
1700 	page->_last_cpupid = -1 & LAST_CPUPID_MASK;  in page_cpupid_reset_last()
1703 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
1705 	return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;  in page_cpupid_last()
1708 extern int page_cpupid_xchg_last(struct page *page, int cpupid);
1710 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
1712 	page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;  in page_cpupid_reset_last()
1716 static inline int xchg_page_access_time(struct page *page, int time)  in xchg_page_access_time()  argument
1720 	last_time = page_cpupid_xchg_last(page, time >> PAGE_ACCESS_TIME_BUCKETS);  in xchg_page_access_time()
1734 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)  in page_cpupid_xchg_last()  argument
1736 	return page_to_nid(page); /* XXX */  in page_cpupid_xchg_last()
1739 static inline int xchg_page_access_time(struct page *page, int time)  in xchg_page_access_time()  argument
1744 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
1746 	return page_to_nid(page); /* XXX */  in page_cpupid_last()
1774 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
1796 static inline u8 page_kasan_tag(const struct page *page)  in page_kasan_tag()  argument
1801 		tag = (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK;  in page_kasan_tag()
1808 static inline void page_kasan_tag_set(struct page *page, u8 tag)  in page_kasan_tag_set()  argument
1816 	old_flags = READ_ONCE(page->flags);  in page_kasan_tag_set()
1821 	} while (unlikely(!try_cmpxchg(&page->flags, &old_flags, flags)));  in page_kasan_tag_set()
1824 static inline void page_kasan_tag_reset(struct page *page)  in page_kasan_tag_reset()  argument
1827 		page_kasan_tag_set(page, 0xff);  in page_kasan_tag_reset()
1832 static inline u8 page_kasan_tag(const struct page *page)  in page_kasan_tag()  argument
1837 static inline void page_kasan_tag_set(struct page *page, u8 tag) { }  in page_kasan_tag_set()  argument
1838 static inline void page_kasan_tag_reset(struct page *page) { }  in page_kasan_tag_reset()  argument
1842 static inline struct zone *page_zone(const struct page *page)  in page_zone()  argument
1844 	return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];  in page_zone()
1847 static inline pg_data_t *page_pgdat(const struct page *page)  in page_pgdat()  argument
1849 	return NODE_DATA(page_to_nid(page));  in page_pgdat()
1854 	return page_zone(&folio->page);  in folio_zone()
1859 	return page_pgdat(&folio->page);  in folio_pgdat()
1863 static inline void set_page_section(struct page *page, unsigned long section)  in set_page_section()  argument
1865 	page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);  in set_page_section()
1866 	page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;  in set_page_section()
1869 static inline unsigned long page_to_section(const struct page *page)  in page_to_section()  argument
1871 	return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;  in page_to_section()
1886 	return page_to_pfn(&folio->page);  in folio_pfn()
1936 static inline bool page_maybe_dma_pinned(struct page *page)  in page_maybe_dma_pinned()  argument
1938 	return folio_maybe_dma_pinned(page_folio(page));  in page_maybe_dma_pinned()
1948 					  struct page *page)  in page_needs_cow_for_dma()  argument
1955 	return page_maybe_dma_pinned(page);  in page_needs_cow_for_dma()
1964 static inline bool is_zero_page(const struct page *page)  in is_zero_page()  argument
1966 	return is_zero_pfn(page_to_pfn(page));  in is_zero_page()
1977 	return is_zero_page(&folio->page);  in is_zero_folio()
2009 static inline void set_page_zone(struct page *page, enum zone_type zone)  in set_page_zone()  argument
2011 	page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);  in set_page_zone()
2012 	page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;  in set_page_zone()
2015 static inline void set_page_node(struct page *page, unsigned long node)  in set_page_node()  argument
2017 	page->flags &= ~(NODES_MASK << NODES_PGSHIFT);  in set_page_node()
2018 	page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;  in set_page_node()
2021 static inline void set_page_links(struct page *page, enum zone_type zone,  in set_page_links()  argument
2024 	set_page_zone(page, zone);  in set_page_links()
2025 	set_page_node(page, node);  in set_page_links()
2027 	set_page_section(page, pfn_to_section_nr(pfn));  in set_page_links()
2053 static inline unsigned long compound_nr(struct page *page)  in compound_nr()  argument
2055 	struct folio *folio = (struct folio *)page;  in compound_nr()
2070 static inline int thp_nr_pages(struct page *page)  in thp_nr_pages()  argument
2072 	return folio_nr_pages((struct folio *)page);  in thp_nr_pages()
2143 static inline int arch_make_page_accessible(struct page *page)  in arch_make_page_accessible()  argument
2170 static __always_inline void *lowmem_page_address(const struct page *page)  in lowmem_page_address()  argument
2172 	return page_to_virt(page);  in lowmem_page_address()
2180 static inline void *page_address(const struct page *page)  in page_address()  argument
2182 	return page->virtual;  in page_address()
2184 static inline void set_page_address(struct page *page, void *address)  in set_page_address()  argument
2186 	page->virtual = address;  in set_page_address()
2192 void *page_address(const struct page *page);
2193 void set_page_address(struct page *page, void *virtual);
2198 #define page_address(page) lowmem_page_address(page)  argument
2199 #define set_page_address(page, address)  do { } while(0)  argument
2205 	return page_address(&folio->page);  in folio_address()
2208 extern pgoff_t __page_file_index(struct page *page);
2214 static inline pgoff_t page_index(struct page *page)  in page_index()  argument
2216 	if (unlikely(PageSwapCache(page)))  in page_index()
2217 		return __page_file_index(page);  in page_index()
2218 	return page->index;  in page_index()
2226 static inline bool page_is_pfmemalloc(const struct page *page)  in page_is_pfmemalloc()  argument
2233 	return (uintptr_t)page->lru.next & BIT(1);  in page_is_pfmemalloc()
2255 static inline void set_page_pfmemalloc(struct page *page)  in set_page_pfmemalloc()  argument
2257 	page->lru.next = (void *)BIT(1);  in set_page_pfmemalloc()
2260 static inline void clear_page_pfmemalloc(struct page *page)  in clear_page_pfmemalloc()  argument
2262 	page->lru.next = NULL;  in clear_page_pfmemalloc()
2271 #define offset_in_thp(page, p)	((unsigned long)(p) & (thp_size(page) - 1))  argument
2328 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
2330 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
2365 int generic_error_remove_page(struct address_space *mapping, struct page *page);
2421 			   unsigned int gup_flags, struct page **pages,
2425 			   unsigned int gup_flags, struct page **pages,
2428 static inline struct page *get_user_page_vma_remote(struct mm_struct *mm,  in get_user_page_vma_remote()
2433 	struct page *page;  in get_user_page_vma_remote()  local
2435 	int got = get_user_pages_remote(mm, addr, 1, gup_flags, &page, NULL);  in get_user_page_vma_remote()
2444 		put_page(page);  in get_user_page_vma_remote()
2449 	return page;  in get_user_page_vma_remote()
2453 		    unsigned int gup_flags, struct page **pages);
2455 		    unsigned int gup_flags, struct page **pages);
2457 		    struct page **pages, unsigned int gup_flags);
2459 		    struct page **pages, unsigned int gup_flags);
2462 			unsigned int gup_flags, struct page **pages);
2464 			unsigned int gup_flags, struct page **pages);
2472 struct page *get_dump_page(unsigned long addr);
2475 bool set_page_dirty(struct page *page);
2476 int set_page_dirty_lock(struct page *page);
2533 			     unsigned int gup_flags, struct page **pages);
2536 			unsigned int gup_flags, struct page **pagep)  in get_user_page_fast_only()
2572 static inline int mm_counter_file(struct page *page)  in mm_counter_file()  argument
2574 	if (PageSwapBacked(page))  in mm_counter_file()
2579 static inline int mm_counter(struct page *page)  in mm_counter()  argument
2581 	if (PageAnon(page))  in mm_counter()
2583 	return mm_counter_file(page);  in mm_counter()
2823 	struct page *page = alloc_pages(gfp | __GFP_COMP, order);  in pagetable_alloc()  local
2825 	return page_ptdesc(page);  in pagetable_alloc()
2837 	struct page *page = ptdesc_page(pt);  in pagetable_free()  local
2839 	__free_pages(page, compound_order(page));  in pagetable_free()
2966 static inline struct page *pmd_pgtable_page(pmd_t *pmd)  in pmd_pgtable_page()
3072 extern void adjust_managed_page_count(struct page *page, long count);
3078 static inline void free_reserved_page(struct page *page)  in free_reserved_page()  argument
3080 	ClearPageReserved(page);  in free_reserved_page()
3081 	init_page_count(page);  in free_reserved_page()
3082 	__free_page(page);  in free_reserved_page()
3083 	adjust_managed_page_count(page, 1);  in free_reserved_page()
3085 #define free_highmem_page(page) free_reserved_page(page)  argument
3087 static inline void mark_page_reserved(struct page *page)  in mark_page_reserved()  argument
3089 	SetPageReserved(page);  in mark_page_reserved()
3090 	adjust_managed_page_count(page, -1);  in mark_page_reserved()
3275 				   unsigned long flags, struct page **pages);
3461 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
3463 			struct page **pages, unsigned long *num);
3464 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
3466 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
3479 				unsigned long addr, struct page *page)  in vmf_insert_page()  argument
3481 	int err = vm_insert_page(vma, addr, page);  in vmf_insert_page()
3527 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
3573 extern void __kernel_poison_pages(struct page *page, int numpages);
3574 extern void __kernel_unpoison_pages(struct page *page, int numpages);
3589 static inline void kernel_poison_pages(struct page *page, int numpages)  in kernel_poison_pages()  argument
3592 		__kernel_poison_pages(page, numpages);  in kernel_poison_pages()
3594 static inline void kernel_unpoison_pages(struct page *page, int numpages)  in kernel_unpoison_pages()  argument
3597 		__kernel_unpoison_pages(page, numpages);  in kernel_unpoison_pages()
3602 static inline void __kernel_poison_pages(struct page *page, int nunmpages) { }  in __kernel_poison_pages()  argument
3603 static inline void kernel_poison_pages(struct page *page, int numpages) { }  in kernel_poison_pages()  argument
3604 static inline void kernel_unpoison_pages(struct page *page, int numpages) { }  in kernel_unpoison_pages()  argument
3648 extern void __kernel_map_pages(struct page *page, int numpages, int enable);
3650 static inline void debug_pagealloc_map_pages(struct page *page, int numpages)  in debug_pagealloc_map_pages()  argument
3653 		__kernel_map_pages(page, numpages, 1);  in debug_pagealloc_map_pages()
3656 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages)  in debug_pagealloc_unmap_pages()  argument
3659 		__kernel_map_pages(page, numpages, 0);  in debug_pagealloc_unmap_pages()
3675 static inline bool page_is_guard(struct page *page)  in page_is_guard()  argument
3680 	return PageGuard(page);  in page_is_guard()
3683 bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order,
3685 static inline bool set_page_guard(struct zone *zone, struct page *page,  in set_page_guard()  argument
3690 	return __set_page_guard(zone, page, order, migratetype);  in set_page_guard()
3693 void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order,
3695 static inline void clear_page_guard(struct zone *zone, struct page *page,  in clear_page_guard()  argument
3700 	__clear_page_guard(zone, page, order, migratetype);  in clear_page_guard()
3704 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {}  in debug_pagealloc_map_pages()  argument
3705 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {}  in debug_pagealloc_unmap_pages()  argument
3708 static inline bool page_is_guard(struct page *page) { return false; }  in page_is_guard()  argument
3709 static inline bool set_page_guard(struct zone *zone, struct page *page,  in set_page_guard()  argument
3711 static inline void clear_page_guard(struct zone *zone, struct page *page,  in clear_page_guard()  argument
3757 struct page * __populate_section_memmap(unsigned long pfn,
3767 			    struct vmem_altmap *altmap, struct page *reuse);
3797 	if (!pgmap || !is_power_of_2(sizeof(struct page)))  in __vmemmap_can_optimize()
3801 	nr_vmemmap_pages = ((nr_pages * sizeof(struct page)) >> PAGE_SHIFT);  in __vmemmap_can_optimize()
3823 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
3840 extern void shake_page(struct page *p);
3875 void add_to_kill_ksm(struct task_struct *tsk, struct page *p,
3941 extern void clear_huge_page(struct page *page,
3975 extern int memcmp_pages(struct page *page1, struct page *page2);
3977 static inline int pages_identical(struct page *page1, struct page *page2)  in pages_identical()