150 static inline void __mm_zero_struct_page(struct page *page)  in __mm_zero_struct_page()  argument
152 	unsigned long *_pp = (void *)page;  in __mm_zero_struct_page()
155 	BUILD_BUG_ON(sizeof(struct page) & 7);  in __mm_zero_struct_page()
156 	BUILD_BUG_ON(sizeof(struct page) < 56);  in __mm_zero_struct_page()
157 	BUILD_BUG_ON(sizeof(struct page) > 80);  in __mm_zero_struct_page()
159 	switch (sizeof(struct page)) {  in __mm_zero_struct_page()
177 #define mm_zero_struct_page(pp)  ((void)memset((pp), 0, sizeof(struct page)))
213 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))  argument
221 #define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
430 	struct page *cow_page;		/* Page handler may use for COW fault */
432 	struct page *page;		/* ->fault handlers should return a  member
525 	struct page *(*find_special_page)(struct vm_area_struct *vma,
605 static inline int put_page_testzero(struct page *page)  in put_page_testzero()  argument
607 	VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);  in put_page_testzero()
608 	return page_ref_dec_and_test(page);  in put_page_testzero()
617 static inline int get_page_unless_zero(struct page *page)  in get_page_unless_zero()  argument
619 	return page_ref_add_unless(page, 1, 0);  in get_page_unless_zero()
634 struct page *vmalloc_to_page(const void *addr);
698 static inline int compound_mapcount(struct page *page)  in compound_mapcount()  argument
700 	VM_BUG_ON_PAGE(!PageCompound(page), page);  in compound_mapcount()
701 	page = compound_head(page);  in compound_mapcount()
702 	return atomic_read(compound_mapcount_ptr(page)) + 1;  in compound_mapcount()
710 static inline void page_mapcount_reset(struct page *page)  in page_mapcount_reset()  argument
712 	atomic_set(&(page)->_mapcount, -1);  in page_mapcount_reset()
715 int __page_mapcount(struct page *page);
717 static inline int page_mapcount(struct page *page)  in page_mapcount()  argument
719 	VM_BUG_ON_PAGE(PageSlab(page), page);  in page_mapcount()
721 	if (unlikely(PageCompound(page)))  in page_mapcount()
722 		return __page_mapcount(page);  in page_mapcount()
723 	return atomic_read(&page->_mapcount) + 1;  in page_mapcount()
727 int total_mapcount(struct page *page);
728 int page_trans_huge_mapcount(struct page *page, int *total_mapcount);
730 static inline int total_mapcount(struct page *page)  in total_mapcount()  argument
732 	return page_mapcount(page);  in total_mapcount()
734 static inline int page_trans_huge_mapcount(struct page *page,  in page_trans_huge_mapcount()  argument
737 	int mapcount = page_mapcount(page);  in page_trans_huge_mapcount()
744 static inline struct page *virt_to_head_page(const void *x)  in virt_to_head_page()
746 	struct page *page = virt_to_page(x);  in virt_to_head_page()  local
748 	return compound_head(page);  in virt_to_head_page()
751 void __put_page(struct page *page);
755 void split_page(struct page *page, unsigned int order);
762 typedef void compound_page_dtor(struct page *);
778 static inline void set_compound_page_dtor(struct page *page,  in set_compound_page_dtor()  argument
781 	VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page);  in set_compound_page_dtor()
782 	page[1].compound_dtor = compound_dtor;  in set_compound_page_dtor()
785 static inline compound_page_dtor *get_compound_page_dtor(struct page *page)  in get_compound_page_dtor()  argument
787 	VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page);  in get_compound_page_dtor()
788 	return compound_page_dtors[page[1].compound_dtor];  in get_compound_page_dtor()
791 static inline unsigned int compound_order(struct page *page)  in compound_order()  argument
793 	if (!PageHead(page))  in compound_order()
795 	return page[1].compound_order;  in compound_order()
798 static inline void set_compound_order(struct page *page, unsigned int order)  in set_compound_order()  argument
800 	page[1].compound_order = order;  in set_compound_order()
804 static inline unsigned long compound_nr(struct page *page)  in compound_nr()  argument
806 	return 1UL << compound_order(page);  in compound_nr()
810 static inline unsigned long page_size(struct page *page)  in page_size()  argument
812 	return PAGE_SIZE << compound_order(page);  in page_size()
816 static inline unsigned int page_shift(struct page *page)  in page_shift()  argument
818 	return PAGE_SHIFT + compound_order(page);  in page_shift()
821 void free_compound_page(struct page *page);
838 		struct page *page);
950 static inline enum zone_type page_zonenum(const struct page *page)  in page_zonenum()  argument
952 	return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;  in page_zonenum()
956 static inline bool is_zone_device_page(const struct page *page)  in is_zone_device_page()  argument
958 	return page_zonenum(page) == ZONE_DEVICE;  in is_zone_device_page()
963 static inline bool is_zone_device_page(const struct page *page)  in is_zone_device_page()  argument
970 void __put_devmap_managed_page(struct page *page);
972 static inline bool put_devmap_managed_page(struct page *page)  in put_devmap_managed_page()  argument
976 	if (!is_zone_device_page(page))  in put_devmap_managed_page()
978 	switch (page->pgmap->type) {  in put_devmap_managed_page()
981 		__put_devmap_managed_page(page);  in put_devmap_managed_page()
990 static inline bool put_devmap_managed_page(struct page *page)  in put_devmap_managed_page()  argument
996 static inline bool is_device_private_page(const struct page *page)  in is_device_private_page()  argument
1000 		is_zone_device_page(page) &&  in is_device_private_page()
1001 		page->pgmap->type == MEMORY_DEVICE_PRIVATE;  in is_device_private_page()
1004 static inline bool is_pci_p2pdma_page(const struct page *page)  in is_pci_p2pdma_page()  argument
1008 		is_zone_device_page(page) &&  in is_pci_p2pdma_page()
1009 		page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA;  in is_pci_p2pdma_page()
1013 #define page_ref_zero_or_close_to_overflow(page) \  argument
1014 	((unsigned int) page_ref_count(page) + 127u <= 127u)
1016 static inline void get_page(struct page *page)  in get_page()  argument
1018 	page = compound_head(page);  in get_page()
1023 	VM_BUG_ON_PAGE(page_ref_zero_or_close_to_overflow(page), page);  in get_page()
1024 	page_ref_inc(page);  in get_page()
1027 static inline __must_check bool try_get_page(struct page *page)  in try_get_page()  argument
1029 	page = compound_head(page);  in try_get_page()
1030 	if (WARN_ON_ONCE(page_ref_count(page) <= 0))  in try_get_page()
1032 	page_ref_inc(page);  in try_get_page()
1036 static inline void put_page(struct page *page)  in put_page()  argument
1038 	page = compound_head(page);  in put_page()
1046 	if (put_devmap_managed_page(page))  in put_page()
1049 	if (put_page_testzero(page))  in put_page()
1050 		__put_page(page);  in put_page()
1068 static inline void put_user_page(struct page *page)  in put_user_page()  argument
1070 	put_page(page);  in put_user_page()
1073 void put_user_pages_dirty_lock(struct page **pages, unsigned long npages,
1076 void put_user_pages(struct page **pages, unsigned long npages);
1090 static inline int page_zone_id(struct page *page)  in page_zone_id()  argument
1092 	return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;  in page_zone_id()
1096 extern int page_to_nid(const struct page *page);
1098 static inline int page_to_nid(const struct page *page)  in page_to_nid()  argument
1100 	struct page *p = (struct page *)page;  in page_to_nid()
1144 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)  in page_cpupid_xchg_last()  argument
1146 	return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);  in page_cpupid_xchg_last()
1149 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
1151 	return page->_last_cpupid;  in page_cpupid_last()
1153 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
1155 	page->_last_cpupid = -1 & LAST_CPUPID_MASK;  in page_cpupid_reset_last()
1158 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
1160 	return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;  in page_cpupid_last()
1163 extern int page_cpupid_xchg_last(struct page *page, int cpupid);
1165 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
1167 	page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;  in page_cpupid_reset_last()
1171 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)  in page_cpupid_xchg_last()  argument
1173 	return page_to_nid(page); /* XXX */  in page_cpupid_xchg_last()
1176 static inline int page_cpupid_last(struct page *page)  in page_cpupid_last()  argument
1178 	return page_to_nid(page); /* XXX */  in page_cpupid_last()
1206 static inline void page_cpupid_reset_last(struct page *page)  in page_cpupid_reset_last()  argument
1217 static inline u8 page_kasan_tag(const struct page *page)  in page_kasan_tag()  argument
1219 	return (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK;  in page_kasan_tag()
1222 static inline void page_kasan_tag_set(struct page *page, u8 tag)  in page_kasan_tag_set()  argument
1224 	page->flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT);  in page_kasan_tag_set()
1225 	page->flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT;  in page_kasan_tag_set()
1228 static inline void page_kasan_tag_reset(struct page *page)  in page_kasan_tag_reset()  argument
1230 	page_kasan_tag_set(page, 0xff);  in page_kasan_tag_reset()
1233 static inline u8 page_kasan_tag(const struct page *page)  in page_kasan_tag()  argument
1238 static inline void page_kasan_tag_set(struct page *page, u8 tag) { }  in page_kasan_tag_set()  argument
1239 static inline void page_kasan_tag_reset(struct page *page) { }  in page_kasan_tag_reset()  argument
1242 static inline struct zone *page_zone(const struct page *page)  in page_zone()  argument
1244 	return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];  in page_zone()
1247 static inline pg_data_t *page_pgdat(const struct page *page)  in page_pgdat()  argument
1249 	return NODE_DATA(page_to_nid(page));  in page_pgdat()
1253 static inline void set_page_section(struct page *page, unsigned long section)  in set_page_section()  argument
1255 	page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);  in set_page_section()
1256 	page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;  in set_page_section()
1259 static inline unsigned long page_to_section(const struct page *page)  in page_to_section()  argument
1261 	return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;  in page_to_section()
1265 static inline void set_page_zone(struct page *page, enum zone_type zone)  in set_page_zone()  argument
1267 	page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);  in set_page_zone()
1268 	page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;  in set_page_zone()
1271 static inline void set_page_node(struct page *page, unsigned long node)  in set_page_node()  argument
1273 	page->flags &= ~(NODES_MASK << NODES_PGSHIFT);  in set_page_node()
1274 	page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;  in set_page_node()
1277 static inline void set_page_links(struct page *page, enum zone_type zone,  in set_page_links()  argument
1280 	set_page_zone(page, zone);  in set_page_links()
1281 	set_page_node(page, node);  in set_page_links()
1283 	set_page_section(page, pfn_to_section_nr(pfn));  in set_page_links()
1288 static inline struct mem_cgroup *page_memcg(struct page *page)  in page_memcg()  argument
1290 	return page->mem_cgroup;  in page_memcg()
1292 static inline struct mem_cgroup *page_memcg_rcu(struct page *page)  in page_memcg_rcu()  argument
1295 	return READ_ONCE(page->mem_cgroup);  in page_memcg_rcu()
1298 static inline struct mem_cgroup *page_memcg(struct page *page)  in page_memcg()  argument
1302 static inline struct mem_cgroup *page_memcg_rcu(struct page *page)  in page_memcg_rcu()  argument
1314 static __always_inline void *lowmem_page_address(const struct page *page)  in lowmem_page_address()  argument
1316 	return page_to_virt(page);  in lowmem_page_address()
1324 static inline void *page_address(const struct page *page)  in page_address()  argument
1326 	return page->virtual;  in page_address()
1328 static inline void set_page_address(struct page *page, void *address)  in set_page_address()  argument
1330 	page->virtual = address;  in set_page_address()
1336 void *page_address(const struct page *page);
1337 void set_page_address(struct page *page, void *virtual);
1342 #define page_address(page) lowmem_page_address(page)  argument
1343 #define set_page_address(page, address)  do { } while(0)  argument
1347 extern void *page_rmapping(struct page *page);
1348 extern struct anon_vma *page_anon_vma(struct page *page);
1349 extern struct address_space *page_mapping(struct page *page);
1351 extern struct address_space *__page_file_mapping(struct page *);
1354 struct address_space *page_file_mapping(struct page *page)  in page_file_mapping()  argument
1356 	if (unlikely(PageSwapCache(page)))  in page_file_mapping()
1357 		return __page_file_mapping(page);  in page_file_mapping()
1359 	return page->mapping;  in page_file_mapping()
1362 extern pgoff_t __page_file_index(struct page *page);
1368 static inline pgoff_t page_index(struct page *page)  in page_index()  argument
1370 	if (unlikely(PageSwapCache(page)))  in page_index()
1371 		return __page_file_index(page);  in page_index()
1372 	return page->index;  in page_index()
1375 bool page_mapped(struct page *page);
1376 struct address_space *page_mapping(struct page *page);
1377 struct address_space *page_mapping_file(struct page *page);
1384 static inline bool page_is_pfmemalloc(struct page *page)  in page_is_pfmemalloc()  argument
1390 	return page->index == -1UL;  in page_is_pfmemalloc()
1397 static inline void set_page_pfmemalloc(struct page *page)  in set_page_pfmemalloc()  argument
1399 	page->index = -1UL;  in set_page_pfmemalloc()
1402 static inline void clear_page_pfmemalloc(struct page *page)  in clear_page_pfmemalloc()  argument
1404 	page->index = 0;  in clear_page_pfmemalloc()
1439 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1441 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
1471 int truncate_inode_page(struct address_space *mapping, struct page *page);
1472 int generic_error_remove_page(struct address_space *mapping, struct page *page);
1473 int invalidate_inode_page(struct page *page);
1522 			    unsigned int gup_flags, struct page **pages,
1525 			    unsigned int gup_flags, struct page **pages,
1528 		    unsigned int gup_flags, struct page **pages, int *locked);
1530 		    struct page **pages, unsigned int gup_flags);
1533 			unsigned int gup_flags, struct page **pages);
1563 static inline struct page **frame_vector_pages(struct frame_vector *vec)  in frame_vector_pages()
1571 	return (struct page **)(vec->ptrs);  in frame_vector_pages()
1583 			struct page **pages);
1584 int get_kernel_page(unsigned long start, int write, struct page **pages);
1585 struct page *get_dump_page(unsigned long addr);
1587 extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
1588 extern void do_invalidatepage(struct page *page, unsigned int offset,
1591 void __set_page_dirty(struct page *, struct address_space *, int warn);
1592 int __set_page_dirty_nobuffers(struct page *page);
1593 int __set_page_dirty_no_writeback(struct page *page);
1595 				struct page *page);
1596 void account_page_dirtied(struct page *page, struct address_space *mapping);
1597 void account_page_cleaned(struct page *page, struct address_space *mapping,
1599 int set_page_dirty(struct page *page);
1600 int set_page_dirty_lock(struct page *page);
1601 void __cancel_dirty_page(struct page *page);
1602 static inline void cancel_dirty_page(struct page *page)  in cancel_dirty_page()  argument
1605 	if (PageDirty(page))  in cancel_dirty_page()
1606 		__cancel_dirty_page(page);  in cancel_dirty_page()
1608 int clear_page_dirty_for_io(struct page *page);
1627 			  struct page **pages);
1662 static inline int mm_counter_file(struct page *page)  in mm_counter_file()  argument
1664 	if (PageSwapBacked(page))  in mm_counter_file()
1669 static inline int mm_counter(struct page *page)  in mm_counter()  argument
1671 	if (PageAnon(page))  in mm_counter()
1673 	return mm_counter_file(page);  in mm_counter()
1880 extern bool ptlock_alloc(struct page *page);
1881 extern void ptlock_free(struct page *page);
1883 static inline spinlock_t *ptlock_ptr(struct page *page)  in ptlock_ptr()  argument
1885 	return page->ptl;  in ptlock_ptr()
1892 static inline bool ptlock_alloc(struct page *page)  in ptlock_alloc()  argument
1897 static inline void ptlock_free(struct page *page)  in ptlock_free()  argument
1901 static inline spinlock_t *ptlock_ptr(struct page *page)  in ptlock_ptr()  argument
1903 	return &page->ptl;  in ptlock_ptr()
1912 static inline bool ptlock_init(struct page *page)  in ptlock_init()  argument
1921 	VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);  in ptlock_init()
1922 	if (!ptlock_alloc(page))  in ptlock_init()
1924 	spin_lock_init(ptlock_ptr(page));  in ptlock_init()
1937 static inline bool ptlock_init(struct page *page) { return true; }  in ptlock_init()  argument
1938 static inline void ptlock_free(struct page *page) {}  in ptlock_free()  argument
1947 static inline bool pgtable_pte_page_ctor(struct page *page)  in pgtable_pte_page_ctor()  argument
1949 	if (!ptlock_init(page))  in pgtable_pte_page_ctor()
1951 	__SetPageTable(page);  in pgtable_pte_page_ctor()
1952 	inc_zone_page_state(page, NR_PAGETABLE);  in pgtable_pte_page_ctor()
1956 static inline void pgtable_pte_page_dtor(struct page *page)  in pgtable_pte_page_dtor()  argument
1958 	ptlock_free(page);  in pgtable_pte_page_dtor()
1959 	__ClearPageTable(page);  in pgtable_pte_page_dtor()
1960 	dec_zone_page_state(page, NR_PAGETABLE);  in pgtable_pte_page_dtor()
1992 static struct page *pmd_to_page(pmd_t *pmd)  in pmd_to_page()
2003 static inline bool pgtable_pmd_page_ctor(struct page *page)  in pgtable_pmd_page_ctor()  argument
2006 	page->pmd_huge_pte = NULL;  in pgtable_pmd_page_ctor()
2008 	return ptlock_init(page);  in pgtable_pmd_page_ctor()
2011 static inline void pgtable_pmd_page_dtor(struct page *page)  in pgtable_pmd_page_dtor()  argument
2014 	VM_BUG_ON_PAGE(page->pmd_huge_pte, page);  in pgtable_pmd_page_dtor()
2016 	ptlock_free(page);  in pgtable_pmd_page_dtor()
2028 static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }  in pgtable_pmd_page_ctor()  argument
2029 static inline void pgtable_pmd_page_dtor(struct page *page) {}  in pgtable_pmd_page_dtor()  argument
2081 extern void free_highmem_page(struct page *page);
2084 extern void adjust_managed_page_count(struct page *page, long count);
2090 static inline void __free_reserved_page(struct page *page)  in __free_reserved_page()  argument
2092 	ClearPageReserved(page);  in __free_reserved_page()
2093 	init_page_count(page);  in __free_reserved_page()
2094 	__free_page(page);  in __free_reserved_page()
2097 static inline void free_reserved_page(struct page *page)  in free_reserved_page()  argument
2099 	__free_reserved_page(page);  in free_reserved_page()
2100 	adjust_managed_page_count(page, 1);  in free_reserved_page()
2103 static inline void mark_page_reserved(struct page *page)  in mark_page_reserved()  argument
2105 	SetPageReserved(page);  in mark_page_reserved()
2106 	adjust_managed_page_count(page, -1);  in mark_page_reserved()
2324 				   unsigned long flags, struct page **pages);
2415 int __must_check write_one_page(struct page *page);
2433 				struct page *pg,
2535 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
2536 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2538 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
2551 				unsigned long addr, struct page *page)  in vmf_insert_page()  argument
2553 	int err = vm_insert_page(vma, addr, page);  in vmf_insert_page()
2570 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
2638 extern void kernel_poison_pages(struct page *page, int numpages, int enable);
2641 static inline void kernel_poison_pages(struct page *page, int numpages,  in kernel_poison_pages()  argument
2684 extern void __kernel_map_pages(struct page *page, int numpages, int enable);
2687 kernel_map_pages(struct page *page, int numpages, int enable)  in kernel_map_pages()  argument
2689 	__kernel_map_pages(page, numpages, enable);  in kernel_map_pages()
2692 extern bool kernel_page_present(struct page *page);
2696 kernel_map_pages(struct page *page, int numpages, int enable) {}  in kernel_map_pages()  argument
2698 static inline bool kernel_page_present(struct page *page) { return true; }  in kernel_page_present()  argument
2745 struct page * __populate_section_memmap(unsigned long pfn,
2766 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
2778 extern int get_hwpoison_page(struct page *page);
2779 #define put_hwpoison_page(page)	put_page(page)  argument
2782 extern void shake_page(struct page *p, int access);
2784 extern int soft_offline_page(struct page *page, int flags);
2823 extern void clear_huge_page(struct page *page,
2826 extern void copy_user_huge_page(struct page *dst, struct page *src,
2830 extern long copy_huge_page_from_user(struct page *dst_page,
2850 static inline bool page_is_guard(struct page *page)  in page_is_guard()  argument
2855 	return PageGuard(page);  in page_is_guard()
2860 static inline bool page_is_guard(struct page *page) { return false; }  in page_is_guard()  argument
2869 extern int memcmp_pages(struct page *page1, struct page *page2);
2871 static inline int pages_identical(struct page *page1, struct page *page2)  in pages_identical()