3  * Macros for manipulating and testing page->flags
18  * Various page->flags bits:
20  * PG_reserved is set for special pages. The "struct page" of such a page
25  * - Pages reserved or allocated early during boot (before the page allocator
27  *   initial vmemmap, initial page tables, crashkernel, elfcorehdr, and much
29  *   be given to the page allocator.
32  * - The zero page(s)
33  * - Pages not added to the page allocator when onlining a section because
49  * Consequently, PG_reserved for a page mapped into user space can indicate
50  * the zero page, the vDSO, MMIO pages or device memory.
53  * specific data (which is normally at page->private). It can be used by
60  * PG_locked also pins a page in pagecache, and blocks truncation of the file
63  * page_waitqueue(page) is a wait queue of all tasks waiting for the page
66  * PG_swapbacked is set when a page uses swap as a backing storage.  This are
71  * PG_uptodate tells whether the page's contents is valid.  When a read
72  * completes, the page becomes uptodate, unless a disk I/O error happened.
74  * PG_referenced, PG_reclaim are used for page reclaim for anonymous and
77  * PG_error is set to indicate that an I/O error occurred on this page.
79  * PG_arch_1 is an architecture specific page state bit.  The generic code
80  * guarantees that this bit is cleared for a page when it first is entered into
81  * the page cache.
83  * PG_hwpoison indicates that a page got corrupted in hardware and contains
91  * The page flags field is split into two parts, the main flags area
104 	PG_locked,		/* Page is locked. Don't touch. */
111 …PG_waiters,		/* Page has waiters, check its waitqueue. Must be bit #7 and in the same byte as "PG_…
119 	PG_writeback,		/* Page is under writeback */
120 	PG_head,		/* A head page */
123 	PG_swapbacked,		/* Page is backed by RAM/swap */
124 	PG_unevictable,		/* Page is "unevictable"  */
126 	PG_mlocked,		/* Page is vma mlocked */
129 	PG_uncached,		/* Page has been mapped as uncached */
132 	PG_hwpoison,		/* hardware poisoned page. Don't touch */
150 	PG_swapcache = PG_owner_priv_1,	/* Swap page: swp_entry_t in private */
152 	/* Two page bits are conscripted by FS-Cache to maintain local caching
156 	PG_fscache = PG_private_2,	/* page backed by cache */
159 	/* Pinned in Xen as a read-only pagetable page. */
163 	/* Has a grant mapping of another (foreign) domain's page. */
171 	/* Compound pages. Stored in first tail page's flags */
176 	 * Compound pages. Stored in first tail page's flags.
183 	/* non-lru isolated movable page */
194 static inline unsigned long _compound_head(const struct page *page)  in _compound_head()  argument
196 	unsigned long head = READ_ONCE(page->compound_head);  in _compound_head()
200 	return (unsigned long)page;  in _compound_head()
203 #define compound_head(page)	((typeof(page))_compound_head(page))  argument
205 static __always_inline int PageTail(struct page *page)  in PageTail()  argument
207 	return READ_ONCE(page->compound_head) & 1;  in PageTail()
210 static __always_inline int PageCompound(struct page *page)  in PageCompound()  argument
212 	return test_bit(PG_head, &page->flags) || PageTail(page);  in PageCompound()
216 static inline int PagePoisoned(const struct page *page)  in PagePoisoned()  argument
218 	return page->flags == PAGE_POISON_PATTERN;  in PagePoisoned()
222 void page_init_poison(struct page *page, size_t size);
224 static inline void page_init_poison(struct page *page, size_t size)  in page_init_poison()  argument
230  * Page flags policies wrt compound pages
233  *     check if this struct page poisoned/uninitialized
236  *     the page flag is relevant for small, head and tail pages.
239  *     for compound page all operations related to the page flag applied to
240  *     head page.
243  *     for compound page, callers only ever operate on the head page.
246  *     modifications of the page flag must be done on small or head pages,
250  *     the page flag is not relevant for compound pages.
253  *     the page flag is stored in the first tail page.
255 #define PF_POISONED_CHECK(page) ({					\  argument
256 		VM_BUG_ON_PGFLAGS(PagePoisoned(page), page);		\
257 		page; })
258 #define PF_ANY(page, enforce)	PF_POISONED_CHECK(page)  argument
259 #define PF_HEAD(page, enforce)	PF_POISONED_CHECK(compound_head(page))  argument
260 #define PF_ONLY_HEAD(page, enforce) ({					\  argument
261 		VM_BUG_ON_PGFLAGS(PageTail(page), page);		\
262 		PF_POISONED_CHECK(page); })
263 #define PF_NO_TAIL(page, enforce) ({					\  argument
264 		VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page);	\
265 		PF_POISONED_CHECK(compound_head(page)); })
266 #define PF_NO_COMPOUND(page, enforce) ({				\  argument
267 		VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page);	\
268 		PF_POISONED_CHECK(page); })
269 #define PF_SECOND(page, enforce) ({					\  argument
270 		VM_BUG_ON_PGFLAGS(!PageHead(page), page);		\
271 		PF_POISONED_CHECK(&page[1]); })
274  * Macros to create function definitions for page flags
277 static __always_inline int Page##uname(struct page *page)		\
278 	{ return test_bit(PG_##lname, &policy(page, 0)->flags); }
281 static __always_inline void SetPage##uname(struct page *page)		\
282 	{ set_bit(PG_##lname, &policy(page, 1)->flags); }
285 static __always_inline void ClearPage##uname(struct page *page)		\
286 	{ clear_bit(PG_##lname, &policy(page, 1)->flags); }
289 static __always_inline void __SetPage##uname(struct page *page)		\
290 	{ __set_bit(PG_##lname, &policy(page, 1)->flags); }
293 static __always_inline void __ClearPage##uname(struct page *page)	\
294 	{ __clear_bit(PG_##lname, &policy(page, 1)->flags); }
297 static __always_inline int TestSetPage##uname(struct page *page)	\
298 	{ return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
301 static __always_inline int TestClearPage##uname(struct page *page)	\
302 	{ return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
319 static inline int Page##uname(const struct page *page) { return 0; }
322 static inline void SetPage##uname(struct page *page) {  }
325 static inline void ClearPage##uname(struct page *page) {  }
328 static inline void __ClearPage##uname(struct page *page) {  }
331 static inline int TestSetPage##uname(struct page *page) { return 0; }
334 static inline int TestClearPage##uname(struct page *page) { return 0; }
376  * Private page markings that may be used by the filesystem that owns the page  in PAGEFLAG()
387  * risky: they bypass page accounting.  in PAGEFLAG()
410 static __always_inline int PageSwapCache(struct page *page)  in PAGEFLAG()
413 	page = compound_head(page);  in PAGEFLAG()
415 	return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags);  in PAGEFLAG()
447 extern bool take_page_off_buddy(struct page *page);
475  * On an anonymous page mapped into a user virtual memory area,  in TESTPAGEFLAG()
476  * page->mapping points to its anon_vma, not to a struct address_space;  in TESTPAGEFLAG()
479  * On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled,  in TESTPAGEFLAG()
481  * bit; and then page->mapping points, not to an anon_vma, but to a private  in TESTPAGEFLAG()
482  * structure which KSM associates with that merged page.  See ksm.h.  in TESTPAGEFLAG()
485  * page and then page->mapping points a struct address_space.  in TESTPAGEFLAG()
488  * address_space which maps the page from disk; whereas "page_mapped"  in TESTPAGEFLAG()
489  * refers to user virtual address space into which the page is mapped.  in TESTPAGEFLAG()
496 static __always_inline int PageMappingFlags(struct page *page)  in TESTPAGEFLAG()
498 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;  in TESTPAGEFLAG()
501 static __always_inline int PageAnon(struct page *page)  in PageAnon()  argument
503 	page = compound_head(page);  in PageAnon()
504 	return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;  in PageAnon()
507 static __always_inline int __PageMovable(struct page *page)  in __PageMovable()  argument
509 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==  in __PageMovable()
515  * A KSM page is one of those write-protected "shared pages" or "merged pages"
516  * which KSM maps into multiple mms, wherever identical anonymous page content
517  * is found in VM_MERGEABLE vmas.  It's a PageAnon page, pointing not to any
518  * anon_vma, but to that page's node of the stable tree.
520 static __always_inline int PageKsm(struct page *page)  in PageKsm()  argument
522 	page = compound_head(page);  in PageKsm()
523 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==  in PageKsm()
530 u64 stable_page_flags(struct page *page);
532 static inline int PageUptodate(struct page *page)  in PageUptodate()  argument
535 	page = compound_head(page);  in PageUptodate()
536 	ret = test_bit(PG_uptodate, &(page)->flags);  in PageUptodate()
538 	 * Must ensure that the data we read out of the page is loaded  in PageUptodate()
539 	 * _after_ we've loaded page->flags to check for PageUptodate.  in PageUptodate()
540 	 * We can skip the barrier if the page is not uptodate, because  in PageUptodate()
551 static __always_inline void __SetPageUptodate(struct page *page)  in __SetPageUptodate()  argument
553 	VM_BUG_ON_PAGE(PageTail(page), page);  in __SetPageUptodate()
555 	__set_bit(PG_uptodate, &page->flags);  in __SetPageUptodate()
558 static __always_inline void SetPageUptodate(struct page *page)  in SetPageUptodate()  argument
560 	VM_BUG_ON_PAGE(PageTail(page), page);  in SetPageUptodate()
563 	 * so that all previous stores issued in order to bring the page  in SetPageUptodate()
567 	set_bit(PG_uptodate, &page->flags);  in SetPageUptodate()
572 int test_clear_page_writeback(struct page *page);
573 int __test_set_page_writeback(struct page *page, bool keep_write);
575 #define test_set_page_writeback(page)			\  argument
576 	__test_set_page_writeback(page, false)
577 #define test_set_page_writeback_keepwrite(page)	\  argument
578 	__test_set_page_writeback(page, true)
580 static inline void set_page_writeback(struct page *page)  in set_page_writeback()  argument
582 	test_set_page_writeback(page);  in set_page_writeback()
585 static inline void set_page_writeback_keepwrite(struct page *page)  in set_page_writeback_keepwrite()  argument
587 	test_set_page_writeback_keepwrite(page);  in set_page_writeback_keepwrite()
592 static __always_inline void set_compound_head(struct page *page, struct page *head)  in __PAGEFLAG()
594 	WRITE_ONCE(page->compound_head, (unsigned long)head + 1);  in __PAGEFLAG()
597 static __always_inline void clear_compound_head(struct page *page)  in clear_compound_head()  argument
599 	WRITE_ONCE(page->compound_head, 0);  in clear_compound_head()
603 static inline void ClearPageCompound(struct page *page)  in ClearPageCompound()  argument
605 	BUG_ON(!PageHead(page));  in ClearPageCompound()
606 	ClearPageHead(page);  in ClearPageCompound()
613 int PageHuge(struct page *page);
614 int PageHeadHuge(struct page *page);
630 static inline int PageTransHuge(struct page *page)  in TESTPAGEFLAG_FALSE()
632 	VM_BUG_ON_PAGE(PageTail(page), page);  in TESTPAGEFLAG_FALSE()
633 	return PageHead(page);  in TESTPAGEFLAG_FALSE()
641 static inline int PageTransCompound(struct page *page)  in PageTransCompound()  argument
643 	return PageCompound(page);  in PageTransCompound()
651 static inline int PageTransTail(struct page *page)  in PageTransTail()  argument
653 	return PageTail(page);  in PageTransTail()
657  * PageDoubleMap indicates that the compound page is mapped with PTEs as well
661  * per small page mapcount accounting (and its overhead from atomic operations)
664  * For the page PageDoubleMap means ->_mapcount in all sub-pages is offset up
683  * compound page.  in PAGEFLAG()
685  * This flag is set by hwpoison handler.  Cleared by THP split or free page.  in PAGEFLAG()
695  * Check if a page is currently marked HWPoisoned. Note that this check is  in PAGEFLAG()
699 static inline bool is_page_hwpoison(struct page *page)  in PAGEFLAG()
701 	if (PageHWPoison(page))  in PAGEFLAG()
703 	return PageHuge(page) && PageHWPoison(compound_head(page));  in PAGEFLAG()
712  * mistaken for a page type value.
723 #define PageType(page, flag)						\  argument
724 	((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
726 static inline int page_has_type(struct page *page)  in page_has_type()  argument
728 	return (int)page->page_type < PAGE_MAPCOUNT_RESERVE;  in page_has_type()
732 static __always_inline int Page##uname(struct page *page)		\
734 	return PageType(page, PG_##lname);				\
736 static __always_inline void __SetPage##uname(struct page *page)		\
738 	VM_BUG_ON_PAGE(!PageType(page, 0), page);			\
739 	page->page_type &= ~PG_##lname;					\
741 static __always_inline void __ClearPage##uname(struct page *page)	\
743 	VM_BUG_ON_PAGE(!Page##uname(page), page);			\
744 	page->page_type |= PG_##lname;					\
748  * PageBuddy() indicates that the page is free and in the buddy system
754  * PageOffline() indicates that the page is logically offline although the
770  * There are drivers that mark a page PageOffline() and expect there won't be
771  * any further access to page content. PFN walkers that read content of random
783  * Marks pages in use as page tables.
792 extern bool is_free_buddy_page(struct page *page);
800 static inline int PageSlabPfmemalloc(struct page *page)  in PageSlabPfmemalloc()  argument
802 	VM_BUG_ON_PAGE(!PageSlab(page), page);  in PageSlabPfmemalloc()
803 	return PageActive(page);  in PageSlabPfmemalloc()
807  * A version of PageSlabPfmemalloc() for opportunistic checks where the page
810 static inline int __PageSlabPfmemalloc(struct page *page)  in __PageSlabPfmemalloc()  argument
812 	return PageActive(page);  in __PageSlabPfmemalloc()
815 static inline void SetPageSlabPfmemalloc(struct page *page)  in SetPageSlabPfmemalloc()  argument
817 	VM_BUG_ON_PAGE(!PageSlab(page), page);  in SetPageSlabPfmemalloc()
818 	SetPageActive(page);  in SetPageSlabPfmemalloc()
821 static inline void __ClearPageSlabPfmemalloc(struct page *page)  in __ClearPageSlabPfmemalloc()  argument
823 	VM_BUG_ON_PAGE(!PageSlab(page), page);  in __ClearPageSlabPfmemalloc()
824 	__ClearPageActive(page);  in __ClearPageSlabPfmemalloc()
827 static inline void ClearPageSlabPfmemalloc(struct page *page)  in ClearPageSlabPfmemalloc()  argument
829 	VM_BUG_ON_PAGE(!PageSlab(page), page);  in ClearPageSlabPfmemalloc()
830 	ClearPageActive(page);  in ClearPageSlabPfmemalloc()
840  * Flags checked when a page is freed.  Pages being freed should not have
851  * Flags checked when a page is prepped for return by the page allocator.
853  * there has been a kernel bug or struct page corruption.
855  * __PG_HWPOISON is exceptional because it needs to be kept beyond page's
856  * alloc-free cycle to prevent from reusing the page.
864  * page_has_private - Determine if page has private stuff
865  * @page: The page to be checked
867  * Determine if a page has private stuff, indicating that release routines
870 static inline int page_has_private(struct page *page)  in page_has_private()  argument
872 	return !!(page->flags & PAGE_FLAGS_PRIVATE);  in page_has_private()