208 static __always_inline const struct page *page_fixed_fake_head(const struct page *page)  in page_fixed_fake_head()  argument
211 		return page;  in page_fixed_fake_head()
219 	if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&  in page_fixed_fake_head()
220 	    test_bit(PG_head, &page->flags)) {  in page_fixed_fake_head()
226 		unsigned long head = READ_ONCE(page[1].compound_head);  in page_fixed_fake_head()
229 			return (const struct page *)(head - 1);  in page_fixed_fake_head()
231 	return page;  in page_fixed_fake_head()
234 static inline const struct page *page_fixed_fake_head(const struct page *page)  in page_fixed_fake_head()  argument
236 	return page;  in page_fixed_fake_head()
240 static __always_inline int page_is_fake_head(struct page *page)  in page_is_fake_head()  argument
242 	return page_fixed_fake_head(page) != page;  in page_is_fake_head()
245 static inline unsigned long _compound_head(const struct page *page)  in _compound_head()  argument
247 	unsigned long head = READ_ONCE(page->compound_head);  in _compound_head()
251 	return (unsigned long)page_fixed_fake_head(page);  in _compound_head()
254 #define compound_head(page)	((typeof(page))_compound_head(page))  argument
270 	const struct page *:	(const struct folio *)_compound_head(p), \
271 	struct page *:		(struct folio *)_compound_head(p)))
282 #define folio_page(folio, n)	nth_page(&(folio)->page, n)
284 static __always_inline int PageTail(struct page *page)  in PageTail()  argument
286 	return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page);  in PageTail()
289 static __always_inline int PageCompound(struct page *page)  in PageCompound()  argument
291 	return test_bit(PG_head, &page->flags) ||  in PageCompound()
292 	       READ_ONCE(page->compound_head) & 1;  in PageCompound()
296 static inline int PagePoisoned(const struct page *page)  in PagePoisoned()  argument
298 	return READ_ONCE(page->flags) == PAGE_POISON_PATTERN;  in PagePoisoned()
302 void page_init_poison(struct page *page, size_t size);
304 static inline void page_init_poison(struct page *page, size_t size)  in page_init_poison()  argument
311 	struct page *page = &folio->page;  in folio_flags()  local
313 	VM_BUG_ON_PGFLAGS(PageTail(page), page);  in folio_flags()
314 	VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);  in folio_flags()
315 	return &page[n].flags;  in folio_flags()
344 #define PF_POISONED_CHECK(page) ({					\  argument
345 		VM_BUG_ON_PGFLAGS(PagePoisoned(page), page);		\
346 		page; })
347 #define PF_ANY(page, enforce)	PF_POISONED_CHECK(page)  argument
348 #define PF_HEAD(page, enforce)	PF_POISONED_CHECK(compound_head(page))  argument
349 #define PF_ONLY_HEAD(page, enforce) ({					\  argument
350 		VM_BUG_ON_PGFLAGS(PageTail(page), page);		\
351 		PF_POISONED_CHECK(page); })
352 #define PF_NO_TAIL(page, enforce) ({					\  argument
353 		VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page);	\
354 		PF_POISONED_CHECK(compound_head(page)); })
355 #define PF_NO_COMPOUND(page, enforce) ({				\  argument
356 		VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page);	\
357 		PF_POISONED_CHECK(page); })
358 #define PF_SECOND(page, enforce) ({					\  argument
359 		VM_BUG_ON_PGFLAGS(!PageHead(page), page);		\
360 		PF_POISONED_CHECK(&page[1]); })
376 static __always_inline int Page##uname(struct page *page)		\
377 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
383 static __always_inline void SetPage##uname(struct page *page)		\
384 { set_bit(PG_##lname, &policy(page, 1)->flags); }
390 static __always_inline void ClearPage##uname(struct page *page)		\
391 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
397 static __always_inline void __SetPage##uname(struct page *page)		\
398 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
404 static __always_inline void __ClearPage##uname(struct page *page)	\
405 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
411 static __always_inline int TestSetPage##uname(struct page *page)	\
412 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
418 static __always_inline int TestClearPage##uname(struct page *page)	\
419 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
437 static inline int Page##uname(const struct page *page) { return 0; }
441 static inline void SetPage##uname(struct page *page) {  }
445 static inline void ClearPage##uname(struct page *page) {  }
449 static inline void __ClearPage##uname(struct page *page) {  }
454 static inline int TestSetPage##uname(struct page *page) { return 0; }
459 static inline int TestClearPage##uname(struct page *page) { return 0; }
541 static __always_inline bool PageSwapCache(struct page *page)  in PageSwapCache()  argument
543 	return folio_test_swapcache(page_folio(page));  in PageSwapCache()
576 extern void SetPageHWPoisonTakenOff(struct page *page);
577 extern void ClearPageHWPoisonTakenOff(struct page *page);
578 extern bool take_page_off_buddy(struct page *page);
579 extern bool put_page_back_buddy(struct page *page);
645 static __always_inline int PageMappingFlags(struct page *page)  in PageMappingFlags()  argument
647 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;  in PageMappingFlags()
655 static __always_inline bool PageAnon(struct page *page)  in PageAnon()  argument
657 	return folio_test_anon(page_folio(page));  in PageAnon()
666 static __always_inline int __PageMovable(struct page *page)  in __PageMovable()  argument
668 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==  in __PageMovable()
685 static __always_inline bool PageKsm(struct page *page)  in PageKsm()  argument
687 	return folio_test_ksm(page_folio(page));  in PageKsm()
693 u64 stable_page_flags(struct page *page);
722 static inline int PageUptodate(struct page *page)  in PageUptodate()  argument
724 	return folio_test_uptodate(page_folio(page));  in PageUptodate()
744 static __always_inline void __SetPageUptodate(struct page *page)  in __SetPageUptodate()  argument
746 	__folio_mark_uptodate((struct folio *)page);  in __SetPageUptodate()
749 static __always_inline void SetPageUptodate(struct page *page)  in SetPageUptodate()  argument
751 	folio_mark_uptodate((struct folio *)page);  in SetPageUptodate()
757 bool set_page_writeback(struct page *page);
764 static inline bool test_set_page_writeback(struct page *page)  in test_set_page_writeback()  argument
766 	return set_page_writeback(page);  in test_set_page_writeback()
774 static __always_inline int PageHead(struct page *page)  in PageHead()  argument
776 	PF_POISONED_CHECK(page);  in PageHead()
777 	return test_bit(PG_head, &page->flags) && !page_is_fake_head(page);  in PageHead()
795 static __always_inline void set_compound_head(struct page *page, struct page *head)  in set_compound_head()  argument
797 	WRITE_ONCE(page->compound_head, (unsigned long)head + 1);  in set_compound_head()
800 static __always_inline void clear_compound_head(struct page *page)  in clear_compound_head()  argument
802 	WRITE_ONCE(page->compound_head, 0);  in clear_compound_head()
806 static inline void ClearPageCompound(struct page *page)  in ClearPageCompound()  argument
808 	BUG_ON(!PageHead(page));  in ClearPageCompound()
809 	ClearPageHead(page);  in ClearPageCompound()
819 int PageHuge(struct page *page);
850 static inline int PageTransHuge(struct page *page)  in PageTransHuge()  argument
852 	VM_BUG_ON_PAGE(PageTail(page), page);  in PageTransHuge()
853 	return PageHead(page);  in PageTransHuge()
861 static inline int PageTransCompound(struct page *page)  in PageTransCompound()  argument
863 	return PageCompound(page);  in PageTransCompound()
871 static inline int PageTransTail(struct page *page)  in PageTransTail()  argument
873 	return PageTail(page);  in PageTransTail()
901 static inline bool is_page_hwpoison(struct page *page)  in TESTPAGEFLAG_FALSE()
903 	if (PageHWPoison(page))  in TESTPAGEFLAG_FALSE()
905 	return PageHuge(page) && PageHWPoison(compound_head(page));  in TESTPAGEFLAG_FALSE()
925 #define PageType(page, flag)						\  argument
926 	((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
928 	((folio->page.page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
935 static inline int page_has_type(struct page *page)  in page_has_type()  argument
937 	return page_type_has_type(page->page_type);  in page_has_type()
941 static __always_inline int Page##uname(const struct page *page)		\
943 	return PageType(page, PG_##lname);				\
949 static __always_inline void __SetPage##uname(struct page *page)		\
951 	VM_BUG_ON_PAGE(!PageType(page, 0), page);			\
952 	page->page_type &= ~PG_##lname;					\
957 	folio->page.page_type &= ~PG_##lname;				\
959 static __always_inline void __ClearPage##uname(struct page *page)	\
961 	VM_BUG_ON_PAGE(!Page##uname(page), page);			\
962 	page->page_type |= PG_##lname;					\
967 	folio->page.page_type |= PG_##lname;				\
1015 extern bool is_free_buddy_page(struct page *page);
1019 static __always_inline int PageAnonExclusive(struct page *page)  in PageAnonExclusive()  argument
1021 	VM_BUG_ON_PGFLAGS(!PageAnon(page), page);  in PageAnonExclusive()
1022 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in PageAnonExclusive()
1023 	return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in PageAnonExclusive()
1026 static __always_inline void SetPageAnonExclusive(struct page *page)  in SetPageAnonExclusive()  argument
1028 	VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page);  in SetPageAnonExclusive()
1029 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in SetPageAnonExclusive()
1030 	set_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in SetPageAnonExclusive()
1033 static __always_inline void ClearPageAnonExclusive(struct page *page)  in ClearPageAnonExclusive()  argument
1035 	VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page);  in ClearPageAnonExclusive()
1036 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in ClearPageAnonExclusive()
1037 	clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in ClearPageAnonExclusive()
1040 static __always_inline void __ClearPageAnonExclusive(struct page *page)  in __ClearPageAnonExclusive()  argument
1042 	VM_BUG_ON_PGFLAGS(!PageAnon(page), page);  in __ClearPageAnonExclusive()
1043 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in __ClearPageAnonExclusive()
1044 	__clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in __ClearPageAnonExclusive()
1092 static inline int page_has_private(struct page *page)  in page_has_private()  argument
1094 	return !!(page->flags & PAGE_FLAGS_PRIVATE);  in page_has_private()
1099 	return page_has_private(&folio->page);  in folio_has_private()