214 static __always_inline const struct page *page_fixed_fake_head(const struct page *page)  in page_fixed_fake_head()  argument
217 		return page;  in page_fixed_fake_head()
225 	if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&  in page_fixed_fake_head()
226 	    test_bit(PG_head, &page->flags)) {  in page_fixed_fake_head()
232 		unsigned long head = READ_ONCE(page[1].compound_head);  in page_fixed_fake_head()
235 			return (const struct page *)(head - 1);  in page_fixed_fake_head()
237 	return page;  in page_fixed_fake_head()
240 static inline const struct page *page_fixed_fake_head(const struct page *page)  in page_fixed_fake_head()  argument
242 	return page;  in page_fixed_fake_head()
246 static __always_inline int page_is_fake_head(struct page *page)  in page_is_fake_head()  argument
248 	return page_fixed_fake_head(page) != page;  in page_is_fake_head()
251 static inline unsigned long _compound_head(const struct page *page)  in _compound_head()  argument
253 	unsigned long head = READ_ONCE(page->compound_head);  in _compound_head()
257 	return (unsigned long)page_fixed_fake_head(page);  in _compound_head()
260 #define compound_head(page)	((typeof(page))_compound_head(page))  argument
276 	const struct page *:	(const struct folio *)_compound_head(p), \
277 	struct page *:		(struct folio *)_compound_head(p)))
288 #define folio_page(folio, n)	nth_page(&(folio)->page, n)
290 static __always_inline int PageTail(struct page *page)  in PageTail()  argument
292 	return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page);  in PageTail()
295 static __always_inline int PageCompound(struct page *page)  in PageCompound()  argument
297 	return test_bit(PG_head, &page->flags) ||  in PageCompound()
298 	       READ_ONCE(page->compound_head) & 1;  in PageCompound()
302 static inline int PagePoisoned(const struct page *page)  in PagePoisoned()  argument
304 	return READ_ONCE(page->flags) == PAGE_POISON_PATTERN;  in PagePoisoned()
308 void page_init_poison(struct page *page, size_t size);
310 static inline void page_init_poison(struct page *page, size_t size)  in page_init_poison()  argument
317 	struct page *page = &folio->page;  in folio_flags()  local
319 	VM_BUG_ON_PGFLAGS(PageTail(page), page);  in folio_flags()
320 	VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);  in folio_flags()
321 	return &page[n].flags;  in folio_flags()
350 #define PF_POISONED_CHECK(page) ({					\  argument
351 		VM_BUG_ON_PGFLAGS(PagePoisoned(page), page);		\
352 		page; })
353 #define PF_ANY(page, enforce)	PF_POISONED_CHECK(page)  argument
354 #define PF_HEAD(page, enforce)	PF_POISONED_CHECK(compound_head(page))  argument
355 #define PF_ONLY_HEAD(page, enforce) ({					\  argument
356 		VM_BUG_ON_PGFLAGS(PageTail(page), page);		\
357 		PF_POISONED_CHECK(page); })
358 #define PF_NO_TAIL(page, enforce) ({					\  argument
359 		VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page);	\
360 		PF_POISONED_CHECK(compound_head(page)); })
361 #define PF_NO_COMPOUND(page, enforce) ({				\  argument
362 		VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page);	\
363 		PF_POISONED_CHECK(page); })
364 #define PF_SECOND(page, enforce) ({					\  argument
365 		VM_BUG_ON_PGFLAGS(!PageHead(page), page);		\
366 		PF_POISONED_CHECK(&page[1]); })
382 static __always_inline int Page##uname(struct page *page)		\
383 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
389 static __always_inline void SetPage##uname(struct page *page)		\
390 { set_bit(PG_##lname, &policy(page, 1)->flags); }
396 static __always_inline void ClearPage##uname(struct page *page)		\
397 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
403 static __always_inline void __SetPage##uname(struct page *page)		\
404 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
410 static __always_inline void __ClearPage##uname(struct page *page)	\
411 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
417 static __always_inline int TestSetPage##uname(struct page *page)	\
418 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
424 static __always_inline int TestClearPage##uname(struct page *page)	\
425 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
443 static inline int Page##uname(const struct page *page) { return 0; }
447 static inline void SetPage##uname(struct page *page) {  }
451 static inline void ClearPage##uname(struct page *page) {  }
455 static inline void __ClearPage##uname(struct page *page) {  }
460 static inline int TestSetPage##uname(struct page *page) { return 0; }
465 static inline int TestClearPage##uname(struct page *page) { return 0; }
547 static __always_inline bool PageSwapCache(struct page *page)  in PageSwapCache()  argument
549 	return folio_test_swapcache(page_folio(page));  in PageSwapCache()
582 extern void SetPageHWPoisonTakenOff(struct page *page);
583 extern void ClearPageHWPoisonTakenOff(struct page *page);
584 extern bool take_page_off_buddy(struct page *page);
585 extern bool put_page_back_buddy(struct page *page);
651 static __always_inline int PageMappingFlags(struct page *page)  in PageMappingFlags()  argument
653 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;  in PageMappingFlags()
661 static __always_inline bool PageAnon(struct page *page)  in PageAnon()  argument
663 	return folio_test_anon(page_folio(page));  in PageAnon()
672 static __always_inline int __PageMovable(struct page *page)  in __PageMovable()  argument
674 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==  in __PageMovable()
691 static __always_inline bool PageKsm(struct page *page)  in PageKsm()  argument
693 	return folio_test_ksm(page_folio(page));  in PageKsm()
699 u64 stable_page_flags(struct page *page);
728 static inline int PageUptodate(struct page *page)  in PageUptodate()  argument
730 	return folio_test_uptodate(page_folio(page));  in PageUptodate()
750 static __always_inline void __SetPageUptodate(struct page *page)  in __SetPageUptodate()  argument
752 	__folio_mark_uptodate((struct folio *)page);  in __SetPageUptodate()
755 static __always_inline void SetPageUptodate(struct page *page)  in SetPageUptodate()  argument
757 	folio_mark_uptodate((struct folio *)page);  in SetPageUptodate()
763 bool set_page_writeback(struct page *page);
770 static inline void set_page_writeback_keepwrite(struct page *page)  in set_page_writeback_keepwrite()  argument
772 	folio_start_writeback_keepwrite(page_folio(page));  in set_page_writeback_keepwrite()
775 static inline bool test_set_page_writeback(struct page *page)  in test_set_page_writeback()  argument
777 	return set_page_writeback(page);  in test_set_page_writeback()
785 static __always_inline int PageHead(struct page *page)  in PageHead()  argument
787 	PF_POISONED_CHECK(page);  in PageHead()
788 	return test_bit(PG_head, &page->flags) && !page_is_fake_head(page);  in PageHead()
806 static __always_inline void set_compound_head(struct page *page, struct page *head)  in set_compound_head()  argument
808 	WRITE_ONCE(page->compound_head, (unsigned long)head + 1);  in set_compound_head()
811 static __always_inline void clear_compound_head(struct page *page)  in clear_compound_head()  argument
813 	WRITE_ONCE(page->compound_head, 0);  in clear_compound_head()
817 static inline void ClearPageCompound(struct page *page)  in ClearPageCompound()  argument
819 	BUG_ON(!PageHead(page));  in ClearPageCompound()
820 	ClearPageHead(page);  in ClearPageCompound()
827 int PageHuge(struct page *page);
828 int PageHeadHuge(struct page *page);
831 	return PageHeadHuge(&folio->page);  in folio_test_hugetlb()
847 static inline int PageTransHuge(struct page *page)  in TESTPAGEFLAG_FALSE()
849 	VM_BUG_ON_PAGE(PageTail(page), page);  in TESTPAGEFLAG_FALSE()
850 	return PageHead(page);  in TESTPAGEFLAG_FALSE()
863 static inline int PageTransCompound(struct page *page)  in PageTransCompound()  argument
865 	return PageCompound(page);  in PageTransCompound()
873 static inline int PageTransTail(struct page *page)  in PageTransTail()  argument
875 	return PageTail(page);  in PageTransTail()
921 static inline bool is_page_hwpoison(struct page *page)  in PAGEFLAG()
923 	if (PageHWPoison(page))  in PAGEFLAG()
925 	return PageHuge(page) && PageHWPoison(compound_head(page));  in PAGEFLAG()
945 #define PageType(page, flag)						\  argument
946 	((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE)
948 static inline int page_has_type(struct page *page)  in page_has_type()  argument
950 	return (int)page->page_type < PAGE_MAPCOUNT_RESERVE;  in page_has_type()
954 static __always_inline int Page##uname(struct page *page)		\
956 	return PageType(page, PG_##lname);				\
958 static __always_inline void __SetPage##uname(struct page *page)		\
960 	VM_BUG_ON_PAGE(!PageType(page, 0), page);			\
961 	page->page_type &= ~PG_##lname;					\
963 static __always_inline void __ClearPage##uname(struct page *page)	\
965 	VM_BUG_ON_PAGE(!Page##uname(page), page);			\
966 	page->page_type |= PG_##lname;					\
1014 extern bool is_free_buddy_page(struct page *page);
1018 static __always_inline int PageAnonExclusive(struct page *page)  in PageAnonExclusive()  argument
1020 	VM_BUG_ON_PGFLAGS(!PageAnon(page), page);  in PageAnonExclusive()
1021 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in PageAnonExclusive()
1022 	return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in PageAnonExclusive()
1025 static __always_inline void SetPageAnonExclusive(struct page *page)  in SetPageAnonExclusive()  argument
1027 	VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page);  in SetPageAnonExclusive()
1028 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in SetPageAnonExclusive()
1029 	set_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in SetPageAnonExclusive()
1032 static __always_inline void ClearPageAnonExclusive(struct page *page)  in ClearPageAnonExclusive()  argument
1034 	VM_BUG_ON_PGFLAGS(!PageAnon(page) || PageKsm(page), page);  in ClearPageAnonExclusive()
1035 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in ClearPageAnonExclusive()
1036 	clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in ClearPageAnonExclusive()
1039 static __always_inline void __ClearPageAnonExclusive(struct page *page)  in __ClearPageAnonExclusive()  argument
1041 	VM_BUG_ON_PGFLAGS(!PageAnon(page), page);  in __ClearPageAnonExclusive()
1042 	VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);  in __ClearPageAnonExclusive()
1043 	__clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags);  in __ClearPageAnonExclusive()
1083 static inline int page_has_private(struct page *page)  in page_has_private()  argument
1085 	return !!(page->flags & PAGE_FLAGS_PRIVATE);  in page_has_private()
1090 	return page_has_private(&folio->page);  in folio_has_private()