351 static inline bool folio_memcg_kmem(struct folio *folio);
375 static inline struct mem_cgroup *__folio_memcg(struct folio *folio)  in __folio_memcg()  argument
377 	unsigned long memcg_data = folio->memcg_data;  in __folio_memcg()
379 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);  in __folio_memcg()
380 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio);  in __folio_memcg()
381 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio);  in __folio_memcg()
396 static inline struct obj_cgroup *__folio_objcg(struct folio *folio)  in __folio_objcg()  argument
398 	unsigned long memcg_data = folio->memcg_data;  in __folio_objcg()
400 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);  in __folio_objcg()
401 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio);  in __folio_objcg()
402 	VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio);  in __folio_objcg()
428 static inline struct mem_cgroup *folio_memcg(struct folio *folio)  in folio_memcg()  argument
430 	if (folio_memcg_kmem(folio))  in folio_memcg()
431 		return obj_cgroup_memcg(__folio_objcg(folio));  in folio_memcg()
432 	return __folio_memcg(folio);  in folio_memcg()
451 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)  in folio_memcg_rcu()  argument
453 	unsigned long memcg_data = READ_ONCE(folio->memcg_data);  in folio_memcg_rcu()
455 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);  in folio_memcg_rcu()
534 static inline bool folio_memcg_kmem(struct folio *folio)  in folio_memcg_kmem()  argument
536 	VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page);  in folio_memcg_kmem()
537 	VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJCGS, folio);  in folio_memcg_kmem()
538 	return folio->memcg_data & MEMCG_DATA_KMEM;  in folio_memcg_kmem()
543 static inline bool folio_memcg_kmem(struct folio *folio)  in folio_memcg_kmem()  argument
646 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp);
662 static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm,  in mem_cgroup_charge()  argument
667 	return __mem_cgroup_charge(folio, mm, gfp);  in mem_cgroup_charge()
670 int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm,
674 void __mem_cgroup_uncharge(struct folio *folio);
682 static inline void mem_cgroup_uncharge(struct folio *folio)  in mem_cgroup_uncharge()  argument
686 	__mem_cgroup_uncharge(folio);  in mem_cgroup_uncharge()
697 void mem_cgroup_migrate(struct folio *old, struct folio *new);
741 static inline struct lruvec *folio_lruvec(struct folio *folio)  in folio_lruvec()  argument
743 	struct mem_cgroup *memcg = folio_memcg(folio);  in folio_lruvec()
745 	VM_WARN_ON_ONCE_FOLIO(!memcg && !mem_cgroup_disabled(), folio);  in folio_lruvec()
746 	return mem_cgroup_lruvec(memcg, folio_pgdat(folio));  in folio_lruvec()
753 struct lruvec *folio_lruvec_lock(struct folio *folio);
754 struct lruvec *folio_lruvec_lock_irq(struct folio *folio);
755 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
759 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio);
762 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)  in lruvec_memcg_debug()  argument
933 void folio_memcg_lock(struct folio *folio);
934 void folio_memcg_unlock(struct folio *folio);
1073 static inline void count_memcg_folio_events(struct folio *folio,  in count_memcg_folio_events()  argument
1076 	struct mem_cgroup *memcg = folio_memcg(folio);  in count_memcg_folio_events()
1148 static inline struct mem_cgroup *folio_memcg(struct folio *folio)  in folio_memcg()  argument
1158 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)  in folio_memcg_rcu()  argument
1169 static inline bool folio_memcg_kmem(struct folio *folio)  in folio_memcg_kmem()  argument
1222 static inline int mem_cgroup_charge(struct folio *folio,  in mem_cgroup_charge()  argument
1228 static inline int mem_cgroup_swapin_charge_folio(struct folio *folio,  in mem_cgroup_swapin_charge_folio()  argument
1238 static inline void mem_cgroup_uncharge(struct folio *folio)  in mem_cgroup_uncharge()  argument
1246 static inline void mem_cgroup_migrate(struct folio *old, struct folio *new)  in mem_cgroup_migrate()
1256 static inline struct lruvec *folio_lruvec(struct folio *folio)  in folio_lruvec()  argument
1258 	struct pglist_data *pgdat = folio_pgdat(folio);  in folio_lruvec()
1263 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)  in lruvec_memcg_debug()  argument
1297 static inline struct lruvec *folio_lruvec_lock(struct folio *folio)  in folio_lruvec_lock()  argument
1299 	struct pglist_data *pgdat = folio_pgdat(folio);  in folio_lruvec_lock()
1305 static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio)  in folio_lruvec_lock_irq()  argument
1307 	struct pglist_data *pgdat = folio_pgdat(folio);  in folio_lruvec_lock_irq()
1313 static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,  in folio_lruvec_lock_irqsave()  argument
1316 	struct pglist_data *pgdat = folio_pgdat(folio);  in folio_lruvec_lock_irqsave()
1415 static inline void folio_memcg_lock(struct folio *folio)  in folio_memcg_lock()  argument
1419 static inline void folio_memcg_unlock(struct folio *folio)  in folio_memcg_unlock()  argument
1547 static inline void count_memcg_folio_events(struct folio *folio,  in count_memcg_folio_events()  argument
1610 static inline bool folio_matches_lruvec(struct folio *folio,  in folio_matches_lruvec()  argument
1613 	return lruvec_pgdat(lruvec) == folio_pgdat(folio) &&  in folio_matches_lruvec()
1614 	       lruvec_memcg(lruvec) == folio_memcg(folio);  in folio_matches_lruvec()
1618 static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio,  in folio_lruvec_relock_irq()  argument
1622 		if (folio_matches_lruvec(folio, locked_lruvec))  in folio_lruvec_relock_irq()
1628 	return folio_lruvec_lock_irq(folio);  in folio_lruvec_relock_irq()
1632 static inline struct lruvec *folio_lruvec_relock_irqsave(struct folio *folio,  in folio_lruvec_relock_irqsave()  argument
1636 		if (folio_matches_lruvec(folio, locked_lruvec))  in folio_lruvec_relock_irqsave()
1642 	return folio_lruvec_lock_irqsave(folio, flags);  in folio_lruvec_relock_irqsave()
1652 void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio,
1655 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,  in mem_cgroup_track_foreign_dirty()  argument
1661 	if (unlikely(&folio_memcg(folio)->css != wb->memcg_css))  in mem_cgroup_track_foreign_dirty()
1662 		mem_cgroup_track_foreign_dirty_slowpath(folio, wb);  in mem_cgroup_track_foreign_dirty()
1682 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,  in mem_cgroup_track_foreign_dirty()  argument