359 static inline bool folio_memcg_kmem(struct folio *folio);
383 static inline struct mem_cgroup *__folio_memcg(struct folio *folio)  in __folio_memcg()  argument
385 	unsigned long memcg_data = folio->memcg_data;  in __folio_memcg()
387 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);  in __folio_memcg()
388 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio);  in __folio_memcg()
389 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio);  in __folio_memcg()
404 static inline struct obj_cgroup *__folio_objcg(struct folio *folio)  in __folio_objcg()  argument
406 	unsigned long memcg_data = folio->memcg_data;  in __folio_objcg()
408 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);  in __folio_objcg()
409 	VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJCGS, folio);  in __folio_objcg()
410 	VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio);  in __folio_objcg()
436 static inline struct mem_cgroup *folio_memcg(struct folio *folio)  in folio_memcg()  argument
438 	if (folio_memcg_kmem(folio))  in folio_memcg()
439 		return obj_cgroup_memcg(__folio_objcg(folio));  in folio_memcg()
440 	return __folio_memcg(folio);  in folio_memcg()
459 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)  in folio_memcg_rcu()  argument
461 	unsigned long memcg_data = READ_ONCE(folio->memcg_data);  in folio_memcg_rcu()
463 	VM_BUG_ON_FOLIO(folio_test_slab(folio), folio);  in folio_memcg_rcu()
498 static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)  in folio_memcg_check()  argument
504 	unsigned long memcg_data = READ_ONCE(folio->memcg_data);  in folio_memcg_check()
523 	return folio_memcg_check((struct folio *)page);  in page_memcg_check()
549 static inline bool folio_memcg_kmem(struct folio *folio)  in folio_memcg_kmem()  argument
551 	VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page);  in folio_memcg_kmem()
552 	VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJCGS, folio);  in folio_memcg_kmem()
553 	return folio->memcg_data & MEMCG_DATA_KMEM;  in folio_memcg_kmem()
558 static inline bool folio_memcg_kmem(struct folio *folio)  in folio_memcg_kmem()  argument
665 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp);
681 static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm,  in mem_cgroup_charge()  argument
686 	return __mem_cgroup_charge(folio, mm, gfp);  in mem_cgroup_charge()
689 int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm,
693 void __mem_cgroup_uncharge(struct folio *folio);
701 static inline void mem_cgroup_uncharge(struct folio *folio)  in mem_cgroup_uncharge()  argument
705 	__mem_cgroup_uncharge(folio);  in mem_cgroup_uncharge()
716 void mem_cgroup_migrate(struct folio *old, struct folio *new);
760 static inline struct lruvec *folio_lruvec(struct folio *folio)  in folio_lruvec()  argument
762 	struct mem_cgroup *memcg = folio_memcg(folio);  in folio_lruvec()
764 	VM_WARN_ON_ONCE_FOLIO(!memcg && !mem_cgroup_disabled(), folio);  in folio_lruvec()
765 	return mem_cgroup_lruvec(memcg, folio_pgdat(folio));  in folio_lruvec()
772 struct lruvec *folio_lruvec_lock(struct folio *folio);
773 struct lruvec *folio_lruvec_lock_irq(struct folio *folio);
774 struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
778 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio);
781 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)  in lruvec_memcg_debug()  argument
900 struct cgroup_subsys_state *mem_cgroup_css_from_folio(struct folio *folio);
956 void folio_memcg_lock(struct folio *folio);
957 void folio_memcg_unlock(struct folio *folio);
1092 static inline void count_memcg_folio_events(struct folio *folio,  in count_memcg_folio_events()  argument
1095 	struct mem_cgroup *memcg = folio_memcg(folio);  in count_memcg_folio_events()
1166 static inline struct mem_cgroup *folio_memcg(struct folio *folio)  in folio_memcg()  argument
1176 static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio)  in folio_memcg_rcu()  argument
1182 static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)  in folio_memcg_check()  argument
1192 static inline bool folio_memcg_kmem(struct folio *folio)  in folio_memcg_kmem()  argument
1252 static inline int mem_cgroup_charge(struct folio *folio,  in mem_cgroup_charge()  argument
1258 static inline int mem_cgroup_swapin_charge_folio(struct folio *folio,  in mem_cgroup_swapin_charge_folio()  argument
1268 static inline void mem_cgroup_uncharge(struct folio *folio)  in mem_cgroup_uncharge()  argument
1276 static inline void mem_cgroup_migrate(struct folio *old, struct folio *new)  in mem_cgroup_migrate()
1286 static inline struct lruvec *folio_lruvec(struct folio *folio)  in folio_lruvec()  argument
1288 	struct pglist_data *pgdat = folio_pgdat(folio);  in folio_lruvec()
1293 void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)  in lruvec_memcg_debug()  argument
1332 static inline struct lruvec *folio_lruvec_lock(struct folio *folio)  in folio_lruvec_lock()  argument
1334 	struct pglist_data *pgdat = folio_pgdat(folio);  in folio_lruvec_lock()
1340 static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio)  in folio_lruvec_lock_irq()  argument
1342 	struct pglist_data *pgdat = folio_pgdat(folio);  in folio_lruvec_lock_irq()
1348 static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,  in folio_lruvec_lock_irqsave()  argument
1351 	struct pglist_data *pgdat = folio_pgdat(folio);  in folio_lruvec_lock_irqsave()
1441 static inline void folio_memcg_lock(struct folio *folio)  in folio_memcg_lock()  argument
1445 static inline void folio_memcg_unlock(struct folio *folio)  in folio_memcg_unlock()  argument
1573 static inline void count_memcg_folio_events(struct folio *folio,  in count_memcg_folio_events()  argument
1636 static inline bool folio_matches_lruvec(struct folio *folio,  in folio_matches_lruvec()  argument
1639 	return lruvec_pgdat(lruvec) == folio_pgdat(folio) &&  in folio_matches_lruvec()
1640 	       lruvec_memcg(lruvec) == folio_memcg(folio);  in folio_matches_lruvec()
1644 static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio,  in folio_lruvec_relock_irq()  argument
1648 		if (folio_matches_lruvec(folio, locked_lruvec))  in folio_lruvec_relock_irq()
1654 	return folio_lruvec_lock_irq(folio);  in folio_lruvec_relock_irq()
1658 static inline struct lruvec *folio_lruvec_relock_irqsave(struct folio *folio,  in folio_lruvec_relock_irqsave()  argument
1662 		if (folio_matches_lruvec(folio, locked_lruvec))  in folio_lruvec_relock_irqsave()
1668 	return folio_lruvec_lock_irqsave(folio, flags);  in folio_lruvec_relock_irqsave()
1678 void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio,
1681 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,  in mem_cgroup_track_foreign_dirty()  argument
1689 	memcg = folio_memcg(folio);  in mem_cgroup_track_foreign_dirty()
1691 		mem_cgroup_track_foreign_dirty_slowpath(folio, wb);  in mem_cgroup_track_foreign_dirty()
1711 static inline void mem_cgroup_track_foreign_dirty(struct folio *folio,  in mem_cgroup_track_foreign_dirty()  argument
1767 struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio);
1851 static inline struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio)  in get_obj_cgroup_from_folio()  argument