37 static __always_inline void __update_lru_size(struct lruvec *lruvec,  in __update_lru_size()  argument
41 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);  in __update_lru_size()
43 	lockdep_assert_held(&lruvec->lru_lock);  in __update_lru_size()
46 	__mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages);  in __update_lru_size()
51 static __always_inline void update_lru_size(struct lruvec *lruvec,  in update_lru_size()  argument
55 	__update_lru_size(lruvec, lru, zid, nr_pages);  in update_lru_size()
57 	mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages);  in update_lru_size()
164 static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen)  in lru_gen_is_active()  argument
166 	unsigned long max_seq = lruvec->lrugen.max_seq;  in lru_gen_is_active()
174 static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,  in lru_gen_update_size()  argument
181 	struct lru_gen_struct *lrugen = &lruvec->lrugen;  in lru_gen_update_size()
196 		if (lru_gen_is_active(lruvec, new_gen))  in lru_gen_update_size()
198 		__update_lru_size(lruvec, lru, zone, delta);  in lru_gen_update_size()
204 		if (lru_gen_is_active(lruvec, old_gen))  in lru_gen_update_size()
206 		__update_lru_size(lruvec, lru, zone, -delta);  in lru_gen_update_size()
211 	if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) {  in lru_gen_update_size()
212 		__update_lru_size(lruvec, lru, zone, -delta);  in lru_gen_update_size()
213 		__update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta);  in lru_gen_update_size()
217 	VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen));  in lru_gen_update_size()
220 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_add_folio()  argument
227 	struct lru_gen_struct *lrugen = &lruvec->lrugen;  in lru_gen_add_folio()
256 	lru_gen_update_size(lruvec, folio, -1, gen);  in lru_gen_add_folio()
266 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_del_folio()  argument
278 	flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;  in lru_gen_del_folio()
282 	lru_gen_update_size(lruvec, folio, gen, -1);  in lru_gen_del_folio()
300 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_add_folio()  argument
305 static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)  in lru_gen_del_folio()  argument
313 void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)  in lruvec_add_folio()  argument
317 	if (lru_gen_add_folio(lruvec, folio, false))  in lruvec_add_folio()
320 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_add_folio()
323 		list_add(&folio->lru, &lruvec->lists[lru]);  in lruvec_add_folio()
327 				struct lruvec *lruvec)  in add_page_to_lru_list()  argument
329 	lruvec_add_folio(lruvec, page_folio(page));  in add_page_to_lru_list()
333 void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)  in lruvec_add_folio_tail()  argument
337 	if (lru_gen_add_folio(lruvec, folio, true))  in lruvec_add_folio_tail()
340 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_add_folio_tail()
343 	list_add_tail(&folio->lru, &lruvec->lists[lru]);  in lruvec_add_folio_tail()
347 void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)  in lruvec_del_folio()  argument
351 	if (lru_gen_del_folio(lruvec, folio, false))  in lruvec_del_folio()
356 	update_lru_size(lruvec, lru, folio_zonenum(folio),  in lruvec_del_folio()
361 				struct lruvec *lruvec)  in del_page_from_lru_list()  argument
363 	lruvec_del_folio(lruvec, page_folio(page));  in del_page_from_lru_list()