48 list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)  in list_lru_from_memcg_idx()  argument
55 	memcg_lrus = rcu_dereference_check(nlru->memcg_lrus,  in list_lru_from_memcg_idx()
56 					   lockdep_is_held(&nlru->lock));  in list_lru_from_memcg_idx()
59 	return &nlru->lru;  in list_lru_from_memcg_idx()
73 list_lru_from_kmem(struct list_lru_node *nlru, void *ptr,  in list_lru_from_kmem()  argument
76 	struct list_lru_one *l = &nlru->lru;  in list_lru_from_kmem()
79 	if (!nlru->memcg_lrus)  in list_lru_from_kmem()
86 	l = list_lru_from_memcg_idx(nlru, memcg_cache_id(memcg));  in list_lru_from_kmem()
112 list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)  in list_lru_from_memcg_idx()  argument
114 	return &nlru->lru;  in list_lru_from_memcg_idx()
118 list_lru_from_kmem(struct list_lru_node *nlru, void *ptr,  in list_lru_from_kmem()  argument
123 	return &nlru->lru;  in list_lru_from_kmem()
130 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_add()  local
134 	spin_lock(&nlru->lock);  in list_lru_add()
136 		l = list_lru_from_kmem(nlru, item, &memcg);  in list_lru_add()
142 		nlru->nr_items++;  in list_lru_add()
143 		spin_unlock(&nlru->lock);  in list_lru_add()
146 	spin_unlock(&nlru->lock);  in list_lru_add()
154 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_del()  local
157 	spin_lock(&nlru->lock);  in list_lru_del()
159 		l = list_lru_from_kmem(nlru, item, NULL);  in list_lru_del()
162 		nlru->nr_items--;  in list_lru_del()
163 		spin_unlock(&nlru->lock);  in list_lru_del()
166 	spin_unlock(&nlru->lock);  in list_lru_del()
189 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_count_one()  local
194 	l = list_lru_from_memcg_idx(nlru, memcg_cache_id(memcg));  in list_lru_count_one()
204 	struct list_lru_node *nlru;  in list_lru_count_node()  local
206 	nlru = &lru->node[nid];  in list_lru_count_node()
207 	return nlru->nr_items;  in list_lru_count_node()
212 __list_lru_walk_one(struct list_lru_node *nlru, int memcg_idx,  in __list_lru_walk_one()  argument
221 	l = list_lru_from_memcg_idx(nlru, memcg_idx);  in __list_lru_walk_one()
234 		ret = isolate(item, l, &nlru->lock, cb_arg);  in __list_lru_walk_one()
237 			assert_spin_locked(&nlru->lock);  in __list_lru_walk_one()
241 			nlru->nr_items--;  in __list_lru_walk_one()
260 			assert_spin_locked(&nlru->lock);  in __list_lru_walk_one()
274 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_walk_one()  local
277 	spin_lock(&nlru->lock);  in list_lru_walk_one()
278 	ret = __list_lru_walk_one(nlru, memcg_cache_id(memcg), isolate, cb_arg,  in list_lru_walk_one()
280 	spin_unlock(&nlru->lock);  in list_lru_walk_one()
290 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_walk_one_irq()  local
293 	spin_lock_irq(&nlru->lock);  in list_lru_walk_one_irq()
294 	ret = __list_lru_walk_one(nlru, memcg_cache_id(memcg), isolate, cb_arg,  in list_lru_walk_one_irq()
296 	spin_unlock_irq(&nlru->lock);  in list_lru_walk_one_irq()
311 			struct list_lru_node *nlru = &lru->node[nid];  in list_lru_walk_node()  local
313 			spin_lock(&nlru->lock);  in list_lru_walk_node()
314 			isolated += __list_lru_walk_one(nlru, memcg_idx,  in list_lru_walk_node()
317 			spin_unlock(&nlru->lock);  in list_lru_walk_node()
364 static int memcg_init_list_lru_node(struct list_lru_node *nlru)  in memcg_init_list_lru_node()  argument
378 	RCU_INIT_POINTER(nlru->memcg_lrus, memcg_lrus);  in memcg_init_list_lru_node()
383 static void memcg_destroy_list_lru_node(struct list_lru_node *nlru)  in memcg_destroy_list_lru_node()  argument
390 	memcg_lrus = rcu_dereference_protected(nlru->memcg_lrus, true);  in memcg_destroy_list_lru_node()
403 static int memcg_update_list_lru_node(struct list_lru_node *nlru,  in memcg_update_list_lru_node()  argument
410 	old = rcu_dereference_protected(nlru->memcg_lrus,  in memcg_update_list_lru_node()
430 	spin_lock_irq(&nlru->lock);  in memcg_update_list_lru_node()
431 	rcu_assign_pointer(nlru->memcg_lrus, new);  in memcg_update_list_lru_node()
432 	spin_unlock_irq(&nlru->lock);  in memcg_update_list_lru_node()
438 static void memcg_cancel_update_list_lru_node(struct list_lru_node *nlru,  in memcg_cancel_update_list_lru_node()  argument
443 	memcg_lrus = rcu_dereference_protected(nlru->memcg_lrus,  in memcg_cancel_update_list_lru_node()
544 	struct list_lru_node *nlru = &lru->node[nid];  in memcg_drain_list_lru_node()  local
553 	spin_lock_irq(&nlru->lock);  in memcg_drain_list_lru_node()
555 	src = list_lru_from_memcg_idx(nlru, src_idx);  in memcg_drain_list_lru_node()
556 	dst = list_lru_from_memcg_idx(nlru, dst_idx);  in memcg_drain_list_lru_node()
565 	spin_unlock_irq(&nlru->lock);  in memcg_drain_list_lru_node()