46 list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)  in list_lru_from_memcg_idx()  argument
53 	memcg_lrus = rcu_dereference_check(nlru->memcg_lrus,  in list_lru_from_memcg_idx()
54 					   lockdep_is_held(&nlru->lock));  in list_lru_from_memcg_idx()
57 	return &nlru->lru;  in list_lru_from_memcg_idx()
61 list_lru_from_kmem(struct list_lru_node *nlru, void *ptr,  in list_lru_from_kmem()  argument
64 	struct list_lru_one *l = &nlru->lru;  in list_lru_from_kmem()
67 	if (!nlru->memcg_lrus)  in list_lru_from_kmem()
74 	l = list_lru_from_memcg_idx(nlru, memcg_cache_id(memcg));  in list_lru_from_kmem()
100 list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)  in list_lru_from_memcg_idx()  argument
102 	return &nlru->lru;  in list_lru_from_memcg_idx()
106 list_lru_from_kmem(struct list_lru_node *nlru, void *ptr,  in list_lru_from_kmem()  argument
111 	return &nlru->lru;  in list_lru_from_kmem()
118 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_add()  local
122 	spin_lock(&nlru->lock);  in list_lru_add()
124 		l = list_lru_from_kmem(nlru, item, &memcg);  in list_lru_add()
130 		nlru->nr_items++;  in list_lru_add()
131 		spin_unlock(&nlru->lock);  in list_lru_add()
134 	spin_unlock(&nlru->lock);  in list_lru_add()
142 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_del()  local
145 	spin_lock(&nlru->lock);  in list_lru_del()
147 		l = list_lru_from_kmem(nlru, item, NULL);  in list_lru_del()
150 		nlru->nr_items--;  in list_lru_del()
151 		spin_unlock(&nlru->lock);  in list_lru_del()
154 	spin_unlock(&nlru->lock);  in list_lru_del()
177 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_count_one()  local
182 	l = list_lru_from_memcg_idx(nlru, memcg_cache_id(memcg));  in list_lru_count_one()
192 	struct list_lru_node *nlru;  in list_lru_count_node()  local
194 	nlru = &lru->node[nid];  in list_lru_count_node()
195 	return nlru->nr_items;  in list_lru_count_node()
200 __list_lru_walk_one(struct list_lru_node *nlru, int memcg_idx,  in __list_lru_walk_one()  argument
209 	l = list_lru_from_memcg_idx(nlru, memcg_idx);  in __list_lru_walk_one()
222 		ret = isolate(item, l, &nlru->lock, cb_arg);  in __list_lru_walk_one()
225 			assert_spin_locked(&nlru->lock);  in __list_lru_walk_one()
229 			nlru->nr_items--;  in __list_lru_walk_one()
248 			assert_spin_locked(&nlru->lock);  in __list_lru_walk_one()
262 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_walk_one()  local
265 	spin_lock(&nlru->lock);  in list_lru_walk_one()
266 	ret = __list_lru_walk_one(nlru, memcg_cache_id(memcg), isolate, cb_arg,  in list_lru_walk_one()
268 	spin_unlock(&nlru->lock);  in list_lru_walk_one()
278 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_walk_one_irq()  local
281 	spin_lock_irq(&nlru->lock);  in list_lru_walk_one_irq()
282 	ret = __list_lru_walk_one(nlru, memcg_cache_id(memcg), isolate, cb_arg,  in list_lru_walk_one_irq()
284 	spin_unlock_irq(&nlru->lock);  in list_lru_walk_one_irq()
299 			struct list_lru_node *nlru = &lru->node[nid];  in list_lru_walk_node()  local
301 			spin_lock(&nlru->lock);  in list_lru_walk_node()
302 			isolated += __list_lru_walk_one(nlru, memcg_idx,  in list_lru_walk_node()
305 			spin_unlock(&nlru->lock);  in list_lru_walk_node()
352 static int memcg_init_list_lru_node(struct list_lru_node *nlru)  in memcg_init_list_lru_node()  argument
366 	RCU_INIT_POINTER(nlru->memcg_lrus, memcg_lrus);  in memcg_init_list_lru_node()
371 static void memcg_destroy_list_lru_node(struct list_lru_node *nlru)  in memcg_destroy_list_lru_node()  argument
378 	memcg_lrus = rcu_dereference_protected(nlru->memcg_lrus, true);  in memcg_destroy_list_lru_node()
383 static int memcg_update_list_lru_node(struct list_lru_node *nlru,  in memcg_update_list_lru_node()  argument
390 	old = rcu_dereference_protected(nlru->memcg_lrus,  in memcg_update_list_lru_node()
410 	spin_lock_irq(&nlru->lock);  in memcg_update_list_lru_node()
411 	rcu_assign_pointer(nlru->memcg_lrus, new);  in memcg_update_list_lru_node()
412 	spin_unlock_irq(&nlru->lock);  in memcg_update_list_lru_node()
418 static void memcg_cancel_update_list_lru_node(struct list_lru_node *nlru,  in memcg_cancel_update_list_lru_node()  argument
423 	memcg_lrus = rcu_dereference_protected(nlru->memcg_lrus,  in memcg_cancel_update_list_lru_node()
526 	struct list_lru_node *nlru = &lru->node[nid];  in memcg_drain_list_lru_node()  local
534 	spin_lock_irq(&nlru->lock);  in memcg_drain_list_lru_node()
536 	src = list_lru_from_memcg_idx(nlru, src_idx);  in memcg_drain_list_lru_node()
537 	dst = list_lru_from_memcg_idx(nlru, dst_idx);  in memcg_drain_list_lru_node()
547 	spin_unlock_irq(&nlru->lock);  in memcg_drain_list_lru_node()