256 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
261 	struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);  in obj_cgroup_release()  local
286 	nr_bytes = atomic_read(&objcg->nr_charged_bytes);  in obj_cgroup_release()
291 		obj_cgroup_uncharge_pages(objcg, nr_pages);  in obj_cgroup_release()
294 	list_del(&objcg->list);  in obj_cgroup_release()
298 	kfree_rcu(objcg, rcu);  in obj_cgroup_release()
303 	struct obj_cgroup *objcg;  in obj_cgroup_alloc()  local
306 	objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL);  in obj_cgroup_alloc()
307 	if (!objcg)  in obj_cgroup_alloc()
310 	ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,  in obj_cgroup_alloc()
313 		kfree(objcg);  in obj_cgroup_alloc()
316 	INIT_LIST_HEAD(&objcg->list);  in obj_cgroup_alloc()
317 	return objcg;  in obj_cgroup_alloc()
323 	struct obj_cgroup *objcg, *iter;  in memcg_reparent_objcgs()  local
325 	objcg = rcu_replace_pointer(memcg->objcg, NULL, true);  in memcg_reparent_objcgs()
330 	list_add(&objcg->list, &memcg->objcg_list);  in memcg_reparent_objcgs()
339 	percpu_ref_kill(&objcg->refcnt);  in memcg_reparent_objcgs()
2863 static inline void mod_objcg_mlstate(struct obj_cgroup *objcg,  in mod_objcg_mlstate()  argument
2871 	memcg = obj_cgroup_memcg(objcg);  in mod_objcg_mlstate()
2996 	struct obj_cgroup *objcg = NULL;  in __get_obj_cgroup_from_memcg()  local
2999 		objcg = rcu_dereference(memcg->objcg);  in __get_obj_cgroup_from_memcg()
3000 		if (objcg && obj_cgroup_tryget(objcg))  in __get_obj_cgroup_from_memcg()
3002 		objcg = NULL;  in __get_obj_cgroup_from_memcg()
3004 	return objcg;  in __get_obj_cgroup_from_memcg()
3009 	struct obj_cgroup *objcg = NULL;  in get_obj_cgroup_from_current()  local
3020 	objcg = __get_obj_cgroup_from_memcg(memcg);  in get_obj_cgroup_from_current()
3022 	return objcg;  in get_obj_cgroup_from_current()
3027 	struct obj_cgroup *objcg;  in get_obj_cgroup_from_page()  local
3033 		objcg = __folio_objcg(page_folio(page));  in get_obj_cgroup_from_page()
3034 		obj_cgroup_get(objcg);  in get_obj_cgroup_from_page()
3041 			objcg = __get_obj_cgroup_from_memcg(memcg);  in get_obj_cgroup_from_page()
3043 			objcg = NULL;  in get_obj_cgroup_from_page()
3046 	return objcg;  in get_obj_cgroup_from_page()
3066 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,  in obj_cgroup_uncharge_pages()  argument
3071 	memcg = get_mem_cgroup_from_objcg(objcg);  in obj_cgroup_uncharge_pages()
3087 static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp,  in obj_cgroup_charge_pages()  argument
3093 	memcg = get_mem_cgroup_from_objcg(objcg);  in obj_cgroup_charge_pages()
3116 	struct obj_cgroup *objcg;  in __memcg_kmem_charge_page()  local
3119 	objcg = get_obj_cgroup_from_current();  in __memcg_kmem_charge_page()
3120 	if (objcg) {  in __memcg_kmem_charge_page()
3121 		ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order);  in __memcg_kmem_charge_page()
3123 			page->memcg_data = (unsigned long)objcg |  in __memcg_kmem_charge_page()
3127 		obj_cgroup_put(objcg);  in __memcg_kmem_charge_page()
3140 	struct obj_cgroup *objcg;  in __memcg_kmem_uncharge_page()  local
3146 	objcg = __folio_objcg(folio);  in __memcg_kmem_uncharge_page()
3147 	obj_cgroup_uncharge_pages(objcg, nr_pages);  in __memcg_kmem_uncharge_page()
3149 	obj_cgroup_put(objcg);  in __memcg_kmem_uncharge_page()
3152 void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat,  in mod_objcg_state()  argument
3168 	if (stock->cached_objcg != objcg) {  in mod_objcg_state()
3170 		obj_cgroup_get(objcg);  in mod_objcg_state()
3171 		stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)  in mod_objcg_state()
3172 				? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;  in mod_objcg_state()
3173 		stock->cached_objcg = objcg;  in mod_objcg_state()
3180 			mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B,  in mod_objcg_state()
3185 			mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B,  in mod_objcg_state()
3211 		mod_objcg_mlstate(objcg, pgdat, idx, nr);  in mod_objcg_state()
3218 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)  in consume_obj_stock()  argument
3227 	if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {  in consume_obj_stock()
3314 static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes,  in refill_obj_stock()  argument
3325 	if (stock->cached_objcg != objcg) { /* reset if necessary */  in refill_obj_stock()
3327 		obj_cgroup_get(objcg);  in refill_obj_stock()
3328 		stock->cached_objcg = objcg;  in refill_obj_stock()
3329 		stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)  in refill_obj_stock()
3330 				? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;  in refill_obj_stock()
3345 		obj_cgroup_uncharge_pages(objcg, nr_pages);  in refill_obj_stock()
3348 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)  in obj_cgroup_charge()  argument
3353 	if (consume_obj_stock(objcg, size))  in obj_cgroup_charge()
3385 	ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages);  in obj_cgroup_charge()
3387 		refill_obj_stock(objcg, PAGE_SIZE - nr_bytes, false);  in obj_cgroup_charge()
3392 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)  in obj_cgroup_uncharge()  argument
3394 	refill_obj_stock(objcg, size, true);  in obj_cgroup_uncharge()
3723 	struct obj_cgroup *objcg;  in memcg_online_kmem()  local
3731 	objcg = obj_cgroup_alloc();  in memcg_online_kmem()
3732 	if (!objcg)  in memcg_online_kmem()
3735 	objcg->memcg = memcg;  in memcg_online_kmem()
3736 	rcu_assign_pointer(memcg->objcg, objcg);  in memcg_online_kmem()
7025 	struct obj_cgroup *objcg;  in uncharge_folio()  local
7035 		objcg = __folio_objcg(folio);  in uncharge_folio()
7040 		memcg = get_mem_cgroup_from_objcg(objcg);  in uncharge_folio()
7067 		obj_cgroup_put(objcg);  in uncharge_folio()
7653 bool obj_cgroup_may_zswap(struct obj_cgroup *objcg)  in obj_cgroup_may_zswap()  argument
7661 	original_memcg = get_mem_cgroup_from_objcg(objcg);  in obj_cgroup_may_zswap()
7693 void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size)  in obj_cgroup_charge_zswap()  argument
7703 	if (obj_cgroup_charge(objcg, GFP_KERNEL, size))  in obj_cgroup_charge_zswap()
7707 	memcg = obj_cgroup_memcg(objcg);  in obj_cgroup_charge_zswap()
7720 void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size)  in obj_cgroup_uncharge_zswap()  argument
7727 	obj_cgroup_uncharge(objcg, size);  in obj_cgroup_uncharge_zswap()
7730 	memcg = obj_cgroup_memcg(objcg);  in obj_cgroup_uncharge_zswap()