2223 static void drain_obj_stock(struct memcg_stock_pcp *stock);
2224 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2228 static inline void drain_obj_stock(struct memcg_stock_pcp *stock)  in drain_obj_stock()  argument
2231 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,  in obj_stock_flush_required()  argument
2251 	struct memcg_stock_pcp *stock;  in consume_stock()  local
2260 	stock = this_cpu_ptr(&memcg_stock);  in consume_stock()
2261 	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {  in consume_stock()
2262 		stock->nr_pages -= nr_pages;  in consume_stock()
2274 static void drain_stock(struct memcg_stock_pcp *stock)  in drain_stock()  argument
2276 	struct mem_cgroup *old = stock->cached;  in drain_stock()
2281 	if (stock->nr_pages) {  in drain_stock()
2282 		page_counter_uncharge(&old->memory, stock->nr_pages);  in drain_stock()
2284 			page_counter_uncharge(&old->memsw, stock->nr_pages);  in drain_stock()
2285 		stock->nr_pages = 0;  in drain_stock()
2289 	stock->cached = NULL;  in drain_stock()
2294 	struct memcg_stock_pcp *stock;  in drain_local_stock()  local
2303 	stock = this_cpu_ptr(&memcg_stock);  in drain_local_stock()
2304 	drain_obj_stock(stock);  in drain_local_stock()
2305 	drain_stock(stock);  in drain_local_stock()
2306 	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);  in drain_local_stock()
2317 	struct memcg_stock_pcp *stock;  in refill_stock()  local
2322 	stock = this_cpu_ptr(&memcg_stock);  in refill_stock()
2323 	if (stock->cached != memcg) { /* reset if necessary */  in refill_stock()
2324 		drain_stock(stock);  in refill_stock()
2326 		stock->cached = memcg;  in refill_stock()
2328 	stock->nr_pages += nr_pages;  in refill_stock()
2330 	if (stock->nr_pages > MEMCG_CHARGE_BATCH)  in refill_stock()
2331 		drain_stock(stock);  in refill_stock()
2355 		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);  in drain_all_stock()  local
2360 		memcg = stock->cached;  in drain_all_stock()
2361 		if (memcg && stock->nr_pages &&  in drain_all_stock()
2364 		if (obj_stock_flush_required(stock, root_memcg))  in drain_all_stock()
2369 		    !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {  in drain_all_stock()
2371 				drain_local_stock(&stock->work);  in drain_all_stock()
2373 				schedule_work_on(cpu, &stock->work);  in drain_all_stock()
2382 	struct memcg_stock_pcp *stock;  in memcg_hotplug_cpu_dead()  local
2385 	stock = &per_cpu(memcg_stock, cpu);  in memcg_hotplug_cpu_dead()
2386 	drain_stock(stock);  in memcg_hotplug_cpu_dead()
3141 	struct memcg_stock_pcp *stock;  in consume_obj_stock()  local
3147 	stock = this_cpu_ptr(&memcg_stock);  in consume_obj_stock()
3148 	if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {  in consume_obj_stock()
3149 		stock->nr_bytes -= nr_bytes;  in consume_obj_stock()
3158 static void drain_obj_stock(struct memcg_stock_pcp *stock)  in drain_obj_stock()  argument
3160 	struct obj_cgroup *old = stock->cached_objcg;  in drain_obj_stock()
3165 	if (stock->nr_bytes) {  in drain_obj_stock()
3166 		unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;  in drain_obj_stock()
3167 		unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1);  in drain_obj_stock()
3186 		stock->nr_bytes = 0;  in drain_obj_stock()
3190 	stock->cached_objcg = NULL;  in drain_obj_stock()
3193 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,  in obj_stock_flush_required()  argument
3198 	if (stock->cached_objcg) {  in obj_stock_flush_required()
3199 		memcg = obj_cgroup_memcg(stock->cached_objcg);  in obj_stock_flush_required()
3209 	struct memcg_stock_pcp *stock;  in refill_obj_stock()  local
3214 	stock = this_cpu_ptr(&memcg_stock);  in refill_obj_stock()
3215 	if (stock->cached_objcg != objcg) { /* reset if necessary */  in refill_obj_stock()
3216 		drain_obj_stock(stock);  in refill_obj_stock()
3218 		stock->cached_objcg = objcg;  in refill_obj_stock()
3219 		stock->nr_bytes = atomic_xchg(&objcg->nr_charged_bytes, 0);  in refill_obj_stock()
3221 	stock->nr_bytes += nr_bytes;  in refill_obj_stock()
3223 	if (stock->nr_bytes > PAGE_SIZE)  in refill_obj_stock()
3224 		drain_obj_stock(stock);  in refill_obj_stock()