Lines Matching refs:stock
2192 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock);
2193 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2198 static inline struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) in drain_obj_stock() argument
2202 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, in obj_stock_flush_required() argument
2225 struct memcg_stock_pcp *stock; in consume_stock() local
2234 stock = this_cpu_ptr(&memcg_stock); in consume_stock()
2235 if (memcg == stock->cached && stock->nr_pages >= nr_pages) { in consume_stock()
2236 stock->nr_pages -= nr_pages; in consume_stock()
2248 static void drain_stock(struct memcg_stock_pcp *stock) in drain_stock() argument
2250 struct mem_cgroup *old = stock->cached; in drain_stock()
2255 if (stock->nr_pages) { in drain_stock()
2256 page_counter_uncharge(&old->memory, stock->nr_pages); in drain_stock()
2258 page_counter_uncharge(&old->memsw, stock->nr_pages); in drain_stock()
2259 stock->nr_pages = 0; in drain_stock()
2263 stock->cached = NULL; in drain_stock()
2268 struct memcg_stock_pcp *stock; in drain_local_stock() local
2279 stock = this_cpu_ptr(&memcg_stock); in drain_local_stock()
2280 old = drain_obj_stock(stock); in drain_local_stock()
2281 drain_stock(stock); in drain_local_stock()
2282 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); in drain_local_stock()
2295 struct memcg_stock_pcp *stock; in __refill_stock() local
2297 stock = this_cpu_ptr(&memcg_stock); in __refill_stock()
2298 if (stock->cached != memcg) { /* reset if necessary */ in __refill_stock()
2299 drain_stock(stock); in __refill_stock()
2301 stock->cached = memcg; in __refill_stock()
2303 stock->nr_pages += nr_pages; in __refill_stock()
2305 if (stock->nr_pages > MEMCG_CHARGE_BATCH) in __refill_stock()
2306 drain_stock(stock); in __refill_stock()
2338 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); in drain_all_stock() local
2343 memcg = stock->cached; in drain_all_stock()
2344 if (memcg && stock->nr_pages && in drain_all_stock()
2347 else if (obj_stock_flush_required(stock, root_memcg)) in drain_all_stock()
2352 !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) { in drain_all_stock()
2354 drain_local_stock(&stock->work); in drain_all_stock()
2356 schedule_work_on(cpu, &stock->work); in drain_all_stock()
2365 struct memcg_stock_pcp *stock; in memcg_hotplug_cpu_dead() local
2367 stock = &per_cpu(memcg_stock, cpu); in memcg_hotplug_cpu_dead()
2368 drain_stock(stock); in memcg_hotplug_cpu_dead()
3155 struct memcg_stock_pcp *stock; in mod_objcg_state() local
3161 stock = this_cpu_ptr(&memcg_stock); in mod_objcg_state()
3168 if (stock->cached_objcg != objcg) { in mod_objcg_state()
3169 old = drain_obj_stock(stock); in mod_objcg_state()
3171 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in mod_objcg_state()
3173 stock->cached_objcg = objcg; in mod_objcg_state()
3174 stock->cached_pgdat = pgdat; in mod_objcg_state()
3175 } else if (stock->cached_pgdat != pgdat) { in mod_objcg_state()
3177 struct pglist_data *oldpg = stock->cached_pgdat; in mod_objcg_state()
3179 if (stock->nr_slab_reclaimable_b) { in mod_objcg_state()
3181 stock->nr_slab_reclaimable_b); in mod_objcg_state()
3182 stock->nr_slab_reclaimable_b = 0; in mod_objcg_state()
3184 if (stock->nr_slab_unreclaimable_b) { in mod_objcg_state()
3186 stock->nr_slab_unreclaimable_b); in mod_objcg_state()
3187 stock->nr_slab_unreclaimable_b = 0; in mod_objcg_state()
3189 stock->cached_pgdat = pgdat; in mod_objcg_state()
3192 bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b in mod_objcg_state()
3193 : &stock->nr_slab_unreclaimable_b; in mod_objcg_state()
3220 struct memcg_stock_pcp *stock; in consume_obj_stock() local
3226 stock = this_cpu_ptr(&memcg_stock); in consume_obj_stock()
3227 if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) { in consume_obj_stock()
3228 stock->nr_bytes -= nr_bytes; in consume_obj_stock()
3237 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) in drain_obj_stock() argument
3239 struct obj_cgroup *old = stock->cached_objcg; in drain_obj_stock()
3244 if (stock->nr_bytes) { in drain_obj_stock()
3245 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT; in drain_obj_stock()
3246 unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1); in drain_obj_stock()
3270 stock->nr_bytes = 0; in drain_obj_stock()
3276 if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) { in drain_obj_stock()
3277 if (stock->nr_slab_reclaimable_b) { in drain_obj_stock()
3278 mod_objcg_mlstate(old, stock->cached_pgdat, in drain_obj_stock()
3280 stock->nr_slab_reclaimable_b); in drain_obj_stock()
3281 stock->nr_slab_reclaimable_b = 0; in drain_obj_stock()
3283 if (stock->nr_slab_unreclaimable_b) { in drain_obj_stock()
3284 mod_objcg_mlstate(old, stock->cached_pgdat, in drain_obj_stock()
3286 stock->nr_slab_unreclaimable_b); in drain_obj_stock()
3287 stock->nr_slab_unreclaimable_b = 0; in drain_obj_stock()
3289 stock->cached_pgdat = NULL; in drain_obj_stock()
3292 stock->cached_objcg = NULL; in drain_obj_stock()
3300 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, in obj_stock_flush_required() argument
3305 if (stock->cached_objcg) { in obj_stock_flush_required()
3306 memcg = obj_cgroup_memcg(stock->cached_objcg); in obj_stock_flush_required()
3317 struct memcg_stock_pcp *stock; in refill_obj_stock() local
3324 stock = this_cpu_ptr(&memcg_stock); in refill_obj_stock()
3325 if (stock->cached_objcg != objcg) { /* reset if necessary */ in refill_obj_stock()
3326 old = drain_obj_stock(stock); in refill_obj_stock()
3328 stock->cached_objcg = objcg; in refill_obj_stock()
3329 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in refill_obj_stock()
3333 stock->nr_bytes += nr_bytes; in refill_obj_stock()
3335 if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) { in refill_obj_stock()
3336 nr_pages = stock->nr_bytes >> PAGE_SHIFT; in refill_obj_stock()
3337 stock->nr_bytes &= (PAGE_SIZE - 1); in refill_obj_stock()