Lines Matching refs:stock
2066 static void drain_obj_stock(struct obj_stock *stock);
2067 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2071 static inline void drain_obj_stock(struct obj_stock *stock) in drain_obj_stock() argument
2074 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, in obj_stock_flush_required() argument
2094 struct memcg_stock_pcp *stock; in get_obj_stock() local
2099 stock = this_cpu_ptr(&memcg_stock); in get_obj_stock()
2100 return &stock->task_obj; in get_obj_stock()
2104 stock = this_cpu_ptr(&memcg_stock); in get_obj_stock()
2105 return &stock->irq_obj; in get_obj_stock()
2129 struct memcg_stock_pcp *stock; in consume_stock() local
2138 stock = this_cpu_ptr(&memcg_stock); in consume_stock()
2139 if (memcg == stock->cached && stock->nr_pages >= nr_pages) { in consume_stock()
2140 stock->nr_pages -= nr_pages; in consume_stock()
2152 static void drain_stock(struct memcg_stock_pcp *stock) in drain_stock() argument
2154 struct mem_cgroup *old = stock->cached; in drain_stock()
2159 if (stock->nr_pages) { in drain_stock()
2160 page_counter_uncharge(&old->memory, stock->nr_pages); in drain_stock()
2162 page_counter_uncharge(&old->memsw, stock->nr_pages); in drain_stock()
2163 stock->nr_pages = 0; in drain_stock()
2167 stock->cached = NULL; in drain_stock()
2172 struct memcg_stock_pcp *stock; in drain_local_stock() local
2182 stock = this_cpu_ptr(&memcg_stock); in drain_local_stock()
2183 drain_obj_stock(&stock->irq_obj); in drain_local_stock()
2185 drain_obj_stock(&stock->task_obj); in drain_local_stock()
2186 drain_stock(stock); in drain_local_stock()
2187 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); in drain_local_stock()
2198 struct memcg_stock_pcp *stock; in refill_stock() local
2203 stock = this_cpu_ptr(&memcg_stock); in refill_stock()
2204 if (stock->cached != memcg) { /* reset if necessary */ in refill_stock()
2205 drain_stock(stock); in refill_stock()
2207 stock->cached = memcg; in refill_stock()
2209 stock->nr_pages += nr_pages; in refill_stock()
2211 if (stock->nr_pages > MEMCG_CHARGE_BATCH) in refill_stock()
2212 drain_stock(stock); in refill_stock()
2236 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); in drain_all_stock() local
2241 memcg = stock->cached; in drain_all_stock()
2242 if (memcg && stock->nr_pages && in drain_all_stock()
2245 else if (obj_stock_flush_required(stock, root_memcg)) in drain_all_stock()
2250 !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) { in drain_all_stock()
2252 drain_local_stock(&stock->work); in drain_all_stock()
2254 schedule_work_on(cpu, &stock->work); in drain_all_stock()
2263 struct memcg_stock_pcp *stock; in memcg_hotplug_cpu_dead() local
2265 stock = &per_cpu(memcg_stock, cpu); in memcg_hotplug_cpu_dead()
2266 drain_stock(stock); in memcg_hotplug_cpu_dead()
3034 struct obj_stock *stock = get_obj_stock(&flags); in mod_objcg_state() local
3042 if (stock->cached_objcg != objcg) { in mod_objcg_state()
3043 drain_obj_stock(stock); in mod_objcg_state()
3045 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in mod_objcg_state()
3047 stock->cached_objcg = objcg; in mod_objcg_state()
3048 stock->cached_pgdat = pgdat; in mod_objcg_state()
3049 } else if (stock->cached_pgdat != pgdat) { in mod_objcg_state()
3051 struct pglist_data *oldpg = stock->cached_pgdat; in mod_objcg_state()
3053 if (stock->nr_slab_reclaimable_b) { in mod_objcg_state()
3055 stock->nr_slab_reclaimable_b); in mod_objcg_state()
3056 stock->nr_slab_reclaimable_b = 0; in mod_objcg_state()
3058 if (stock->nr_slab_unreclaimable_b) { in mod_objcg_state()
3060 stock->nr_slab_unreclaimable_b); in mod_objcg_state()
3061 stock->nr_slab_unreclaimable_b = 0; in mod_objcg_state()
3063 stock->cached_pgdat = pgdat; in mod_objcg_state()
3066 bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b in mod_objcg_state()
3067 : &stock->nr_slab_unreclaimable_b; in mod_objcg_state()
3093 struct obj_stock *stock = get_obj_stock(&flags); in consume_obj_stock() local
3096 if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) { in consume_obj_stock()
3097 stock->nr_bytes -= nr_bytes; in consume_obj_stock()
3106 static void drain_obj_stock(struct obj_stock *stock) in drain_obj_stock() argument
3108 struct obj_cgroup *old = stock->cached_objcg; in drain_obj_stock()
3113 if (stock->nr_bytes) { in drain_obj_stock()
3114 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT; in drain_obj_stock()
3115 unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1); in drain_obj_stock()
3131 stock->nr_bytes = 0; in drain_obj_stock()
3137 if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) { in drain_obj_stock()
3138 if (stock->nr_slab_reclaimable_b) { in drain_obj_stock()
3139 mod_objcg_mlstate(old, stock->cached_pgdat, in drain_obj_stock()
3141 stock->nr_slab_reclaimable_b); in drain_obj_stock()
3142 stock->nr_slab_reclaimable_b = 0; in drain_obj_stock()
3144 if (stock->nr_slab_unreclaimable_b) { in drain_obj_stock()
3145 mod_objcg_mlstate(old, stock->cached_pgdat, in drain_obj_stock()
3147 stock->nr_slab_unreclaimable_b); in drain_obj_stock()
3148 stock->nr_slab_unreclaimable_b = 0; in drain_obj_stock()
3150 stock->cached_pgdat = NULL; in drain_obj_stock()
3154 stock->cached_objcg = NULL; in drain_obj_stock()
3157 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, in obj_stock_flush_required() argument
3162 if (in_task() && stock->task_obj.cached_objcg) { in obj_stock_flush_required()
3163 memcg = obj_cgroup_memcg(stock->task_obj.cached_objcg); in obj_stock_flush_required()
3167 if (stock->irq_obj.cached_objcg) { in obj_stock_flush_required()
3168 memcg = obj_cgroup_memcg(stock->irq_obj.cached_objcg); in obj_stock_flush_required()
3180 struct obj_stock *stock = get_obj_stock(&flags); in refill_obj_stock() local
3183 if (stock->cached_objcg != objcg) { /* reset if necessary */ in refill_obj_stock()
3184 drain_obj_stock(stock); in refill_obj_stock()
3186 stock->cached_objcg = objcg; in refill_obj_stock()
3187 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in refill_obj_stock()
3191 stock->nr_bytes += nr_bytes; in refill_obj_stock()
3193 if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) { in refill_obj_stock()
3194 nr_pages = stock->nr_bytes >> PAGE_SHIFT; in refill_obj_stock()
3195 stock->nr_bytes &= (PAGE_SIZE - 1); in refill_obj_stock()