Lines Matching refs:stock
2202 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock);
2203 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2208 static inline struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) in drain_obj_stock() argument
2212 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, in obj_stock_flush_required() argument
2235 struct memcg_stock_pcp *stock; in consume_stock() local
2244 stock = this_cpu_ptr(&memcg_stock); in consume_stock()
2245 if (memcg == READ_ONCE(stock->cached) && stock->nr_pages >= nr_pages) { in consume_stock()
2246 stock->nr_pages -= nr_pages; in consume_stock()
2258 static void drain_stock(struct memcg_stock_pcp *stock) in drain_stock() argument
2260 struct mem_cgroup *old = READ_ONCE(stock->cached); in drain_stock()
2265 if (stock->nr_pages) { in drain_stock()
2266 page_counter_uncharge(&old->memory, stock->nr_pages); in drain_stock()
2268 page_counter_uncharge(&old->memsw, stock->nr_pages); in drain_stock()
2269 stock->nr_pages = 0; in drain_stock()
2273 WRITE_ONCE(stock->cached, NULL); in drain_stock()
2278 struct memcg_stock_pcp *stock; in drain_local_stock() local
2289 stock = this_cpu_ptr(&memcg_stock); in drain_local_stock()
2290 old = drain_obj_stock(stock); in drain_local_stock()
2291 drain_stock(stock); in drain_local_stock()
2292 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); in drain_local_stock()
2305 struct memcg_stock_pcp *stock; in __refill_stock() local
2307 stock = this_cpu_ptr(&memcg_stock); in __refill_stock()
2308 if (READ_ONCE(stock->cached) != memcg) { /* reset if necessary */ in __refill_stock()
2309 drain_stock(stock); in __refill_stock()
2311 WRITE_ONCE(stock->cached, memcg); in __refill_stock()
2313 stock->nr_pages += nr_pages; in __refill_stock()
2315 if (stock->nr_pages > MEMCG_CHARGE_BATCH) in __refill_stock()
2316 drain_stock(stock); in __refill_stock()
2348 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); in drain_all_stock() local
2353 memcg = READ_ONCE(stock->cached); in drain_all_stock()
2354 if (memcg && stock->nr_pages && in drain_all_stock()
2357 else if (obj_stock_flush_required(stock, root_memcg)) in drain_all_stock()
2362 !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) { in drain_all_stock()
2364 drain_local_stock(&stock->work); in drain_all_stock()
2366 schedule_work_on(cpu, &stock->work); in drain_all_stock()
2375 struct memcg_stock_pcp *stock; in memcg_hotplug_cpu_dead() local
2377 stock = &per_cpu(memcg_stock, cpu); in memcg_hotplug_cpu_dead()
2378 drain_stock(stock); in memcg_hotplug_cpu_dead()
3166 struct memcg_stock_pcp *stock; in mod_objcg_state() local
3172 stock = this_cpu_ptr(&memcg_stock); in mod_objcg_state()
3179 if (READ_ONCE(stock->cached_objcg) != objcg) { in mod_objcg_state()
3180 old = drain_obj_stock(stock); in mod_objcg_state()
3182 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in mod_objcg_state()
3184 WRITE_ONCE(stock->cached_objcg, objcg); in mod_objcg_state()
3185 stock->cached_pgdat = pgdat; in mod_objcg_state()
3186 } else if (stock->cached_pgdat != pgdat) { in mod_objcg_state()
3188 struct pglist_data *oldpg = stock->cached_pgdat; in mod_objcg_state()
3190 if (stock->nr_slab_reclaimable_b) { in mod_objcg_state()
3192 stock->nr_slab_reclaimable_b); in mod_objcg_state()
3193 stock->nr_slab_reclaimable_b = 0; in mod_objcg_state()
3195 if (stock->nr_slab_unreclaimable_b) { in mod_objcg_state()
3197 stock->nr_slab_unreclaimable_b); in mod_objcg_state()
3198 stock->nr_slab_unreclaimable_b = 0; in mod_objcg_state()
3200 stock->cached_pgdat = pgdat; in mod_objcg_state()
3203 bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b in mod_objcg_state()
3204 : &stock->nr_slab_unreclaimable_b; in mod_objcg_state()
3231 struct memcg_stock_pcp *stock; in consume_obj_stock() local
3237 stock = this_cpu_ptr(&memcg_stock); in consume_obj_stock()
3238 if (objcg == READ_ONCE(stock->cached_objcg) && stock->nr_bytes >= nr_bytes) { in consume_obj_stock()
3239 stock->nr_bytes -= nr_bytes; in consume_obj_stock()
3248 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) in drain_obj_stock() argument
3250 struct obj_cgroup *old = READ_ONCE(stock->cached_objcg); in drain_obj_stock()
3255 if (stock->nr_bytes) { in drain_obj_stock()
3256 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT; in drain_obj_stock()
3257 unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1); in drain_obj_stock()
3281 stock->nr_bytes = 0; in drain_obj_stock()
3287 if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) { in drain_obj_stock()
3288 if (stock->nr_slab_reclaimable_b) { in drain_obj_stock()
3289 mod_objcg_mlstate(old, stock->cached_pgdat, in drain_obj_stock()
3291 stock->nr_slab_reclaimable_b); in drain_obj_stock()
3292 stock->nr_slab_reclaimable_b = 0; in drain_obj_stock()
3294 if (stock->nr_slab_unreclaimable_b) { in drain_obj_stock()
3295 mod_objcg_mlstate(old, stock->cached_pgdat, in drain_obj_stock()
3297 stock->nr_slab_unreclaimable_b); in drain_obj_stock()
3298 stock->nr_slab_unreclaimable_b = 0; in drain_obj_stock()
3300 stock->cached_pgdat = NULL; in drain_obj_stock()
3303 WRITE_ONCE(stock->cached_objcg, NULL); in drain_obj_stock()
3311 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, in obj_stock_flush_required() argument
3314 struct obj_cgroup *objcg = READ_ONCE(stock->cached_objcg); in obj_stock_flush_required()
3329 struct memcg_stock_pcp *stock; in refill_obj_stock() local
3336 stock = this_cpu_ptr(&memcg_stock); in refill_obj_stock()
3337 if (READ_ONCE(stock->cached_objcg) != objcg) { /* reset if necessary */ in refill_obj_stock()
3338 old = drain_obj_stock(stock); in refill_obj_stock()
3340 WRITE_ONCE(stock->cached_objcg, objcg); in refill_obj_stock()
3341 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in refill_obj_stock()
3345 stock->nr_bytes += nr_bytes; in refill_obj_stock()
3347 if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) { in refill_obj_stock()
3348 nr_pages = stock->nr_bytes >> PAGE_SHIFT; in refill_obj_stock()
3349 stock->nr_bytes &= (PAGE_SIZE - 1); in refill_obj_stock()