Lines Matching refs:blkg
72 static void blkg_free(struct blkcg_gq *blkg) in blkg_free() argument
76 if (!blkg) in blkg_free()
80 if (blkg->pd[i]) in blkg_free()
81 blkcg_policy[i]->pd_free_fn(blkg->pd[i]); in blkg_free()
83 blkg_rwstat_exit(&blkg->stat_ios); in blkg_free()
84 blkg_rwstat_exit(&blkg->stat_bytes); in blkg_free()
85 percpu_ref_exit(&blkg->refcnt); in blkg_free()
86 kfree(blkg); in blkg_free()
91 struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head); in __blkg_release() local
93 WARN_ON(!bio_list_empty(&blkg->async_bios)); in __blkg_release()
96 css_put(&blkg->blkcg->css); in __blkg_release()
97 if (blkg->parent) in __blkg_release()
98 blkg_put(blkg->parent); in __blkg_release()
100 wb_congested_put(blkg->wb_congested); in __blkg_release()
102 blkg_free(blkg); in __blkg_release()
115 struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt); in blkg_release() local
117 call_rcu(&blkg->rcu_head, __blkg_release); in blkg_release()
122 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq, in blkg_async_bio_workfn() local
128 spin_lock_bh(&blkg->async_bio_lock); in blkg_async_bio_workfn()
129 bio_list_merge(&bios, &blkg->async_bios); in blkg_async_bio_workfn()
130 bio_list_init(&blkg->async_bios); in blkg_async_bio_workfn()
131 spin_unlock_bh(&blkg->async_bio_lock); in blkg_async_bio_workfn()
148 struct blkcg_gq *blkg; in blkg_alloc() local
152 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node); in blkg_alloc()
153 if (!blkg) in blkg_alloc()
156 if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask)) in blkg_alloc()
159 if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) || in blkg_alloc()
160 blkg_rwstat_init(&blkg->stat_ios, gfp_mask)) in blkg_alloc()
163 blkg->q = q; in blkg_alloc()
164 INIT_LIST_HEAD(&blkg->q_node); in blkg_alloc()
165 spin_lock_init(&blkg->async_bio_lock); in blkg_alloc()
166 bio_list_init(&blkg->async_bios); in blkg_alloc()
167 INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn); in blkg_alloc()
168 blkg->blkcg = blkcg; in blkg_alloc()
182 blkg->pd[i] = pd; in blkg_alloc()
183 pd->blkg = blkg; in blkg_alloc()
187 return blkg; in blkg_alloc()
190 blkg_free(blkg); in blkg_alloc()
197 struct blkcg_gq *blkg; in blkg_lookup_slowpath() local
205 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id); in blkg_lookup_slowpath()
206 if (blkg && blkg->q == q) { in blkg_lookup_slowpath()
209 rcu_assign_pointer(blkcg->blkg_hint, blkg); in blkg_lookup_slowpath()
211 return blkg; in blkg_lookup_slowpath()
226 struct blkcg_gq *blkg; in blkg_create() local
261 blkg = new_blkg; in blkg_create()
262 blkg->wb_congested = wb_congested; in blkg_create()
266 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false); in blkg_create()
267 if (WARN_ON_ONCE(!blkg->parent)) { in blkg_create()
271 blkg_get(blkg->parent); in blkg_create()
278 if (blkg->pd[i] && pol->pd_init_fn) in blkg_create()
279 pol->pd_init_fn(blkg->pd[i]); in blkg_create()
284 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg); in blkg_create()
286 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); in blkg_create()
287 list_add(&blkg->q_node, &q->blkg_list); in blkg_create()
292 if (blkg->pd[i] && pol->pd_online_fn) in blkg_create()
293 pol->pd_online_fn(blkg->pd[i]); in blkg_create()
296 blkg->online = true; in blkg_create()
300 return blkg; in blkg_create()
303 blkg_put(blkg); in blkg_create()
331 struct blkcg_gq *blkg; in __blkg_lookup_create() local
336 blkg = __blkg_lookup(blkcg, q, true); in __blkg_lookup_create()
337 if (blkg) in __blkg_lookup_create()
338 return blkg; in __blkg_lookup_create()
351 blkg = __blkg_lookup(parent, q, false); in __blkg_lookup_create()
352 if (blkg) { in __blkg_lookup_create()
354 ret_blkg = blkg; in __blkg_lookup_create()
361 blkg = blkg_create(pos, q, NULL); in __blkg_lookup_create()
362 if (IS_ERR(blkg)) in __blkg_lookup_create()
365 return blkg; in __blkg_lookup_create()
380 struct blkcg_gq *blkg = blkg_lookup(blkcg, q); in blkg_lookup_create() local
382 if (unlikely(!blkg)) { in blkg_lookup_create()
386 blkg = __blkg_lookup_create(blkcg, q); in blkg_lookup_create()
390 return blkg; in blkg_lookup_create()
393 static void blkg_destroy(struct blkcg_gq *blkg) in blkg_destroy() argument
395 struct blkcg *blkcg = blkg->blkcg; in blkg_destroy()
396 struct blkcg_gq *parent = blkg->parent; in blkg_destroy()
399 lockdep_assert_held(&blkg->q->queue_lock); in blkg_destroy()
403 WARN_ON_ONCE(list_empty(&blkg->q_node)); in blkg_destroy()
404 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node)); in blkg_destroy()
409 if (blkg->pd[i] && pol->pd_offline_fn) in blkg_destroy()
410 pol->pd_offline_fn(blkg->pd[i]); in blkg_destroy()
414 blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes); in blkg_destroy()
415 blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios); in blkg_destroy()
418 blkg->online = false; in blkg_destroy()
420 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id); in blkg_destroy()
421 list_del_init(&blkg->q_node); in blkg_destroy()
422 hlist_del_init_rcu(&blkg->blkcg_node); in blkg_destroy()
429 if (rcu_access_pointer(blkcg->blkg_hint) == blkg) in blkg_destroy()
436 percpu_ref_kill(&blkg->refcnt); in blkg_destroy()
447 struct blkcg_gq *blkg, *n; in blkg_destroy_all() local
450 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) { in blkg_destroy_all()
451 struct blkcg *blkcg = blkg->blkcg; in blkg_destroy_all()
454 blkg_destroy(blkg); in blkg_destroy_all()
466 struct blkcg_gq *blkg; in blkcg_reset_stats() local
477 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_reset_stats()
478 blkg_rwstat_reset(&blkg->stat_bytes); in blkcg_reset_stats()
479 blkg_rwstat_reset(&blkg->stat_ios); in blkcg_reset_stats()
484 if (blkg->pd[i] && pol->pd_reset_stats_fn) in blkcg_reset_stats()
485 pol->pd_reset_stats_fn(blkg->pd[i]); in blkcg_reset_stats()
494 const char *blkg_dev_name(struct blkcg_gq *blkg) in blkg_dev_name() argument
497 if (blkg->q->backing_dev_info->dev) in blkg_dev_name()
498 return dev_name(blkg->q->backing_dev_info->dev); in blkg_dev_name()
526 struct blkcg_gq *blkg; in blkcg_print_blkgs() local
530 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_print_blkgs()
531 spin_lock_irq(&blkg->q->queue_lock); in blkcg_print_blkgs()
532 if (blkcg_policy_enabled(blkg->q, pol)) in blkcg_print_blkgs()
533 total += prfill(sf, blkg->pd[pol->plid], data); in blkcg_print_blkgs()
534 spin_unlock_irq(&blkg->q->queue_lock); in blkcg_print_blkgs()
553 const char *dname = blkg_dev_name(pd->blkg); in __blkg_prfill_u64()
581 const char *dname = blkg_dev_name(pd->blkg); in __blkg_prfill_rwstat()
623 blkg_rwstat_read((void *)pd->blkg + off, &rwstat); in blkg_prfill_rwstat_field()
667 blkg_rwstat_recursive_sum(pd->blkg, NULL, off, &rwstat); in blkg_prfill_rwstat_field_recursive()
715 void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol, in blkg_rwstat_recursive_sum() argument
722 lockdep_assert_held(&blkg->q->queue_lock); in blkg_rwstat_recursive_sum()
725 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) { in blkg_rwstat_recursive_sum()
812 struct blkcg_gq *blkg; in blkg_conf_prep() local
824 blkg = blkg_lookup_check(blkcg, pol, q); in blkg_conf_prep()
825 if (IS_ERR(blkg)) { in blkg_conf_prep()
826 ret = PTR_ERR(blkg); in blkg_conf_prep()
830 if (blkg) in blkg_conf_prep()
861 blkg = blkg_lookup_check(pos, pol, q); in blkg_conf_prep()
862 if (IS_ERR(blkg)) { in blkg_conf_prep()
863 ret = PTR_ERR(blkg); in blkg_conf_prep()
867 if (blkg) { in blkg_conf_prep()
870 blkg = blkg_create(pos, q, new_blkg); in blkg_conf_prep()
871 if (IS_ERR(blkg)) { in blkg_conf_prep()
872 ret = PTR_ERR(blkg); in blkg_conf_prep()
882 ctx->blkg = blkg; in blkg_conf_prep()
924 struct blkcg_gq *blkg; in blkcg_print_stat() local
928 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_print_stat()
937 spin_lock_irq(&blkg->q->queue_lock); in blkcg_print_stat()
939 if (!blkg->online) in blkcg_print_stat()
942 dname = blkg_dev_name(blkg); in blkcg_print_stat()
954 blkg_rwstat_recursive_sum(blkg, NULL, in blkcg_print_stat()
960 blkg_rwstat_recursive_sum(blkg, NULL, in blkcg_print_stat()
974 if (blkcg_debug_stats && atomic_read(&blkg->use_delay)) { in blkcg_print_stat()
978 atomic_read(&blkg->use_delay), in blkcg_print_stat()
979 (unsigned long long)atomic64_read(&blkg->delay_nsec)); in blkcg_print_stat()
986 if (!blkg->pd[i] || !pol->pd_stat_fn) in blkcg_print_stat()
989 written = pol->pd_stat_fn(blkg->pd[i], buf+off, size-off); in blkcg_print_stat()
1004 spin_unlock_irq(&blkg->q->queue_lock); in blkcg_print_stat()
1084 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first, in blkcg_destroy_blkgs() local
1086 struct request_queue *q = blkg->q; in blkcg_destroy_blkgs()
1089 blkg_destroy(blkg); in blkcg_destroy_blkgs()
1199 struct blkcg_gq *new_blkg, *blkg; in blkcg_init_queue() local
1212 blkg = blkg_create(&blkcg_root, q, new_blkg); in blkcg_init_queue()
1213 if (IS_ERR(blkg)) in blkcg_init_queue()
1215 q->root_blkg = blkg; in blkcg_init_queue()
1239 return PTR_ERR(blkg); in blkcg_init_queue()
1368 struct blkcg_gq *blkg, *pinned_blkg = NULL; in blkcg_activate_policy() local
1380 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) { in blkcg_activate_policy()
1383 if (blkg->pd[pol->plid]) in blkcg_activate_policy()
1387 if (blkg == pinned_blkg) { in blkcg_activate_policy()
1392 blkg->blkcg); in blkcg_activate_policy()
1402 blkg_get(blkg); in blkcg_activate_policy()
1403 pinned_blkg = blkg; in blkcg_activate_policy()
1410 blkg->blkcg); in blkcg_activate_policy()
1417 blkg->pd[pol->plid] = pd; in blkcg_activate_policy()
1418 pd->blkg = blkg; in blkcg_activate_policy()
1424 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) in blkcg_activate_policy()
1425 pol->pd_init_fn(blkg->pd[pol->plid]); in blkcg_activate_policy()
1443 list_for_each_entry(blkg, &q->blkg_list, q_node) { in blkcg_activate_policy()
1444 if (blkg->pd[pol->plid]) { in blkcg_activate_policy()
1445 pol->pd_free_fn(blkg->pd[pol->plid]); in blkcg_activate_policy()
1446 blkg->pd[pol->plid] = NULL; in blkcg_activate_policy()
1466 struct blkcg_gq *blkg; in blkcg_deactivate_policy() local
1478 list_for_each_entry(blkg, &q->blkg_list, q_node) { in blkcg_deactivate_policy()
1479 if (blkg->pd[pol->plid]) { in blkcg_deactivate_policy()
1481 pol->pd_offline_fn(blkg->pd[pol->plid]); in blkcg_deactivate_policy()
1482 pol->pd_free_fn(blkg->pd[pol->plid]); in blkcg_deactivate_policy()
1483 blkg->pd[pol->plid] = NULL; in blkcg_deactivate_policy()
1616 struct blkcg_gq *blkg = bio->bi_blkg; in __blkcg_punt_bio_submit() local
1622 if (!blkg->parent) in __blkcg_punt_bio_submit()
1625 spin_lock_bh(&blkg->async_bio_lock); in __blkcg_punt_bio_submit()
1626 bio_list_add(&blkg->async_bios, bio); in __blkcg_punt_bio_submit()
1627 spin_unlock_bh(&blkg->async_bio_lock); in __blkcg_punt_bio_submit()
1629 queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work); in __blkcg_punt_bio_submit()
1639 static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now) in blkcg_scale_delay() argument
1641 u64 old = atomic64_read(&blkg->delay_start); in blkcg_scale_delay()
1657 atomic64_cmpxchg(&blkg->delay_start, old, now) == old) { in blkcg_scale_delay()
1658 u64 cur = atomic64_read(&blkg->delay_nsec); in blkcg_scale_delay()
1659 u64 sub = min_t(u64, blkg->last_delay, now - old); in blkcg_scale_delay()
1660 int cur_use = atomic_read(&blkg->use_delay); in blkcg_scale_delay()
1666 if (cur_use < blkg->last_use) in blkcg_scale_delay()
1667 sub = max_t(u64, sub, blkg->last_delay >> 1); in blkcg_scale_delay()
1676 atomic64_set(&blkg->delay_nsec, 0); in blkcg_scale_delay()
1677 blkg->last_delay = 0; in blkcg_scale_delay()
1679 atomic64_sub(sub, &blkg->delay_nsec); in blkcg_scale_delay()
1680 blkg->last_delay = cur - sub; in blkcg_scale_delay()
1682 blkg->last_use = cur_use; in blkcg_scale_delay()
1692 static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay) in blkcg_maybe_throttle_blkg() argument
1700 while (blkg->parent) { in blkcg_maybe_throttle_blkg()
1701 if (atomic_read(&blkg->use_delay)) { in blkcg_maybe_throttle_blkg()
1702 blkcg_scale_delay(blkg, now); in blkcg_maybe_throttle_blkg()
1704 atomic64_read(&blkg->delay_nsec)); in blkcg_maybe_throttle_blkg()
1706 blkg = blkg->parent; in blkcg_maybe_throttle_blkg()
1752 struct blkcg_gq *blkg; in blkcg_maybe_throttle_current() local
1770 blkg = blkg_lookup(blkcg, q); in blkcg_maybe_throttle_current()
1771 if (!blkg) in blkcg_maybe_throttle_current()
1773 if (!blkg_tryget(blkg)) in blkcg_maybe_throttle_current()
1777 blkcg_maybe_throttle_blkg(blkg, use_memdelay); in blkcg_maybe_throttle_current()
1778 blkg_put(blkg); in blkcg_maybe_throttle_current()
1828 void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta) in blkcg_add_delay() argument
1830 blkcg_scale_delay(blkg, now); in blkcg_add_delay()
1831 atomic64_add(delta, &blkg->delay_nsec); in blkcg_add_delay()