Lines Matching refs:blkg
87 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq, in blkg_free_workfn() local
92 if (blkg->pd[i]) in blkg_free_workfn()
93 blkcg_policy[i]->pd_free_fn(blkg->pd[i]); in blkg_free_workfn()
95 if (blkg->q) in blkg_free_workfn()
96 blk_put_queue(blkg->q); in blkg_free_workfn()
97 free_percpu(blkg->iostat_cpu); in blkg_free_workfn()
98 percpu_ref_exit(&blkg->refcnt); in blkg_free_workfn()
99 kfree(blkg); in blkg_free_workfn()
108 static void blkg_free(struct blkcg_gq *blkg) in blkg_free() argument
110 if (!blkg) in blkg_free()
117 INIT_WORK(&blkg->free_work, blkg_free_workfn); in blkg_free()
118 schedule_work(&blkg->free_work); in blkg_free()
123 struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head); in __blkg_release() local
125 WARN_ON(!bio_list_empty(&blkg->async_bios)); in __blkg_release()
128 css_put(&blkg->blkcg->css); in __blkg_release()
129 if (blkg->parent) in __blkg_release()
130 blkg_put(blkg->parent); in __blkg_release()
131 blkg_free(blkg); in __blkg_release()
144 struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt); in blkg_release() local
146 call_rcu(&blkg->rcu_head, __blkg_release); in blkg_release()
151 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq, in blkg_async_bio_workfn() local
159 spin_lock_bh(&blkg->async_bio_lock); in blkg_async_bio_workfn()
160 bio_list_merge(&bios, &blkg->async_bios); in blkg_async_bio_workfn()
161 bio_list_init(&blkg->async_bios); in blkg_async_bio_workfn()
162 spin_unlock_bh(&blkg->async_bio_lock); in blkg_async_bio_workfn()
213 struct blkcg_gq *blkg; in blkg_alloc() local
217 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, disk->queue->node); in blkg_alloc()
218 if (!blkg) in blkg_alloc()
221 if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask)) in blkg_alloc()
224 blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask); in blkg_alloc()
225 if (!blkg->iostat_cpu) in blkg_alloc()
231 blkg->q = disk->queue; in blkg_alloc()
232 INIT_LIST_HEAD(&blkg->q_node); in blkg_alloc()
233 spin_lock_init(&blkg->async_bio_lock); in blkg_alloc()
234 bio_list_init(&blkg->async_bios); in blkg_alloc()
235 INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn); in blkg_alloc()
236 blkg->blkcg = blkcg; in blkg_alloc()
238 u64_stats_init(&blkg->iostat.sync); in blkg_alloc()
240 u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync); in blkg_alloc()
254 blkg->pd[i] = pd; in blkg_alloc()
255 pd->blkg = blkg; in blkg_alloc()
259 return blkg; in blkg_alloc()
262 blkg_free(blkg); in blkg_alloc()
273 struct blkcg_gq *blkg; in blkg_create() local
298 blkg = new_blkg; in blkg_create()
302 blkg->parent = blkg_lookup(blkcg_parent(blkcg), disk->queue); in blkg_create()
303 if (WARN_ON_ONCE(!blkg->parent)) { in blkg_create()
307 blkg_get(blkg->parent); in blkg_create()
314 if (blkg->pd[i] && pol->pd_init_fn) in blkg_create()
315 pol->pd_init_fn(blkg->pd[i]); in blkg_create()
320 ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg); in blkg_create()
322 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); in blkg_create()
323 list_add(&blkg->q_node, &disk->queue->blkg_list); in blkg_create()
328 if (blkg->pd[i] && pol->pd_online_fn) in blkg_create()
329 pol->pd_online_fn(blkg->pd[i]); in blkg_create()
332 blkg->online = true; in blkg_create()
336 return blkg; in blkg_create()
339 blkg_put(blkg); in blkg_create()
366 struct blkcg_gq *blkg; in blkg_lookup_create() local
371 blkg = blkg_lookup(blkcg, q); in blkg_lookup_create()
372 if (blkg) in blkg_lookup_create()
373 return blkg; in blkg_lookup_create()
376 blkg = blkg_lookup(blkcg, q); in blkg_lookup_create()
377 if (blkg) { in blkg_lookup_create()
379 blkg != rcu_dereference(blkcg->blkg_hint)) in blkg_lookup_create()
380 rcu_assign_pointer(blkcg->blkg_hint, blkg); in blkg_lookup_create()
395 blkg = blkg_lookup(parent, q); in blkg_lookup_create()
396 if (blkg) { in blkg_lookup_create()
398 ret_blkg = blkg; in blkg_lookup_create()
405 blkg = blkg_create(pos, disk, NULL); in blkg_lookup_create()
406 if (IS_ERR(blkg)) { in blkg_lookup_create()
407 blkg = ret_blkg; in blkg_lookup_create()
416 return blkg; in blkg_lookup_create()
419 static void blkg_destroy(struct blkcg_gq *blkg) in blkg_destroy() argument
421 struct blkcg *blkcg = blkg->blkcg; in blkg_destroy()
424 lockdep_assert_held(&blkg->q->queue_lock); in blkg_destroy()
428 WARN_ON_ONCE(list_empty(&blkg->q_node)); in blkg_destroy()
429 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node)); in blkg_destroy()
434 if (blkg->pd[i] && pol->pd_offline_fn) in blkg_destroy()
435 pol->pd_offline_fn(blkg->pd[i]); in blkg_destroy()
438 blkg->online = false; in blkg_destroy()
440 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id); in blkg_destroy()
441 list_del_init(&blkg->q_node); in blkg_destroy()
442 hlist_del_init_rcu(&blkg->blkcg_node); in blkg_destroy()
449 if (rcu_access_pointer(blkcg->blkg_hint) == blkg) in blkg_destroy()
456 percpu_ref_kill(&blkg->refcnt); in blkg_destroy()
462 struct blkcg_gq *blkg, *n; in blkg_destroy_all() local
467 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) { in blkg_destroy_all()
468 struct blkcg *blkcg = blkg->blkcg; in blkg_destroy_all()
471 blkg_destroy(blkg); in blkg_destroy_all()
494 struct blkcg_gq *blkg; in blkcg_reset_stats() local
505 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_reset_stats()
508 per_cpu_ptr(blkg->iostat_cpu, cpu); in blkcg_reset_stats()
511 memset(&blkg->iostat, 0, sizeof(blkg->iostat)); in blkcg_reset_stats()
516 if (blkg->pd[i] && pol->pd_reset_stats_fn) in blkcg_reset_stats()
517 pol->pd_reset_stats_fn(blkg->pd[i]); in blkcg_reset_stats()
526 const char *blkg_dev_name(struct blkcg_gq *blkg) in blkg_dev_name() argument
528 if (!blkg->q->disk || !blkg->q->disk->bdi->dev) in blkg_dev_name()
530 return bdi_dev_name(blkg->q->disk->bdi); in blkg_dev_name()
557 struct blkcg_gq *blkg; in blkcg_print_blkgs() local
561 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_print_blkgs()
562 spin_lock_irq(&blkg->q->queue_lock); in blkcg_print_blkgs()
563 if (blkcg_policy_enabled(blkg->q, pol)) in blkcg_print_blkgs()
564 total += prfill(sf, blkg->pd[pol->plid], data); in blkcg_print_blkgs()
565 spin_unlock_irq(&blkg->q->queue_lock); in blkcg_print_blkgs()
584 const char *dname = blkg_dev_name(pd->blkg); in __blkg_prfill_u64()
651 struct blkcg_gq *blkg; in blkg_conf_prep() local
676 blkg = blkg_lookup(blkcg, q); in blkg_conf_prep()
677 if (blkg) in blkg_conf_prep()
720 blkg = blkg_lookup(pos, q); in blkg_conf_prep()
721 if (blkg) { in blkg_conf_prep()
724 blkg = blkg_create(pos, disk, new_blkg); in blkg_conf_prep()
725 if (IS_ERR(blkg)) { in blkg_conf_prep()
726 ret = PTR_ERR(blkg); in blkg_conf_prep()
739 ctx->blkg = blkg; in blkg_conf_prep()
812 static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur, in blkcg_iostat_update() argument
819 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync); in blkcg_iostat_update()
822 blkg_iostat_add(&blkg->iostat.cur, &delta); in blkcg_iostat_update()
824 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags); in blkcg_iostat_update()
830 struct blkcg_gq *blkg; in blkcg_rstat_flush() local
838 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_rstat_flush()
839 struct blkcg_gq *parent = blkg->parent; in blkcg_rstat_flush()
840 struct blkg_iostat_set *bisc = per_cpu_ptr(blkg->iostat_cpu, cpu); in blkcg_rstat_flush()
850 blkcg_iostat_update(blkg, &cur, &bisc->last); in blkcg_rstat_flush()
854 blkcg_iostat_update(parent, &blkg->iostat.cur, in blkcg_rstat_flush()
855 &blkg->iostat.last); in blkcg_rstat_flush()
881 struct blkcg_gq *blkg = bdev->bd_disk->queue->root_blkg; in blkcg_fill_root_iostats() local
906 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync); in blkcg_fill_root_iostats()
907 blkg_iostat_set(&blkg->iostat.cur, &tmp); in blkcg_fill_root_iostats()
908 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags); in blkcg_fill_root_iostats()
912 static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s) in blkcg_print_one_stat() argument
914 struct blkg_iostat_set *bis = &blkg->iostat; in blkcg_print_one_stat()
920 if (!blkg->online) in blkcg_print_one_stat()
923 dname = blkg_dev_name(blkg); in blkcg_print_one_stat()
946 if (blkcg_debug_stats && atomic_read(&blkg->use_delay)) { in blkcg_print_one_stat()
948 atomic_read(&blkg->use_delay), in blkcg_print_one_stat()
949 atomic64_read(&blkg->delay_nsec)); in blkcg_print_one_stat()
955 if (!blkg->pd[i] || !pol->pd_stat_fn) in blkcg_print_one_stat()
958 pol->pd_stat_fn(blkg->pd[i], s); in blkcg_print_one_stat()
967 struct blkcg_gq *blkg; in blkcg_print_stat() local
975 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { in blkcg_print_stat()
976 spin_lock_irq(&blkg->q->queue_lock); in blkcg_print_stat()
977 blkcg_print_one_stat(blkg, sf); in blkcg_print_stat()
978 spin_unlock_irq(&blkg->q->queue_lock); in blkcg_print_stat()
1046 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first, in blkcg_destroy_blkgs() local
1048 struct request_queue *q = blkg->q; in blkcg_destroy_blkgs()
1062 blkg_destroy(blkg); in blkcg_destroy_blkgs()
1223 struct blkcg_gq *new_blkg, *blkg; in blkcg_init_disk() local
1238 blkg = blkg_create(&blkcg_root, disk, new_blkg); in blkcg_init_disk()
1239 if (IS_ERR(blkg)) in blkcg_init_disk()
1241 q->root_blkg = blkg; in blkcg_init_disk()
1272 return PTR_ERR(blkg); in blkcg_init_disk()
1350 struct blkcg_gq *blkg, *pinned_blkg = NULL; in blkcg_activate_policy() local
1362 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) { in blkcg_activate_policy()
1365 if (blkg->pd[pol->plid]) in blkcg_activate_policy()
1369 if (blkg == pinned_blkg) { in blkcg_activate_policy()
1374 blkg->blkcg); in blkcg_activate_policy()
1384 blkg_get(blkg); in blkcg_activate_policy()
1385 pinned_blkg = blkg; in blkcg_activate_policy()
1392 blkg->blkcg); in blkcg_activate_policy()
1399 blkg->pd[pol->plid] = pd; in blkcg_activate_policy()
1400 pd->blkg = blkg; in blkcg_activate_policy()
1406 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) in blkcg_activate_policy()
1407 pol->pd_init_fn(blkg->pd[pol->plid]); in blkcg_activate_policy()
1425 list_for_each_entry(blkg, &q->blkg_list, q_node) { in blkcg_activate_policy()
1426 struct blkcg *blkcg = blkg->blkcg; in blkcg_activate_policy()
1429 if (blkg->pd[pol->plid]) { in blkcg_activate_policy()
1430 pol->pd_free_fn(blkg->pd[pol->plid]); in blkcg_activate_policy()
1431 blkg->pd[pol->plid] = NULL; in blkcg_activate_policy()
1452 struct blkcg_gq *blkg; in blkcg_deactivate_policy() local
1464 list_for_each_entry(blkg, &q->blkg_list, q_node) { in blkcg_deactivate_policy()
1465 struct blkcg *blkcg = blkg->blkcg; in blkcg_deactivate_policy()
1468 if (blkg->pd[pol->plid]) { in blkcg_deactivate_policy()
1470 pol->pd_offline_fn(blkg->pd[pol->plid]); in blkcg_deactivate_policy()
1471 pol->pd_free_fn(blkg->pd[pol->plid]); in blkcg_deactivate_policy()
1472 blkg->pd[pol->plid] = NULL; in blkcg_deactivate_policy()
1606 struct blkcg_gq *blkg = bio->bi_blkg; in __blkcg_punt_bio_submit() local
1612 if (!blkg->parent) in __blkcg_punt_bio_submit()
1615 spin_lock_bh(&blkg->async_bio_lock); in __blkcg_punt_bio_submit()
1616 bio_list_add(&blkg->async_bios, bio); in __blkcg_punt_bio_submit()
1617 spin_unlock_bh(&blkg->async_bio_lock); in __blkcg_punt_bio_submit()
1619 queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work); in __blkcg_punt_bio_submit()
1629 static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now) in blkcg_scale_delay() argument
1631 u64 old = atomic64_read(&blkg->delay_start); in blkcg_scale_delay()
1634 if (atomic_read(&blkg->use_delay) < 0) in blkcg_scale_delay()
1651 atomic64_try_cmpxchg(&blkg->delay_start, &old, now)) { in blkcg_scale_delay()
1652 u64 cur = atomic64_read(&blkg->delay_nsec); in blkcg_scale_delay()
1653 u64 sub = min_t(u64, blkg->last_delay, now - old); in blkcg_scale_delay()
1654 int cur_use = atomic_read(&blkg->use_delay); in blkcg_scale_delay()
1660 if (cur_use < blkg->last_use) in blkcg_scale_delay()
1661 sub = max_t(u64, sub, blkg->last_delay >> 1); in blkcg_scale_delay()
1670 atomic64_set(&blkg->delay_nsec, 0); in blkcg_scale_delay()
1671 blkg->last_delay = 0; in blkcg_scale_delay()
1673 atomic64_sub(sub, &blkg->delay_nsec); in blkcg_scale_delay()
1674 blkg->last_delay = cur - sub; in blkcg_scale_delay()
1676 blkg->last_use = cur_use; in blkcg_scale_delay()
1686 static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay) in blkcg_maybe_throttle_blkg() argument
1695 while (blkg->parent) { in blkcg_maybe_throttle_blkg()
1696 int use_delay = atomic_read(&blkg->use_delay); in blkcg_maybe_throttle_blkg()
1701 blkcg_scale_delay(blkg, now); in blkcg_maybe_throttle_blkg()
1702 this_delay = atomic64_read(&blkg->delay_nsec); in blkcg_maybe_throttle_blkg()
1708 blkg = blkg->parent; in blkcg_maybe_throttle_blkg()
1756 struct blkcg_gq *blkg; in blkcg_maybe_throttle_current() local
1769 blkg = blkg_lookup(blkcg, q); in blkcg_maybe_throttle_current()
1770 if (!blkg) in blkcg_maybe_throttle_current()
1772 if (!blkg_tryget(blkg)) in blkcg_maybe_throttle_current()
1776 blkcg_maybe_throttle_blkg(blkg, use_memdelay); in blkcg_maybe_throttle_current()
1777 blkg_put(blkg); in blkcg_maybe_throttle_current()
1832 void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta) in blkcg_add_delay() argument
1834 if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0)) in blkcg_add_delay()
1836 blkcg_scale_delay(blkg, now); in blkcg_add_delay()
1837 atomic64_add(delta, &blkg->delay_nsec); in blkcg_add_delay()
1852 struct blkcg_gq *blkg, *ret_blkg = NULL; in blkg_tryget_closest() local
1855 blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_bdev->bd_disk); in blkg_tryget_closest()
1856 while (blkg) { in blkg_tryget_closest()
1857 if (blkg_tryget(blkg)) { in blkg_tryget_closest()
1858 ret_blkg = blkg; in blkg_tryget_closest()
1861 blkg = blkg->parent; in blkg_tryget_closest()