Lines Matching refs:tg
92 static inline struct blkcg_gq *tg_to_blkg(struct throtl_grp *tg) in tg_to_blkg() argument
94 return pd_to_blkg(&tg->pd); in tg_to_blkg()
121 struct throtl_grp *tg = sq_to_tg(sq); in sq_to_td() local
123 if (tg) in sq_to_td()
124 return tg->td; in sq_to_td()
147 static uint64_t tg_bps_limit(struct throtl_grp *tg, int rw) in tg_bps_limit() argument
149 struct blkcg_gq *blkg = tg_to_blkg(tg); in tg_bps_limit()
156 td = tg->td; in tg_bps_limit()
157 ret = tg->bps[rw][td->limit_index]; in tg_bps_limit()
161 tg->iops[rw][td->limit_index]) in tg_bps_limit()
167 if (td->limit_index == LIMIT_MAX && tg->bps[rw][LIMIT_LOW] && in tg_bps_limit()
168 tg->bps[rw][LIMIT_LOW] != tg->bps[rw][LIMIT_MAX]) { in tg_bps_limit()
171 adjusted = throtl_adjusted_limit(tg->bps[rw][LIMIT_LOW], td); in tg_bps_limit()
172 ret = min(tg->bps[rw][LIMIT_MAX], adjusted); in tg_bps_limit()
177 static unsigned int tg_iops_limit(struct throtl_grp *tg, int rw) in tg_iops_limit() argument
179 struct blkcg_gq *blkg = tg_to_blkg(tg); in tg_iops_limit()
186 td = tg->td; in tg_iops_limit()
187 ret = tg->iops[rw][td->limit_index]; in tg_iops_limit()
188 if (ret == 0 && tg->td->limit_index == LIMIT_LOW) { in tg_iops_limit()
191 tg->bps[rw][td->limit_index]) in tg_iops_limit()
197 if (td->limit_index == LIMIT_MAX && tg->iops[rw][LIMIT_LOW] && in tg_iops_limit()
198 tg->iops[rw][LIMIT_LOW] != tg->iops[rw][LIMIT_MAX]) { in tg_iops_limit()
201 adjusted = throtl_adjusted_limit(tg->iops[rw][LIMIT_LOW], td); in tg_iops_limit()
204 ret = min_t(unsigned int, tg->iops[rw][LIMIT_MAX], adjusted); in tg_iops_limit()
244 static void throtl_qnode_init(struct throtl_qnode *qn, struct throtl_grp *tg) in throtl_qnode_init() argument
248 qn->tg = tg; in throtl_qnode_init()
267 blkg_get(tg_to_blkg(qn->tg)); in throtl_qnode_add_bio()
319 *tg_to_put = qn->tg; in throtl_pop_queued()
321 blkg_put(tg_to_blkg(qn->tg)); in throtl_pop_queued()
341 struct throtl_grp *tg; in throtl_pd_alloc() local
344 tg = kzalloc_node(sizeof(*tg), gfp, disk->node_id); in throtl_pd_alloc()
345 if (!tg) in throtl_pd_alloc()
348 if (blkg_rwstat_init(&tg->stat_bytes, gfp)) in throtl_pd_alloc()
351 if (blkg_rwstat_init(&tg->stat_ios, gfp)) in throtl_pd_alloc()
354 throtl_service_queue_init(&tg->service_queue); in throtl_pd_alloc()
357 throtl_qnode_init(&tg->qnode_on_self[rw], tg); in throtl_pd_alloc()
358 throtl_qnode_init(&tg->qnode_on_parent[rw], tg); in throtl_pd_alloc()
361 RB_CLEAR_NODE(&tg->rb_node); in throtl_pd_alloc()
362 tg->bps[READ][LIMIT_MAX] = U64_MAX; in throtl_pd_alloc()
363 tg->bps[WRITE][LIMIT_MAX] = U64_MAX; in throtl_pd_alloc()
364 tg->iops[READ][LIMIT_MAX] = UINT_MAX; in throtl_pd_alloc()
365 tg->iops[WRITE][LIMIT_MAX] = UINT_MAX; in throtl_pd_alloc()
366 tg->bps_conf[READ][LIMIT_MAX] = U64_MAX; in throtl_pd_alloc()
367 tg->bps_conf[WRITE][LIMIT_MAX] = U64_MAX; in throtl_pd_alloc()
368 tg->iops_conf[READ][LIMIT_MAX] = UINT_MAX; in throtl_pd_alloc()
369 tg->iops_conf[WRITE][LIMIT_MAX] = UINT_MAX; in throtl_pd_alloc()
372 tg->latency_target = DFL_LATENCY_TARGET; in throtl_pd_alloc()
373 tg->latency_target_conf = DFL_LATENCY_TARGET; in throtl_pd_alloc()
374 tg->idletime_threshold = DFL_IDLE_THRESHOLD; in throtl_pd_alloc()
375 tg->idletime_threshold_conf = DFL_IDLE_THRESHOLD; in throtl_pd_alloc()
377 return &tg->pd; in throtl_pd_alloc()
380 blkg_rwstat_exit(&tg->stat_bytes); in throtl_pd_alloc()
382 kfree(tg); in throtl_pd_alloc()
388 struct throtl_grp *tg = pd_to_tg(pd); in throtl_pd_init() local
389 struct blkcg_gq *blkg = tg_to_blkg(tg); in throtl_pd_init()
391 struct throtl_service_queue *sq = &tg->service_queue; in throtl_pd_init()
410 tg->td = td; in throtl_pd_init()
418 static void tg_update_has_rules(struct throtl_grp *tg) in tg_update_has_rules() argument
420 struct throtl_grp *parent_tg = sq_to_tg(tg->service_queue.parent_sq); in tg_update_has_rules()
421 struct throtl_data *td = tg->td; in tg_update_has_rules()
425 tg->has_rules_iops[rw] = in tg_update_has_rules()
428 tg_iops_limit(tg, rw) != UINT_MAX); in tg_update_has_rules()
429 tg->has_rules_bps[rw] = in tg_update_has_rules()
432 (tg_bps_limit(tg, rw) != U64_MAX)); in tg_update_has_rules()
438 struct throtl_grp *tg = pd_to_tg(pd); in throtl_pd_online() local
443 tg_update_has_rules(tg); in throtl_pd_online()
455 struct throtl_grp *tg = blkg_to_tg(blkg); in blk_throtl_update_limit_valid() local
457 if (tg->bps[READ][LIMIT_LOW] || tg->bps[WRITE][LIMIT_LOW] || in blk_throtl_update_limit_valid()
458 tg->iops[READ][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) { in blk_throtl_update_limit_valid()
476 struct throtl_grp *tg = pd_to_tg(pd); in throtl_pd_offline() local
478 tg->bps[READ][LIMIT_LOW] = 0; in throtl_pd_offline()
479 tg->bps[WRITE][LIMIT_LOW] = 0; in throtl_pd_offline()
480 tg->iops[READ][LIMIT_LOW] = 0; in throtl_pd_offline()
481 tg->iops[WRITE][LIMIT_LOW] = 0; in throtl_pd_offline()
483 blk_throtl_update_limit_valid(tg->td); in throtl_pd_offline()
485 if (!tg->td->limit_valid[tg->td->limit_index]) in throtl_pd_offline()
486 throtl_upgrade_state(tg->td); in throtl_pd_offline()
491 struct throtl_grp *tg = pd_to_tg(pd); in throtl_pd_free() local
493 del_timer_sync(&tg->service_queue.pending_timer); in throtl_pd_free()
494 blkg_rwstat_exit(&tg->stat_bytes); in throtl_pd_free()
495 blkg_rwstat_exit(&tg->stat_ios); in throtl_pd_free()
496 kfree(tg); in throtl_pd_free()
520 struct throtl_grp *tg; in update_min_dispatch_time() local
522 tg = throtl_rb_first(parent_sq); in update_min_dispatch_time()
523 if (!tg) in update_min_dispatch_time()
526 parent_sq->first_pending_disptime = tg->disptime; in update_min_dispatch_time()
529 static void tg_service_queue_add(struct throtl_grp *tg) in tg_service_queue_add() argument
531 struct throtl_service_queue *parent_sq = tg->service_queue.parent_sq; in tg_service_queue_add()
535 unsigned long key = tg->disptime; in tg_service_queue_add()
550 rb_link_node(&tg->rb_node, parent, node); in tg_service_queue_add()
551 rb_insert_color_cached(&tg->rb_node, &parent_sq->pending_tree, in tg_service_queue_add()
555 static void throtl_enqueue_tg(struct throtl_grp *tg) in throtl_enqueue_tg() argument
557 if (!(tg->flags & THROTL_TG_PENDING)) { in throtl_enqueue_tg()
558 tg_service_queue_add(tg); in throtl_enqueue_tg()
559 tg->flags |= THROTL_TG_PENDING; in throtl_enqueue_tg()
560 tg->service_queue.parent_sq->nr_pending++; in throtl_enqueue_tg()
564 static void throtl_dequeue_tg(struct throtl_grp *tg) in throtl_dequeue_tg() argument
566 if (tg->flags & THROTL_TG_PENDING) { in throtl_dequeue_tg()
568 tg->service_queue.parent_sq; in throtl_dequeue_tg()
570 throtl_rb_erase(&tg->rb_node, parent_sq); in throtl_dequeue_tg()
572 tg->flags &= ~THROTL_TG_PENDING; in throtl_dequeue_tg()
633 static inline void throtl_start_new_slice_with_credit(struct throtl_grp *tg, in throtl_start_new_slice_with_credit() argument
636 tg->bytes_disp[rw] = 0; in throtl_start_new_slice_with_credit()
637 tg->io_disp[rw] = 0; in throtl_start_new_slice_with_credit()
638 tg->carryover_bytes[rw] = 0; in throtl_start_new_slice_with_credit()
639 tg->carryover_ios[rw] = 0; in throtl_start_new_slice_with_credit()
647 if (time_after(start, tg->slice_start[rw])) in throtl_start_new_slice_with_credit()
648 tg->slice_start[rw] = start; in throtl_start_new_slice_with_credit()
650 tg->slice_end[rw] = jiffies + tg->td->throtl_slice; in throtl_start_new_slice_with_credit()
651 throtl_log(&tg->service_queue, in throtl_start_new_slice_with_credit()
653 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice_with_credit()
654 tg->slice_end[rw], jiffies); in throtl_start_new_slice_with_credit()
657 static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw, in throtl_start_new_slice() argument
660 tg->bytes_disp[rw] = 0; in throtl_start_new_slice()
661 tg->io_disp[rw] = 0; in throtl_start_new_slice()
662 tg->slice_start[rw] = jiffies; in throtl_start_new_slice()
663 tg->slice_end[rw] = jiffies + tg->td->throtl_slice; in throtl_start_new_slice()
665 tg->carryover_bytes[rw] = 0; in throtl_start_new_slice()
666 tg->carryover_ios[rw] = 0; in throtl_start_new_slice()
669 throtl_log(&tg->service_queue, in throtl_start_new_slice()
671 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice()
672 tg->slice_end[rw], jiffies); in throtl_start_new_slice()
675 static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw, in throtl_set_slice_end() argument
678 tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice); in throtl_set_slice_end()
681 static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw, in throtl_extend_slice() argument
684 throtl_set_slice_end(tg, rw, jiffy_end); in throtl_extend_slice()
685 throtl_log(&tg->service_queue, in throtl_extend_slice()
687 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_extend_slice()
688 tg->slice_end[rw], jiffies); in throtl_extend_slice()
692 static bool throtl_slice_used(struct throtl_grp *tg, bool rw) in throtl_slice_used() argument
694 if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw])) in throtl_slice_used()
736 static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw) in throtl_trim_slice() argument
742 BUG_ON(time_before(tg->slice_end[rw], tg->slice_start[rw])); in throtl_trim_slice()
749 if (throtl_slice_used(tg, rw)) in throtl_trim_slice()
760 throtl_set_slice_end(tg, rw, jiffies + tg->td->throtl_slice); in throtl_trim_slice()
762 time_elapsed = rounddown(jiffies - tg->slice_start[rw], in throtl_trim_slice()
763 tg->td->throtl_slice); in throtl_trim_slice()
767 bytes_trim = calculate_bytes_allowed(tg_bps_limit(tg, rw), in throtl_trim_slice()
769 tg->carryover_bytes[rw]; in throtl_trim_slice()
770 io_trim = calculate_io_allowed(tg_iops_limit(tg, rw), time_elapsed) + in throtl_trim_slice()
771 tg->carryover_ios[rw]; in throtl_trim_slice()
775 tg->carryover_bytes[rw] = 0; in throtl_trim_slice()
776 if ((long long)tg->bytes_disp[rw] >= bytes_trim) in throtl_trim_slice()
777 tg->bytes_disp[rw] -= bytes_trim; in throtl_trim_slice()
779 tg->bytes_disp[rw] = 0; in throtl_trim_slice()
781 tg->carryover_ios[rw] = 0; in throtl_trim_slice()
782 if ((int)tg->io_disp[rw] >= io_trim) in throtl_trim_slice()
783 tg->io_disp[rw] -= io_trim; in throtl_trim_slice()
785 tg->io_disp[rw] = 0; in throtl_trim_slice()
787 tg->slice_start[rw] += time_elapsed; in throtl_trim_slice()
789 throtl_log(&tg->service_queue, in throtl_trim_slice()
791 rw == READ ? 'R' : 'W', time_elapsed / tg->td->throtl_slice, in throtl_trim_slice()
792 bytes_trim, io_trim, tg->slice_start[rw], tg->slice_end[rw], in throtl_trim_slice()
796 static void __tg_update_carryover(struct throtl_grp *tg, bool rw) in __tg_update_carryover() argument
798 unsigned long jiffy_elapsed = jiffies - tg->slice_start[rw]; in __tg_update_carryover()
799 u64 bps_limit = tg_bps_limit(tg, rw); in __tg_update_carryover()
800 u32 iops_limit = tg_iops_limit(tg, rw); in __tg_update_carryover()
809 tg->carryover_bytes[rw] += in __tg_update_carryover()
811 tg->bytes_disp[rw]; in __tg_update_carryover()
813 tg->carryover_ios[rw] += in __tg_update_carryover()
815 tg->io_disp[rw]; in __tg_update_carryover()
818 static void tg_update_carryover(struct throtl_grp *tg) in tg_update_carryover() argument
820 if (tg->service_queue.nr_queued[READ]) in tg_update_carryover()
821 __tg_update_carryover(tg, READ); in tg_update_carryover()
822 if (tg->service_queue.nr_queued[WRITE]) in tg_update_carryover()
823 __tg_update_carryover(tg, WRITE); in tg_update_carryover()
826 throtl_log(&tg->service_queue, "%s: %lld %lld %d %d\n", __func__, in tg_update_carryover()
827 tg->carryover_bytes[READ], tg->carryover_bytes[WRITE], in tg_update_carryover()
828 tg->carryover_ios[READ], tg->carryover_ios[WRITE]); in tg_update_carryover()
831 static unsigned long tg_within_iops_limit(struct throtl_grp *tg, struct bio *bio, in tg_within_iops_limit() argument
842 jiffy_elapsed = jiffies - tg->slice_start[rw]; in tg_within_iops_limit()
845 jiffy_elapsed_rnd = roundup(jiffy_elapsed + 1, tg->td->throtl_slice); in tg_within_iops_limit()
847 tg->carryover_ios[rw]; in tg_within_iops_limit()
848 if (io_allowed > 0 && tg->io_disp[rw] + 1 <= io_allowed) in tg_within_iops_limit()
856 static unsigned long tg_within_bps_limit(struct throtl_grp *tg, struct bio *bio, in tg_within_bps_limit() argument
870 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw]; in tg_within_bps_limit()
874 jiffy_elapsed_rnd = tg->td->throtl_slice; in tg_within_bps_limit()
876 jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, tg->td->throtl_slice); in tg_within_bps_limit()
878 tg->carryover_bytes[rw]; in tg_within_bps_limit()
879 if (bytes_allowed > 0 && tg->bytes_disp[rw] + bio_size <= bytes_allowed) in tg_within_bps_limit()
883 extra_bytes = tg->bytes_disp[rw] + bio_size - bytes_allowed; in tg_within_bps_limit()
901 static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio, in tg_may_dispatch() argument
906 u64 bps_limit = tg_bps_limit(tg, rw); in tg_may_dispatch()
907 u32 iops_limit = tg_iops_limit(tg, rw); in tg_may_dispatch()
915 BUG_ON(tg->service_queue.nr_queued[rw] && in tg_may_dispatch()
916 bio != throtl_peek_queued(&tg->service_queue.queued[rw])); in tg_may_dispatch()
920 tg->flags & THROTL_TG_CANCELING) { in tg_may_dispatch()
933 if (throtl_slice_used(tg, rw) && !(tg->service_queue.nr_queued[rw])) in tg_may_dispatch()
934 throtl_start_new_slice(tg, rw, true); in tg_may_dispatch()
936 if (time_before(tg->slice_end[rw], in tg_may_dispatch()
937 jiffies + tg->td->throtl_slice)) in tg_may_dispatch()
938 throtl_extend_slice(tg, rw, in tg_may_dispatch()
939 jiffies + tg->td->throtl_slice); in tg_may_dispatch()
942 bps_wait = tg_within_bps_limit(tg, bio, bps_limit); in tg_may_dispatch()
943 iops_wait = tg_within_iops_limit(tg, bio, iops_limit); in tg_may_dispatch()
955 if (time_before(tg->slice_end[rw], jiffies + max_wait)) in tg_may_dispatch()
956 throtl_extend_slice(tg, rw, jiffies + max_wait); in tg_may_dispatch()
961 static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio) in throtl_charge_bio() argument
968 tg->bytes_disp[rw] += bio_size; in throtl_charge_bio()
969 tg->last_bytes_disp[rw] += bio_size; in throtl_charge_bio()
972 tg->io_disp[rw]++; in throtl_charge_bio()
973 tg->last_io_disp[rw]++; in throtl_charge_bio()
986 struct throtl_grp *tg) in throtl_add_bio_tg() argument
988 struct throtl_service_queue *sq = &tg->service_queue; in throtl_add_bio_tg()
992 qn = &tg->qnode_on_self[rw]; in throtl_add_bio_tg()
1001 tg->flags |= THROTL_TG_WAS_EMPTY; in throtl_add_bio_tg()
1006 throtl_enqueue_tg(tg); in throtl_add_bio_tg()
1009 static void tg_update_disptime(struct throtl_grp *tg) in tg_update_disptime() argument
1011 struct throtl_service_queue *sq = &tg->service_queue; in tg_update_disptime()
1017 tg_may_dispatch(tg, bio, &read_wait); in tg_update_disptime()
1021 tg_may_dispatch(tg, bio, &write_wait); in tg_update_disptime()
1027 throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq); in tg_update_disptime()
1028 tg->disptime = disptime; in tg_update_disptime()
1029 tg_service_queue_add(tg); in tg_update_disptime()
1032 tg->flags &= ~THROTL_TG_WAS_EMPTY; in tg_update_disptime()
1045 static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw) in tg_dispatch_one_bio() argument
1047 struct throtl_service_queue *sq = &tg->service_queue; in tg_dispatch_one_bio()
1062 throtl_charge_bio(tg, bio); in tg_dispatch_one_bio()
1072 throtl_add_bio_tg(bio, &tg->qnode_on_parent[rw], parent_tg); in tg_dispatch_one_bio()
1073 start_parent_slice_with_credit(tg, parent_tg, rw); in tg_dispatch_one_bio()
1076 throtl_qnode_add_bio(bio, &tg->qnode_on_parent[rw], in tg_dispatch_one_bio()
1078 BUG_ON(tg->td->nr_queued[rw] <= 0); in tg_dispatch_one_bio()
1079 tg->td->nr_queued[rw]--; in tg_dispatch_one_bio()
1082 throtl_trim_slice(tg, rw); in tg_dispatch_one_bio()
1088 static int throtl_dispatch_tg(struct throtl_grp *tg) in throtl_dispatch_tg() argument
1090 struct throtl_service_queue *sq = &tg->service_queue; in throtl_dispatch_tg()
1099 tg_may_dispatch(tg, bio, NULL)) { in throtl_dispatch_tg()
1101 tg_dispatch_one_bio(tg, bio_data_dir(bio)); in throtl_dispatch_tg()
1109 tg_may_dispatch(tg, bio, NULL)) { in throtl_dispatch_tg()
1111 tg_dispatch_one_bio(tg, bio_data_dir(bio)); in throtl_dispatch_tg()
1126 struct throtl_grp *tg; in throtl_select_dispatch() local
1132 tg = throtl_rb_first(parent_sq); in throtl_select_dispatch()
1133 if (!tg) in throtl_select_dispatch()
1136 if (time_before(jiffies, tg->disptime)) in throtl_select_dispatch()
1139 nr_disp += throtl_dispatch_tg(tg); in throtl_select_dispatch()
1141 sq = &tg->service_queue; in throtl_select_dispatch()
1143 tg_update_disptime(tg); in throtl_select_dispatch()
1145 throtl_dequeue_tg(tg); in throtl_select_dispatch()
1174 struct throtl_grp *tg = sq_to_tg(sq); in throtl_pending_timer_fn() local
1182 if (tg) in throtl_pending_timer_fn()
1183 q = tg->pd.blkg->q; in throtl_pending_timer_fn()
1224 if (tg->flags & THROTL_TG_WAS_EMPTY) { in throtl_pending_timer_fn()
1225 tg_update_disptime(tg); in throtl_pending_timer_fn()
1229 tg = sq_to_tg(sq); in throtl_pending_timer_fn()
1279 struct throtl_grp *tg = pd_to_tg(pd); in tg_prfill_conf_u64() local
1280 u64 v = *(u64 *)((void *)tg + off); in tg_prfill_conf_u64()
1290 struct throtl_grp *tg = pd_to_tg(pd); in tg_prfill_conf_uint() local
1291 unsigned int v = *(unsigned int *)((void *)tg + off); in tg_prfill_conf_uint()
1312 static void tg_conf_updated(struct throtl_grp *tg, bool global) in tg_conf_updated() argument
1314 struct throtl_service_queue *sq = &tg->service_queue; in tg_conf_updated()
1318 throtl_log(&tg->service_queue, in tg_conf_updated()
1320 tg_bps_limit(tg, READ), tg_bps_limit(tg, WRITE), in tg_conf_updated()
1321 tg_iops_limit(tg, READ), tg_iops_limit(tg, WRITE)); in tg_conf_updated()
1331 global ? tg->td->queue->root_blkg : tg_to_blkg(tg)) { in tg_conf_updated()
1359 throtl_start_new_slice(tg, READ, false); in tg_conf_updated()
1360 throtl_start_new_slice(tg, WRITE, false); in tg_conf_updated()
1362 if (tg->flags & THROTL_TG_PENDING) { in tg_conf_updated()
1363 tg_update_disptime(tg); in tg_conf_updated()
1373 struct throtl_grp *tg; in tg_set_conf() local
1389 tg = blkg_to_tg(ctx.blkg); in tg_set_conf()
1390 tg_update_carryover(tg); in tg_set_conf()
1393 *(u64 *)((void *)tg + of_cft(of)->private) = v; in tg_set_conf()
1395 *(unsigned int *)((void *)tg + of_cft(of)->private) = v; in tg_set_conf()
1397 tg_conf_updated(tg, false); in tg_set_conf()
1493 struct throtl_grp *tg = pd_to_tg(pd); in tg_prfill_limit() local
1512 if (tg->bps_conf[READ][off] == bps_dft && in tg_prfill_limit()
1513 tg->bps_conf[WRITE][off] == bps_dft && in tg_prfill_limit()
1514 tg->iops_conf[READ][off] == iops_dft && in tg_prfill_limit()
1515 tg->iops_conf[WRITE][off] == iops_dft && in tg_prfill_limit()
1517 (tg->idletime_threshold_conf == DFL_IDLE_THRESHOLD && in tg_prfill_limit()
1518 tg->latency_target_conf == DFL_LATENCY_TARGET))) in tg_prfill_limit()
1521 if (tg->bps_conf[READ][off] != U64_MAX) in tg_prfill_limit()
1523 tg->bps_conf[READ][off]); in tg_prfill_limit()
1524 if (tg->bps_conf[WRITE][off] != U64_MAX) in tg_prfill_limit()
1526 tg->bps_conf[WRITE][off]); in tg_prfill_limit()
1527 if (tg->iops_conf[READ][off] != UINT_MAX) in tg_prfill_limit()
1529 tg->iops_conf[READ][off]); in tg_prfill_limit()
1530 if (tg->iops_conf[WRITE][off] != UINT_MAX) in tg_prfill_limit()
1532 tg->iops_conf[WRITE][off]); in tg_prfill_limit()
1534 if (tg->idletime_threshold_conf == ULONG_MAX) in tg_prfill_limit()
1538 tg->idletime_threshold_conf); in tg_prfill_limit()
1540 if (tg->latency_target_conf == ULONG_MAX) in tg_prfill_limit()
1544 " latency=%lu", tg->latency_target_conf); in tg_prfill_limit()
1565 struct throtl_grp *tg; in tg_set_limit() local
1578 tg = blkg_to_tg(ctx.blkg); in tg_set_limit()
1579 tg_update_carryover(tg); in tg_set_limit()
1581 v[0] = tg->bps_conf[READ][index]; in tg_set_limit()
1582 v[1] = tg->bps_conf[WRITE][index]; in tg_set_limit()
1583 v[2] = tg->iops_conf[READ][index]; in tg_set_limit()
1584 v[3] = tg->iops_conf[WRITE][index]; in tg_set_limit()
1586 idle_time = tg->idletime_threshold_conf; in tg_set_limit()
1587 latency_time = tg->latency_target_conf; in tg_set_limit()
1627 tg->bps_conf[READ][index] = v[0]; in tg_set_limit()
1628 tg->bps_conf[WRITE][index] = v[1]; in tg_set_limit()
1629 tg->iops_conf[READ][index] = v[2]; in tg_set_limit()
1630 tg->iops_conf[WRITE][index] = v[3]; in tg_set_limit()
1633 tg->bps[READ][index] = v[0]; in tg_set_limit()
1634 tg->bps[WRITE][index] = v[1]; in tg_set_limit()
1635 tg->iops[READ][index] = v[2]; in tg_set_limit()
1636 tg->iops[WRITE][index] = v[3]; in tg_set_limit()
1638 tg->bps[READ][LIMIT_LOW] = min(tg->bps_conf[READ][LIMIT_LOW], in tg_set_limit()
1639 tg->bps_conf[READ][LIMIT_MAX]); in tg_set_limit()
1640 tg->bps[WRITE][LIMIT_LOW] = min(tg->bps_conf[WRITE][LIMIT_LOW], in tg_set_limit()
1641 tg->bps_conf[WRITE][LIMIT_MAX]); in tg_set_limit()
1642 tg->iops[READ][LIMIT_LOW] = min(tg->iops_conf[READ][LIMIT_LOW], in tg_set_limit()
1643 tg->iops_conf[READ][LIMIT_MAX]); in tg_set_limit()
1644 tg->iops[WRITE][LIMIT_LOW] = min(tg->iops_conf[WRITE][LIMIT_LOW], in tg_set_limit()
1645 tg->iops_conf[WRITE][LIMIT_MAX]); in tg_set_limit()
1646 tg->idletime_threshold_conf = idle_time; in tg_set_limit()
1647 tg->latency_target_conf = latency_time; in tg_set_limit()
1650 if (!(tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW] || in tg_set_limit()
1651 tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) || in tg_set_limit()
1652 tg->idletime_threshold_conf == DFL_IDLE_THRESHOLD || in tg_set_limit()
1653 tg->latency_target_conf == DFL_LATENCY_TARGET) { in tg_set_limit()
1654 tg->bps[READ][LIMIT_LOW] = 0; in tg_set_limit()
1655 tg->bps[WRITE][LIMIT_LOW] = 0; in tg_set_limit()
1656 tg->iops[READ][LIMIT_LOW] = 0; in tg_set_limit()
1657 tg->iops[WRITE][LIMIT_LOW] = 0; in tg_set_limit()
1658 tg->idletime_threshold = DFL_IDLE_THRESHOLD; in tg_set_limit()
1659 tg->latency_target = DFL_LATENCY_TARGET; in tg_set_limit()
1661 tg->idletime_threshold = tg->idletime_threshold_conf; in tg_set_limit()
1662 tg->latency_target = tg->latency_target_conf; in tg_set_limit()
1665 blk_throtl_update_limit_valid(tg->td); in tg_set_limit()
1666 if (tg->td->limit_valid[LIMIT_LOW]) { in tg_set_limit()
1668 tg->td->limit_index = LIMIT_LOW; in tg_set_limit()
1670 tg->td->limit_index = LIMIT_MAX; in tg_set_limit()
1671 tg_conf_updated(tg, index == LIMIT_LOW && in tg_set_limit()
1672 tg->td->limit_valid[LIMIT_LOW]); in tg_set_limit()
1731 struct throtl_grp *tg = blkg_to_tg(blkg); in blk_throtl_cancel_bios() local
1732 struct throtl_service_queue *sq = &tg->service_queue; in blk_throtl_cancel_bios()
1738 tg->flags |= THROTL_TG_CANCELING; in blk_throtl_cancel_bios()
1747 if (!(tg->flags & THROTL_TG_PENDING)) in blk_throtl_cancel_bios()
1754 tg_update_disptime(tg); in blk_throtl_cancel_bios()
1763 static unsigned long __tg_last_low_overflow_time(struct throtl_grp *tg) in __tg_last_low_overflow_time() argument
1767 if (tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW]) in __tg_last_low_overflow_time()
1768 rtime = tg->last_low_overflow_time[READ]; in __tg_last_low_overflow_time()
1769 if (tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) in __tg_last_low_overflow_time()
1770 wtime = tg->last_low_overflow_time[WRITE]; in __tg_last_low_overflow_time()
1774 static unsigned long tg_last_low_overflow_time(struct throtl_grp *tg) in tg_last_low_overflow_time() argument
1777 struct throtl_grp *parent = tg; in tg_last_low_overflow_time()
1778 unsigned long ret = __tg_last_low_overflow_time(tg); in tg_last_low_overflow_time()
1801 static bool throtl_tg_is_idle(struct throtl_grp *tg) in throtl_tg_is_idle() argument
1813 time = min_t(unsigned long, MAX_IDLE_TIME, 4 * tg->idletime_threshold); in throtl_tg_is_idle()
1814 ret = tg->latency_target == DFL_LATENCY_TARGET || in throtl_tg_is_idle()
1815 tg->idletime_threshold == DFL_IDLE_THRESHOLD || in throtl_tg_is_idle()
1816 (ktime_get_ns() >> 10) - tg->last_finish_time > time || in throtl_tg_is_idle()
1817 tg->avg_idletime > tg->idletime_threshold || in throtl_tg_is_idle()
1818 (tg->latency_target && tg->bio_cnt && in throtl_tg_is_idle()
1819 tg->bad_bio_cnt * 5 < tg->bio_cnt); in throtl_tg_is_idle()
1820 throtl_log(&tg->service_queue, in throtl_tg_is_idle()
1822 tg->avg_idletime, tg->idletime_threshold, tg->bad_bio_cnt, in throtl_tg_is_idle()
1823 tg->bio_cnt, ret, tg->td->scale); in throtl_tg_is_idle()
1827 static bool throtl_low_limit_reached(struct throtl_grp *tg, int rw) in throtl_low_limit_reached() argument
1829 struct throtl_service_queue *sq = &tg->service_queue; in throtl_low_limit_reached()
1830 bool limit = tg->bps[rw][LIMIT_LOW] || tg->iops[rw][LIMIT_LOW]; in throtl_low_limit_reached()
1841 static bool throtl_tg_can_upgrade(struct throtl_grp *tg) in throtl_tg_can_upgrade() argument
1848 if (throtl_low_limit_reached(tg, READ) && in throtl_tg_can_upgrade()
1849 throtl_low_limit_reached(tg, WRITE)) in throtl_tg_can_upgrade()
1853 tg_last_low_overflow_time(tg) + tg->td->throtl_slice) && in throtl_tg_can_upgrade()
1854 throtl_tg_is_idle(tg)) in throtl_tg_can_upgrade()
1859 static bool throtl_hierarchy_can_upgrade(struct throtl_grp *tg) in throtl_hierarchy_can_upgrade() argument
1862 if (throtl_tg_can_upgrade(tg)) in throtl_hierarchy_can_upgrade()
1864 tg = sq_to_tg(tg->service_queue.parent_sq); in throtl_hierarchy_can_upgrade()
1865 if (!tg || !tg_to_blkg(tg)->parent) in throtl_hierarchy_can_upgrade()
1885 struct throtl_grp *tg = blkg_to_tg(blkg); in throtl_can_upgrade() local
1887 if (tg == this_tg) in throtl_can_upgrade()
1889 if (!list_empty(&tg_to_blkg(tg)->blkcg->css.children)) in throtl_can_upgrade()
1891 if (!throtl_hierarchy_can_upgrade(tg)) { in throtl_can_upgrade()
1900 static void throtl_upgrade_check(struct throtl_grp *tg) in throtl_upgrade_check() argument
1904 if (tg->td->limit_index != LIMIT_LOW) in throtl_upgrade_check()
1907 if (time_after(tg->last_check_time + tg->td->throtl_slice, now)) in throtl_upgrade_check()
1910 tg->last_check_time = now; in throtl_upgrade_check()
1913 __tg_last_low_overflow_time(tg) + tg->td->throtl_slice)) in throtl_upgrade_check()
1916 if (throtl_can_upgrade(tg->td, NULL)) in throtl_upgrade_check()
1917 throtl_upgrade_state(tg->td); in throtl_upgrade_check()
1931 struct throtl_grp *tg = blkg_to_tg(blkg); in throtl_upgrade_state() local
1932 struct throtl_service_queue *sq = &tg->service_queue; in throtl_upgrade_state()
1934 tg->disptime = jiffies - 1; in throtl_upgrade_state()
1958 static bool throtl_tg_can_downgrade(struct throtl_grp *tg) in throtl_tg_can_downgrade() argument
1960 struct throtl_data *td = tg->td; in throtl_tg_can_downgrade()
1967 if (time_after_eq(now, tg_last_low_overflow_time(tg) + in throtl_tg_can_downgrade()
1969 (!throtl_tg_is_idle(tg) || in throtl_tg_can_downgrade()
1970 !list_empty(&tg_to_blkg(tg)->blkcg->css.children))) in throtl_tg_can_downgrade()
1975 static bool throtl_hierarchy_can_downgrade(struct throtl_grp *tg) in throtl_hierarchy_can_downgrade() argument
1977 struct throtl_data *td = tg->td; in throtl_hierarchy_can_downgrade()
1983 if (!throtl_tg_can_downgrade(tg)) in throtl_hierarchy_can_downgrade()
1985 tg = sq_to_tg(tg->service_queue.parent_sq); in throtl_hierarchy_can_downgrade()
1986 if (!tg || !tg_to_blkg(tg)->parent) in throtl_hierarchy_can_downgrade()
1992 static void throtl_downgrade_check(struct throtl_grp *tg) in throtl_downgrade_check() argument
1999 if (tg->td->limit_index != LIMIT_MAX || in throtl_downgrade_check()
2000 !tg->td->limit_valid[LIMIT_LOW]) in throtl_downgrade_check()
2002 if (!list_empty(&tg_to_blkg(tg)->blkcg->css.children)) in throtl_downgrade_check()
2004 if (time_after(tg->last_check_time + tg->td->throtl_slice, now)) in throtl_downgrade_check()
2007 elapsed_time = now - tg->last_check_time; in throtl_downgrade_check()
2008 tg->last_check_time = now; in throtl_downgrade_check()
2010 if (time_before(now, tg_last_low_overflow_time(tg) + in throtl_downgrade_check()
2011 tg->td->throtl_slice)) in throtl_downgrade_check()
2014 if (tg->bps[READ][LIMIT_LOW]) { in throtl_downgrade_check()
2015 bps = tg->last_bytes_disp[READ] * HZ; in throtl_downgrade_check()
2017 if (bps >= tg->bps[READ][LIMIT_LOW]) in throtl_downgrade_check()
2018 tg->last_low_overflow_time[READ] = now; in throtl_downgrade_check()
2021 if (tg->bps[WRITE][LIMIT_LOW]) { in throtl_downgrade_check()
2022 bps = tg->last_bytes_disp[WRITE] * HZ; in throtl_downgrade_check()
2024 if (bps >= tg->bps[WRITE][LIMIT_LOW]) in throtl_downgrade_check()
2025 tg->last_low_overflow_time[WRITE] = now; in throtl_downgrade_check()
2028 if (tg->iops[READ][LIMIT_LOW]) { in throtl_downgrade_check()
2029 iops = tg->last_io_disp[READ] * HZ / elapsed_time; in throtl_downgrade_check()
2030 if (iops >= tg->iops[READ][LIMIT_LOW]) in throtl_downgrade_check()
2031 tg->last_low_overflow_time[READ] = now; in throtl_downgrade_check()
2034 if (tg->iops[WRITE][LIMIT_LOW]) { in throtl_downgrade_check()
2035 iops = tg->last_io_disp[WRITE] * HZ / elapsed_time; in throtl_downgrade_check()
2036 if (iops >= tg->iops[WRITE][LIMIT_LOW]) in throtl_downgrade_check()
2037 tg->last_low_overflow_time[WRITE] = now; in throtl_downgrade_check()
2044 if (throtl_hierarchy_can_downgrade(tg)) in throtl_downgrade_check()
2045 throtl_downgrade_state(tg->td); in throtl_downgrade_check()
2047 tg->last_bytes_disp[READ] = 0; in throtl_downgrade_check()
2048 tg->last_bytes_disp[WRITE] = 0; in throtl_downgrade_check()
2049 tg->last_io_disp[READ] = 0; in throtl_downgrade_check()
2050 tg->last_io_disp[WRITE] = 0; in throtl_downgrade_check()
2053 static void blk_throtl_update_idletime(struct throtl_grp *tg) in blk_throtl_update_idletime() argument
2056 unsigned long last_finish_time = tg->last_finish_time; in blk_throtl_update_idletime()
2063 last_finish_time == tg->checked_last_finish_time) in blk_throtl_update_idletime()
2066 tg->avg_idletime = (tg->avg_idletime * 7 + now - last_finish_time) >> 3; in blk_throtl_update_idletime()
2067 tg->checked_last_finish_time = last_finish_time; in blk_throtl_update_idletime()
2151 static void blk_throtl_update_idletime(struct throtl_grp *tg) in blk_throtl_update_idletime() argument
2155 static void throtl_downgrade_check(struct throtl_grp *tg) in throtl_downgrade_check() argument
2159 static void throtl_upgrade_check(struct throtl_grp *tg) in throtl_upgrade_check() argument
2179 struct throtl_grp *tg = blkg_to_tg(blkg); in __blk_throtl_bio() local
2183 struct throtl_data *td = tg->td; in __blk_throtl_bio()
2191 blk_throtl_update_idletime(tg); in __blk_throtl_bio()
2193 sq = &tg->service_queue; in __blk_throtl_bio()
2197 if (tg->last_low_overflow_time[rw] == 0) in __blk_throtl_bio()
2198 tg->last_low_overflow_time[rw] = jiffies; in __blk_throtl_bio()
2199 throtl_downgrade_check(tg); in __blk_throtl_bio()
2200 throtl_upgrade_check(tg); in __blk_throtl_bio()
2206 if (!tg_may_dispatch(tg, bio, NULL)) { in __blk_throtl_bio()
2207 tg->last_low_overflow_time[rw] = jiffies; in __blk_throtl_bio()
2208 if (throtl_can_upgrade(td, tg)) { in __blk_throtl_bio()
2216 throtl_charge_bio(tg, bio); in __blk_throtl_bio()
2229 throtl_trim_slice(tg, rw); in __blk_throtl_bio()
2236 qn = &tg->qnode_on_parent[rw]; in __blk_throtl_bio()
2238 tg = sq_to_tg(sq); in __blk_throtl_bio()
2239 if (!tg) { in __blk_throtl_bio()
2248 tg->bytes_disp[rw], bio->bi_iter.bi_size, in __blk_throtl_bio()
2249 tg_bps_limit(tg, rw), in __blk_throtl_bio()
2250 tg->io_disp[rw], tg_iops_limit(tg, rw), in __blk_throtl_bio()
2253 tg->last_low_overflow_time[rw] = jiffies; in __blk_throtl_bio()
2256 throtl_add_bio_tg(bio, qn, tg); in __blk_throtl_bio()
2265 if (tg->flags & THROTL_TG_WAS_EMPTY) { in __blk_throtl_bio()
2266 tg_update_disptime(tg); in __blk_throtl_bio()
2267 throtl_schedule_next_dispatch(tg->service_queue.parent_sq, true); in __blk_throtl_bio()
2314 struct throtl_grp *tg; in blk_throtl_bio_endio() local
2324 tg = blkg_to_tg(blkg); in blk_throtl_bio_endio()
2325 if (!tg->td->limit_valid[LIMIT_LOW]) in blk_throtl_bio_endio()
2329 tg->last_finish_time = finish_time_ns >> 10; in blk_throtl_bio_endio()
2339 throtl_track_latency(tg->td, bio_issue_size(&bio->bi_issue), in blk_throtl_bio_endio()
2342 if (tg->latency_target && lat >= tg->td->filtered_latency) { in blk_throtl_bio_endio()
2347 threshold = tg->td->avg_buckets[rw][bucket].latency + in blk_throtl_bio_endio()
2348 tg->latency_target; in blk_throtl_bio_endio()
2350 tg->bad_bio_cnt++; in blk_throtl_bio_endio()
2355 tg->bio_cnt++; in blk_throtl_bio_endio()
2358 if (time_after(jiffies, tg->bio_cnt_reset_time) || tg->bio_cnt > 1024) { in blk_throtl_bio_endio()
2359 tg->bio_cnt_reset_time = tg->td->throtl_slice + jiffies; in blk_throtl_bio_endio()
2360 tg->bio_cnt /= 2; in blk_throtl_bio_endio()
2361 tg->bad_bio_cnt /= 2; in blk_throtl_bio_endio()