Lines Matching +full:lock +full:- +full:latency +full:- +full:ns
1 // SPDX-License-Identifier: GPL-2.0
13 #include <linux/blk-cgroup.h>
15 #include "blk-cgroup-rwstat.h"
30 #define DFL_LATENCY_TARGET (-1L)
35 * For HD, very small latency comes from sequential IO. Such IO is helpless to
55 * the parent, they're popped in round-robin order so that no single source
60 * throtl_service_queue and then dispatched in round-robin order.
69 struct list_head node; /* service_queue->queued[] */
86 * their ->disptime.
96 THROTL_TG_WAS_EMPTY = 1 << 1, /* bio_lists[] became non-empty */
171 unsigned long last_finish_time; /* ns / 1024 */
172 unsigned long checked_last_finish_time; /* ns / 1024 */
173 unsigned long avg_idletime; /* ns / 1024 */
178 unsigned int bad_bio_cnt; /* bios exceeding latency threshold */
188 /* We measure latency for request size from <= 4k to >= 1M */
192 unsigned long total_latency; /* ns / 1024 */
197 unsigned long latency; /* ns / 1024 */ member
246 return pd_to_blkg(&tg->pd); in tg_to_blkg()
250 * sq_to_tg - return the throl_grp the specified service queue belongs to
253 * Return the throtl_grp @sq belongs to. If @sq is the top-level one
258 if (sq && sq->parent_sq) in sq_to_tg()
265 * sq_to_td - return throtl_data the specified service queue belongs to
276 return tg->td; in sq_to_td()
292 if (td->scale < 4096 && time_after_eq(jiffies, in throtl_adjusted_limit()
293 td->low_upgrade_time + td->scale * td->throtl_slice)) in throtl_adjusted_limit()
294 td->scale = (jiffies - td->low_upgrade_time) / td->throtl_slice; in throtl_adjusted_limit()
296 return low + (low >> 1) * td->scale; in throtl_adjusted_limit()
305 if (cgroup_subsys_on_dfl(io_cgrp_subsys) && !blkg->parent) in tg_bps_limit()
308 td = tg->td; in tg_bps_limit()
309 ret = tg->bps[rw][td->limit_index]; in tg_bps_limit()
310 if (ret == 0 && td->limit_index == LIMIT_LOW) { in tg_bps_limit()
312 if (!list_empty(&blkg->blkcg->css.children) || in tg_bps_limit()
313 tg->iops[rw][td->limit_index]) in tg_bps_limit()
319 if (td->limit_index == LIMIT_MAX && tg->bps[rw][LIMIT_LOW] && in tg_bps_limit()
320 tg->bps[rw][LIMIT_LOW] != tg->bps[rw][LIMIT_MAX]) { in tg_bps_limit()
323 adjusted = throtl_adjusted_limit(tg->bps[rw][LIMIT_LOW], td); in tg_bps_limit()
324 ret = min(tg->bps[rw][LIMIT_MAX], adjusted); in tg_bps_limit()
335 if (cgroup_subsys_on_dfl(io_cgrp_subsys) && !blkg->parent) in tg_iops_limit()
338 td = tg->td; in tg_iops_limit()
339 ret = tg->iops[rw][td->limit_index]; in tg_iops_limit()
340 if (ret == 0 && tg->td->limit_index == LIMIT_LOW) { in tg_iops_limit()
342 if (!list_empty(&blkg->blkcg->css.children) || in tg_iops_limit()
343 tg->bps[rw][td->limit_index]) in tg_iops_limit()
349 if (td->limit_index == LIMIT_MAX && tg->iops[rw][LIMIT_LOW] && in tg_iops_limit()
350 tg->iops[rw][LIMIT_LOW] != tg->iops[rw][LIMIT_MAX]) { in tg_iops_limit()
353 adjusted = throtl_adjusted_limit(tg->iops[rw][LIMIT_LOW], td); in tg_iops_limit()
356 ret = min_t(unsigned int, tg->iops[rw][LIMIT_MAX], adjusted); in tg_iops_limit()
362 clamp_t(int, order_base_2(sectors) - 3, 0, LATENCY_BUCKET_SIZE - 1)
365 * throtl_log - log debug message via blktrace
378 if (likely(!blk_trace_note_message_enabled(__td->queue))) \
381 blk_add_cgroup_trace_msg(__td->queue, \
382 tg_to_blkg(__tg)->blkcg, "throtl " fmt, ##args);\
384 blk_add_trace_msg(__td->queue, "throtl " fmt, ##args); \
393 return bio->bi_iter.bi_size; in throtl_bio_data_size()
398 INIT_LIST_HEAD(&qn->node); in throtl_qnode_init()
399 bio_list_init(&qn->bios); in throtl_qnode_init()
400 qn->tg = tg; in throtl_qnode_init()
404 * throtl_qnode_add_bio - add a bio to a throtl_qnode and activate it
407 * @queued: the service_queue->queued[] list @qn belongs to
410 * @qn->tg's reference count is bumped when @qn is activated. See the
416 bio_list_add(&qn->bios, bio); in throtl_qnode_add_bio()
417 if (list_empty(&qn->node)) { in throtl_qnode_add_bio()
418 list_add_tail(&qn->node, queued); in throtl_qnode_add_bio()
419 blkg_get(tg_to_blkg(qn->tg)); in throtl_qnode_add_bio()
424 * throtl_peek_queued - peek the first bio on a qnode list
436 bio = bio_list_peek(&qn->bios); in throtl_peek_queued()
442 * throtl_pop_queued - pop the first bio form a qnode list
448 * that the popping order is round-robin.
465 bio = bio_list_pop(&qn->bios); in throtl_pop_queued()
468 if (bio_list_empty(&qn->bios)) { in throtl_pop_queued()
469 list_del_init(&qn->node); in throtl_pop_queued()
471 *tg_to_put = qn->tg; in throtl_pop_queued()
473 blkg_put(tg_to_blkg(qn->tg)); in throtl_pop_queued()
475 list_move_tail(&qn->node, queued); in throtl_pop_queued()
484 INIT_LIST_HEAD(&sq->queued[0]); in throtl_service_queue_init()
485 INIT_LIST_HEAD(&sq->queued[1]); in throtl_service_queue_init()
486 sq->pending_tree = RB_ROOT_CACHED; in throtl_service_queue_init()
487 timer_setup(&sq->pending_timer, throtl_pending_timer_fn, 0); in throtl_service_queue_init()
497 tg = kzalloc_node(sizeof(*tg), gfp, q->node); in throtl_pd_alloc()
501 if (blkg_rwstat_init(&tg->stat_bytes, gfp)) in throtl_pd_alloc()
504 if (blkg_rwstat_init(&tg->stat_ios, gfp)) in throtl_pd_alloc()
507 throtl_service_queue_init(&tg->service_queue); in throtl_pd_alloc()
510 throtl_qnode_init(&tg->qnode_on_self[rw], tg); in throtl_pd_alloc()
511 throtl_qnode_init(&tg->qnode_on_parent[rw], tg); in throtl_pd_alloc()
514 RB_CLEAR_NODE(&tg->rb_node); in throtl_pd_alloc()
515 tg->bps[READ][LIMIT_MAX] = U64_MAX; in throtl_pd_alloc()
516 tg->bps[WRITE][LIMIT_MAX] = U64_MAX; in throtl_pd_alloc()
517 tg->iops[READ][LIMIT_MAX] = UINT_MAX; in throtl_pd_alloc()
518 tg->iops[WRITE][LIMIT_MAX] = UINT_MAX; in throtl_pd_alloc()
519 tg->bps_conf[READ][LIMIT_MAX] = U64_MAX; in throtl_pd_alloc()
520 tg->bps_conf[WRITE][LIMIT_MAX] = U64_MAX; in throtl_pd_alloc()
521 tg->iops_conf[READ][LIMIT_MAX] = UINT_MAX; in throtl_pd_alloc()
522 tg->iops_conf[WRITE][LIMIT_MAX] = UINT_MAX; in throtl_pd_alloc()
525 tg->latency_target = DFL_LATENCY_TARGET; in throtl_pd_alloc()
526 tg->latency_target_conf = DFL_LATENCY_TARGET; in throtl_pd_alloc()
527 tg->idletime_threshold = DFL_IDLE_THRESHOLD; in throtl_pd_alloc()
528 tg->idletime_threshold_conf = DFL_IDLE_THRESHOLD; in throtl_pd_alloc()
530 return &tg->pd; in throtl_pd_alloc()
533 blkg_rwstat_exit(&tg->stat_bytes); in throtl_pd_alloc()
543 struct throtl_data *td = blkg->q->td; in throtl_pd_init()
544 struct throtl_service_queue *sq = &tg->service_queue; in throtl_pd_init()
559 sq->parent_sq = &td->service_queue; in throtl_pd_init()
560 if (cgroup_subsys_on_dfl(io_cgrp_subsys) && blkg->parent) in throtl_pd_init()
561 sq->parent_sq = &blkg_to_tg(blkg->parent)->service_queue; in throtl_pd_init()
562 tg->td = td; in throtl_pd_init()
572 struct throtl_grp *parent_tg = sq_to_tg(tg->service_queue.parent_sq); in tg_update_has_rules()
573 struct throtl_data *td = tg->td; in tg_update_has_rules()
577 tg->has_rules[rw] = (parent_tg && parent_tg->has_rules[rw]) || in tg_update_has_rules()
578 (td->limit_valid[td->limit_index] && in tg_update_has_rules()
601 blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { in blk_throtl_update_limit_valid()
604 if (tg->bps[READ][LIMIT_LOW] || tg->bps[WRITE][LIMIT_LOW] || in blk_throtl_update_limit_valid()
605 tg->iops[READ][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) { in blk_throtl_update_limit_valid()
612 td->limit_valid[LIMIT_LOW] = low_valid; in blk_throtl_update_limit_valid()
625 tg->bps[READ][LIMIT_LOW] = 0; in throtl_pd_offline()
626 tg->bps[WRITE][LIMIT_LOW] = 0; in throtl_pd_offline()
627 tg->iops[READ][LIMIT_LOW] = 0; in throtl_pd_offline()
628 tg->iops[WRITE][LIMIT_LOW] = 0; in throtl_pd_offline()
630 blk_throtl_update_limit_valid(tg->td); in throtl_pd_offline()
632 if (!tg->td->limit_valid[tg->td->limit_index]) in throtl_pd_offline()
633 throtl_upgrade_state(tg->td); in throtl_pd_offline()
640 del_timer_sync(&tg->service_queue.pending_timer); in throtl_pd_free()
641 blkg_rwstat_exit(&tg->stat_bytes); in throtl_pd_free()
642 blkg_rwstat_exit(&tg->stat_ios); in throtl_pd_free()
651 n = rb_first_cached(&parent_sq->pending_tree); in throtl_rb_first()
661 rb_erase_cached(n, &parent_sq->pending_tree); in throtl_rb_erase()
663 --parent_sq->nr_pending; in throtl_rb_erase()
674 parent_sq->first_pending_disptime = tg->disptime; in update_min_dispatch_time()
679 struct throtl_service_queue *parent_sq = tg->service_queue.parent_sq; in tg_service_queue_add()
680 struct rb_node **node = &parent_sq->pending_tree.rb_root.rb_node; in tg_service_queue_add()
683 unsigned long key = tg->disptime; in tg_service_queue_add()
690 if (time_before(key, __tg->disptime)) in tg_service_queue_add()
691 node = &parent->rb_left; in tg_service_queue_add()
693 node = &parent->rb_right; in tg_service_queue_add()
698 rb_link_node(&tg->rb_node, parent, node); in tg_service_queue_add()
699 rb_insert_color_cached(&tg->rb_node, &parent_sq->pending_tree, in tg_service_queue_add()
705 if (!(tg->flags & THROTL_TG_PENDING)) { in throtl_enqueue_tg()
707 tg->flags |= THROTL_TG_PENDING; in throtl_enqueue_tg()
708 tg->service_queue.parent_sq->nr_pending++; in throtl_enqueue_tg()
714 if (tg->flags & THROTL_TG_PENDING) { in throtl_dequeue_tg()
715 throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq); in throtl_dequeue_tg()
716 tg->flags &= ~THROTL_TG_PENDING; in throtl_dequeue_tg()
720 /* Call with queue lock held */
724 unsigned long max_expire = jiffies + 8 * sq_to_td(sq)->throtl_slice; in throtl_schedule_pending_timer()
735 mod_timer(&sq->pending_timer, expires); in throtl_schedule_pending_timer()
737 expires - jiffies, jiffies); in throtl_schedule_pending_timer()
741 * throtl_schedule_next_dispatch - schedule the next dispatch cycle
745 * Arm @sq->pending_timer so that the next dispatch cycle starts on the
755 * delay before dispatch starts even if @sq->first_pending_disptime is not
762 if (!sq->nr_pending) in throtl_schedule_next_dispatch()
768 if (force || time_after(sq->first_pending_disptime, jiffies)) { in throtl_schedule_next_dispatch()
769 throtl_schedule_pending_timer(sq, sq->first_pending_disptime); in throtl_schedule_next_dispatch()
780 tg->bytes_disp[rw] = 0; in throtl_start_new_slice_with_credit()
781 tg->io_disp[rw] = 0; in throtl_start_new_slice_with_credit()
783 atomic_set(&tg->io_split_cnt[rw], 0); in throtl_start_new_slice_with_credit()
791 if (time_after_eq(start, tg->slice_start[rw])) in throtl_start_new_slice_with_credit()
792 tg->slice_start[rw] = start; in throtl_start_new_slice_with_credit()
794 tg->slice_end[rw] = jiffies + tg->td->throtl_slice; in throtl_start_new_slice_with_credit()
795 throtl_log(&tg->service_queue, in throtl_start_new_slice_with_credit()
797 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice_with_credit()
798 tg->slice_end[rw], jiffies); in throtl_start_new_slice_with_credit()
803 tg->bytes_disp[rw] = 0; in throtl_start_new_slice()
804 tg->io_disp[rw] = 0; in throtl_start_new_slice()
805 tg->slice_start[rw] = jiffies; in throtl_start_new_slice()
806 tg->slice_end[rw] = jiffies + tg->td->throtl_slice; in throtl_start_new_slice()
808 atomic_set(&tg->io_split_cnt[rw], 0); in throtl_start_new_slice()
810 throtl_log(&tg->service_queue, in throtl_start_new_slice()
812 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice()
813 tg->slice_end[rw], jiffies); in throtl_start_new_slice()
819 tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice); in throtl_set_slice_end()
826 throtl_log(&tg->service_queue, in throtl_extend_slice()
828 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_extend_slice()
829 tg->slice_end[rw], jiffies); in throtl_extend_slice()
835 if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw])) in throtl_slice_used()
847 BUG_ON(time_before(tg->slice_end[rw], tg->slice_start[rw])); in throtl_trim_slice()
850 * If bps are unlimited (-1), then time slice don't get in throtl_trim_slice()
865 throtl_set_slice_end(tg, rw, jiffies + tg->td->throtl_slice); in throtl_trim_slice()
867 time_elapsed = jiffies - tg->slice_start[rw]; in throtl_trim_slice()
869 nr_slices = time_elapsed / tg->td->throtl_slice; in throtl_trim_slice()
873 tmp = tg_bps_limit(tg, rw) * tg->td->throtl_slice * nr_slices; in throtl_trim_slice()
877 io_trim = (tg_iops_limit(tg, rw) * tg->td->throtl_slice * nr_slices) / in throtl_trim_slice()
883 if (tg->bytes_disp[rw] >= bytes_trim) in throtl_trim_slice()
884 tg->bytes_disp[rw] -= bytes_trim; in throtl_trim_slice()
886 tg->bytes_disp[rw] = 0; in throtl_trim_slice()
888 if (tg->io_disp[rw] >= io_trim) in throtl_trim_slice()
889 tg->io_disp[rw] -= io_trim; in throtl_trim_slice()
891 tg->io_disp[rw] = 0; in throtl_trim_slice()
893 tg->slice_start[rw] += nr_slices * tg->td->throtl_slice; in throtl_trim_slice()
895 throtl_log(&tg->service_queue, in throtl_trim_slice()
898 tg->slice_start[rw], tg->slice_end[rw], jiffies); in throtl_trim_slice()
915 jiffy_elapsed = jiffies - tg->slice_start[rw]; in tg_with_in_iops_limit()
918 jiffy_elapsed_rnd = roundup(jiffy_elapsed + 1, tg->td->throtl_slice); in tg_with_in_iops_limit()
935 if (tg->io_disp[rw] + 1 <= io_allowed) { in tg_with_in_iops_limit()
942 jiffy_wait = jiffy_elapsed_rnd - jiffy_elapsed; in tg_with_in_iops_limit()
963 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw]; in tg_with_in_bps_limit()
967 jiffy_elapsed_rnd = tg->td->throtl_slice; in tg_with_in_bps_limit()
969 jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, tg->td->throtl_slice); in tg_with_in_bps_limit()
975 if (tg->bytes_disp[rw] + bio_size <= bytes_allowed) { in tg_with_in_bps_limit()
982 extra_bytes = tg->bytes_disp[rw] + bio_size - bytes_allowed; in tg_with_in_bps_limit()
992 jiffy_wait = jiffy_wait + (jiffy_elapsed_rnd - jiffy_elapsed); in tg_with_in_bps_limit()
1000 * of jiffies to wait before this bio is with-in IO rate and can be dispatched
1016 BUG_ON(tg->service_queue.nr_queued[rw] && in tg_may_dispatch()
1017 bio != throtl_peek_queued(&tg->service_queue.queued[rw])); in tg_may_dispatch()
1019 /* If tg->bps = -1, then BW is unlimited */ in tg_may_dispatch()
1033 if (throtl_slice_used(tg, rw) && !(tg->service_queue.nr_queued[rw])) in tg_may_dispatch()
1036 if (time_before(tg->slice_end[rw], in tg_may_dispatch()
1037 jiffies + tg->td->throtl_slice)) in tg_may_dispatch()
1039 jiffies + tg->td->throtl_slice); in tg_may_dispatch()
1043 tg->io_disp[rw] += atomic_xchg(&tg->io_split_cnt[rw], 0); in tg_may_dispatch()
1057 if (time_before(tg->slice_end[rw], jiffies + max_wait)) in tg_may_dispatch()
1069 tg->bytes_disp[rw] += bio_size; in throtl_charge_bio()
1070 tg->io_disp[rw]++; in throtl_charge_bio()
1071 tg->last_bytes_disp[rw] += bio_size; in throtl_charge_bio()
1072 tg->last_io_disp[rw]++; in throtl_charge_bio()
1076 * more than once as a throttled bio will go through blk-throtl the in throtl_charge_bio()
1085 * throtl_add_bio_tg - add a bio to the specified throtl_grp
1091 * tg->qnode_on_self[] is used.
1096 struct throtl_service_queue *sq = &tg->service_queue; in throtl_add_bio_tg()
1100 qn = &tg->qnode_on_self[rw]; in throtl_add_bio_tg()
1108 if (!sq->nr_queued[rw]) in throtl_add_bio_tg()
1109 tg->flags |= THROTL_TG_WAS_EMPTY; in throtl_add_bio_tg()
1111 throtl_qnode_add_bio(bio, qn, &sq->queued[rw]); in throtl_add_bio_tg()
1113 sq->nr_queued[rw]++; in throtl_add_bio_tg()
1119 struct throtl_service_queue *sq = &tg->service_queue; in tg_update_disptime()
1120 unsigned long read_wait = -1, write_wait = -1, min_wait = -1, disptime; in tg_update_disptime()
1123 bio = throtl_peek_queued(&sq->queued[READ]); in tg_update_disptime()
1127 bio = throtl_peek_queued(&sq->queued[WRITE]); in tg_update_disptime()
1136 tg->disptime = disptime; in tg_update_disptime()
1140 tg->flags &= ~THROTL_TG_WAS_EMPTY; in tg_update_disptime()
1148 child_tg->slice_start[rw]); in start_parent_slice_with_credit()
1155 struct throtl_service_queue *sq = &tg->service_queue; in tg_dispatch_one_bio()
1156 struct throtl_service_queue *parent_sq = sq->parent_sq; in tg_dispatch_one_bio()
1167 bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put); in tg_dispatch_one_bio()
1168 sq->nr_queued[rw]--; in tg_dispatch_one_bio()
1175 * @td->service_queue, @bio is ready to be issued. Put it on its in tg_dispatch_one_bio()
1180 throtl_add_bio_tg(bio, &tg->qnode_on_parent[rw], parent_tg); in tg_dispatch_one_bio()
1183 throtl_qnode_add_bio(bio, &tg->qnode_on_parent[rw], in tg_dispatch_one_bio()
1184 &parent_sq->queued[rw]); in tg_dispatch_one_bio()
1185 BUG_ON(tg->td->nr_queued[rw] <= 0); in tg_dispatch_one_bio()
1186 tg->td->nr_queued[rw]--; in tg_dispatch_one_bio()
1197 struct throtl_service_queue *sq = &tg->service_queue; in throtl_dispatch_tg()
1200 unsigned int max_nr_writes = THROTL_GRP_QUANTUM - max_nr_reads; in throtl_dispatch_tg()
1205 while ((bio = throtl_peek_queued(&sq->queued[READ])) && in throtl_dispatch_tg()
1215 while ((bio = throtl_peek_queued(&sq->queued[WRITE])) && in throtl_dispatch_tg()
1236 if (!parent_sq->nr_pending) in throtl_select_dispatch()
1243 if (time_before(jiffies, tg->disptime)) in throtl_select_dispatch()
1250 sq = &tg->service_queue; in throtl_select_dispatch()
1251 if (sq->nr_queued[0] || sq->nr_queued[1]) in throtl_select_dispatch()
1264 * throtl_pending_timer_fn - timer function for service_queue->pending_timer
1275 * the top-level service_tree is reached, throtl_data->dispatch_work is
1283 struct request_queue *q = td->queue; in throtl_pending_timer_fn()
1288 spin_lock_irq(&q->queue_lock); in throtl_pending_timer_fn()
1293 parent_sq = sq->parent_sq; in throtl_pending_timer_fn()
1298 sq->nr_queued[READ] + sq->nr_queued[WRITE], in throtl_pending_timer_fn()
1299 sq->nr_queued[READ], sq->nr_queued[WRITE]); in throtl_pending_timer_fn()
1311 spin_unlock_irq(&q->queue_lock); in throtl_pending_timer_fn()
1313 spin_lock_irq(&q->queue_lock); in throtl_pending_timer_fn()
1321 if (tg->flags & THROTL_TG_WAS_EMPTY) { in throtl_pending_timer_fn()
1331 /* reached the top-level, queue issuing */ in throtl_pending_timer_fn()
1332 queue_work(kthrotld_workqueue, &td->dispatch_work); in throtl_pending_timer_fn()
1335 spin_unlock_irq(&q->queue_lock); in throtl_pending_timer_fn()
1339 * blk_throtl_dispatch_work_fn - work function for throtl_data->dispatch_work
1343 * of throtl_data->service_queue. Those bios are ready and issued by this
1350 struct throtl_service_queue *td_sq = &td->service_queue; in blk_throtl_dispatch_work_fn()
1351 struct request_queue *q = td->queue; in blk_throtl_dispatch_work_fn()
1359 spin_lock_irq(&q->queue_lock); in blk_throtl_dispatch_work_fn()
1361 while ((bio = throtl_pop_queued(&td_sq->queued[rw], NULL))) in blk_throtl_dispatch_work_fn()
1363 spin_unlock_irq(&q->queue_lock); in blk_throtl_dispatch_work_fn()
1398 &blkcg_policy_throtl, seq_cft(sf)->private, false); in tg_print_conf_u64()
1405 &blkcg_policy_throtl, seq_cft(sf)->private, false); in tg_print_conf_uint()
1411 struct throtl_service_queue *sq = &tg->service_queue; in tg_conf_updated()
1415 throtl_log(&tg->service_queue, in tg_conf_updated()
1425 * blk-throttle. in tg_conf_updated()
1428 global ? tg->td->queue->root_blkg : tg_to_blkg(tg)) { in tg_conf_updated()
1434 if (!cgroup_subsys_on_dfl(io_cgrp_subsys) || !blkg->parent || in tg_conf_updated()
1435 !blkg->parent->parent) in tg_conf_updated()
1437 parent_tg = blkg_to_tg(blkg->parent); in tg_conf_updated()
1440 * higher latency target in tg_conf_updated()
1442 this_tg->idletime_threshold = min(this_tg->idletime_threshold, in tg_conf_updated()
1443 parent_tg->idletime_threshold); in tg_conf_updated()
1444 this_tg->latency_target = max(this_tg->latency_target, in tg_conf_updated()
1445 parent_tg->latency_target); in tg_conf_updated()
1459 if (tg->flags & THROTL_TG_PENDING) { in tg_conf_updated()
1461 throtl_schedule_next_dispatch(sq->parent_sq, true); in tg_conf_updated()
1478 ret = -EINVAL; in tg_set_conf()
1487 *(u64 *)((void *)tg + of_cft(of)->private) = v; in tg_set_conf()
1489 *(unsigned int *)((void *)tg + of_cft(of)->private) = v; in tg_set_conf()
1514 seq_cft(sf)->private, true); in tg_print_rwstat()
1532 seq_cft(sf)->private, true); in tg_print_rwstat_recursive()
1588 const char *dname = blkg_dev_name(pd->blkg); in tg_prfill_limit()
1606 if (tg->bps_conf[READ][off] == bps_dft && in tg_prfill_limit()
1607 tg->bps_conf[WRITE][off] == bps_dft && in tg_prfill_limit()
1608 tg->iops_conf[READ][off] == iops_dft && in tg_prfill_limit()
1609 tg->iops_conf[WRITE][off] == iops_dft && in tg_prfill_limit()
1611 (tg->idletime_threshold_conf == DFL_IDLE_THRESHOLD && in tg_prfill_limit()
1612 tg->latency_target_conf == DFL_LATENCY_TARGET))) in tg_prfill_limit()
1615 if (tg->bps_conf[READ][off] != U64_MAX) in tg_prfill_limit()
1617 tg->bps_conf[READ][off]); in tg_prfill_limit()
1618 if (tg->bps_conf[WRITE][off] != U64_MAX) in tg_prfill_limit()
1620 tg->bps_conf[WRITE][off]); in tg_prfill_limit()
1621 if (tg->iops_conf[READ][off] != UINT_MAX) in tg_prfill_limit()
1623 tg->iops_conf[READ][off]); in tg_prfill_limit()
1624 if (tg->iops_conf[WRITE][off] != UINT_MAX) in tg_prfill_limit()
1626 tg->iops_conf[WRITE][off]); in tg_prfill_limit()
1628 if (tg->idletime_threshold_conf == ULONG_MAX) in tg_prfill_limit()
1632 tg->idletime_threshold_conf); in tg_prfill_limit()
1634 if (tg->latency_target_conf == ULONG_MAX) in tg_prfill_limit()
1635 strcpy(latency_time, " latency=max"); in tg_prfill_limit()
1638 " latency=%lu", tg->latency_target_conf); in tg_prfill_limit()
1650 &blkcg_policy_throtl, seq_cft(sf)->private, false); in tg_print_limit()
1664 int index = of_cft(of)->private; in tg_set_limit()
1672 v[0] = tg->bps_conf[READ][index]; in tg_set_limit()
1673 v[1] = tg->bps_conf[WRITE][index]; in tg_set_limit()
1674 v[2] = tg->iops_conf[READ][index]; in tg_set_limit()
1675 v[3] = tg->iops_conf[WRITE][index]; in tg_set_limit()
1677 idle_time = tg->idletime_threshold_conf; in tg_set_limit()
1678 latency_time = tg->latency_target_conf; in tg_set_limit()
1691 ret = -EINVAL; in tg_set_limit()
1697 ret = -ERANGE; in tg_set_limit()
1701 ret = -EINVAL; in tg_set_limit()
1712 else if (off == LIMIT_LOW && !strcmp(tok, "latency")) in tg_set_limit()
1718 tg->bps_conf[READ][index] = v[0]; in tg_set_limit()
1719 tg->bps_conf[WRITE][index] = v[1]; in tg_set_limit()
1720 tg->iops_conf[READ][index] = v[2]; in tg_set_limit()
1721 tg->iops_conf[WRITE][index] = v[3]; in tg_set_limit()
1724 tg->bps[READ][index] = v[0]; in tg_set_limit()
1725 tg->bps[WRITE][index] = v[1]; in tg_set_limit()
1726 tg->iops[READ][index] = v[2]; in tg_set_limit()
1727 tg->iops[WRITE][index] = v[3]; in tg_set_limit()
1729 tg->bps[READ][LIMIT_LOW] = min(tg->bps_conf[READ][LIMIT_LOW], in tg_set_limit()
1730 tg->bps_conf[READ][LIMIT_MAX]); in tg_set_limit()
1731 tg->bps[WRITE][LIMIT_LOW] = min(tg->bps_conf[WRITE][LIMIT_LOW], in tg_set_limit()
1732 tg->bps_conf[WRITE][LIMIT_MAX]); in tg_set_limit()
1733 tg->iops[READ][LIMIT_LOW] = min(tg->iops_conf[READ][LIMIT_LOW], in tg_set_limit()
1734 tg->iops_conf[READ][LIMIT_MAX]); in tg_set_limit()
1735 tg->iops[WRITE][LIMIT_LOW] = min(tg->iops_conf[WRITE][LIMIT_LOW], in tg_set_limit()
1736 tg->iops_conf[WRITE][LIMIT_MAX]); in tg_set_limit()
1737 tg->idletime_threshold_conf = idle_time; in tg_set_limit()
1738 tg->latency_target_conf = latency_time; in tg_set_limit()
1741 if (!(tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW] || in tg_set_limit()
1742 tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) || in tg_set_limit()
1743 tg->idletime_threshold_conf == DFL_IDLE_THRESHOLD || in tg_set_limit()
1744 tg->latency_target_conf == DFL_LATENCY_TARGET) { in tg_set_limit()
1745 tg->bps[READ][LIMIT_LOW] = 0; in tg_set_limit()
1746 tg->bps[WRITE][LIMIT_LOW] = 0; in tg_set_limit()
1747 tg->iops[READ][LIMIT_LOW] = 0; in tg_set_limit()
1748 tg->iops[WRITE][LIMIT_LOW] = 0; in tg_set_limit()
1749 tg->idletime_threshold = DFL_IDLE_THRESHOLD; in tg_set_limit()
1750 tg->latency_target = DFL_LATENCY_TARGET; in tg_set_limit()
1752 tg->idletime_threshold = tg->idletime_threshold_conf; in tg_set_limit()
1753 tg->latency_target = tg->latency_target_conf; in tg_set_limit()
1756 blk_throtl_update_limit_valid(tg->td); in tg_set_limit()
1757 if (tg->td->limit_valid[LIMIT_LOW]) { in tg_set_limit()
1759 tg->td->limit_index = LIMIT_LOW; in tg_set_limit()
1761 tg->td->limit_index = LIMIT_MAX; in tg_set_limit()
1763 tg->td->limit_valid[LIMIT_LOW]); in tg_set_limit()
1792 struct throtl_data *td = q->td; in throtl_shutdown_wq()
1794 cancel_work_sync(&td->dispatch_work); in throtl_shutdown_wq()
1812 if (tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW]) in __tg_last_low_overflow_time()
1813 rtime = tg->last_low_overflow_time[READ]; in __tg_last_low_overflow_time()
1814 if (tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) in __tg_last_low_overflow_time()
1815 wtime = tg->last_low_overflow_time[WRITE]; in __tg_last_low_overflow_time()
1827 parent_sq = parent->service_queue.parent_sq; in tg_last_low_overflow_time()
1836 if (!parent->bps[READ][LIMIT_LOW] && in tg_last_low_overflow_time()
1837 !parent->iops[READ][LIMIT_LOW] && in tg_last_low_overflow_time()
1838 !parent->bps[WRITE][LIMIT_LOW] && in tg_last_low_overflow_time()
1839 !parent->iops[WRITE][LIMIT_LOW]) in tg_last_low_overflow_time()
1851 * - single idle is too long, longer than a fixed value (in case user in throtl_tg_is_idle()
1853 * - average think time is more than threshold in throtl_tg_is_idle()
1854 * - IO latency is largely below threshold in throtl_tg_is_idle()
1859 time = min_t(unsigned long, MAX_IDLE_TIME, 4 * tg->idletime_threshold); in throtl_tg_is_idle()
1860 ret = tg->latency_target == DFL_LATENCY_TARGET || in throtl_tg_is_idle()
1861 tg->idletime_threshold == DFL_IDLE_THRESHOLD || in throtl_tg_is_idle()
1862 (ktime_get_ns() >> 10) - tg->last_finish_time > time || in throtl_tg_is_idle()
1863 tg->avg_idletime > tg->idletime_threshold || in throtl_tg_is_idle()
1864 (tg->latency_target && tg->bio_cnt && in throtl_tg_is_idle()
1865 tg->bad_bio_cnt * 5 < tg->bio_cnt); in throtl_tg_is_idle()
1866 throtl_log(&tg->service_queue, in throtl_tg_is_idle()
1868 tg->avg_idletime, tg->idletime_threshold, tg->bad_bio_cnt, in throtl_tg_is_idle()
1869 tg->bio_cnt, ret, tg->td->scale); in throtl_tg_is_idle()
1875 struct throtl_service_queue *sq = &tg->service_queue; in throtl_tg_can_upgrade()
1882 read_limit = tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW]; in throtl_tg_can_upgrade()
1883 write_limit = tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]; in throtl_tg_can_upgrade()
1886 if (read_limit && sq->nr_queued[READ] && in throtl_tg_can_upgrade()
1887 (!write_limit || sq->nr_queued[WRITE])) in throtl_tg_can_upgrade()
1889 if (write_limit && sq->nr_queued[WRITE] && in throtl_tg_can_upgrade()
1890 (!read_limit || sq->nr_queued[READ])) in throtl_tg_can_upgrade()
1894 tg_last_low_overflow_time(tg) + tg->td->throtl_slice) && in throtl_tg_can_upgrade()
1905 tg = sq_to_tg(tg->service_queue.parent_sq); in throtl_hierarchy_can_upgrade()
1906 if (!tg || !tg_to_blkg(tg)->parent) in throtl_hierarchy_can_upgrade()
1918 if (td->limit_index != LIMIT_LOW) in throtl_can_upgrade()
1921 if (time_before(jiffies, td->low_downgrade_time + td->throtl_slice)) in throtl_can_upgrade()
1925 blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { in throtl_can_upgrade()
1930 if (!list_empty(&tg_to_blkg(tg)->blkcg->css.children)) in throtl_can_upgrade()
1945 if (tg->td->limit_index != LIMIT_LOW) in throtl_upgrade_check()
1948 if (time_after(tg->last_check_time + tg->td->throtl_slice, now)) in throtl_upgrade_check()
1951 tg->last_check_time = now; in throtl_upgrade_check()
1954 __tg_last_low_overflow_time(tg) + tg->td->throtl_slice)) in throtl_upgrade_check()
1957 if (throtl_can_upgrade(tg->td, NULL)) in throtl_upgrade_check()
1958 throtl_upgrade_state(tg->td); in throtl_upgrade_check()
1966 throtl_log(&td->service_queue, "upgrade to max"); in throtl_upgrade_state()
1967 td->limit_index = LIMIT_MAX; in throtl_upgrade_state()
1968 td->low_upgrade_time = jiffies; in throtl_upgrade_state()
1969 td->scale = 0; in throtl_upgrade_state()
1971 blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { in throtl_upgrade_state()
1973 struct throtl_service_queue *sq = &tg->service_queue; in throtl_upgrade_state()
1975 tg->disptime = jiffies - 1; in throtl_upgrade_state()
1980 throtl_select_dispatch(&td->service_queue); in throtl_upgrade_state()
1981 throtl_schedule_next_dispatch(&td->service_queue, true); in throtl_upgrade_state()
1982 queue_work(kthrotld_workqueue, &td->dispatch_work); in throtl_upgrade_state()
1987 td->scale /= 2; in throtl_downgrade_state()
1989 throtl_log(&td->service_queue, "downgrade, scale %d", td->scale); in throtl_downgrade_state()
1990 if (td->scale) { in throtl_downgrade_state()
1991 td->low_upgrade_time = jiffies - td->scale * td->throtl_slice; in throtl_downgrade_state()
1995 td->limit_index = LIMIT_LOW; in throtl_downgrade_state()
1996 td->low_downgrade_time = jiffies; in throtl_downgrade_state()
2001 struct throtl_data *td = tg->td; in throtl_tg_can_downgrade()
2008 if (time_after_eq(now, td->low_upgrade_time + td->throtl_slice) && in throtl_tg_can_downgrade()
2010 td->throtl_slice) && in throtl_tg_can_downgrade()
2012 !list_empty(&tg_to_blkg(tg)->blkcg->css.children))) in throtl_tg_can_downgrade()
2022 tg = sq_to_tg(tg->service_queue.parent_sq); in throtl_hierarchy_can_downgrade()
2023 if (!tg || !tg_to_blkg(tg)->parent) in throtl_hierarchy_can_downgrade()
2036 if (tg->td->limit_index != LIMIT_MAX || in throtl_downgrade_check()
2037 !tg->td->limit_valid[LIMIT_LOW]) in throtl_downgrade_check()
2039 if (!list_empty(&tg_to_blkg(tg)->blkcg->css.children)) in throtl_downgrade_check()
2041 if (time_after(tg->last_check_time + tg->td->throtl_slice, now)) in throtl_downgrade_check()
2044 elapsed_time = now - tg->last_check_time; in throtl_downgrade_check()
2045 tg->last_check_time = now; in throtl_downgrade_check()
2048 tg->td->throtl_slice)) in throtl_downgrade_check()
2051 if (tg->bps[READ][LIMIT_LOW]) { in throtl_downgrade_check()
2052 bps = tg->last_bytes_disp[READ] * HZ; in throtl_downgrade_check()
2054 if (bps >= tg->bps[READ][LIMIT_LOW]) in throtl_downgrade_check()
2055 tg->last_low_overflow_time[READ] = now; in throtl_downgrade_check()
2058 if (tg->bps[WRITE][LIMIT_LOW]) { in throtl_downgrade_check()
2059 bps = tg->last_bytes_disp[WRITE] * HZ; in throtl_downgrade_check()
2061 if (bps >= tg->bps[WRITE][LIMIT_LOW]) in throtl_downgrade_check()
2062 tg->last_low_overflow_time[WRITE] = now; in throtl_downgrade_check()
2065 if (tg->iops[READ][LIMIT_LOW]) { in throtl_downgrade_check()
2066 tg->last_io_disp[READ] += atomic_xchg(&tg->last_io_split_cnt[READ], 0); in throtl_downgrade_check()
2067 iops = tg->last_io_disp[READ] * HZ / elapsed_time; in throtl_downgrade_check()
2068 if (iops >= tg->iops[READ][LIMIT_LOW]) in throtl_downgrade_check()
2069 tg->last_low_overflow_time[READ] = now; in throtl_downgrade_check()
2072 if (tg->iops[WRITE][LIMIT_LOW]) { in throtl_downgrade_check()
2073 tg->last_io_disp[WRITE] += atomic_xchg(&tg->last_io_split_cnt[WRITE], 0); in throtl_downgrade_check()
2074 iops = tg->last_io_disp[WRITE] * HZ / elapsed_time; in throtl_downgrade_check()
2075 if (iops >= tg->iops[WRITE][LIMIT_LOW]) in throtl_downgrade_check()
2076 tg->last_low_overflow_time[WRITE] = now; in throtl_downgrade_check()
2084 throtl_downgrade_state(tg->td); in throtl_downgrade_check()
2086 tg->last_bytes_disp[READ] = 0; in throtl_downgrade_check()
2087 tg->last_bytes_disp[WRITE] = 0; in throtl_downgrade_check()
2088 tg->last_io_disp[READ] = 0; in throtl_downgrade_check()
2089 tg->last_io_disp[WRITE] = 0; in throtl_downgrade_check()
2095 unsigned long last_finish_time = tg->last_finish_time; in blk_throtl_update_idletime()
2102 last_finish_time == tg->checked_last_finish_time) in blk_throtl_update_idletime()
2105 tg->avg_idletime = (tg->avg_idletime * 7 + now - last_finish_time) >> 3; in blk_throtl_update_idletime()
2106 tg->checked_last_finish_time = last_finish_time; in blk_throtl_update_idletime()
2115 unsigned long latency[2]; in throtl_update_latency_buckets() local
2117 if (!blk_queue_nonrot(td->queue) || !td->limit_valid[LIMIT_LOW]) in throtl_update_latency_buckets()
2119 if (time_before(jiffies, td->last_calculate_time + HZ)) in throtl_update_latency_buckets()
2121 td->last_calculate_time = jiffies; in throtl_update_latency_buckets()
2126 struct latency_bucket *tmp = &td->tmp_buckets[rw][i]; in throtl_update_latency_buckets()
2132 bucket = per_cpu_ptr(td->latency_buckets[rw], in throtl_update_latency_buckets()
2134 tmp->total_latency += bucket[i].total_latency; in throtl_update_latency_buckets()
2135 tmp->samples += bucket[i].samples; in throtl_update_latency_buckets()
2140 if (tmp->samples >= 32) { in throtl_update_latency_buckets()
2141 int samples = tmp->samples; in throtl_update_latency_buckets()
2143 latency[rw] = tmp->total_latency; in throtl_update_latency_buckets()
2145 tmp->total_latency = 0; in throtl_update_latency_buckets()
2146 tmp->samples = 0; in throtl_update_latency_buckets()
2147 latency[rw] /= samples; in throtl_update_latency_buckets()
2148 if (latency[rw] == 0) in throtl_update_latency_buckets()
2150 avg_latency[rw][i].latency = latency[rw]; in throtl_update_latency_buckets()
2157 if (!avg_latency[rw][i].latency) { in throtl_update_latency_buckets()
2158 if (td->avg_buckets[rw][i].latency < last_latency[rw]) in throtl_update_latency_buckets()
2159 td->avg_buckets[rw][i].latency = in throtl_update_latency_buckets()
2164 if (!td->avg_buckets[rw][i].valid) in throtl_update_latency_buckets()
2165 latency[rw] = avg_latency[rw][i].latency; in throtl_update_latency_buckets()
2167 latency[rw] = (td->avg_buckets[rw][i].latency * 7 + in throtl_update_latency_buckets()
2168 avg_latency[rw][i].latency) >> 3; in throtl_update_latency_buckets()
2170 td->avg_buckets[rw][i].latency = max(latency[rw], in throtl_update_latency_buckets()
2172 td->avg_buckets[rw][i].valid = true; in throtl_update_latency_buckets()
2173 last_latency[rw] = td->avg_buckets[rw][i].latency; in throtl_update_latency_buckets()
2178 throtl_log(&td->service_queue, in throtl_update_latency_buckets()
2179 "Latency bucket %d: read latency=%ld, read valid=%d, " in throtl_update_latency_buckets()
2180 "write latency=%ld, write valid=%d", i, in throtl_update_latency_buckets()
2181 td->avg_buckets[READ][i].latency, in throtl_update_latency_buckets()
2182 td->avg_buckets[READ][i].valid, in throtl_update_latency_buckets()
2183 td->avg_buckets[WRITE][i].latency, in throtl_update_latency_buckets()
2184 td->avg_buckets[WRITE][i].valid); in throtl_update_latency_buckets()
2194 struct blkcg_gq *blkg = bio->bi_blkg; in blk_throtl_charge_bio_split()
2200 if (!parent->has_rules[rw]) in blk_throtl_charge_bio_split()
2203 atomic_inc(&parent->io_split_cnt[rw]); in blk_throtl_charge_bio_split()
2204 atomic_inc(&parent->last_io_split_cnt[rw]); in blk_throtl_charge_bio_split()
2206 parent_sq = parent->service_queue.parent_sq; in blk_throtl_charge_bio_split()
2213 struct request_queue *q = bio->bi_bdev->bd_disk->queue; in blk_throtl_bio()
2214 struct blkcg_gq *blkg = bio->bi_blkg; in blk_throtl_bio()
2220 struct throtl_data *td = tg->td; in blk_throtl_bio()
2229 blkg_rwstat_add(&tg->stat_bytes, bio->bi_opf, in blk_throtl_bio()
2230 bio->bi_iter.bi_size); in blk_throtl_bio()
2231 blkg_rwstat_add(&tg->stat_ios, bio->bi_opf, 1); in blk_throtl_bio()
2234 if (!tg->has_rules[rw]) in blk_throtl_bio()
2237 spin_lock_irq(&q->queue_lock); in blk_throtl_bio()
2243 sq = &tg->service_queue; in blk_throtl_bio()
2247 if (tg->last_low_overflow_time[rw] == 0) in blk_throtl_bio()
2248 tg->last_low_overflow_time[rw] = jiffies; in blk_throtl_bio()
2251 /* throtl is FIFO - if bios are already queued, should queue */ in blk_throtl_bio()
2252 if (sq->nr_queued[rw]) in blk_throtl_bio()
2257 tg->last_low_overflow_time[rw] = jiffies; in blk_throtl_bio()
2286 qn = &tg->qnode_on_parent[rw]; in blk_throtl_bio()
2287 sq = sq->parent_sq; in blk_throtl_bio()
2293 /* out-of-limit, queue to @tg */ in blk_throtl_bio()
2296 tg->bytes_disp[rw], bio->bi_iter.bi_size, in blk_throtl_bio()
2298 tg->io_disp[rw], tg_iops_limit(tg, rw), in blk_throtl_bio()
2299 sq->nr_queued[READ], sq->nr_queued[WRITE]); in blk_throtl_bio()
2301 tg->last_low_overflow_time[rw] = jiffies; in blk_throtl_bio()
2303 td->nr_queued[rw]++; in blk_throtl_bio()
2313 if (tg->flags & THROTL_TG_WAS_EMPTY) { in blk_throtl_bio()
2315 throtl_schedule_next_dispatch(tg->service_queue.parent_sq, true); in blk_throtl_bio()
2319 spin_unlock_irq(&q->queue_lock); in blk_throtl_bio()
2324 if (throttled || !td->track_bio_latency) in blk_throtl_bio()
2325 bio->bi_issue.value |= BIO_ISSUE_THROTL_SKIP_LATENCY; in blk_throtl_bio()
2335 struct latency_bucket *latency; in throtl_track_latency() local
2338 if (!td || td->limit_index != LIMIT_LOW || in throtl_track_latency()
2340 !blk_queue_nonrot(td->queue)) in throtl_track_latency()
2345 latency = get_cpu_ptr(td->latency_buckets[op]); in throtl_track_latency()
2346 latency[index].total_latency += time; in throtl_track_latency()
2347 latency[index].samples++; in throtl_track_latency()
2348 put_cpu_ptr(td->latency_buckets[op]); in throtl_track_latency()
2353 struct request_queue *q = rq->q; in blk_throtl_stat_add()
2354 struct throtl_data *td = q->td; in blk_throtl_stat_add()
2370 blkg = bio->bi_blkg; in blk_throtl_bio_endio()
2374 if (!tg->td->limit_valid[LIMIT_LOW]) in blk_throtl_bio_endio()
2378 tg->last_finish_time = finish_time_ns >> 10; in blk_throtl_bio_endio()
2380 start_time = bio_issue_time(&bio->bi_issue) >> 10; in blk_throtl_bio_endio()
2385 lat = finish_time - start_time; in blk_throtl_bio_endio()
2387 if (!(bio->bi_issue.value & BIO_ISSUE_THROTL_SKIP_LATENCY)) in blk_throtl_bio_endio()
2388 throtl_track_latency(tg->td, bio_issue_size(&bio->bi_issue), in blk_throtl_bio_endio()
2391 if (tg->latency_target && lat >= tg->td->filtered_latency) { in blk_throtl_bio_endio()
2395 bucket = request_bucket_index(bio_issue_size(&bio->bi_issue)); in blk_throtl_bio_endio()
2396 threshold = tg->td->avg_buckets[rw][bucket].latency + in blk_throtl_bio_endio()
2397 tg->latency_target; in blk_throtl_bio_endio()
2399 tg->bad_bio_cnt++; in blk_throtl_bio_endio()
2404 tg->bio_cnt++; in blk_throtl_bio_endio()
2407 if (time_after(jiffies, tg->bio_cnt_reset_time) || tg->bio_cnt > 1024) { in blk_throtl_bio_endio()
2408 tg->bio_cnt_reset_time = tg->td->throtl_slice + jiffies; in blk_throtl_bio_endio()
2409 tg->bio_cnt /= 2; in blk_throtl_bio_endio()
2410 tg->bad_bio_cnt /= 2; in blk_throtl_bio_endio()
2420 td = kzalloc_node(sizeof(*td), GFP_KERNEL, q->node); in blk_throtl_init()
2422 return -ENOMEM; in blk_throtl_init()
2423 td->latency_buckets[READ] = __alloc_percpu(sizeof(struct latency_bucket) * in blk_throtl_init()
2425 if (!td->latency_buckets[READ]) { in blk_throtl_init()
2427 return -ENOMEM; in blk_throtl_init()
2429 td->latency_buckets[WRITE] = __alloc_percpu(sizeof(struct latency_bucket) * in blk_throtl_init()
2431 if (!td->latency_buckets[WRITE]) { in blk_throtl_init()
2432 free_percpu(td->latency_buckets[READ]); in blk_throtl_init()
2434 return -ENOMEM; in blk_throtl_init()
2437 INIT_WORK(&td->dispatch_work, blk_throtl_dispatch_work_fn); in blk_throtl_init()
2438 throtl_service_queue_init(&td->service_queue); in blk_throtl_init()
2440 q->td = td; in blk_throtl_init()
2441 td->queue = q; in blk_throtl_init()
2443 td->limit_valid[LIMIT_MAX] = true; in blk_throtl_init()
2444 td->limit_index = LIMIT_MAX; in blk_throtl_init()
2445 td->low_upgrade_time = jiffies; in blk_throtl_init()
2446 td->low_downgrade_time = jiffies; in blk_throtl_init()
2451 free_percpu(td->latency_buckets[READ]); in blk_throtl_init()
2452 free_percpu(td->latency_buckets[WRITE]); in blk_throtl_init()
2460 BUG_ON(!q->td); in blk_throtl_exit()
2461 del_timer_sync(&q->td->service_queue.pending_timer); in blk_throtl_exit()
2464 free_percpu(q->td->latency_buckets[READ]); in blk_throtl_exit()
2465 free_percpu(q->td->latency_buckets[WRITE]); in blk_throtl_exit()
2466 kfree(q->td); in blk_throtl_exit()
2474 td = q->td; in blk_throtl_register_queue()
2478 td->throtl_slice = DFL_THROTL_SLICE_SSD; in blk_throtl_register_queue()
2479 td->filtered_latency = LATENCY_FILTERED_SSD; in blk_throtl_register_queue()
2481 td->throtl_slice = DFL_THROTL_SLICE_HD; in blk_throtl_register_queue()
2482 td->filtered_latency = LATENCY_FILTERED_HD; in blk_throtl_register_queue()
2484 td->avg_buckets[READ][i].latency = DFL_HD_BASELINE_LATENCY; in blk_throtl_register_queue()
2485 td->avg_buckets[WRITE][i].latency = DFL_HD_BASELINE_LATENCY; in blk_throtl_register_queue()
2490 td->throtl_slice = DFL_THROTL_SLICE_HD; in blk_throtl_register_queue()
2493 td->track_bio_latency = !queue_is_mq(q); in blk_throtl_register_queue()
2494 if (!td->track_bio_latency) in blk_throtl_register_queue()
2501 if (!q->td) in blk_throtl_sample_time_show()
2502 return -EINVAL; in blk_throtl_sample_time_show()
2503 return sprintf(page, "%u\n", jiffies_to_msecs(q->td->throtl_slice)); in blk_throtl_sample_time_show()
2512 if (!q->td) in blk_throtl_sample_time_store()
2513 return -EINVAL; in blk_throtl_sample_time_store()
2515 return -EINVAL; in blk_throtl_sample_time_store()
2518 return -EINVAL; in blk_throtl_sample_time_store()
2519 q->td->throtl_slice = t; in blk_throtl_sample_time_store()