Lines Matching refs:td
115 struct throtl_data *td; member
276 return tg->td; in sq_to_td()
289 static uint64_t throtl_adjusted_limit(uint64_t low, struct throtl_data *td) in throtl_adjusted_limit() argument
292 if (td->scale < 4096 && time_after_eq(jiffies, in throtl_adjusted_limit()
293 td->low_upgrade_time + td->scale * td->throtl_slice)) in throtl_adjusted_limit()
294 td->scale = (jiffies - td->low_upgrade_time) / td->throtl_slice; in throtl_adjusted_limit()
296 return low + (low >> 1) * td->scale; in throtl_adjusted_limit()
302 struct throtl_data *td; in tg_bps_limit() local
308 td = tg->td; in tg_bps_limit()
309 ret = tg->bps[rw][td->limit_index]; in tg_bps_limit()
310 if (ret == 0 && td->limit_index == LIMIT_LOW) { in tg_bps_limit()
313 tg->iops[rw][td->limit_index]) in tg_bps_limit()
319 if (td->limit_index == LIMIT_MAX && tg->bps[rw][LIMIT_LOW] && in tg_bps_limit()
323 adjusted = throtl_adjusted_limit(tg->bps[rw][LIMIT_LOW], td); in tg_bps_limit()
332 struct throtl_data *td; in tg_iops_limit() local
338 td = tg->td; in tg_iops_limit()
339 ret = tg->iops[rw][td->limit_index]; in tg_iops_limit()
340 if (ret == 0 && tg->td->limit_index == LIMIT_LOW) { in tg_iops_limit()
343 tg->bps[rw][td->limit_index]) in tg_iops_limit()
349 if (td->limit_index == LIMIT_MAX && tg->iops[rw][LIMIT_LOW] && in tg_iops_limit()
353 adjusted = throtl_adjusted_limit(tg->iops[rw][LIMIT_LOW], td); in tg_iops_limit()
543 struct throtl_data *td = blkg->q->td; in throtl_pd_init() local
559 sq->parent_sq = &td->service_queue; in throtl_pd_init()
562 tg->td = td; in throtl_pd_init()
573 struct throtl_data *td = tg->td; in tg_update_has_rules() local
578 (td->limit_valid[td->limit_index] && in tg_update_has_rules()
594 static void blk_throtl_update_limit_valid(struct throtl_data *td) in blk_throtl_update_limit_valid() argument
601 blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { in blk_throtl_update_limit_valid()
612 td->limit_valid[LIMIT_LOW] = low_valid; in blk_throtl_update_limit_valid()
615 static inline void blk_throtl_update_limit_valid(struct throtl_data *td) in blk_throtl_update_limit_valid() argument
620 static void throtl_upgrade_state(struct throtl_data *td);
630 blk_throtl_update_limit_valid(tg->td); in throtl_pd_offline()
632 if (!tg->td->limit_valid[tg->td->limit_index]) in throtl_pd_offline()
633 throtl_upgrade_state(tg->td); in throtl_pd_offline()
794 tg->slice_end[rw] = jiffies + tg->td->throtl_slice; in throtl_start_new_slice_with_credit()
806 tg->slice_end[rw] = jiffies + tg->td->throtl_slice; in throtl_start_new_slice()
819 tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice); in throtl_set_slice_end()
865 throtl_set_slice_end(tg, rw, jiffies + tg->td->throtl_slice); in throtl_trim_slice()
869 nr_slices = time_elapsed / tg->td->throtl_slice; in throtl_trim_slice()
873 tmp = tg_bps_limit(tg, rw) * tg->td->throtl_slice * nr_slices; in throtl_trim_slice()
877 io_trim = (tg_iops_limit(tg, rw) * tg->td->throtl_slice * nr_slices) / in throtl_trim_slice()
893 tg->slice_start[rw] += nr_slices * tg->td->throtl_slice; in throtl_trim_slice()
918 jiffy_elapsed_rnd = roundup(jiffy_elapsed + 1, tg->td->throtl_slice); in tg_with_in_iops_limit()
967 jiffy_elapsed_rnd = tg->td->throtl_slice; in tg_with_in_bps_limit()
969 jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, tg->td->throtl_slice); in tg_with_in_bps_limit()
1037 jiffies + tg->td->throtl_slice)) in tg_may_dispatch()
1039 jiffies + tg->td->throtl_slice); in tg_may_dispatch()
1185 BUG_ON(tg->td->nr_queued[rw] <= 0); in tg_dispatch_one_bio()
1186 tg->td->nr_queued[rw]--; in tg_dispatch_one_bio()
1261 static bool throtl_can_upgrade(struct throtl_data *td,
1282 struct throtl_data *td = sq_to_td(sq); in throtl_pending_timer_fn() local
1283 struct request_queue *q = td->queue; in throtl_pending_timer_fn()
1289 if (throtl_can_upgrade(td, NULL)) in throtl_pending_timer_fn()
1290 throtl_upgrade_state(td); in throtl_pending_timer_fn()
1332 queue_work(kthrotld_workqueue, &td->dispatch_work); in throtl_pending_timer_fn()
1348 struct throtl_data *td = container_of(work, struct throtl_data, in blk_throtl_dispatch_work_fn() local
1350 struct throtl_service_queue *td_sq = &td->service_queue; in blk_throtl_dispatch_work_fn()
1351 struct request_queue *q = td->queue; in blk_throtl_dispatch_work_fn()
1428 global ? tg->td->queue->root_blkg : tg_to_blkg(tg)) { in tg_conf_updated()
1756 blk_throtl_update_limit_valid(tg->td); in tg_set_limit()
1757 if (tg->td->limit_valid[LIMIT_LOW]) { in tg_set_limit()
1759 tg->td->limit_index = LIMIT_LOW; in tg_set_limit()
1761 tg->td->limit_index = LIMIT_MAX; in tg_set_limit()
1763 tg->td->limit_valid[LIMIT_LOW]); in tg_set_limit()
1792 struct throtl_data *td = q->td; in throtl_shutdown_wq() local
1794 cancel_work_sync(&td->dispatch_work); in throtl_shutdown_wq()
1869 tg->bio_cnt, ret, tg->td->scale); in throtl_tg_is_idle()
1894 tg_last_low_overflow_time(tg) + tg->td->throtl_slice) && in throtl_tg_can_upgrade()
1912 static bool throtl_can_upgrade(struct throtl_data *td, in throtl_can_upgrade() argument
1918 if (td->limit_index != LIMIT_LOW) in throtl_can_upgrade()
1921 if (time_before(jiffies, td->low_downgrade_time + td->throtl_slice)) in throtl_can_upgrade()
1925 blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { in throtl_can_upgrade()
1945 if (tg->td->limit_index != LIMIT_LOW) in throtl_upgrade_check()
1948 if (time_after(tg->last_check_time + tg->td->throtl_slice, now)) in throtl_upgrade_check()
1954 __tg_last_low_overflow_time(tg) + tg->td->throtl_slice)) in throtl_upgrade_check()
1957 if (throtl_can_upgrade(tg->td, NULL)) in throtl_upgrade_check()
1958 throtl_upgrade_state(tg->td); in throtl_upgrade_check()
1961 static void throtl_upgrade_state(struct throtl_data *td) in throtl_upgrade_state() argument
1966 throtl_log(&td->service_queue, "upgrade to max"); in throtl_upgrade_state()
1967 td->limit_index = LIMIT_MAX; in throtl_upgrade_state()
1968 td->low_upgrade_time = jiffies; in throtl_upgrade_state()
1969 td->scale = 0; in throtl_upgrade_state()
1971 blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { in throtl_upgrade_state()
1980 throtl_select_dispatch(&td->service_queue); in throtl_upgrade_state()
1981 throtl_schedule_next_dispatch(&td->service_queue, true); in throtl_upgrade_state()
1982 queue_work(kthrotld_workqueue, &td->dispatch_work); in throtl_upgrade_state()
1985 static void throtl_downgrade_state(struct throtl_data *td) in throtl_downgrade_state() argument
1987 td->scale /= 2; in throtl_downgrade_state()
1989 throtl_log(&td->service_queue, "downgrade, scale %d", td->scale); in throtl_downgrade_state()
1990 if (td->scale) { in throtl_downgrade_state()
1991 td->low_upgrade_time = jiffies - td->scale * td->throtl_slice; in throtl_downgrade_state()
1995 td->limit_index = LIMIT_LOW; in throtl_downgrade_state()
1996 td->low_downgrade_time = jiffies; in throtl_downgrade_state()
2001 struct throtl_data *td = tg->td; in throtl_tg_can_downgrade() local
2008 if (time_after_eq(now, td->low_upgrade_time + td->throtl_slice) && in throtl_tg_can_downgrade()
2010 td->throtl_slice) && in throtl_tg_can_downgrade()
2036 if (tg->td->limit_index != LIMIT_MAX || in throtl_downgrade_check()
2037 !tg->td->limit_valid[LIMIT_LOW]) in throtl_downgrade_check()
2041 if (time_after(tg->last_check_time + tg->td->throtl_slice, now)) in throtl_downgrade_check()
2048 tg->td->throtl_slice)) in throtl_downgrade_check()
2084 throtl_downgrade_state(tg->td); in throtl_downgrade_check()
2110 static void throtl_update_latency_buckets(struct throtl_data *td) in throtl_update_latency_buckets() argument
2117 if (!blk_queue_nonrot(td->queue) || !td->limit_valid[LIMIT_LOW]) in throtl_update_latency_buckets()
2119 if (time_before(jiffies, td->last_calculate_time + HZ)) in throtl_update_latency_buckets()
2121 td->last_calculate_time = jiffies; in throtl_update_latency_buckets()
2126 struct latency_bucket *tmp = &td->tmp_buckets[rw][i]; in throtl_update_latency_buckets()
2132 bucket = per_cpu_ptr(td->latency_buckets[rw], in throtl_update_latency_buckets()
2158 if (td->avg_buckets[rw][i].latency < last_latency[rw]) in throtl_update_latency_buckets()
2159 td->avg_buckets[rw][i].latency = in throtl_update_latency_buckets()
2164 if (!td->avg_buckets[rw][i].valid) in throtl_update_latency_buckets()
2167 latency[rw] = (td->avg_buckets[rw][i].latency * 7 + in throtl_update_latency_buckets()
2170 td->avg_buckets[rw][i].latency = max(latency[rw], in throtl_update_latency_buckets()
2172 td->avg_buckets[rw][i].valid = true; in throtl_update_latency_buckets()
2173 last_latency[rw] = td->avg_buckets[rw][i].latency; in throtl_update_latency_buckets()
2178 throtl_log(&td->service_queue, in throtl_update_latency_buckets()
2181 td->avg_buckets[READ][i].latency, in throtl_update_latency_buckets()
2182 td->avg_buckets[READ][i].valid, in throtl_update_latency_buckets()
2183 td->avg_buckets[WRITE][i].latency, in throtl_update_latency_buckets()
2184 td->avg_buckets[WRITE][i].valid); in throtl_update_latency_buckets()
2187 static inline void throtl_update_latency_buckets(struct throtl_data *td) in throtl_update_latency_buckets() argument
2220 struct throtl_data *td = tg->td; in blk_throtl_bio() local
2239 throtl_update_latency_buckets(td); in blk_throtl_bio()
2258 if (throtl_can_upgrade(td, tg)) { in blk_throtl_bio()
2259 throtl_upgrade_state(td); in blk_throtl_bio()
2303 td->nr_queued[rw]++; in blk_throtl_bio()
2324 if (throttled || !td->track_bio_latency) in blk_throtl_bio()
2332 static void throtl_track_latency(struct throtl_data *td, sector_t size, in throtl_track_latency() argument
2338 if (!td || td->limit_index != LIMIT_LOW || in throtl_track_latency()
2340 !blk_queue_nonrot(td->queue)) in throtl_track_latency()
2345 latency = get_cpu_ptr(td->latency_buckets[op]); in throtl_track_latency()
2348 put_cpu_ptr(td->latency_buckets[op]); in throtl_track_latency()
2354 struct throtl_data *td = q->td; in blk_throtl_stat_add() local
2356 throtl_track_latency(td, blk_rq_stats_sectors(rq), req_op(rq), in blk_throtl_stat_add()
2374 if (!tg->td->limit_valid[LIMIT_LOW]) in blk_throtl_bio_endio()
2388 throtl_track_latency(tg->td, bio_issue_size(&bio->bi_issue), in blk_throtl_bio_endio()
2391 if (tg->latency_target && lat >= tg->td->filtered_latency) { in blk_throtl_bio_endio()
2396 threshold = tg->td->avg_buckets[rw][bucket].latency + in blk_throtl_bio_endio()
2408 tg->bio_cnt_reset_time = tg->td->throtl_slice + jiffies; in blk_throtl_bio_endio()
2417 struct throtl_data *td; in blk_throtl_init() local
2420 td = kzalloc_node(sizeof(*td), GFP_KERNEL, q->node); in blk_throtl_init()
2421 if (!td) in blk_throtl_init()
2423 td->latency_buckets[READ] = __alloc_percpu(sizeof(struct latency_bucket) * in blk_throtl_init()
2425 if (!td->latency_buckets[READ]) { in blk_throtl_init()
2426 kfree(td); in blk_throtl_init()
2429 td->latency_buckets[WRITE] = __alloc_percpu(sizeof(struct latency_bucket) * in blk_throtl_init()
2431 if (!td->latency_buckets[WRITE]) { in blk_throtl_init()
2432 free_percpu(td->latency_buckets[READ]); in blk_throtl_init()
2433 kfree(td); in blk_throtl_init()
2437 INIT_WORK(&td->dispatch_work, blk_throtl_dispatch_work_fn); in blk_throtl_init()
2438 throtl_service_queue_init(&td->service_queue); in blk_throtl_init()
2440 q->td = td; in blk_throtl_init()
2441 td->queue = q; in blk_throtl_init()
2443 td->limit_valid[LIMIT_MAX] = true; in blk_throtl_init()
2444 td->limit_index = LIMIT_MAX; in blk_throtl_init()
2445 td->low_upgrade_time = jiffies; in blk_throtl_init()
2446 td->low_downgrade_time = jiffies; in blk_throtl_init()
2451 free_percpu(td->latency_buckets[READ]); in blk_throtl_init()
2452 free_percpu(td->latency_buckets[WRITE]); in blk_throtl_init()
2453 kfree(td); in blk_throtl_init()
2460 BUG_ON(!q->td); in blk_throtl_exit()
2461 del_timer_sync(&q->td->service_queue.pending_timer); in blk_throtl_exit()
2464 free_percpu(q->td->latency_buckets[READ]); in blk_throtl_exit()
2465 free_percpu(q->td->latency_buckets[WRITE]); in blk_throtl_exit()
2466 kfree(q->td); in blk_throtl_exit()
2471 struct throtl_data *td; in blk_throtl_register_queue() local
2474 td = q->td; in blk_throtl_register_queue()
2475 BUG_ON(!td); in blk_throtl_register_queue()
2478 td->throtl_slice = DFL_THROTL_SLICE_SSD; in blk_throtl_register_queue()
2479 td->filtered_latency = LATENCY_FILTERED_SSD; in blk_throtl_register_queue()
2481 td->throtl_slice = DFL_THROTL_SLICE_HD; in blk_throtl_register_queue()
2482 td->filtered_latency = LATENCY_FILTERED_HD; in blk_throtl_register_queue()
2484 td->avg_buckets[READ][i].latency = DFL_HD_BASELINE_LATENCY; in blk_throtl_register_queue()
2485 td->avg_buckets[WRITE][i].latency = DFL_HD_BASELINE_LATENCY; in blk_throtl_register_queue()
2490 td->throtl_slice = DFL_THROTL_SLICE_HD; in blk_throtl_register_queue()
2493 td->track_bio_latency = !queue_is_mq(q); in blk_throtl_register_queue()
2494 if (!td->track_bio_latency) in blk_throtl_register_queue()
2501 if (!q->td) in blk_throtl_sample_time_show()
2503 return sprintf(page, "%u\n", jiffies_to_msecs(q->td->throtl_slice)); in blk_throtl_sample_time_show()
2512 if (!q->td) in blk_throtl_sample_time_store()
2519 q->td->throtl_slice = t; in blk_throtl_sample_time_store()