Lines Matching refs:bfqd
232 #define BFQ_RQ_SEEKY(bfqd, last_pos, rq) \ argument
235 (!blk_queue_nonrot(bfqd->queue) || \
440 static struct bfq_io_cq *bfq_bic_lookup(struct bfq_data *bfqd, in bfq_bic_lookup() argument
462 void bfq_schedule_dispatch(struct bfq_data *bfqd) in bfq_schedule_dispatch() argument
464 if (bfqd->queued != 0) { in bfq_schedule_dispatch()
465 bfq_log(bfqd, "schedule dispatch"); in bfq_schedule_dispatch()
466 blk_mq_run_hw_queues(bfqd->queue, true); in bfq_schedule_dispatch()
479 static struct request *bfq_choose_req(struct bfq_data *bfqd, in bfq_choose_req() argument
510 back_max = bfqd->bfq_back_max * 2; in bfq_choose_req()
520 d1 = (last - s1) * bfqd->bfq_back_penalty; in bfq_choose_req()
527 d2 = (last - s2) * bfqd->bfq_back_penalty; in bfq_choose_req()
577 struct bfq_data *bfqd = data->q->elevator->elevator_data; in bfq_limit_depth() local
583 bfqd->word_depths[!!bfqd->wr_busy_queues][op_is_sync(op)]; in bfq_limit_depth()
585 bfq_log(bfqd, "[%s] wr_busy %d sync %d depth %u", in bfq_limit_depth()
586 __func__, bfqd->wr_busy_queues, op_is_sync(op), in bfq_limit_depth()
591 bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root, in bfq_rq_pos_tree_lookup() argument
624 bfq_log(bfqd, "rq_pos_tree_lookup %llu: returning %d", in bfq_rq_pos_tree_lookup()
647 bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_pos_tree_add_move() argument
658 if (bfqq == &bfqd->oom_bfqq) in bfq_pos_tree_add_move()
675 __bfqq = bfq_rq_pos_tree_lookup(bfqd, bfqq->pos_root, in bfq_pos_tree_add_move()
717 static bool bfq_asymmetric_scenario(struct bfq_data *bfqd, in bfq_asymmetric_scenario() argument
724 rb_first_cached(&bfqd->queue_weights_tree), in bfq_asymmetric_scenario()
733 !RB_EMPTY_ROOT(&bfqd->queue_weights_tree.rb_root) && in bfq_asymmetric_scenario()
734 (bfqd->queue_weights_tree.rb_root.rb_node->rb_left || in bfq_asymmetric_scenario()
735 bfqd->queue_weights_tree.rb_root.rb_node->rb_right); in bfq_asymmetric_scenario()
738 (bfqd->busy_queues[0] && bfqd->busy_queues[1]) || in bfq_asymmetric_scenario()
739 (bfqd->busy_queues[0] && bfqd->busy_queues[2]) || in bfq_asymmetric_scenario()
740 (bfqd->busy_queues[1] && bfqd->busy_queues[2]); in bfq_asymmetric_scenario()
744 || bfqd->num_groups_with_pending_reqs > 0 in bfq_asymmetric_scenario()
762 void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_weights_tree_add() argument
836 void __bfq_weights_tree_remove(struct bfq_data *bfqd, in __bfq_weights_tree_remove() argument
859 void bfq_weights_tree_remove(struct bfq_data *bfqd, in bfq_weights_tree_remove() argument
894 bfqd->num_groups_with_pending_reqs--; in bfq_weights_tree_remove()
904 __bfq_weights_tree_remove(bfqd, bfqq, in bfq_weights_tree_remove()
905 &bfqd->queue_weights_tree); in bfq_weights_tree_remove()
926 bfq_log_bfqq(bfqq->bfqd, bfqq, "check_fifo: returned %p", rq); in bfq_check_fifo()
930 static struct request *bfq_find_next_rq(struct bfq_data *bfqd, in bfq_find_next_rq() argument
954 return bfq_choose_req(bfqd, next, prev, blk_rq_pos(last)); in bfq_find_next_rq()
962 bfq_asymmetric_scenario(bfqq->bfqd, bfqq)) in bfq_serv_to_charge()
979 static void bfq_updated_next_req(struct bfq_data *bfqd, in bfq_updated_next_req() argument
989 if (bfqq == bfqd->in_service_queue) in bfq_updated_next_req()
1002 bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu", in bfq_updated_next_req()
1004 bfq_requeue_bfqq(bfqd, bfqq, false); in bfq_updated_next_req()
1008 static unsigned int bfq_wr_duration(struct bfq_data *bfqd) in bfq_wr_duration() argument
1012 if (bfqd->bfq_wr_max_time > 0) in bfq_wr_duration()
1013 return bfqd->bfq_wr_max_time; in bfq_wr_duration()
1015 dur = bfqd->rate_dur_prod; in bfq_wr_duration()
1016 do_div(dur, bfqd->peak_rate); in bfq_wr_duration()
1043 struct bfq_data *bfqd) in switch_back_to_interactive_wr() argument
1045 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in switch_back_to_interactive_wr()
1046 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in switch_back_to_interactive_wr()
1051 bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, in bfq_bfqq_resume_state() argument
1078 if (bfqd->low_latency) { in bfq_bfqq_resume_state()
1090 if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && in bfq_bfqq_resume_state()
1093 bfq_wr_duration(bfqd))) { in bfq_bfqq_resume_state()
1094 switch_back_to_interactive_wr(bfqq, bfqd); in bfq_bfqq_resume_state()
1097 bfq_log_bfqq(bfqq->bfqd, bfqq, in bfq_bfqq_resume_state()
1109 bfqd->wr_busy_queues++; in bfq_bfqq_resume_state()
1111 bfqd->wr_busy_queues--; in bfq_bfqq_resume_state()
1121 static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_reset_burst_list() argument
1126 hlist_for_each_entry_safe(item, n, &bfqd->burst_list, burst_list_node) in bfq_reset_burst_list()
1134 if (bfq_tot_busy_queues(bfqd) == 0) { in bfq_reset_burst_list()
1135 hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); in bfq_reset_burst_list()
1136 bfqd->burst_size = 1; in bfq_reset_burst_list()
1138 bfqd->burst_size = 0; in bfq_reset_burst_list()
1140 bfqd->burst_parent_entity = bfqq->entity.parent; in bfq_reset_burst_list()
1144 static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_add_to_burst() argument
1147 bfqd->burst_size++; in bfq_add_to_burst()
1149 if (bfqd->burst_size == bfqd->bfq_large_burst_thresh) { in bfq_add_to_burst()
1157 bfqd->large_burst = true; in bfq_add_to_burst()
1163 hlist_for_each_entry(bfqq_item, &bfqd->burst_list, in bfq_add_to_burst()
1175 hlist_for_each_entry_safe(pos, n, &bfqd->burst_list, in bfq_add_to_burst()
1184 hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); in bfq_add_to_burst()
1296 static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_handle_burst() argument
1326 if (time_is_before_jiffies(bfqd->last_ins_in_burst + in bfq_handle_burst()
1327 bfqd->bfq_burst_interval) || in bfq_handle_burst()
1328 bfqq->entity.parent != bfqd->burst_parent_entity) { in bfq_handle_burst()
1329 bfqd->large_burst = false; in bfq_handle_burst()
1330 bfq_reset_burst_list(bfqd, bfqq); in bfq_handle_burst()
1339 if (bfqd->large_burst) { in bfq_handle_burst()
1349 bfq_add_to_burst(bfqd, bfqq); in bfq_handle_burst()
1359 bfqd->last_ins_in_burst = jiffies; in bfq_handle_burst()
1374 static int bfq_max_budget(struct bfq_data *bfqd) in bfq_max_budget() argument
1376 if (bfqd->budgets_assigned < bfq_stats_min_budgets) in bfq_max_budget()
1379 return bfqd->bfq_max_budget; in bfq_max_budget()
1386 static int bfq_min_budget(struct bfq_data *bfqd) in bfq_min_budget() argument
1388 if (bfqd->budgets_assigned < bfq_stats_min_budgets) in bfq_min_budget()
1391 return bfqd->bfq_max_budget / 32; in bfq_min_budget()
1498 static bool bfq_bfqq_update_budg_for_activation(struct bfq_data *bfqd, in bfq_bfqq_update_budg_for_activation() argument
1567 static void bfq_update_bfqq_wr_on_rq_arrival(struct bfq_data *bfqd, in bfq_update_bfqq_wr_on_rq_arrival() argument
1579 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in bfq_update_bfqq_wr_on_rq_arrival()
1580 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in bfq_update_bfqq_wr_on_rq_arrival()
1595 bfqq->wr_coeff = bfqd->bfq_wr_coeff * in bfq_update_bfqq_wr_on_rq_arrival()
1598 bfqd->bfq_wr_rt_max_time; in bfq_update_bfqq_wr_on_rq_arrival()
1612 2 * bfq_min_budget(bfqd)); in bfq_update_bfqq_wr_on_rq_arrival()
1615 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in bfq_update_bfqq_wr_on_rq_arrival()
1616 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in bfq_update_bfqq_wr_on_rq_arrival()
1650 bfqd->bfq_wr_rt_max_time) { in bfq_update_bfqq_wr_on_rq_arrival()
1655 bfqd->bfq_wr_rt_max_time; in bfq_update_bfqq_wr_on_rq_arrival()
1656 bfqq->wr_coeff = bfqd->bfq_wr_coeff * in bfq_update_bfqq_wr_on_rq_arrival()
1664 static bool bfq_bfqq_idle_for_long_time(struct bfq_data *bfqd, in bfq_bfqq_idle_for_long_time() argument
1670 bfqd->bfq_wr_min_idle_time); in bfq_bfqq_idle_for_long_time()
1705 static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, in bfq_bfqq_handle_idle_busy_switch() argument
1713 idle_for_long_time = bfq_bfqq_idle_for_long_time(bfqd, bfqq), in bfq_bfqq_handle_idle_busy_switch()
1721 bfqd->bfq_slice_idle * 3; in bfq_bfqq_handle_idle_busy_switch()
1734 soft_rt = bfqd->bfq_wr_max_softrt_rate > 0 && in bfq_bfqq_handle_idle_busy_switch()
1754 wr_or_deserves_wr = bfqd->low_latency && in bfq_bfqq_handle_idle_busy_switch()
1765 bfq_bfqq_update_budg_for_activation(bfqd, bfqq, in bfq_bfqq_handle_idle_busy_switch()
1792 if (bfqd->low_latency) { in bfq_bfqq_handle_idle_busy_switch()
1796 jiffies - bfqd->bfq_wr_min_idle_time - 1; in bfq_bfqq_handle_idle_busy_switch()
1799 bfqd->bfq_wr_min_idle_time)) { in bfq_bfqq_handle_idle_busy_switch()
1800 bfq_update_bfqq_wr_on_rq_arrival(bfqd, bfqq, in bfq_bfqq_handle_idle_busy_switch()
1816 bfq_add_bfqq_busy(bfqd, bfqq); in bfq_bfqq_handle_idle_busy_switch()
1861 if (bfqd->in_service_queue && in bfq_bfqq_handle_idle_busy_switch()
1863 bfqq->wr_coeff >= bfqd->in_service_queue->wr_coeff) || in bfq_bfqq_handle_idle_busy_switch()
1864 bfq_bfqq_higher_class_or_weight(bfqq, bfqd->in_service_queue) || in bfq_bfqq_handle_idle_busy_switch()
1865 !bfq_better_to_idle(bfqd->in_service_queue)) && in bfq_bfqq_handle_idle_busy_switch()
1866 next_queue_may_preempt(bfqd)) in bfq_bfqq_handle_idle_busy_switch()
1867 bfq_bfqq_expire(bfqd, bfqd->in_service_queue, in bfq_bfqq_handle_idle_busy_switch()
1871 static void bfq_reset_inject_limit(struct bfq_data *bfqd, in bfq_reset_inject_limit() argument
1881 bfqd->waited_rq = NULL; in bfq_reset_inject_limit()
2019 static void bfq_check_waker(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_check_waker() argument
2022 if (!bfqd->last_completed_rq_bfqq || in bfq_check_waker()
2023 bfqd->last_completed_rq_bfqq == bfqq || in bfq_check_waker()
2026 now_ns - bfqd->last_completion >= 4 * NSEC_PER_MSEC || in bfq_check_waker()
2027 bfqd->last_completed_rq_bfqq == bfqq->waker_bfqq) in bfq_check_waker()
2030 if (bfqd->last_completed_rq_bfqq != in bfq_check_waker()
2038 bfqd->last_completed_rq_bfqq; in bfq_check_waker()
2044 bfqq->waker_bfqq = bfqd->last_completed_rq_bfqq; in bfq_check_waker()
2070 &bfqd->last_completed_rq_bfqq->woken_list); in bfq_check_waker()
2077 struct bfq_data *bfqd = bfqq->bfqd; in bfq_add_request() local
2083 bfq_log_bfqq(bfqd, bfqq, "add_request %d", rq_is_sync(rq)); in bfq_add_request()
2085 bfqd->queued++; in bfq_add_request()
2088 bfq_check_waker(bfqd, bfqq, now_ns); in bfq_add_request()
2098 bfq_reset_inject_limit(bfqd, bfqq); in bfq_add_request()
2125 if (bfqq == bfqd->in_service_queue && in bfq_add_request()
2126 (bfqd->rq_in_driver == 0 || in bfq_add_request()
2128 bfqd->rqs_injected && bfqd->rq_in_driver > 0)) && in bfq_add_request()
2131 bfqd->last_empty_occupied_ns = ktime_get_ns(); in bfq_add_request()
2138 bfqd->wait_dispatch = true; in bfq_add_request()
2152 if (bfqd->rq_in_driver == 0) in bfq_add_request()
2153 bfqd->rqs_injected = false; in bfq_add_request()
2166 next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position); in bfq_add_request()
2173 if (unlikely(!bfqd->nonrot_with_queueing && prev != bfqq->next_rq)) in bfq_add_request()
2174 bfq_pos_tree_add_move(bfqd, bfqq); in bfq_add_request()
2177 bfq_bfqq_handle_idle_busy_switch(bfqd, bfqq, old_wr_coeff, in bfq_add_request()
2180 if (bfqd->low_latency && old_wr_coeff == 1 && !rq_is_sync(rq) && in bfq_add_request()
2183 bfqd->bfq_wr_min_inter_arr_async)) { in bfq_add_request()
2184 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in bfq_add_request()
2185 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in bfq_add_request()
2187 bfqd->wr_busy_queues++; in bfq_add_request()
2191 bfq_updated_next_req(bfqd, bfqq); in bfq_add_request()
2220 if (bfqd->low_latency && in bfq_add_request()
2225 static struct request *bfq_find_rq_fmerge(struct bfq_data *bfqd, in bfq_find_rq_fmerge() argument
2229 struct bfq_queue *bfqq = bfqd->bio_bfqq; in bfq_find_rq_fmerge()
2249 struct bfq_data *bfqd = q->elevator->elevator_data;
2251 bfqd->rq_in_driver++;
2256 struct bfq_data *bfqd = q->elevator->elevator_data;
2258 bfqd->rq_in_driver--;
2266 struct bfq_data *bfqd = bfqq->bfqd; in bfq_remove_request() local
2270 bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq); in bfq_remove_request()
2271 bfq_updated_next_req(bfqd, bfqq); in bfq_remove_request()
2277 bfqd->queued--; in bfq_remove_request()
2287 if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) { in bfq_remove_request()
2288 bfq_del_bfqq_busy(bfqd, bfqq, false); in bfq_remove_request()
2314 if (unlikely(!bfqd->nonrot_with_queueing)) in bfq_remove_request()
2315 bfq_pos_tree_add_move(bfqd, bfqq); in bfq_remove_request()
2326 struct bfq_data *bfqd = q->elevator->elevator_data; in bfq_bio_merge() local
2335 struct bfq_io_cq *bic = bfq_bic_lookup(bfqd, current->io_context, q); in bfq_bio_merge()
2338 spin_lock_irq(&bfqd->lock); in bfq_bio_merge()
2341 bfqd->bio_bfqq = bic_to_bfqq(bic, op_is_sync(bio->bi_opf)); in bfq_bio_merge()
2343 bfqd->bio_bfqq = NULL; in bfq_bio_merge()
2344 bfqd->bio_bic = bic; in bfq_bio_merge()
2348 spin_unlock_irq(&bfqd->lock); in bfq_bio_merge()
2358 struct bfq_data *bfqd = q->elevator->elevator_data; in bfq_request_merge() local
2361 __rq = bfq_find_rq_fmerge(bfqd, bio, q); in bfq_request_merge()
2384 struct bfq_data *bfqd; in bfq_request_merged() local
2390 bfqd = bfqq->bfqd; in bfq_request_merged()
2398 next_rq = bfq_choose_req(bfqd, bfqq->next_rq, req, in bfq_request_merged()
2399 bfqd->last_position); in bfq_request_merged()
2407 bfq_updated_next_req(bfqd, bfqq); in bfq_request_merged()
2412 if (unlikely(!bfqd->nonrot_with_queueing)) in bfq_request_merged()
2413 bfq_pos_tree_add_move(bfqd, bfqq); in bfq_request_merged()
2490 bfqq->bfqd->bfq_wr_rt_max_time) in bfq_bfqq_end_wr()
2494 bfqq->bfqd->wr_busy_queues--; in bfq_bfqq_end_wr()
2505 void bfq_end_wr_async_queues(struct bfq_data *bfqd, in bfq_end_wr_async_queues() argument
2518 static void bfq_end_wr(struct bfq_data *bfqd) in bfq_end_wr() argument
2522 spin_lock_irq(&bfqd->lock); in bfq_end_wr()
2524 list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) in bfq_end_wr()
2526 list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) in bfq_end_wr()
2528 bfq_end_wr_async(bfqd); in bfq_end_wr()
2530 spin_unlock_irq(&bfqd->lock); in bfq_end_wr()
2548 static struct bfq_queue *bfqq_find_close(struct bfq_data *bfqd, in bfqq_find_close() argument
2563 __bfqq = bfq_rq_pos_tree_lookup(bfqd, root, sector, &parent, NULL); in bfqq_find_close()
2590 static struct bfq_queue *bfq_find_close_cooperator(struct bfq_data *bfqd, in bfq_find_close_cooperator() argument
2603 bfqq = bfqq_find_close(bfqd, cur_bfqq, sector); in bfq_find_close_cooperator()
2641 bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d", in bfq_setup_merge()
2698 static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd,
2722 bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_setup_cooperator() argument
2736 if (unlikely(!bfqd->nonrot_with_queueing)) { in bfq_setup_cooperator()
2759 if (!idling_boosts_thr_without_issues(bfqd, bfqq) && in bfq_setup_cooperator()
2811 if (likely(bfqd->nonrot_with_queueing)) in bfq_setup_cooperator()
2831 if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq)) in bfq_setup_cooperator()
2835 if (bfq_tot_busy_queues(bfqd) == 1) in bfq_setup_cooperator()
2838 in_service_bfqq = bfqd->in_service_queue; in bfq_setup_cooperator()
2841 likely(in_service_bfqq != &bfqd->oom_bfqq) && in bfq_setup_cooperator()
2843 bfqd->in_serv_last_pos) && in bfq_setup_cooperator()
2855 new_bfqq = bfq_find_close_cooperator(bfqd, bfqq, in bfq_setup_cooperator()
2858 if (new_bfqq && likely(new_bfqq != &bfqd->oom_bfqq) && in bfq_setup_cooperator()
2891 bfqq->bfqd->low_latency)) { in bfq_bfqq_save_state()
2901 bic->saved_wr_coeff = bfqq->bfqd->bfq_wr_coeff; in bfq_bfqq_save_state()
2903 bic->saved_wr_cur_max_time = bfq_wr_duration(bfqq->bfqd); in bfq_bfqq_save_state()
2922 else if (cur_bfqq->bfqd && cur_bfqq->bfqd->last_bfqq_created == cur_bfqq) in bfq_reassign_last_bfqq()
2923 cur_bfqq->bfqd->last_bfqq_created = new_bfqq; in bfq_reassign_last_bfqq()
2926 void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_release_process_ref() argument
2940 bfqq != bfqd->in_service_queue) in bfq_release_process_ref()
2941 bfq_del_bfqq_busy(bfqd, bfqq, false); in bfq_release_process_ref()
2949 bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, in bfq_merge_bfqqs() argument
2952 bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", in bfq_merge_bfqqs()
3000 bfqd->wr_busy_queues++; in bfq_merge_bfqqs()
3008 bfqd->wr_busy_queues--; in bfq_merge_bfqqs()
3011 bfq_log_bfqq(bfqd, new_bfqq, "merge_bfqqs: wr_busy %d", in bfq_merge_bfqqs()
3012 bfqd->wr_busy_queues); in bfq_merge_bfqqs()
3044 bfq_release_process_ref(bfqd, bfqq); in bfq_merge_bfqqs()
3050 struct bfq_data *bfqd = q->elevator->elevator_data; in bfq_allow_bio_merge() local
3052 struct bfq_queue *bfqq = bfqd->bio_bfqq, *new_bfqq; in bfq_allow_bio_merge()
3071 new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false, bfqd->bio_bic); in bfq_allow_bio_merge()
3080 bfq_merge_bfqqs(bfqd, bfqd->bio_bic, bfqq, in bfq_allow_bio_merge()
3095 bfqd->bio_bfqq = bfqq; in bfq_allow_bio_merge()
3107 static void bfq_set_budget_timeout(struct bfq_data *bfqd, in bfq_set_budget_timeout() argument
3112 if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time) in bfq_set_budget_timeout()
3117 bfqd->last_budget_start = ktime_get(); in bfq_set_budget_timeout()
3120 bfqd->bfq_timeout * timeout_coeff; in bfq_set_budget_timeout()
3123 static void __bfq_set_in_service_queue(struct bfq_data *bfqd, in __bfq_set_in_service_queue() argument
3129 bfqd->budgets_assigned = (bfqd->budgets_assigned * 7 + 256) / 8; in __bfq_set_in_service_queue()
3133 bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && in __bfq_set_in_service_queue()
3167 bfq_set_budget_timeout(bfqd, bfqq); in __bfq_set_in_service_queue()
3168 bfq_log_bfqq(bfqd, bfqq, in __bfq_set_in_service_queue()
3173 bfqd->in_service_queue = bfqq; in __bfq_set_in_service_queue()
3174 bfqd->in_serv_last_pos = 0; in __bfq_set_in_service_queue()
3180 static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd) in bfq_set_in_service_queue() argument
3182 struct bfq_queue *bfqq = bfq_get_next_queue(bfqd); in bfq_set_in_service_queue()
3184 __bfq_set_in_service_queue(bfqd, bfqq); in bfq_set_in_service_queue()
3188 static void bfq_arm_slice_timer(struct bfq_data *bfqd) in bfq_arm_slice_timer() argument
3190 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_arm_slice_timer()
3200 sl = bfqd->bfq_slice_idle; in bfq_arm_slice_timer()
3212 !bfq_asymmetric_scenario(bfqd, bfqq)) in bfq_arm_slice_timer()
3217 bfqd->last_idling_start = ktime_get(); in bfq_arm_slice_timer()
3218 bfqd->last_idling_start_jiffies = jiffies; in bfq_arm_slice_timer()
3220 hrtimer_start(&bfqd->idle_slice_timer, ns_to_ktime(sl), in bfq_arm_slice_timer()
3232 static unsigned long bfq_calc_max_budget(struct bfq_data *bfqd) in bfq_calc_max_budget() argument
3234 return (u64)bfqd->peak_rate * USEC_PER_MSEC * in bfq_calc_max_budget()
3235 jiffies_to_msecs(bfqd->bfq_timeout)>>BFQ_RATE_SHIFT; in bfq_calc_max_budget()
3243 static void update_thr_responsiveness_params(struct bfq_data *bfqd) in update_thr_responsiveness_params() argument
3245 if (bfqd->bfq_user_max_budget == 0) { in update_thr_responsiveness_params()
3246 bfqd->bfq_max_budget = in update_thr_responsiveness_params()
3247 bfq_calc_max_budget(bfqd); in update_thr_responsiveness_params()
3248 bfq_log(bfqd, "new max_budget = %d", bfqd->bfq_max_budget); in update_thr_responsiveness_params()
3252 static void bfq_reset_rate_computation(struct bfq_data *bfqd, in bfq_reset_rate_computation() argument
3256 bfqd->last_dispatch = bfqd->first_dispatch = ktime_get_ns(); in bfq_reset_rate_computation()
3257 bfqd->peak_rate_samples = 1; in bfq_reset_rate_computation()
3258 bfqd->sequential_samples = 0; in bfq_reset_rate_computation()
3259 bfqd->tot_sectors_dispatched = bfqd->last_rq_max_size = in bfq_reset_rate_computation()
3262 bfqd->peak_rate_samples = 0; /* full re-init on next disp. */ in bfq_reset_rate_computation()
3264 bfq_log(bfqd, in bfq_reset_rate_computation()
3266 bfqd->peak_rate_samples, bfqd->sequential_samples, in bfq_reset_rate_computation()
3267 bfqd->tot_sectors_dispatched); in bfq_reset_rate_computation()
3270 static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) in bfq_update_rate_reset() argument
3282 if (bfqd->peak_rate_samples < BFQ_RATE_MIN_SAMPLES || in bfq_update_rate_reset()
3283 bfqd->delta_from_first < BFQ_RATE_MIN_INTERVAL) in bfq_update_rate_reset()
3292 bfqd->delta_from_first = in bfq_update_rate_reset()
3293 max_t(u64, bfqd->delta_from_first, in bfq_update_rate_reset()
3294 bfqd->last_completion - bfqd->first_dispatch); in bfq_update_rate_reset()
3300 rate = div64_ul(bfqd->tot_sectors_dispatched<<BFQ_RATE_SHIFT, in bfq_update_rate_reset()
3301 div_u64(bfqd->delta_from_first, NSEC_PER_USEC)); in bfq_update_rate_reset()
3309 if ((bfqd->sequential_samples < (3 * bfqd->peak_rate_samples)>>2 && in bfq_update_rate_reset()
3310 rate <= bfqd->peak_rate) || in bfq_update_rate_reset()
3337 weight = (9 * bfqd->sequential_samples) / bfqd->peak_rate_samples; in bfq_update_rate_reset()
3344 div_u64(weight * bfqd->delta_from_first, in bfq_update_rate_reset()
3358 bfqd->peak_rate *= divisor-1; in bfq_update_rate_reset()
3359 bfqd->peak_rate /= divisor; in bfq_update_rate_reset()
3362 bfqd->peak_rate += rate; in bfq_update_rate_reset()
3371 bfqd->peak_rate = max_t(u32, 1, bfqd->peak_rate); in bfq_update_rate_reset()
3373 update_thr_responsiveness_params(bfqd); in bfq_update_rate_reset()
3376 bfq_reset_rate_computation(bfqd, rq); in bfq_update_rate_reset()
3411 static void bfq_update_peak_rate(struct bfq_data *bfqd, struct request *rq) in bfq_update_peak_rate() argument
3415 if (bfqd->peak_rate_samples == 0) { /* first dispatch */ in bfq_update_peak_rate()
3416 bfq_log(bfqd, "update_peak_rate: goto reset, samples %d", in bfq_update_peak_rate()
3417 bfqd->peak_rate_samples); in bfq_update_peak_rate()
3418 bfq_reset_rate_computation(bfqd, rq); in bfq_update_peak_rate()
3434 if (now_ns - bfqd->last_dispatch > 100*NSEC_PER_MSEC && in bfq_update_peak_rate()
3435 bfqd->rq_in_driver == 0) in bfq_update_peak_rate()
3439 bfqd->peak_rate_samples++; in bfq_update_peak_rate()
3441 if ((bfqd->rq_in_driver > 0 || in bfq_update_peak_rate()
3442 now_ns - bfqd->last_completion < BFQ_MIN_TT) in bfq_update_peak_rate()
3443 && !BFQ_RQ_SEEKY(bfqd, bfqd->last_position, rq)) in bfq_update_peak_rate()
3444 bfqd->sequential_samples++; in bfq_update_peak_rate()
3446 bfqd->tot_sectors_dispatched += blk_rq_sectors(rq); in bfq_update_peak_rate()
3449 if (likely(bfqd->peak_rate_samples % 32)) in bfq_update_peak_rate()
3450 bfqd->last_rq_max_size = in bfq_update_peak_rate()
3451 max_t(u32, blk_rq_sectors(rq), bfqd->last_rq_max_size); in bfq_update_peak_rate()
3453 bfqd->last_rq_max_size = blk_rq_sectors(rq); in bfq_update_peak_rate()
3455 bfqd->delta_from_first = now_ns - bfqd->first_dispatch; in bfq_update_peak_rate()
3458 if (bfqd->delta_from_first < BFQ_RATE_REF_INTERVAL) in bfq_update_peak_rate()
3462 bfq_update_rate_reset(bfqd, rq); in bfq_update_peak_rate()
3464 bfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq); in bfq_update_peak_rate()
3465 if (RQ_BFQQ(rq) == bfqd->in_service_queue) in bfq_update_peak_rate()
3466 bfqd->in_serv_last_pos = bfqd->last_position; in bfq_update_peak_rate()
3467 bfqd->last_dispatch = now_ns; in bfq_update_peak_rate()
3696 static bool idling_needed_for_service_guarantees(struct bfq_data *bfqd, in idling_needed_for_service_guarantees() argument
3699 int tot_busy_queues = bfq_tot_busy_queues(bfqd); in idling_needed_for_service_guarantees()
3706 (bfqd->wr_busy_queues < in idling_needed_for_service_guarantees()
3708 bfqd->rq_in_driver >= in idling_needed_for_service_guarantees()
3710 bfq_asymmetric_scenario(bfqd, bfqq) || in idling_needed_for_service_guarantees()
3714 static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, in __bfq_bfqq_expire() argument
3741 idling_needed_for_service_guarantees(bfqd, bfqq))) { in __bfq_bfqq_expire()
3751 bfq_del_bfqq_busy(bfqd, bfqq, true); in __bfq_bfqq_expire()
3753 bfq_requeue_bfqq(bfqd, bfqq, true); in __bfq_bfqq_expire()
3758 if (unlikely(!bfqd->nonrot_with_queueing && in __bfq_bfqq_expire()
3760 bfq_pos_tree_add_move(bfqd, bfqq); in __bfq_bfqq_expire()
3770 return __bfq_bfqd_reset_in_service(bfqd); in __bfq_bfqq_expire()
3782 static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd, in __bfq_bfqq_recalc_budget() argument
3789 min_budget = bfq_min_budget(bfqd); in __bfq_bfqq_recalc_budget()
3801 bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %d, budg left %d", in __bfq_bfqq_recalc_budget()
3803 bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last max_budg %d, min budg %d", in __bfq_bfqq_recalc_budget()
3804 budget, bfq_min_budget(bfqd)); in __bfq_bfqq_recalc_budget()
3805 bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d", in __bfq_bfqq_recalc_budget()
3806 bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue)); in __bfq_bfqq_recalc_budget()
3840 budget = min(budget * 2, bfqd->bfq_max_budget); in __bfq_bfqq_recalc_budget()
3855 budget = min(budget * 2, bfqd->bfq_max_budget); in __bfq_bfqq_recalc_budget()
3867 budget = min(budget * 4, bfqd->bfq_max_budget); in __bfq_bfqq_recalc_budget()
3914 budget = bfqd->bfq_max_budget; in __bfq_bfqq_recalc_budget()
3919 if (bfqd->budgets_assigned >= bfq_stats_min_budgets && in __bfq_bfqq_recalc_budget()
3920 !bfqd->bfq_user_max_budget) in __bfq_bfqq_recalc_budget()
3921 bfqq->max_budget = min(bfqq->max_budget, bfqd->bfq_max_budget); in __bfq_bfqq_recalc_budget()
3938 bfq_log_bfqq(bfqd, bfqq, "head sect: %u, new budget %d", in __bfq_bfqq_recalc_budget()
3974 static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_bfqq_is_slow() argument
3986 delta_ktime = bfqd->last_idling_start; in bfq_bfqq_is_slow()
3989 delta_ktime = ktime_sub(delta_ktime, bfqd->last_budget_start); in bfq_bfqq_is_slow()
3994 if (blk_queue_nonrot(bfqd->queue)) in bfq_bfqq_is_slow()
4023 slow = bfqq->entity.service < bfqd->bfq_max_budget / 2; in bfq_bfqq_is_slow()
4026 bfq_log_bfqq(bfqd, bfqq, "bfq_bfqq_is_slow: slow %d", slow); in bfq_bfqq_is_slow()
4124 static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, in bfq_bfqq_softrt_next_start() argument
4130 bfqd->bfq_wr_max_softrt_rate, in bfq_bfqq_softrt_next_start()
4131 jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); in bfq_bfqq_softrt_next_start()
4160 void bfq_bfqq_expire(struct bfq_data *bfqd, in bfq_bfqq_expire() argument
4172 slow = bfq_bfqq_is_slow(bfqd, bfqq, compensate, reason, &delta); in bfq_bfqq_expire()
4193 bfq_bfqq_charge_time(bfqd, bfqq, delta); in bfq_bfqq_expire()
4195 if (bfqd->low_latency && bfqq->wr_coeff == 1) in bfq_bfqq_expire()
4198 if (bfqd->low_latency && bfqd->bfq_wr_max_softrt_rate > 0 && in bfq_bfqq_expire()
4214 bfq_bfqq_softrt_next_start(bfqd, bfqq); in bfq_bfqq_expire()
4224 bfq_log_bfqq(bfqd, bfqq, in bfq_bfqq_expire()
4233 bfqd->rqs_injected = bfqd->wait_dispatch = false; in bfq_bfqq_expire()
4234 bfqd->waited_rq = NULL; in bfq_bfqq_expire()
4240 __bfq_bfqq_recalc_budget(bfqd, bfqq, reason); in bfq_bfqq_expire()
4241 if (__bfq_bfqq_expire(bfqd, bfqq, reason)) in bfq_bfqq_expire()
4300 bfq_log_bfqq(bfqq->bfqd, bfqq, in bfq_may_expire_for_budg_timeout()
4312 static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd, in idling_boosts_thr_without_issues() argument
4316 !blk_queue_nonrot(bfqd->queue) && !bfqd->hw_tag, in idling_boosts_thr_without_issues()
4350 ((!blk_queue_nonrot(bfqd->queue) || !bfqd->hw_tag) && in idling_boosts_thr_without_issues()
4388 bfqd->wr_busy_queues == 0; in idling_boosts_thr_without_issues()
4414 struct bfq_data *bfqd = bfqq->bfqd; in bfq_better_to_idle() local
4421 if (unlikely(bfqd->strict_guarantees)) in bfq_better_to_idle()
4432 if (bfqd->bfq_slice_idle == 0 || !bfq_bfqq_sync(bfqq) || in bfq_better_to_idle()
4437 idling_boosts_thr_without_issues(bfqd, bfqq); in bfq_better_to_idle()
4440 idling_needed_for_service_guarantees(bfqd, bfqq); in bfq_better_to_idle()
4476 bfq_choose_bfqq_for_injection(struct bfq_data *bfqd) in bfq_choose_bfqq_for_injection() argument
4478 struct bfq_queue *bfqq, *in_serv_bfqq = bfqd->in_service_queue; in bfq_choose_bfqq_for_injection()
4506 time_is_before_eq_jiffies(bfqd->last_idling_start_jiffies + in bfq_choose_bfqq_for_injection()
4507 bfqd->bfq_slice_idle) in bfq_choose_bfqq_for_injection()
4511 if (bfqd->rq_in_driver >= limit) in bfq_choose_bfqq_for_injection()
4526 list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) in bfq_choose_bfqq_for_injection()
4548 if (blk_queue_nonrot(bfqd->queue) && in bfq_choose_bfqq_for_injection()
4555 if (bfqd->rq_in_driver < limit) { in bfq_choose_bfqq_for_injection()
4556 bfqd->rqs_injected = true; in bfq_choose_bfqq_for_injection()
4568 static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) in bfq_select_queue() argument
4574 bfqq = bfqd->in_service_queue; in bfq_select_queue()
4578 bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue"); in bfq_select_queue()
4635 hrtimer_try_to_cancel(&bfqd->idle_slice_timer); in bfq_select_queue()
4764 else if (!idling_boosts_thr_without_issues(bfqd, bfqq) && in bfq_select_queue()
4765 (bfqq->wr_coeff == 1 || bfqd->wr_busy_queues > 1 || in bfq_select_queue()
4767 bfqq = bfq_choose_bfqq_for_injection(bfqd); in bfq_select_queue()
4776 bfq_bfqq_expire(bfqd, bfqq, false, reason); in bfq_select_queue()
4778 bfqq = bfq_set_in_service_queue(bfqd); in bfq_select_queue()
4780 bfq_log_bfqq(bfqd, bfqq, "select_queue: checking new queue"); in bfq_select_queue()
4785 bfq_log_bfqq(bfqd, bfqq, "select_queue: returned this queue"); in bfq_select_queue()
4787 bfq_log(bfqd, "select_queue: no queue returned"); in bfq_select_queue()
4792 static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_update_wr_data() argument
4797 bfq_log_bfqq(bfqd, bfqq, in bfq_update_wr_data()
4805 bfq_log_bfqq(bfqd, bfqq, "WARN: pending prio change"); in bfq_update_wr_data()
4816 if (bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time || in bfq_update_wr_data()
4818 bfq_wr_duration(bfqd))) { in bfq_update_wr_data()
4833 switch_back_to_interactive_wr(bfqq, bfqd); in bfq_update_wr_data()
4838 bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time && in bfq_update_wr_data()
4860 static struct request *bfq_dispatch_rq_from_bfqq(struct bfq_data *bfqd, in bfq_dispatch_rq_from_bfqq() argument
4870 if (bfqq == bfqd->in_service_queue && bfqd->wait_dispatch) { in bfq_dispatch_rq_from_bfqq()
4871 bfqd->wait_dispatch = false; in bfq_dispatch_rq_from_bfqq()
4872 bfqd->waited_rq = rq; in bfq_dispatch_rq_from_bfqq()
4875 bfq_dispatch_remove(bfqd->queue, rq); in bfq_dispatch_rq_from_bfqq()
4877 if (bfqq != bfqd->in_service_queue) in bfq_dispatch_rq_from_bfqq()
4891 bfq_update_wr_data(bfqd, bfqq); in bfq_dispatch_rq_from_bfqq()
4898 if (!(bfq_tot_busy_queues(bfqd) > 1 && bfq_class_idle(bfqq))) in bfq_dispatch_rq_from_bfqq()
4901 bfq_bfqq_expire(bfqd, bfqq, false, BFQQE_BUDGET_EXHAUSTED); in bfq_dispatch_rq_from_bfqq()
4909 struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; in bfq_has_work() local
4915 return !list_empty_careful(&bfqd->dispatch) || in bfq_has_work()
4916 bfq_tot_busy_queues(bfqd) > 0; in bfq_has_work()
4921 struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; in __bfq_dispatch_request() local
4925 if (!list_empty(&bfqd->dispatch)) { in __bfq_dispatch_request()
4926 rq = list_first_entry(&bfqd->dispatch, struct request, in __bfq_dispatch_request()
4970 bfq_log(bfqd, "dispatch requests: %d busy queues", in __bfq_dispatch_request()
4971 bfq_tot_busy_queues(bfqd)); in __bfq_dispatch_request()
4973 if (bfq_tot_busy_queues(bfqd) == 0) in __bfq_dispatch_request()
4988 if (bfqd->strict_guarantees && bfqd->rq_in_driver > 0) in __bfq_dispatch_request()
4991 bfqq = bfq_select_queue(bfqd); in __bfq_dispatch_request()
4995 rq = bfq_dispatch_rq_from_bfqq(bfqd, bfqq); in __bfq_dispatch_request()
4999 bfqd->rq_in_driver++; in __bfq_dispatch_request()
5061 struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; in bfq_dispatch_request() local
5066 spin_lock_irq(&bfqd->lock); in bfq_dispatch_request()
5068 in_serv_queue = bfqd->in_service_queue; in bfq_dispatch_request()
5076 spin_unlock_irq(&bfqd->lock); in bfq_dispatch_request()
5097 if (bfqq->bfqd) in bfq_put_queue()
5098 bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", in bfq_put_queue()
5133 if (bfqq->bic && bfqq->bfqd->burst_size > 0) in bfq_put_queue()
5134 bfqq->bfqd->burst_size--; in bfq_put_queue()
5163 if (bfqq->bfqd && bfqq->bfqd->last_completed_rq_bfqq == bfqq) in bfq_put_queue()
5164 bfqq->bfqd->last_completed_rq_bfqq = NULL; in bfq_put_queue()
5195 static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_exit_bfqq() argument
5197 if (bfqq == bfqd->in_service_queue) { in bfq_exit_bfqq()
5198 __bfq_bfqq_expire(bfqd, bfqq, BFQQE_BUDGET_TIMEOUT); in bfq_exit_bfqq()
5199 bfq_schedule_dispatch(bfqd); in bfq_exit_bfqq()
5202 bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq, bfqq->ref); in bfq_exit_bfqq()
5206 bfq_release_process_ref(bfqd, bfqq); in bfq_exit_bfqq()
5212 struct bfq_data *bfqd; in bfq_exit_icq_bfqq() local
5215 bfqd = bfqq->bfqd; /* NULL if scheduler already exited */ in bfq_exit_icq_bfqq()
5217 if (bfqq && bfqd) { in bfq_exit_icq_bfqq()
5220 spin_lock_irqsave(&bfqd->lock, flags); in bfq_exit_icq_bfqq()
5222 bfq_exit_bfqq(bfqd, bfqq); in bfq_exit_icq_bfqq()
5224 spin_unlock_irqrestore(&bfqd->lock, flags); in bfq_exit_icq_bfqq()
5233 struct bfq_data *bfqd = bic->stable_merge_bfqq->bfqd; in bfq_exit_icq() local
5239 if (bfqd) { in bfq_exit_icq()
5242 spin_lock_irqsave(&bfqd->lock, flags); in bfq_exit_icq()
5244 spin_unlock_irqrestore(&bfqd->lock, flags); in bfq_exit_icq()
5263 struct bfq_data *bfqd = bfqq->bfqd; in bfq_set_next_ioprio_data() local
5265 if (!bfqd) in bfq_set_next_ioprio_data()
5272 bdi_dev_name(bfqq->bfqd->queue->disk->bdi), in bfq_set_next_ioprio_data()
5303 bfq_log_bfqq(bfqd, bfqq, "new_ioprio %d new_weight %d", in bfq_set_next_ioprio_data()
5308 static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd,
5315 struct bfq_data *bfqd = bic_to_bfqd(bic); in bfq_check_ioprio_change() local
5323 if (unlikely(!bfqd) || likely(bic->ioprio == ioprio)) in bfq_check_ioprio_change()
5330 bfq_release_process_ref(bfqd, bfqq); in bfq_check_ioprio_change()
5331 bfqq = bfq_get_queue(bfqd, bio, BLK_RW_ASYNC, bic, true); in bfq_check_ioprio_change()
5340 static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_init_bfqq() argument
5352 bfqq->bfqd = bfqd; in bfq_init_bfqq()
5383 bfqq->max_budget = (2 * bfq_max_budget(bfqd)) / 3; in bfq_init_bfqq()
5406 static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd, in bfq_async_queue_prio() argument
5426 bfq_do_early_stable_merge(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_do_early_stable_merge() argument
5447 bfq_merge_bfqqs(bfqd, bic, bfqq, new_bfqq); in bfq_do_early_stable_merge()
5498 static struct bfq_queue *bfq_do_or_sched_stable_merge(struct bfq_data *bfqd, in bfq_do_or_sched_stable_merge() argument
5504 &bfqd->last_bfqq_created; in bfq_do_or_sched_stable_merge()
5536 bfqd->bfq_burst_interval, in bfq_do_or_sched_stable_merge()
5538 if (likely(bfqd->nonrot_with_queueing)) in bfq_do_or_sched_stable_merge()
5545 bfqq = bfq_do_early_stable_merge(bfqd, bfqq, in bfq_do_or_sched_stable_merge()
5571 static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, in bfq_get_queue() argument
5584 bfqg = bfq_find_set_group(bfqd, __bio_blkcg(bio)); in bfq_get_queue()
5586 bfqq = &bfqd->oom_bfqq; in bfq_get_queue()
5591 async_bfqq = bfq_async_queue_prio(bfqd, bfqg, ioprio_class, in bfq_get_queue()
5600 bfqd->queue->node); in bfq_get_queue()
5603 bfq_init_bfqq(bfqd, bfqq, bic, current->pid, in bfq_get_queue()
5606 bfq_log_bfqq(bfqd, bfqq, "allocated"); in bfq_get_queue()
5608 bfqq = &bfqd->oom_bfqq; in bfq_get_queue()
5609 bfq_log_bfqq(bfqd, bfqq, "using oom bfqq"); in bfq_get_queue()
5625 bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d", in bfq_get_queue()
5633 if (bfqq != &bfqd->oom_bfqq && is_sync && !respawn) in bfq_get_queue()
5634 bfqq = bfq_do_or_sched_stable_merge(bfqd, bfqq, bic); in bfq_get_queue()
5640 static void bfq_update_io_thinktime(struct bfq_data *bfqd, in bfq_update_io_thinktime() argument
5654 elapsed = min_t(u64, elapsed, 2ULL * bfqd->bfq_slice_idle); in bfq_update_io_thinktime()
5663 bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_update_io_seektime() argument
5667 bfqq->seek_history |= BFQ_RQ_SEEKY(bfqd, bfqq->last_request_pos, rq); in bfq_update_io_seektime()
5670 bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && in bfq_update_io_seektime()
5673 bfq_wr_duration(bfqd))) { in bfq_update_io_seektime()
5687 switch_back_to_interactive_wr(bfqq, bfqd); in bfq_update_io_seektime()
5693 static void bfq_update_has_short_ttime(struct bfq_data *bfqd, in bfq_update_has_short_ttime() argument
5705 bfqd->bfq_slice_idle == 0) in bfq_update_has_short_ttime()
5710 bfqd->bfq_wr_min_idle_time)) in bfq_update_has_short_ttime()
5720 bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle>>1)) in bfq_update_has_short_ttime()
5818 bfq_reset_inject_limit(bfqd, bfqq); in bfq_update_has_short_ttime()
5825 static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_rq_enqueued() argument
5833 if (bfqq == bfqd->in_service_queue && bfq_bfqq_wait_request(bfqq)) { in bfq_rq_enqueued()
5854 if (small_req && idling_boosts_thr_without_issues(bfqd, bfqq) && in bfq_rq_enqueued()
5866 hrtimer_try_to_cancel(&bfqd->idle_slice_timer); in bfq_rq_enqueued()
5876 bfq_bfqq_expire(bfqd, bfqq, false, in bfq_rq_enqueued()
5882 static bool __bfq_insert_request(struct bfq_data *bfqd, struct request *rq) in __bfq_insert_request() argument
5885 *new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true, in __bfq_insert_request()
5906 bfq_merge_bfqqs(bfqd, RQ_BIC(rq), in __bfq_insert_request()
5919 bfq_update_io_thinktime(bfqd, bfqq); in __bfq_insert_request()
5920 bfq_update_has_short_ttime(bfqd, bfqq, RQ_BIC(rq)); in __bfq_insert_request()
5921 bfq_update_io_seektime(bfqd, bfqq, rq); in __bfq_insert_request()
5927 rq->fifo_time = ktime_get_ns() + bfqd->bfq_fifo_expire[rq_is_sync(rq)]; in __bfq_insert_request()
5930 bfq_rq_enqueued(bfqd, bfqq, rq); in __bfq_insert_request()
5971 struct bfq_data *bfqd = q->elevator->elevator_data; in bfq_insert_request() local
5981 spin_lock_irq(&bfqd->lock); in bfq_insert_request()
5983 spin_unlock_irq(&bfqd->lock); in bfq_insert_request()
5988 spin_unlock_irq(&bfqd->lock); in bfq_insert_request()
5992 spin_lock_irq(&bfqd->lock); in bfq_insert_request()
6031 (bfqq != bfqd->in_service_queue && in bfq_insert_request()
6032 bfqd->in_service_queue != NULL && in bfq_insert_request()
6033 bfq_tot_busy_queues(bfqd) == 1 + bfq_bfqq_busy(bfqq) && in bfq_insert_request()
6034 (bfqq->waker_bfqq == bfqd->in_service_queue || in bfq_insert_request()
6035 bfqd->in_service_queue->waker_bfqq == bfqq)) || at_head) { in bfq_insert_request()
6037 list_add(&rq->queuelist, &bfqd->dispatch); in bfq_insert_request()
6039 list_add_tail(&rq->queuelist, &bfqd->dispatch); in bfq_insert_request()
6041 idle_timer_disabled = __bfq_insert_request(bfqd, rq); in bfq_insert_request()
6063 spin_unlock_irq(&bfqd->lock); in bfq_insert_request()
6081 static void bfq_update_hw_tag(struct bfq_data *bfqd) in bfq_update_hw_tag() argument
6083 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_update_hw_tag()
6085 bfqd->max_rq_in_driver = max_t(int, bfqd->max_rq_in_driver, in bfq_update_hw_tag()
6086 bfqd->rq_in_driver); in bfq_update_hw_tag()
6088 if (bfqd->hw_tag == 1) in bfq_update_hw_tag()
6097 if (bfqd->rq_in_driver + bfqd->queued <= BFQ_HW_QUEUE_THRESHOLD) in bfq_update_hw_tag()
6108 bfqd->rq_in_driver < BFQ_HW_QUEUE_THRESHOLD) in bfq_update_hw_tag()
6111 if (bfqd->hw_tag_samples++ < BFQ_HW_QUEUE_SAMPLES) in bfq_update_hw_tag()
6114 bfqd->hw_tag = bfqd->max_rq_in_driver > BFQ_HW_QUEUE_THRESHOLD; in bfq_update_hw_tag()
6115 bfqd->max_rq_in_driver = 0; in bfq_update_hw_tag()
6116 bfqd->hw_tag_samples = 0; in bfq_update_hw_tag()
6118 bfqd->nonrot_with_queueing = in bfq_update_hw_tag()
6119 blk_queue_nonrot(bfqd->queue) && bfqd->hw_tag; in bfq_update_hw_tag()
6122 static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) in bfq_completed_request() argument
6127 bfq_update_hw_tag(bfqd); in bfq_completed_request()
6129 bfqd->rq_in_driver--; in bfq_completed_request()
6141 bfq_weights_tree_remove(bfqd, bfqq); in bfq_completed_request()
6152 delta_us = div_u64(now_ns - bfqd->last_completion, NSEC_PER_USEC); in bfq_completed_request()
6171 (bfqd->last_rq_max_size<<BFQ_RATE_SHIFT)/delta_us < in bfq_completed_request()
6173 bfq_update_rate_reset(bfqd, NULL); in bfq_completed_request()
6174 bfqd->last_completion = now_ns; in bfq_completed_request()
6185 bfqd->last_completed_rq_bfqq = bfqq; in bfq_completed_request()
6187 bfqd->last_completed_rq_bfqq = NULL; in bfq_completed_request()
6202 bfqq->wr_coeff != bfqd->bfq_wr_coeff) in bfq_completed_request()
6204 bfq_bfqq_softrt_next_start(bfqd, bfqq); in bfq_completed_request()
6210 if (bfqd->in_service_queue == bfqq) { in bfq_completed_request()
6213 bfq_arm_slice_timer(bfqd); in bfq_completed_request()
6239 bfq_bfqq_expire(bfqd, bfqq, false, in bfq_completed_request()
6244 bfq_bfqq_expire(bfqd, bfqq, false, in bfq_completed_request()
6248 if (!bfqd->rq_in_driver) in bfq_completed_request()
6249 bfq_schedule_dispatch(bfqd); in bfq_completed_request()
6363 static void bfq_update_inject_limit(struct bfq_data *bfqd, in bfq_update_inject_limit() argument
6366 u64 tot_time_ns = ktime_get_ns() - bfqd->last_empty_occupied_ns; in bfq_update_inject_limit()
6369 if (bfqq->last_serv_time_ns > 0 && bfqd->rqs_injected) { in bfq_update_inject_limit()
6376 old_limit <= bfqd->max_rq_in_driver) in bfq_update_inject_limit()
6392 if ((bfqq->last_serv_time_ns == 0 && bfqd->rq_in_driver == 1) || in bfq_update_inject_limit()
6402 } else if (!bfqd->rqs_injected && bfqd->rq_in_driver == 1) in bfq_update_inject_limit()
6416 bfqd->waited_rq = NULL; in bfq_update_inject_limit()
6417 bfqd->rqs_injected = false; in bfq_update_inject_limit()
6429 struct bfq_data *bfqd; in bfq_finish_requeue_request() local
6440 bfqd = bfqq->bfqd; in bfq_finish_requeue_request()
6448 spin_lock_irqsave(&bfqd->lock, flags); in bfq_finish_requeue_request()
6450 if (rq == bfqd->waited_rq) in bfq_finish_requeue_request()
6451 bfq_update_inject_limit(bfqd, bfqq); in bfq_finish_requeue_request()
6453 bfq_completed_request(bfqq, bfqd); in bfq_finish_requeue_request()
6456 spin_unlock_irqrestore(&bfqd->lock, flags); in bfq_finish_requeue_request()
6488 bfq_log_bfqq(bfqq->bfqd, bfqq, "splitting queue"); in bfq_split_bfqq()
6501 bfq_release_process_ref(bfqq->bfqd, bfqq); in bfq_split_bfqq()
6505 static struct bfq_queue *bfq_get_bfqq_handle_split(struct bfq_data *bfqd, in bfq_get_bfqq_handle_split() argument
6513 if (likely(bfqq && bfqq != &bfqd->oom_bfqq)) in bfq_get_bfqq_handle_split()
6521 bfqq = bfq_get_queue(bfqd, bio, is_sync, bic, split); in bfq_get_bfqq_handle_split()
6525 if ((bic->was_in_burst_list && bfqd->large_burst) || in bfq_get_bfqq_handle_split()
6560 &bfqd->burst_list); in bfq_get_bfqq_handle_split()
6611 struct bfq_data *bfqd = q->elevator->elevator_data; in bfq_init_rq() local
6637 bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync, in bfq_init_rq()
6654 bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, in bfq_init_rq()
6677 bfq_log_bfqq(bfqd, bfqq, "get_request %p: bfqq %p, %d", in bfq_init_rq()
6689 if (likely(bfqq != &bfqd->oom_bfqq) && bfqq_process_refs(bfqq) == 1) { in bfq_init_rq()
6697 bfq_bfqq_resume_state(bfqq, bfqd, bic, in bfq_init_rq()
6723 (bfqd->burst_size > 0 || in bfq_init_rq()
6724 bfq_tot_busy_queues(bfqd) == 0))) in bfq_init_rq()
6725 bfq_handle_burst(bfqd, bfqq); in bfq_init_rq()
6731 bfq_idle_slice_timer_body(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_idle_slice_timer_body() argument
6736 spin_lock_irqsave(&bfqd->lock, flags); in bfq_idle_slice_timer_body()
6745 if (bfqq != bfqd->in_service_queue) { in bfq_idle_slice_timer_body()
6746 spin_unlock_irqrestore(&bfqd->lock, flags); in bfq_idle_slice_timer_body()
6770 bfq_bfqq_expire(bfqd, bfqq, true, reason); in bfq_idle_slice_timer_body()
6773 spin_unlock_irqrestore(&bfqd->lock, flags); in bfq_idle_slice_timer_body()
6774 bfq_schedule_dispatch(bfqd); in bfq_idle_slice_timer_body()
6783 struct bfq_data *bfqd = container_of(timer, struct bfq_data, in bfq_idle_slice_timer() local
6785 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_idle_slice_timer()
6796 bfq_idle_slice_timer_body(bfqd, bfqq); in bfq_idle_slice_timer()
6801 static void __bfq_put_async_bfqq(struct bfq_data *bfqd, in __bfq_put_async_bfqq() argument
6806 bfq_log(bfqd, "put_async_bfqq: %p", bfqq); in __bfq_put_async_bfqq()
6808 bfq_bfqq_move(bfqd, bfqq, bfqd->root_group); in __bfq_put_async_bfqq()
6810 bfq_log_bfqq(bfqd, bfqq, "put_async_bfqq: putting %p, %d", in __bfq_put_async_bfqq()
6823 void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg) in bfq_put_async_queues() argument
6829 __bfq_put_async_bfqq(bfqd, &bfqg->async_bfqq[i][j]); in bfq_put_async_queues()
6831 __bfq_put_async_bfqq(bfqd, &bfqg->async_idle_bfqq); in bfq_put_async_queues()
6838 static unsigned int bfq_update_depths(struct bfq_data *bfqd, in bfq_update_depths() argument
6854 bfqd->word_depths[0][0] = max((1U << bt->sb.shift) >> 1, 1U); in bfq_update_depths()
6860 bfqd->word_depths[0][1] = max(((1U << bt->sb.shift) * 3) >> 2, 1U); in bfq_update_depths()
6870 bfqd->word_depths[1][0] = max(((1U << bt->sb.shift) * 3) >> 4, 1U); in bfq_update_depths()
6872 bfqd->word_depths[1][1] = max(((1U << bt->sb.shift) * 6) >> 4, 1U); in bfq_update_depths()
6876 min_shallow = min(min_shallow, bfqd->word_depths[i][j]); in bfq_update_depths()
6883 struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; in bfq_depth_updated() local
6887 min_shallow = bfq_update_depths(bfqd, tags->bitmap_tags); in bfq_depth_updated()
6899 struct bfq_data *bfqd = e->elevator_data; in bfq_exit_queue() local
6902 hrtimer_cancel(&bfqd->idle_slice_timer); in bfq_exit_queue()
6904 spin_lock_irq(&bfqd->lock); in bfq_exit_queue()
6905 list_for_each_entry_safe(bfqq, n, &bfqd->idle_list, bfqq_list) in bfq_exit_queue()
6906 bfq_deactivate_bfqq(bfqd, bfqq, false, false); in bfq_exit_queue()
6907 spin_unlock_irq(&bfqd->lock); in bfq_exit_queue()
6909 hrtimer_cancel(&bfqd->idle_slice_timer); in bfq_exit_queue()
6912 bfqg_and_blkg_put(bfqd->root_group); in bfq_exit_queue()
6915 blkcg_deactivate_policy(bfqd->queue, &blkcg_policy_bfq); in bfq_exit_queue()
6917 spin_lock_irq(&bfqd->lock); in bfq_exit_queue()
6918 bfq_put_async_queues(bfqd, bfqd->root_group); in bfq_exit_queue()
6919 kfree(bfqd->root_group); in bfq_exit_queue()
6920 spin_unlock_irq(&bfqd->lock); in bfq_exit_queue()
6923 kfree(bfqd); in bfq_exit_queue()
6927 struct bfq_data *bfqd) in bfq_init_root_group() argument
6934 root_group->bfqd = bfqd; in bfq_init_root_group()
6944 struct bfq_data *bfqd; in bfq_init_queue() local
6951 bfqd = kzalloc_node(sizeof(*bfqd), GFP_KERNEL, q->node); in bfq_init_queue()
6952 if (!bfqd) { in bfq_init_queue()
6956 eq->elevator_data = bfqd; in bfq_init_queue()
6967 bfq_init_bfqq(bfqd, &bfqd->oom_bfqq, NULL, 1, 0); in bfq_init_queue()
6968 bfqd->oom_bfqq.ref++; in bfq_init_queue()
6969 bfqd->oom_bfqq.new_ioprio = BFQ_DEFAULT_QUEUE_IOPRIO; in bfq_init_queue()
6970 bfqd->oom_bfqq.new_ioprio_class = IOPRIO_CLASS_BE; in bfq_init_queue()
6971 bfqd->oom_bfqq.entity.new_weight = in bfq_init_queue()
6972 bfq_ioprio_to_weight(bfqd->oom_bfqq.new_ioprio); in bfq_init_queue()
6975 bfq_clear_bfqq_just_created(&bfqd->oom_bfqq); in bfq_init_queue()
6982 bfqd->oom_bfqq.entity.prio_changed = 1; in bfq_init_queue()
6984 bfqd->queue = q; in bfq_init_queue()
6986 INIT_LIST_HEAD(&bfqd->dispatch); in bfq_init_queue()
6988 hrtimer_init(&bfqd->idle_slice_timer, CLOCK_MONOTONIC, in bfq_init_queue()
6990 bfqd->idle_slice_timer.function = bfq_idle_slice_timer; in bfq_init_queue()
6992 bfqd->queue_weights_tree = RB_ROOT_CACHED; in bfq_init_queue()
6993 bfqd->num_groups_with_pending_reqs = 0; in bfq_init_queue()
6995 INIT_LIST_HEAD(&bfqd->active_list); in bfq_init_queue()
6996 INIT_LIST_HEAD(&bfqd->idle_list); in bfq_init_queue()
6997 INIT_HLIST_HEAD(&bfqd->burst_list); in bfq_init_queue()
6999 bfqd->hw_tag = -1; in bfq_init_queue()
7000 bfqd->nonrot_with_queueing = blk_queue_nonrot(bfqd->queue); in bfq_init_queue()
7002 bfqd->bfq_max_budget = bfq_default_max_budget; in bfq_init_queue()
7004 bfqd->bfq_fifo_expire[0] = bfq_fifo_expire[0]; in bfq_init_queue()
7005 bfqd->bfq_fifo_expire[1] = bfq_fifo_expire[1]; in bfq_init_queue()
7006 bfqd->bfq_back_max = bfq_back_max; in bfq_init_queue()
7007 bfqd->bfq_back_penalty = bfq_back_penalty; in bfq_init_queue()
7008 bfqd->bfq_slice_idle = bfq_slice_idle; in bfq_init_queue()
7009 bfqd->bfq_timeout = bfq_timeout; in bfq_init_queue()
7011 bfqd->bfq_large_burst_thresh = 8; in bfq_init_queue()
7012 bfqd->bfq_burst_interval = msecs_to_jiffies(180); in bfq_init_queue()
7014 bfqd->low_latency = true; in bfq_init_queue()
7019 bfqd->bfq_wr_coeff = 30; in bfq_init_queue()
7020 bfqd->bfq_wr_rt_max_time = msecs_to_jiffies(300); in bfq_init_queue()
7021 bfqd->bfq_wr_max_time = 0; in bfq_init_queue()
7022 bfqd->bfq_wr_min_idle_time = msecs_to_jiffies(2000); in bfq_init_queue()
7023 bfqd->bfq_wr_min_inter_arr_async = msecs_to_jiffies(500); in bfq_init_queue()
7024 bfqd->bfq_wr_max_softrt_rate = 7000; /* in bfq_init_queue()
7030 bfqd->wr_busy_queues = 0; in bfq_init_queue()
7036 bfqd->rate_dur_prod = ref_rate[blk_queue_nonrot(bfqd->queue)] * in bfq_init_queue()
7037 ref_wr_duration[blk_queue_nonrot(bfqd->queue)]; in bfq_init_queue()
7038 bfqd->peak_rate = ref_rate[blk_queue_nonrot(bfqd->queue)] * 2 / 3; in bfq_init_queue()
7040 spin_lock_init(&bfqd->lock); in bfq_init_queue()
7057 bfqd->root_group = bfq_create_group_hierarchy(bfqd, q->node); in bfq_init_queue()
7058 if (!bfqd->root_group) in bfq_init_queue()
7060 bfq_init_root_group(bfqd->root_group, bfqd); in bfq_init_queue()
7061 bfq_init_entity(&bfqd->oom_bfqq.entity, bfqd->root_group); in bfq_init_queue()
7067 kfree(bfqd); in bfq_init_queue()
7104 struct bfq_data *bfqd = e->elevator_data; \
7112 SHOW_FUNCTION(bfq_fifo_expire_sync_show, bfqd->bfq_fifo_expire[1], 2);
7113 SHOW_FUNCTION(bfq_fifo_expire_async_show, bfqd->bfq_fifo_expire[0], 2);
7114 SHOW_FUNCTION(bfq_back_seek_max_show, bfqd->bfq_back_max, 0);
7115 SHOW_FUNCTION(bfq_back_seek_penalty_show, bfqd->bfq_back_penalty, 0);
7116 SHOW_FUNCTION(bfq_slice_idle_show, bfqd->bfq_slice_idle, 2);
7117 SHOW_FUNCTION(bfq_max_budget_show, bfqd->bfq_user_max_budget, 0);
7118 SHOW_FUNCTION(bfq_timeout_sync_show, bfqd->bfq_timeout, 1);
7119 SHOW_FUNCTION(bfq_strict_guarantees_show, bfqd->strict_guarantees, 0);
7120 SHOW_FUNCTION(bfq_low_latency_show, bfqd->low_latency, 0);
7126 struct bfq_data *bfqd = e->elevator_data; \
7131 USEC_SHOW_FUNCTION(bfq_slice_idle_us_show, bfqd->bfq_slice_idle);
7138 struct bfq_data *bfqd = e->elevator_data; \
7157 STORE_FUNCTION(bfq_fifo_expire_sync_store, &bfqd->bfq_fifo_expire[1], 1,
7159 STORE_FUNCTION(bfq_fifo_expire_async_store, &bfqd->bfq_fifo_expire[0], 1,
7161 STORE_FUNCTION(bfq_back_seek_max_store, &bfqd->bfq_back_max, 0, INT_MAX, 0);
7162 STORE_FUNCTION(bfq_back_seek_penalty_store, &bfqd->bfq_back_penalty, 1,
7164 STORE_FUNCTION(bfq_slice_idle_store, &bfqd->bfq_slice_idle, 0, INT_MAX, 2);
7170 struct bfq_data *bfqd = e->elevator_data; \
7184 USEC_STORE_FUNCTION(bfq_slice_idle_us_store, &bfqd->bfq_slice_idle, 0,
7191 struct bfq_data *bfqd = e->elevator_data; in bfq_max_budget_store() local
7200 bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd); in bfq_max_budget_store()
7204 bfqd->bfq_max_budget = __data; in bfq_max_budget_store()
7207 bfqd->bfq_user_max_budget = __data; in bfq_max_budget_store()
7219 struct bfq_data *bfqd = e->elevator_data; in bfq_timeout_sync_store() local
7232 bfqd->bfq_timeout = msecs_to_jiffies(__data); in bfq_timeout_sync_store()
7233 if (bfqd->bfq_user_max_budget == 0) in bfq_timeout_sync_store()
7234 bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd); in bfq_timeout_sync_store()
7242 struct bfq_data *bfqd = e->elevator_data; in bfq_strict_guarantees_store() local
7252 if (!bfqd->strict_guarantees && __data == 1 in bfq_strict_guarantees_store()
7253 && bfqd->bfq_slice_idle < 8 * NSEC_PER_MSEC) in bfq_strict_guarantees_store()
7254 bfqd->bfq_slice_idle = 8 * NSEC_PER_MSEC; in bfq_strict_guarantees_store()
7256 bfqd->strict_guarantees = __data; in bfq_strict_guarantees_store()
7264 struct bfq_data *bfqd = e->elevator_data; in bfq_low_latency_store() local
7274 if (__data == 0 && bfqd->low_latency != 0) in bfq_low_latency_store()
7275 bfq_end_wr(bfqd); in bfq_low_latency_store()
7276 bfqd->low_latency = __data; in bfq_low_latency_store()