Lines Matching refs:bfqd
230 #define BFQ_RQ_SEEKY(bfqd, last_pos, rq) \ argument
233 (!blk_queue_nonrot(bfqd->queue) || \
399 static struct bfq_io_cq *bfq_bic_lookup(struct bfq_data *bfqd, in bfq_bic_lookup() argument
421 void bfq_schedule_dispatch(struct bfq_data *bfqd) in bfq_schedule_dispatch() argument
423 if (bfqd->queued != 0) { in bfq_schedule_dispatch()
424 bfq_log(bfqd, "schedule dispatch"); in bfq_schedule_dispatch()
425 blk_mq_run_hw_queues(bfqd->queue, true); in bfq_schedule_dispatch()
439 static struct request *bfq_choose_req(struct bfq_data *bfqd, in bfq_choose_req() argument
470 back_max = bfqd->bfq_back_max * 2; in bfq_choose_req()
480 d1 = (last - s1) * bfqd->bfq_back_penalty; in bfq_choose_req()
487 d2 = (last - s2) * bfqd->bfq_back_penalty; in bfq_choose_req()
537 struct bfq_data *bfqd = data->q->elevator->elevator_data; in bfq_limit_depth() local
543 bfqd->word_depths[!!bfqd->wr_busy_queues][op_is_sync(op)]; in bfq_limit_depth()
545 bfq_log(bfqd, "[%s] wr_busy %d sync %d depth %u", in bfq_limit_depth()
546 __func__, bfqd->wr_busy_queues, op_is_sync(op), in bfq_limit_depth()
551 bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root, in bfq_rq_pos_tree_lookup() argument
584 bfq_log(bfqd, "rq_pos_tree_lookup %llu: returning %d", in bfq_rq_pos_tree_lookup()
607 bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_pos_tree_add_move() argument
631 __bfqq = bfq_rq_pos_tree_lookup(bfqd, bfqq->pos_root, in bfq_pos_tree_add_move()
673 static bool bfq_asymmetric_scenario(struct bfq_data *bfqd, in bfq_asymmetric_scenario() argument
680 rb_first_cached(&bfqd->queue_weights_tree), in bfq_asymmetric_scenario()
689 !RB_EMPTY_ROOT(&bfqd->queue_weights_tree.rb_root) && in bfq_asymmetric_scenario()
690 (bfqd->queue_weights_tree.rb_root.rb_node->rb_left || in bfq_asymmetric_scenario()
691 bfqd->queue_weights_tree.rb_root.rb_node->rb_right); in bfq_asymmetric_scenario()
694 (bfqd->busy_queues[0] && bfqd->busy_queues[1]) || in bfq_asymmetric_scenario()
695 (bfqd->busy_queues[0] && bfqd->busy_queues[2]) || in bfq_asymmetric_scenario()
696 (bfqd->busy_queues[1] && bfqd->busy_queues[2]); in bfq_asymmetric_scenario()
700 || bfqd->num_groups_with_pending_reqs > 0 in bfq_asymmetric_scenario()
718 void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_weights_tree_add() argument
792 void __bfq_weights_tree_remove(struct bfq_data *bfqd, in __bfq_weights_tree_remove() argument
815 void bfq_weights_tree_remove(struct bfq_data *bfqd, in bfq_weights_tree_remove() argument
850 bfqd->num_groups_with_pending_reqs--; in bfq_weights_tree_remove()
860 __bfq_weights_tree_remove(bfqd, bfqq, in bfq_weights_tree_remove()
861 &bfqd->queue_weights_tree); in bfq_weights_tree_remove()
882 bfq_log_bfqq(bfqq->bfqd, bfqq, "check_fifo: returned %p", rq); in bfq_check_fifo()
886 static struct request *bfq_find_next_rq(struct bfq_data *bfqd, in bfq_find_next_rq() argument
910 return bfq_choose_req(bfqd, next, prev, blk_rq_pos(last)); in bfq_find_next_rq()
918 bfq_asymmetric_scenario(bfqq->bfqd, bfqq)) in bfq_serv_to_charge()
935 static void bfq_updated_next_req(struct bfq_data *bfqd, in bfq_updated_next_req() argument
945 if (bfqq == bfqd->in_service_queue) in bfq_updated_next_req()
958 bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu", in bfq_updated_next_req()
960 bfq_requeue_bfqq(bfqd, bfqq, false); in bfq_updated_next_req()
964 static unsigned int bfq_wr_duration(struct bfq_data *bfqd) in bfq_wr_duration() argument
968 if (bfqd->bfq_wr_max_time > 0) in bfq_wr_duration()
969 return bfqd->bfq_wr_max_time; in bfq_wr_duration()
971 dur = bfqd->rate_dur_prod; in bfq_wr_duration()
972 do_div(dur, bfqd->peak_rate); in bfq_wr_duration()
999 struct bfq_data *bfqd) in switch_back_to_interactive_wr() argument
1001 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in switch_back_to_interactive_wr()
1002 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in switch_back_to_interactive_wr()
1007 bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, in bfq_bfqq_resume_state() argument
1033 if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && in bfq_bfqq_resume_state()
1036 bfq_wr_duration(bfqd))) { in bfq_bfqq_resume_state()
1037 switch_back_to_interactive_wr(bfqq, bfqd); in bfq_bfqq_resume_state()
1040 bfq_log_bfqq(bfqq->bfqd, bfqq, in bfq_bfqq_resume_state()
1052 bfqd->wr_busy_queues++; in bfq_bfqq_resume_state()
1054 bfqd->wr_busy_queues--; in bfq_bfqq_resume_state()
1064 static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_reset_burst_list() argument
1069 hlist_for_each_entry_safe(item, n, &bfqd->burst_list, burst_list_node) in bfq_reset_burst_list()
1077 if (bfq_tot_busy_queues(bfqd) == 0) { in bfq_reset_burst_list()
1078 hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); in bfq_reset_burst_list()
1079 bfqd->burst_size = 1; in bfq_reset_burst_list()
1081 bfqd->burst_size = 0; in bfq_reset_burst_list()
1083 bfqd->burst_parent_entity = bfqq->entity.parent; in bfq_reset_burst_list()
1087 static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_add_to_burst() argument
1090 bfqd->burst_size++; in bfq_add_to_burst()
1092 if (bfqd->burst_size == bfqd->bfq_large_burst_thresh) { in bfq_add_to_burst()
1100 bfqd->large_burst = true; in bfq_add_to_burst()
1106 hlist_for_each_entry(bfqq_item, &bfqd->burst_list, in bfq_add_to_burst()
1118 hlist_for_each_entry_safe(pos, n, &bfqd->burst_list, in bfq_add_to_burst()
1127 hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); in bfq_add_to_burst()
1239 static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_handle_burst() argument
1269 if (time_is_before_jiffies(bfqd->last_ins_in_burst + in bfq_handle_burst()
1270 bfqd->bfq_burst_interval) || in bfq_handle_burst()
1271 bfqq->entity.parent != bfqd->burst_parent_entity) { in bfq_handle_burst()
1272 bfqd->large_burst = false; in bfq_handle_burst()
1273 bfq_reset_burst_list(bfqd, bfqq); in bfq_handle_burst()
1282 if (bfqd->large_burst) { in bfq_handle_burst()
1292 bfq_add_to_burst(bfqd, bfqq); in bfq_handle_burst()
1302 bfqd->last_ins_in_burst = jiffies; in bfq_handle_burst()
1317 static int bfq_max_budget(struct bfq_data *bfqd) in bfq_max_budget() argument
1319 if (bfqd->budgets_assigned < bfq_stats_min_budgets) in bfq_max_budget()
1322 return bfqd->bfq_max_budget; in bfq_max_budget()
1329 static int bfq_min_budget(struct bfq_data *bfqd) in bfq_min_budget() argument
1331 if (bfqd->budgets_assigned < bfq_stats_min_budgets) in bfq_min_budget()
1334 return bfqd->bfq_max_budget / 32; in bfq_min_budget()
1441 static bool bfq_bfqq_update_budg_for_activation(struct bfq_data *bfqd, in bfq_bfqq_update_budg_for_activation() argument
1510 static void bfq_update_bfqq_wr_on_rq_arrival(struct bfq_data *bfqd, in bfq_update_bfqq_wr_on_rq_arrival() argument
1522 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in bfq_update_bfqq_wr_on_rq_arrival()
1523 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in bfq_update_bfqq_wr_on_rq_arrival()
1538 bfqq->wr_coeff = bfqd->bfq_wr_coeff * in bfq_update_bfqq_wr_on_rq_arrival()
1541 bfqd->bfq_wr_rt_max_time; in bfq_update_bfqq_wr_on_rq_arrival()
1555 2 * bfq_min_budget(bfqd)); in bfq_update_bfqq_wr_on_rq_arrival()
1558 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in bfq_update_bfqq_wr_on_rq_arrival()
1559 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in bfq_update_bfqq_wr_on_rq_arrival()
1593 bfqd->bfq_wr_rt_max_time) { in bfq_update_bfqq_wr_on_rq_arrival()
1598 bfqd->bfq_wr_rt_max_time; in bfq_update_bfqq_wr_on_rq_arrival()
1599 bfqq->wr_coeff = bfqd->bfq_wr_coeff * in bfq_update_bfqq_wr_on_rq_arrival()
1607 static bool bfq_bfqq_idle_for_long_time(struct bfq_data *bfqd, in bfq_bfqq_idle_for_long_time() argument
1613 bfqd->bfq_wr_min_idle_time); in bfq_bfqq_idle_for_long_time()
1646 static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, in bfq_bfqq_handle_idle_busy_switch() argument
1654 idle_for_long_time = bfq_bfqq_idle_for_long_time(bfqd, bfqq), in bfq_bfqq_handle_idle_busy_switch()
1662 bfqd->bfq_slice_idle * 3; in bfq_bfqq_handle_idle_busy_switch()
1673 soft_rt = bfqd->bfq_wr_max_softrt_rate > 0 && in bfq_bfqq_handle_idle_busy_switch()
1679 wr_or_deserves_wr = bfqd->low_latency && in bfq_bfqq_handle_idle_busy_switch()
1689 bfq_bfqq_update_budg_for_activation(bfqd, bfqq, in bfq_bfqq_handle_idle_busy_switch()
1721 bfqd->bfq_requests_within_timer) in bfq_bfqq_handle_idle_busy_switch()
1727 if (bfqd->low_latency) { in bfq_bfqq_handle_idle_busy_switch()
1731 jiffies - bfqd->bfq_wr_min_idle_time - 1; in bfq_bfqq_handle_idle_busy_switch()
1734 bfqd->bfq_wr_min_idle_time)) { in bfq_bfqq_handle_idle_busy_switch()
1735 bfq_update_bfqq_wr_on_rq_arrival(bfqd, bfqq, in bfq_bfqq_handle_idle_busy_switch()
1751 bfq_add_bfqq_busy(bfqd, bfqq); in bfq_bfqq_handle_idle_busy_switch()
1785 if (bfqd->in_service_queue && in bfq_bfqq_handle_idle_busy_switch()
1787 bfqq->wr_coeff >= bfqd->in_service_queue->wr_coeff) || in bfq_bfqq_handle_idle_busy_switch()
1788 bfq_bfqq_higher_class_or_weight(bfqq, bfqd->in_service_queue)) && in bfq_bfqq_handle_idle_busy_switch()
1789 next_queue_may_preempt(bfqd)) in bfq_bfqq_handle_idle_busy_switch()
1790 bfq_bfqq_expire(bfqd, bfqd->in_service_queue, in bfq_bfqq_handle_idle_busy_switch()
1794 static void bfq_reset_inject_limit(struct bfq_data *bfqd, in bfq_reset_inject_limit() argument
1804 bfqd->waited_rq = NULL; in bfq_reset_inject_limit()
1863 struct bfq_data *bfqd = bfqq->bfqd; in bfq_add_request() local
1868 bfq_log_bfqq(bfqd, bfqq, "add_request %d", rq_is_sync(rq)); in bfq_add_request()
1870 bfqd->queued++; in bfq_add_request()
1927 if (bfqd->last_completed_rq_bfqq && in bfq_add_request()
1929 ktime_get_ns() - bfqd->last_completion < in bfq_add_request()
1931 if (bfqd->last_completed_rq_bfqq != bfqq && in bfq_add_request()
1932 bfqd->last_completed_rq_bfqq != in bfq_add_request()
1940 bfqq->waker_bfqq = bfqd->last_completed_rq_bfqq; in bfq_add_request()
1965 &bfqd->last_completed_rq_bfqq->woken_list); in bfq_add_request()
1968 } else if (bfqd->last_completed_rq_bfqq == in bfq_add_request()
1987 bfq_reset_inject_limit(bfqd, bfqq); in bfq_add_request()
2014 if (bfqq == bfqd->in_service_queue && in bfq_add_request()
2015 (bfqd->rq_in_driver == 0 || in bfq_add_request()
2017 bfqd->rqs_injected && bfqd->rq_in_driver > 0)) && in bfq_add_request()
2020 bfqd->last_empty_occupied_ns = ktime_get_ns(); in bfq_add_request()
2027 bfqd->wait_dispatch = true; in bfq_add_request()
2041 if (bfqd->rq_in_driver == 0) in bfq_add_request()
2042 bfqd->rqs_injected = false; in bfq_add_request()
2052 next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position); in bfq_add_request()
2059 if (unlikely(!bfqd->nonrot_with_queueing && prev != bfqq->next_rq)) in bfq_add_request()
2060 bfq_pos_tree_add_move(bfqd, bfqq); in bfq_add_request()
2063 bfq_bfqq_handle_idle_busy_switch(bfqd, bfqq, old_wr_coeff, in bfq_add_request()
2066 if (bfqd->low_latency && old_wr_coeff == 1 && !rq_is_sync(rq) && in bfq_add_request()
2069 bfqd->bfq_wr_min_inter_arr_async)) { in bfq_add_request()
2070 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in bfq_add_request()
2071 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in bfq_add_request()
2073 bfqd->wr_busy_queues++; in bfq_add_request()
2077 bfq_updated_next_req(bfqd, bfqq); in bfq_add_request()
2106 if (bfqd->low_latency && in bfq_add_request()
2111 static struct request *bfq_find_rq_fmerge(struct bfq_data *bfqd, in bfq_find_rq_fmerge() argument
2115 struct bfq_queue *bfqq = bfqd->bio_bfqq; in bfq_find_rq_fmerge()
2135 struct bfq_data *bfqd = q->elevator->elevator_data;
2137 bfqd->rq_in_driver++;
2142 struct bfq_data *bfqd = q->elevator->elevator_data;
2144 bfqd->rq_in_driver--;
2152 struct bfq_data *bfqd = bfqq->bfqd; in bfq_remove_request() local
2156 bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq); in bfq_remove_request()
2157 bfq_updated_next_req(bfqd, bfqq); in bfq_remove_request()
2163 bfqd->queued--; in bfq_remove_request()
2173 if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) { in bfq_remove_request()
2174 bfq_del_bfqq_busy(bfqd, bfqq, false); in bfq_remove_request()
2200 if (unlikely(!bfqd->nonrot_with_queueing)) in bfq_remove_request()
2201 bfq_pos_tree_add_move(bfqd, bfqq); in bfq_remove_request()
2213 struct bfq_data *bfqd = q->elevator->elevator_data; in bfq_bio_merge() local
2222 struct bfq_io_cq *bic = bfq_bic_lookup(bfqd, current->io_context, q); in bfq_bio_merge()
2225 spin_lock_irq(&bfqd->lock); in bfq_bio_merge()
2228 bfqd->bio_bfqq = bic_to_bfqq(bic, op_is_sync(bio->bi_opf)); in bfq_bio_merge()
2230 bfqd->bio_bfqq = NULL; in bfq_bio_merge()
2231 bfqd->bio_bic = bic; in bfq_bio_merge()
2237 spin_unlock_irq(&bfqd->lock); in bfq_bio_merge()
2245 struct bfq_data *bfqd = q->elevator->elevator_data; in bfq_request_merge() local
2248 __rq = bfq_find_rq_fmerge(bfqd, bio, q); in bfq_request_merge()
2268 struct bfq_data *bfqd; in bfq_request_merged() local
2274 bfqd = bfqq->bfqd; in bfq_request_merged()
2282 next_rq = bfq_choose_req(bfqd, bfqq->next_rq, req, in bfq_request_merged()
2283 bfqd->last_position); in bfq_request_merged()
2291 bfq_updated_next_req(bfqd, bfqq); in bfq_request_merged()
2296 if (unlikely(!bfqd->nonrot_with_queueing)) in bfq_request_merged()
2297 bfq_pos_tree_add_move(bfqd, bfqq); in bfq_request_merged()
2352 bfqq->bfqd->wr_busy_queues--; in bfq_bfqq_end_wr()
2363 void bfq_end_wr_async_queues(struct bfq_data *bfqd, in bfq_end_wr_async_queues() argument
2376 static void bfq_end_wr(struct bfq_data *bfqd) in bfq_end_wr() argument
2380 spin_lock_irq(&bfqd->lock); in bfq_end_wr()
2382 list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) in bfq_end_wr()
2384 list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) in bfq_end_wr()
2386 bfq_end_wr_async(bfqd); in bfq_end_wr()
2388 spin_unlock_irq(&bfqd->lock); in bfq_end_wr()
2406 static struct bfq_queue *bfqq_find_close(struct bfq_data *bfqd, in bfqq_find_close() argument
2421 __bfqq = bfq_rq_pos_tree_lookup(bfqd, root, sector, &parent, NULL); in bfqq_find_close()
2448 static struct bfq_queue *bfq_find_close_cooperator(struct bfq_data *bfqd, in bfq_find_close_cooperator() argument
2461 bfqq = bfqq_find_close(bfqd, cur_bfqq, sector); in bfq_find_close_cooperator()
2499 bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d", in bfq_setup_merge()
2577 bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_setup_cooperator() argument
2619 if (likely(bfqd->nonrot_with_queueing)) in bfq_setup_cooperator()
2639 if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq)) in bfq_setup_cooperator()
2643 if (bfq_tot_busy_queues(bfqd) == 1) in bfq_setup_cooperator()
2646 in_service_bfqq = bfqd->in_service_queue; in bfq_setup_cooperator()
2649 likely(in_service_bfqq != &bfqd->oom_bfqq) && in bfq_setup_cooperator()
2651 bfqd->in_serv_last_pos) && in bfq_setup_cooperator()
2663 new_bfqq = bfq_find_close_cooperator(bfqd, bfqq, in bfq_setup_cooperator()
2666 if (new_bfqq && likely(new_bfqq != &bfqd->oom_bfqq) && in bfq_setup_cooperator()
2693 bfqq->bfqd->low_latency)) { in bfq_bfqq_save_state()
2703 bic->saved_wr_coeff = bfqq->bfqd->bfq_wr_coeff; in bfq_bfqq_save_state()
2705 bic->saved_wr_cur_max_time = bfq_wr_duration(bfqq->bfqd); in bfq_bfqq_save_state()
2718 void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_release_process_ref() argument
2732 bfqq != bfqd->in_service_queue) in bfq_release_process_ref()
2733 bfq_del_bfqq_busy(bfqd, bfqq, false); in bfq_release_process_ref()
2739 bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, in bfq_merge_bfqqs() argument
2742 bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", in bfq_merge_bfqqs()
2767 bfqd->wr_busy_queues++; in bfq_merge_bfqqs()
2775 bfqd->wr_busy_queues--; in bfq_merge_bfqqs()
2778 bfq_log_bfqq(bfqd, new_bfqq, "merge_bfqqs: wr_busy %d", in bfq_merge_bfqqs()
2779 bfqd->wr_busy_queues); in bfq_merge_bfqqs()
2808 bfq_release_process_ref(bfqd, bfqq); in bfq_merge_bfqqs()
2814 struct bfq_data *bfqd = q->elevator->elevator_data; in bfq_allow_bio_merge() local
2816 struct bfq_queue *bfqq = bfqd->bio_bfqq, *new_bfqq; in bfq_allow_bio_merge()
2835 new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false); in bfq_allow_bio_merge()
2844 bfq_merge_bfqqs(bfqd, bfqd->bio_bic, bfqq, in bfq_allow_bio_merge()
2859 bfqd->bio_bfqq = bfqq; in bfq_allow_bio_merge()
2871 static void bfq_set_budget_timeout(struct bfq_data *bfqd, in bfq_set_budget_timeout() argument
2876 if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time) in bfq_set_budget_timeout()
2881 bfqd->last_budget_start = ktime_get(); in bfq_set_budget_timeout()
2884 bfqd->bfq_timeout * timeout_coeff; in bfq_set_budget_timeout()
2887 static void __bfq_set_in_service_queue(struct bfq_data *bfqd, in __bfq_set_in_service_queue() argument
2893 bfqd->budgets_assigned = (bfqd->budgets_assigned * 7 + 256) / 8; in __bfq_set_in_service_queue()
2897 bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && in __bfq_set_in_service_queue()
2931 bfq_set_budget_timeout(bfqd, bfqq); in __bfq_set_in_service_queue()
2932 bfq_log_bfqq(bfqd, bfqq, in __bfq_set_in_service_queue()
2937 bfqd->in_service_queue = bfqq; in __bfq_set_in_service_queue()
2943 static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd) in bfq_set_in_service_queue() argument
2945 struct bfq_queue *bfqq = bfq_get_next_queue(bfqd); in bfq_set_in_service_queue()
2947 __bfq_set_in_service_queue(bfqd, bfqq); in bfq_set_in_service_queue()
2951 static void bfq_arm_slice_timer(struct bfq_data *bfqd) in bfq_arm_slice_timer() argument
2953 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_arm_slice_timer()
2963 sl = bfqd->bfq_slice_idle; in bfq_arm_slice_timer()
2975 !bfq_asymmetric_scenario(bfqd, bfqq)) in bfq_arm_slice_timer()
2980 bfqd->last_idling_start = ktime_get(); in bfq_arm_slice_timer()
2981 bfqd->last_idling_start_jiffies = jiffies; in bfq_arm_slice_timer()
2983 hrtimer_start(&bfqd->idle_slice_timer, ns_to_ktime(sl), in bfq_arm_slice_timer()
2995 static unsigned long bfq_calc_max_budget(struct bfq_data *bfqd) in bfq_calc_max_budget() argument
2997 return (u64)bfqd->peak_rate * USEC_PER_MSEC * in bfq_calc_max_budget()
2998 jiffies_to_msecs(bfqd->bfq_timeout)>>BFQ_RATE_SHIFT; in bfq_calc_max_budget()
3006 static void update_thr_responsiveness_params(struct bfq_data *bfqd) in update_thr_responsiveness_params() argument
3008 if (bfqd->bfq_user_max_budget == 0) { in update_thr_responsiveness_params()
3009 bfqd->bfq_max_budget = in update_thr_responsiveness_params()
3010 bfq_calc_max_budget(bfqd); in update_thr_responsiveness_params()
3011 bfq_log(bfqd, "new max_budget = %d", bfqd->bfq_max_budget); in update_thr_responsiveness_params()
3015 static void bfq_reset_rate_computation(struct bfq_data *bfqd, in bfq_reset_rate_computation() argument
3019 bfqd->last_dispatch = bfqd->first_dispatch = ktime_get_ns(); in bfq_reset_rate_computation()
3020 bfqd->peak_rate_samples = 1; in bfq_reset_rate_computation()
3021 bfqd->sequential_samples = 0; in bfq_reset_rate_computation()
3022 bfqd->tot_sectors_dispatched = bfqd->last_rq_max_size = in bfq_reset_rate_computation()
3025 bfqd->peak_rate_samples = 0; /* full re-init on next disp. */ in bfq_reset_rate_computation()
3027 bfq_log(bfqd, in bfq_reset_rate_computation()
3029 bfqd->peak_rate_samples, bfqd->sequential_samples, in bfq_reset_rate_computation()
3030 bfqd->tot_sectors_dispatched); in bfq_reset_rate_computation()
3033 static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) in bfq_update_rate_reset() argument
3045 if (bfqd->peak_rate_samples < BFQ_RATE_MIN_SAMPLES || in bfq_update_rate_reset()
3046 bfqd->delta_from_first < BFQ_RATE_MIN_INTERVAL) in bfq_update_rate_reset()
3055 bfqd->delta_from_first = in bfq_update_rate_reset()
3056 max_t(u64, bfqd->delta_from_first, in bfq_update_rate_reset()
3057 bfqd->last_completion - bfqd->first_dispatch); in bfq_update_rate_reset()
3063 rate = div64_ul(bfqd->tot_sectors_dispatched<<BFQ_RATE_SHIFT, in bfq_update_rate_reset()
3064 div_u64(bfqd->delta_from_first, NSEC_PER_USEC)); in bfq_update_rate_reset()
3072 if ((bfqd->sequential_samples < (3 * bfqd->peak_rate_samples)>>2 && in bfq_update_rate_reset()
3073 rate <= bfqd->peak_rate) || in bfq_update_rate_reset()
3100 weight = (9 * bfqd->sequential_samples) / bfqd->peak_rate_samples; in bfq_update_rate_reset()
3107 div_u64(weight * bfqd->delta_from_first, in bfq_update_rate_reset()
3121 bfqd->peak_rate *= divisor-1; in bfq_update_rate_reset()
3122 bfqd->peak_rate /= divisor; in bfq_update_rate_reset()
3125 bfqd->peak_rate += rate; in bfq_update_rate_reset()
3134 bfqd->peak_rate = max_t(u32, 1, bfqd->peak_rate); in bfq_update_rate_reset()
3136 update_thr_responsiveness_params(bfqd); in bfq_update_rate_reset()
3139 bfq_reset_rate_computation(bfqd, rq); in bfq_update_rate_reset()
3174 static void bfq_update_peak_rate(struct bfq_data *bfqd, struct request *rq) in bfq_update_peak_rate() argument
3178 if (bfqd->peak_rate_samples == 0) { /* first dispatch */ in bfq_update_peak_rate()
3179 bfq_log(bfqd, "update_peak_rate: goto reset, samples %d", in bfq_update_peak_rate()
3180 bfqd->peak_rate_samples); in bfq_update_peak_rate()
3181 bfq_reset_rate_computation(bfqd, rq); in bfq_update_peak_rate()
3197 if (now_ns - bfqd->last_dispatch > 100*NSEC_PER_MSEC && in bfq_update_peak_rate()
3198 bfqd->rq_in_driver == 0) in bfq_update_peak_rate()
3202 bfqd->peak_rate_samples++; in bfq_update_peak_rate()
3204 if ((bfqd->rq_in_driver > 0 || in bfq_update_peak_rate()
3205 now_ns - bfqd->last_completion < BFQ_MIN_TT) in bfq_update_peak_rate()
3206 && !BFQ_RQ_SEEKY(bfqd, bfqd->last_position, rq)) in bfq_update_peak_rate()
3207 bfqd->sequential_samples++; in bfq_update_peak_rate()
3209 bfqd->tot_sectors_dispatched += blk_rq_sectors(rq); in bfq_update_peak_rate()
3212 if (likely(bfqd->peak_rate_samples % 32)) in bfq_update_peak_rate()
3213 bfqd->last_rq_max_size = in bfq_update_peak_rate()
3214 max_t(u32, blk_rq_sectors(rq), bfqd->last_rq_max_size); in bfq_update_peak_rate()
3216 bfqd->last_rq_max_size = blk_rq_sectors(rq); in bfq_update_peak_rate()
3218 bfqd->delta_from_first = now_ns - bfqd->first_dispatch; in bfq_update_peak_rate()
3221 if (bfqd->delta_from_first < BFQ_RATE_REF_INTERVAL) in bfq_update_peak_rate()
3225 bfq_update_rate_reset(bfqd, rq); in bfq_update_peak_rate()
3227 bfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq); in bfq_update_peak_rate()
3228 if (RQ_BFQQ(rq) == bfqd->in_service_queue) in bfq_update_peak_rate()
3229 bfqd->in_serv_last_pos = bfqd->last_position; in bfq_update_peak_rate()
3230 bfqd->last_dispatch = now_ns; in bfq_update_peak_rate()
3444 static bool idling_needed_for_service_guarantees(struct bfq_data *bfqd, in idling_needed_for_service_guarantees() argument
3448 (bfqd->wr_busy_queues < in idling_needed_for_service_guarantees()
3449 bfq_tot_busy_queues(bfqd) || in idling_needed_for_service_guarantees()
3450 bfqd->rq_in_driver >= in idling_needed_for_service_guarantees()
3452 bfq_asymmetric_scenario(bfqd, bfqq); in idling_needed_for_service_guarantees()
3455 static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, in __bfq_bfqq_expire() argument
3482 idling_needed_for_service_guarantees(bfqd, bfqq))) { in __bfq_bfqq_expire()
3492 bfq_del_bfqq_busy(bfqd, bfqq, true); in __bfq_bfqq_expire()
3494 bfq_requeue_bfqq(bfqd, bfqq, true); in __bfq_bfqq_expire()
3499 if (unlikely(!bfqd->nonrot_with_queueing && in __bfq_bfqq_expire()
3501 bfq_pos_tree_add_move(bfqd, bfqq); in __bfq_bfqq_expire()
3511 return __bfq_bfqd_reset_in_service(bfqd); in __bfq_bfqq_expire()
3523 static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd, in __bfq_bfqq_recalc_budget() argument
3530 min_budget = bfq_min_budget(bfqd); in __bfq_bfqq_recalc_budget()
3542 bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %d, budg left %d", in __bfq_bfqq_recalc_budget()
3544 bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last max_budg %d, min budg %d", in __bfq_bfqq_recalc_budget()
3545 budget, bfq_min_budget(bfqd)); in __bfq_bfqq_recalc_budget()
3546 bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d", in __bfq_bfqq_recalc_budget()
3547 bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue)); in __bfq_bfqq_recalc_budget()
3581 budget = min(budget * 2, bfqd->bfq_max_budget); in __bfq_bfqq_recalc_budget()
3596 budget = min(budget * 2, bfqd->bfq_max_budget); in __bfq_bfqq_recalc_budget()
3608 budget = min(budget * 4, bfqd->bfq_max_budget); in __bfq_bfqq_recalc_budget()
3655 budget = bfqd->bfq_max_budget; in __bfq_bfqq_recalc_budget()
3660 if (bfqd->budgets_assigned >= bfq_stats_min_budgets && in __bfq_bfqq_recalc_budget()
3661 !bfqd->bfq_user_max_budget) in __bfq_bfqq_recalc_budget()
3662 bfqq->max_budget = min(bfqq->max_budget, bfqd->bfq_max_budget); in __bfq_bfqq_recalc_budget()
3679 bfq_log_bfqq(bfqd, bfqq, "head sect: %u, new budget %d", in __bfq_bfqq_recalc_budget()
3715 static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_bfqq_is_slow() argument
3727 delta_ktime = bfqd->last_idling_start; in bfq_bfqq_is_slow()
3730 delta_ktime = ktime_sub(delta_ktime, bfqd->last_budget_start); in bfq_bfqq_is_slow()
3735 if (blk_queue_nonrot(bfqd->queue)) in bfq_bfqq_is_slow()
3764 slow = bfqq->entity.service < bfqd->bfq_max_budget / 2; in bfq_bfqq_is_slow()
3767 bfq_log_bfqq(bfqd, bfqq, "bfq_bfqq_is_slow: slow %d", slow); in bfq_bfqq_is_slow()
3865 static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, in bfq_bfqq_softrt_next_start() argument
3871 bfqd->bfq_wr_max_softrt_rate, in bfq_bfqq_softrt_next_start()
3872 jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); in bfq_bfqq_softrt_next_start()
3901 void bfq_bfqq_expire(struct bfq_data *bfqd, in bfq_bfqq_expire() argument
3913 slow = bfq_bfqq_is_slow(bfqd, bfqq, compensate, reason, &delta); in bfq_bfqq_expire()
3934 bfq_bfqq_charge_time(bfqd, bfqq, delta); in bfq_bfqq_expire()
3940 if (bfqd->low_latency && bfqq->wr_coeff == 1) in bfq_bfqq_expire()
3943 if (bfqd->low_latency && bfqd->bfq_wr_max_softrt_rate > 0 && in bfq_bfqq_expire()
3972 bfqq->wr_coeff != bfqd->bfq_wr_coeff) in bfq_bfqq_expire()
3974 bfq_bfqq_softrt_next_start(bfqd, bfqq); in bfq_bfqq_expire()
3984 bfq_log_bfqq(bfqd, bfqq, in bfq_bfqq_expire()
3993 bfqd->rqs_injected = bfqd->wait_dispatch = false; in bfq_bfqq_expire()
3994 bfqd->waited_rq = NULL; in bfq_bfqq_expire()
4000 __bfq_bfqq_recalc_budget(bfqd, bfqq, reason); in bfq_bfqq_expire()
4001 if (__bfq_bfqq_expire(bfqd, bfqq, reason)) in bfq_bfqq_expire()
4060 bfq_log_bfqq(bfqq->bfqd, bfqq, in bfq_may_expire_for_budg_timeout()
4072 static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd, in idling_boosts_thr_without_issues() argument
4076 !blk_queue_nonrot(bfqd->queue) && !bfqd->hw_tag, in idling_boosts_thr_without_issues()
4106 ((!blk_queue_nonrot(bfqd->queue) || !bfqd->hw_tag) && in idling_boosts_thr_without_issues()
4144 bfqd->wr_busy_queues == 0; in idling_boosts_thr_without_issues()
4170 struct bfq_data *bfqd = bfqq->bfqd; in bfq_better_to_idle() local
4173 if (unlikely(bfqd->strict_guarantees)) in bfq_better_to_idle()
4184 if (bfqd->bfq_slice_idle == 0 || !bfq_bfqq_sync(bfqq) || in bfq_better_to_idle()
4189 idling_boosts_thr_without_issues(bfqd, bfqq); in bfq_better_to_idle()
4192 idling_needed_for_service_guarantees(bfqd, bfqq); in bfq_better_to_idle()
4228 bfq_choose_bfqq_for_injection(struct bfq_data *bfqd) in bfq_choose_bfqq_for_injection() argument
4230 struct bfq_queue *bfqq, *in_serv_bfqq = bfqd->in_service_queue; in bfq_choose_bfqq_for_injection()
4258 time_is_before_eq_jiffies(bfqd->last_idling_start_jiffies + in bfq_choose_bfqq_for_injection()
4259 bfqd->bfq_slice_idle) in bfq_choose_bfqq_for_injection()
4263 if (bfqd->rq_in_driver >= limit) in bfq_choose_bfqq_for_injection()
4278 list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) in bfq_choose_bfqq_for_injection()
4300 if (blk_queue_nonrot(bfqd->queue) && in bfq_choose_bfqq_for_injection()
4307 if (bfqd->rq_in_driver < limit) { in bfq_choose_bfqq_for_injection()
4308 bfqd->rqs_injected = true; in bfq_choose_bfqq_for_injection()
4320 static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) in bfq_select_queue() argument
4326 bfqq = bfqd->in_service_queue; in bfq_select_queue()
4330 bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue"); in bfq_select_queue()
4387 hrtimer_try_to_cancel(&bfqd->idle_slice_timer); in bfq_select_queue()
4494 else if (!idling_boosts_thr_without_issues(bfqd, bfqq) && in bfq_select_queue()
4495 (bfqq->wr_coeff == 1 || bfqd->wr_busy_queues > 1 || in bfq_select_queue()
4497 bfqq = bfq_choose_bfqq_for_injection(bfqd); in bfq_select_queue()
4506 bfq_bfqq_expire(bfqd, bfqq, false, reason); in bfq_select_queue()
4508 bfqq = bfq_set_in_service_queue(bfqd); in bfq_select_queue()
4510 bfq_log_bfqq(bfqd, bfqq, "select_queue: checking new queue"); in bfq_select_queue()
4515 bfq_log_bfqq(bfqd, bfqq, "select_queue: returned this queue"); in bfq_select_queue()
4517 bfq_log(bfqd, "select_queue: no queue returned"); in bfq_select_queue()
4522 static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_update_wr_data() argument
4527 bfq_log_bfqq(bfqd, bfqq, in bfq_update_wr_data()
4535 bfq_log_bfqq(bfqd, bfqq, "WARN: pending prio change"); in bfq_update_wr_data()
4546 if (bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time || in bfq_update_wr_data()
4548 bfq_wr_duration(bfqd))) in bfq_update_wr_data()
4551 switch_back_to_interactive_wr(bfqq, bfqd); in bfq_update_wr_data()
4556 bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time && in bfq_update_wr_data()
4578 static struct request *bfq_dispatch_rq_from_bfqq(struct bfq_data *bfqd, in bfq_dispatch_rq_from_bfqq() argument
4588 if (bfqq == bfqd->in_service_queue && bfqd->wait_dispatch) { in bfq_dispatch_rq_from_bfqq()
4589 bfqd->wait_dispatch = false; in bfq_dispatch_rq_from_bfqq()
4590 bfqd->waited_rq = rq; in bfq_dispatch_rq_from_bfqq()
4593 bfq_dispatch_remove(bfqd->queue, rq); in bfq_dispatch_rq_from_bfqq()
4595 if (bfqq != bfqd->in_service_queue) in bfq_dispatch_rq_from_bfqq()
4609 bfq_update_wr_data(bfqd, bfqq); in bfq_dispatch_rq_from_bfqq()
4616 if (!(bfq_tot_busy_queues(bfqd) > 1 && bfq_class_idle(bfqq))) in bfq_dispatch_rq_from_bfqq()
4619 bfq_bfqq_expire(bfqd, bfqq, false, BFQQE_BUDGET_EXHAUSTED); in bfq_dispatch_rq_from_bfqq()
4627 struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; in bfq_has_work() local
4633 return !list_empty_careful(&bfqd->dispatch) || in bfq_has_work()
4634 bfq_tot_busy_queues(bfqd) > 0; in bfq_has_work()
4639 struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; in __bfq_dispatch_request() local
4643 if (!list_empty(&bfqd->dispatch)) { in __bfq_dispatch_request()
4644 rq = list_first_entry(&bfqd->dispatch, struct request, in __bfq_dispatch_request()
4688 bfq_log(bfqd, "dispatch requests: %d busy queues", in __bfq_dispatch_request()
4689 bfq_tot_busy_queues(bfqd)); in __bfq_dispatch_request()
4691 if (bfq_tot_busy_queues(bfqd) == 0) in __bfq_dispatch_request()
4706 if (bfqd->strict_guarantees && bfqd->rq_in_driver > 0) in __bfq_dispatch_request()
4709 bfqq = bfq_select_queue(bfqd); in __bfq_dispatch_request()
4713 rq = bfq_dispatch_rq_from_bfqq(bfqd, bfqq); in __bfq_dispatch_request()
4717 bfqd->rq_in_driver++; in __bfq_dispatch_request()
4779 struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; in bfq_dispatch_request() local
4784 spin_lock_irq(&bfqd->lock); in bfq_dispatch_request()
4786 in_serv_queue = bfqd->in_service_queue; in bfq_dispatch_request()
4794 spin_unlock_irq(&bfqd->lock); in bfq_dispatch_request()
4817 if (bfqq->bfqd) in bfq_put_queue()
4818 bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", in bfq_put_queue()
4853 if (bfqq->bic && bfqq->bfqd->burst_size > 0) in bfq_put_queue()
4854 bfqq->bfqd->burst_size--; in bfq_put_queue()
4884 if (bfqq->bfqd && bfqq->bfqd->last_completed_rq_bfqq == bfqq) in bfq_put_queue()
4885 bfqq->bfqd->last_completed_rq_bfqq = NULL; in bfq_put_queue()
4912 static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_exit_bfqq() argument
4914 if (bfqq == bfqd->in_service_queue) { in bfq_exit_bfqq()
4915 __bfq_bfqq_expire(bfqd, bfqq, BFQQE_BUDGET_TIMEOUT); in bfq_exit_bfqq()
4916 bfq_schedule_dispatch(bfqd); in bfq_exit_bfqq()
4919 bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq, bfqq->ref); in bfq_exit_bfqq()
4923 bfq_release_process_ref(bfqd, bfqq); in bfq_exit_bfqq()
4929 struct bfq_data *bfqd; in bfq_exit_icq_bfqq() local
4932 bfqd = bfqq->bfqd; /* NULL if scheduler already exited */ in bfq_exit_icq_bfqq()
4934 if (bfqq && bfqd) { in bfq_exit_icq_bfqq()
4937 spin_lock_irqsave(&bfqd->lock, flags); in bfq_exit_icq_bfqq()
4939 bfq_exit_bfqq(bfqd, bfqq); in bfq_exit_icq_bfqq()
4941 spin_unlock_irqrestore(&bfqd->lock, flags); in bfq_exit_icq_bfqq()
4962 struct bfq_data *bfqd = bfqq->bfqd; in bfq_set_next_ioprio_data() local
4964 if (!bfqd) in bfq_set_next_ioprio_data()
4970 dev_err(bfqq->bfqd->queue->backing_dev_info->dev, in bfq_set_next_ioprio_data()
5004 static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd,
5010 struct bfq_data *bfqd = bic_to_bfqd(bic); in bfq_check_ioprio_change() local
5018 if (unlikely(!bfqd) || likely(bic->ioprio == ioprio)) in bfq_check_ioprio_change()
5025 bfq_release_process_ref(bfqd, bfqq); in bfq_check_ioprio_change()
5026 bfqq = bfq_get_queue(bfqd, bio, BLK_RW_ASYNC, bic); in bfq_check_ioprio_change()
5035 static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_init_bfqq() argument
5045 bfqq->bfqd = bfqd; in bfq_init_bfqq()
5072 bfqq->max_budget = (2 * bfq_max_budget(bfqd)) / 3; in bfq_init_bfqq()
5095 static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd, in bfq_async_queue_prio() argument
5114 static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, in bfq_get_queue() argument
5126 bfqg = bfq_find_set_group(bfqd, __bio_blkcg(bio)); in bfq_get_queue()
5128 bfqq = &bfqd->oom_bfqq; in bfq_get_queue()
5133 async_bfqq = bfq_async_queue_prio(bfqd, bfqg, ioprio_class, in bfq_get_queue()
5142 bfqd->queue->node); in bfq_get_queue()
5145 bfq_init_bfqq(bfqd, bfqq, bic, current->pid, in bfq_get_queue()
5148 bfq_log_bfqq(bfqd, bfqq, "allocated"); in bfq_get_queue()
5150 bfqq = &bfqd->oom_bfqq; in bfq_get_queue()
5151 bfq_log_bfqq(bfqd, bfqq, "using oom bfqq"); in bfq_get_queue()
5167 bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d", in bfq_get_queue()
5174 bfq_log_bfqq(bfqd, bfqq, "get_queue, at end: %p, %d", bfqq, bfqq->ref); in bfq_get_queue()
5179 static void bfq_update_io_thinktime(struct bfq_data *bfqd, in bfq_update_io_thinktime() argument
5185 elapsed = min_t(u64, elapsed, 2ULL * bfqd->bfq_slice_idle); in bfq_update_io_thinktime()
5194 bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_update_io_seektime() argument
5198 bfqq->seek_history |= BFQ_RQ_SEEKY(bfqd, bfqq->last_request_pos, rq); in bfq_update_io_seektime()
5201 bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && in bfq_update_io_seektime()
5206 static void bfq_update_has_short_ttime(struct bfq_data *bfqd, in bfq_update_has_short_ttime() argument
5218 bfqd->bfq_slice_idle == 0) in bfq_update_has_short_ttime()
5223 bfqd->bfq_wr_min_idle_time)) in bfq_update_has_short_ttime()
5232 bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle)) in bfq_update_has_short_ttime()
5330 bfq_reset_inject_limit(bfqd, bfqq); in bfq_update_has_short_ttime()
5337 static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_rq_enqueued() argument
5345 if (bfqq == bfqd->in_service_queue && bfq_bfqq_wait_request(bfqq)) { in bfq_rq_enqueued()
5366 if (small_req && idling_boosts_thr_without_issues(bfqd, bfqq) && in bfq_rq_enqueued()
5378 hrtimer_try_to_cancel(&bfqd->idle_slice_timer); in bfq_rq_enqueued()
5388 bfq_bfqq_expire(bfqd, bfqq, false, in bfq_rq_enqueued()
5394 static bool __bfq_insert_request(struct bfq_data *bfqd, struct request *rq) in __bfq_insert_request() argument
5397 *new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true); in __bfq_insert_request()
5417 bfq_merge_bfqqs(bfqd, RQ_BIC(rq), in __bfq_insert_request()
5430 bfq_update_io_thinktime(bfqd, bfqq); in __bfq_insert_request()
5431 bfq_update_has_short_ttime(bfqd, bfqq, RQ_BIC(rq)); in __bfq_insert_request()
5432 bfq_update_io_seektime(bfqd, bfqq, rq); in __bfq_insert_request()
5438 rq->fifo_time = ktime_get_ns() + bfqd->bfq_fifo_expire[rq_is_sync(rq)]; in __bfq_insert_request()
5441 bfq_rq_enqueued(bfqd, bfqq, rq); in __bfq_insert_request()
5482 struct bfq_data *bfqd = q->elevator->elevator_data; in bfq_insert_request() local
5487 spin_lock_irq(&bfqd->lock); in bfq_insert_request()
5489 spin_unlock_irq(&bfqd->lock); in bfq_insert_request()
5493 spin_unlock_irq(&bfqd->lock); in bfq_insert_request()
5497 spin_lock_irq(&bfqd->lock); in bfq_insert_request()
5501 list_add(&rq->queuelist, &bfqd->dispatch); in bfq_insert_request()
5503 list_add_tail(&rq->queuelist, &bfqd->dispatch); in bfq_insert_request()
5505 idle_timer_disabled = __bfq_insert_request(bfqd, rq); in bfq_insert_request()
5527 spin_unlock_irq(&bfqd->lock); in bfq_insert_request()
5545 static void bfq_update_hw_tag(struct bfq_data *bfqd) in bfq_update_hw_tag() argument
5547 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_update_hw_tag()
5549 bfqd->max_rq_in_driver = max_t(int, bfqd->max_rq_in_driver, in bfq_update_hw_tag()
5550 bfqd->rq_in_driver); in bfq_update_hw_tag()
5552 if (bfqd->hw_tag == 1) in bfq_update_hw_tag()
5561 if (bfqd->rq_in_driver + bfqd->queued <= BFQ_HW_QUEUE_THRESHOLD) in bfq_update_hw_tag()
5572 bfqd->rq_in_driver < BFQ_HW_QUEUE_THRESHOLD) in bfq_update_hw_tag()
5575 if (bfqd->hw_tag_samples++ < BFQ_HW_QUEUE_SAMPLES) in bfq_update_hw_tag()
5578 bfqd->hw_tag = bfqd->max_rq_in_driver > BFQ_HW_QUEUE_THRESHOLD; in bfq_update_hw_tag()
5579 bfqd->max_rq_in_driver = 0; in bfq_update_hw_tag()
5580 bfqd->hw_tag_samples = 0; in bfq_update_hw_tag()
5582 bfqd->nonrot_with_queueing = in bfq_update_hw_tag()
5583 blk_queue_nonrot(bfqd->queue) && bfqd->hw_tag; in bfq_update_hw_tag()
5586 static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) in bfq_completed_request() argument
5591 bfq_update_hw_tag(bfqd); in bfq_completed_request()
5593 bfqd->rq_in_driver--; in bfq_completed_request()
5605 bfq_weights_tree_remove(bfqd, bfqq); in bfq_completed_request()
5616 delta_us = div_u64(now_ns - bfqd->last_completion, NSEC_PER_USEC); in bfq_completed_request()
5635 (bfqd->last_rq_max_size<<BFQ_RATE_SHIFT)/delta_us < in bfq_completed_request()
5637 bfq_update_rate_reset(bfqd, NULL); in bfq_completed_request()
5638 bfqd->last_completion = now_ns; in bfq_completed_request()
5639 bfqd->last_completed_rq_bfqq = bfqq; in bfq_completed_request()
5654 bfqq->wr_coeff != bfqd->bfq_wr_coeff) in bfq_completed_request()
5656 bfq_bfqq_softrt_next_start(bfqd, bfqq); in bfq_completed_request()
5662 if (bfqd->in_service_queue == bfqq) { in bfq_completed_request()
5665 bfq_arm_slice_timer(bfqd); in bfq_completed_request()
5691 bfq_bfqq_expire(bfqd, bfqq, false, in bfq_completed_request()
5696 bfq_bfqq_expire(bfqd, bfqq, false, in bfq_completed_request()
5700 if (!bfqd->rq_in_driver) in bfq_completed_request()
5701 bfq_schedule_dispatch(bfqd); in bfq_completed_request()
5815 static void bfq_update_inject_limit(struct bfq_data *bfqd, in bfq_update_inject_limit() argument
5818 u64 tot_time_ns = ktime_get_ns() - bfqd->last_empty_occupied_ns; in bfq_update_inject_limit()
5821 if (bfqq->last_serv_time_ns > 0 && bfqd->rqs_injected) { in bfq_update_inject_limit()
5828 old_limit <= bfqd->max_rq_in_driver) in bfq_update_inject_limit()
5844 if ((bfqq->last_serv_time_ns == 0 && bfqd->rq_in_driver == 1) || in bfq_update_inject_limit()
5854 } else if (!bfqd->rqs_injected && bfqd->rq_in_driver == 1) in bfq_update_inject_limit()
5868 bfqd->waited_rq = NULL; in bfq_update_inject_limit()
5869 bfqd->rqs_injected = false; in bfq_update_inject_limit()
5881 struct bfq_data *bfqd; in bfq_finish_requeue_request() local
5903 bfqd = bfqq->bfqd; in bfq_finish_requeue_request()
5914 spin_lock_irqsave(&bfqd->lock, flags); in bfq_finish_requeue_request()
5916 if (rq == bfqd->waited_rq) in bfq_finish_requeue_request()
5917 bfq_update_inject_limit(bfqd, bfqq); in bfq_finish_requeue_request()
5919 bfq_completed_request(bfqq, bfqd); in bfq_finish_requeue_request()
5922 spin_unlock_irqrestore(&bfqd->lock, flags); in bfq_finish_requeue_request()
5973 bfq_log_bfqq(bfqq->bfqd, bfqq, "splitting queue"); in bfq_split_bfqq()
5986 bfq_release_process_ref(bfqq->bfqd, bfqq); in bfq_split_bfqq()
5990 static struct bfq_queue *bfq_get_bfqq_handle_split(struct bfq_data *bfqd, in bfq_get_bfqq_handle_split() argument
5998 if (likely(bfqq && bfqq != &bfqd->oom_bfqq)) in bfq_get_bfqq_handle_split()
6006 bfqq = bfq_get_queue(bfqd, bio, is_sync, bic); in bfq_get_bfqq_handle_split()
6010 if ((bic->was_in_burst_list && bfqd->large_burst) || in bfq_get_bfqq_handle_split()
6045 &bfqd->burst_list); in bfq_get_bfqq_handle_split()
6096 struct bfq_data *bfqd = q->elevator->elevator_data; in bfq_init_rq() local
6122 bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync, in bfq_init_rq()
6128 bfq_log_bfqq(bfqd, bfqq, "breaking apart bfqq"); in bfq_init_rq()
6138 bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, in bfq_init_rq()
6148 bfq_log_bfqq(bfqd, bfqq, "get_request %p: bfqq %p, %d", in bfq_init_rq()
6160 if (likely(bfqq != &bfqd->oom_bfqq) && bfqq_process_refs(bfqq) == 1) { in bfq_init_rq()
6168 bfq_bfqq_resume_state(bfqq, bfqd, bic, in bfq_init_rq()
6194 (bfqd->burst_size > 0 || in bfq_init_rq()
6195 bfq_tot_busy_queues(bfqd) == 0))) in bfq_init_rq()
6196 bfq_handle_burst(bfqd, bfqq); in bfq_init_rq()
6203 struct bfq_data *bfqd = bfqq->bfqd; in bfq_idle_slice_timer_body() local
6207 spin_lock_irqsave(&bfqd->lock, flags); in bfq_idle_slice_timer_body()
6210 if (bfqq != bfqd->in_service_queue) { in bfq_idle_slice_timer_body()
6211 spin_unlock_irqrestore(&bfqd->lock, flags); in bfq_idle_slice_timer_body()
6233 bfq_bfqq_expire(bfqd, bfqq, true, reason); in bfq_idle_slice_timer_body()
6236 spin_unlock_irqrestore(&bfqd->lock, flags); in bfq_idle_slice_timer_body()
6237 bfq_schedule_dispatch(bfqd); in bfq_idle_slice_timer_body()
6246 struct bfq_data *bfqd = container_of(timer, struct bfq_data, in bfq_idle_slice_timer() local
6248 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_idle_slice_timer()
6264 static void __bfq_put_async_bfqq(struct bfq_data *bfqd, in __bfq_put_async_bfqq() argument
6269 bfq_log(bfqd, "put_async_bfqq: %p", bfqq); in __bfq_put_async_bfqq()
6271 bfq_bfqq_move(bfqd, bfqq, bfqd->root_group); in __bfq_put_async_bfqq()
6273 bfq_log_bfqq(bfqd, bfqq, "put_async_bfqq: putting %p, %d", in __bfq_put_async_bfqq()
6286 void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg) in bfq_put_async_queues() argument
6292 __bfq_put_async_bfqq(bfqd, &bfqg->async_bfqq[i][j]); in bfq_put_async_queues()
6294 __bfq_put_async_bfqq(bfqd, &bfqg->async_idle_bfqq); in bfq_put_async_queues()
6301 static unsigned int bfq_update_depths(struct bfq_data *bfqd, in bfq_update_depths() argument
6317 bfqd->word_depths[0][0] = max((1U << bt->sb.shift) >> 1, 1U); in bfq_update_depths()
6323 bfqd->word_depths[0][1] = max(((1U << bt->sb.shift) * 3) >> 2, 1U); in bfq_update_depths()
6333 bfqd->word_depths[1][0] = max(((1U << bt->sb.shift) * 3) >> 4, 1U); in bfq_update_depths()
6335 bfqd->word_depths[1][1] = max(((1U << bt->sb.shift) * 6) >> 4, 1U); in bfq_update_depths()
6339 min_shallow = min(min_shallow, bfqd->word_depths[i][j]); in bfq_update_depths()
6346 struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; in bfq_depth_updated() local
6350 min_shallow = bfq_update_depths(bfqd, &tags->bitmap_tags); in bfq_depth_updated()
6362 struct bfq_data *bfqd = e->elevator_data; in bfq_exit_queue() local
6365 hrtimer_cancel(&bfqd->idle_slice_timer); in bfq_exit_queue()
6367 spin_lock_irq(&bfqd->lock); in bfq_exit_queue()
6368 list_for_each_entry_safe(bfqq, n, &bfqd->idle_list, bfqq_list) in bfq_exit_queue()
6369 bfq_deactivate_bfqq(bfqd, bfqq, false, false); in bfq_exit_queue()
6370 spin_unlock_irq(&bfqd->lock); in bfq_exit_queue()
6372 hrtimer_cancel(&bfqd->idle_slice_timer); in bfq_exit_queue()
6376 bfqg_and_blkg_put(bfqd->root_group); in bfq_exit_queue()
6378 blkcg_deactivate_policy(bfqd->queue, &blkcg_policy_bfq); in bfq_exit_queue()
6380 spin_lock_irq(&bfqd->lock); in bfq_exit_queue()
6381 bfq_put_async_queues(bfqd, bfqd->root_group); in bfq_exit_queue()
6382 kfree(bfqd->root_group); in bfq_exit_queue()
6383 spin_unlock_irq(&bfqd->lock); in bfq_exit_queue()
6386 kfree(bfqd); in bfq_exit_queue()
6390 struct bfq_data *bfqd) in bfq_init_root_group() argument
6397 root_group->bfqd = bfqd; in bfq_init_root_group()
6407 struct bfq_data *bfqd; in bfq_init_queue() local
6414 bfqd = kzalloc_node(sizeof(*bfqd), GFP_KERNEL, q->node); in bfq_init_queue()
6415 if (!bfqd) { in bfq_init_queue()
6419 eq->elevator_data = bfqd; in bfq_init_queue()
6430 bfq_init_bfqq(bfqd, &bfqd->oom_bfqq, NULL, 1, 0); in bfq_init_queue()
6431 bfqd->oom_bfqq.ref++; in bfq_init_queue()
6432 bfqd->oom_bfqq.new_ioprio = BFQ_DEFAULT_QUEUE_IOPRIO; in bfq_init_queue()
6433 bfqd->oom_bfqq.new_ioprio_class = IOPRIO_CLASS_BE; in bfq_init_queue()
6434 bfqd->oom_bfqq.entity.new_weight = in bfq_init_queue()
6435 bfq_ioprio_to_weight(bfqd->oom_bfqq.new_ioprio); in bfq_init_queue()
6438 bfq_clear_bfqq_just_created(&bfqd->oom_bfqq); in bfq_init_queue()
6445 bfqd->oom_bfqq.entity.prio_changed = 1; in bfq_init_queue()
6447 bfqd->queue = q; in bfq_init_queue()
6449 INIT_LIST_HEAD(&bfqd->dispatch); in bfq_init_queue()
6451 hrtimer_init(&bfqd->idle_slice_timer, CLOCK_MONOTONIC, in bfq_init_queue()
6453 bfqd->idle_slice_timer.function = bfq_idle_slice_timer; in bfq_init_queue()
6455 bfqd->queue_weights_tree = RB_ROOT_CACHED; in bfq_init_queue()
6456 bfqd->num_groups_with_pending_reqs = 0; in bfq_init_queue()
6458 INIT_LIST_HEAD(&bfqd->active_list); in bfq_init_queue()
6459 INIT_LIST_HEAD(&bfqd->idle_list); in bfq_init_queue()
6460 INIT_HLIST_HEAD(&bfqd->burst_list); in bfq_init_queue()
6462 bfqd->hw_tag = -1; in bfq_init_queue()
6463 bfqd->nonrot_with_queueing = blk_queue_nonrot(bfqd->queue); in bfq_init_queue()
6465 bfqd->bfq_max_budget = bfq_default_max_budget; in bfq_init_queue()
6467 bfqd->bfq_fifo_expire[0] = bfq_fifo_expire[0]; in bfq_init_queue()
6468 bfqd->bfq_fifo_expire[1] = bfq_fifo_expire[1]; in bfq_init_queue()
6469 bfqd->bfq_back_max = bfq_back_max; in bfq_init_queue()
6470 bfqd->bfq_back_penalty = bfq_back_penalty; in bfq_init_queue()
6471 bfqd->bfq_slice_idle = bfq_slice_idle; in bfq_init_queue()
6472 bfqd->bfq_timeout = bfq_timeout; in bfq_init_queue()
6474 bfqd->bfq_requests_within_timer = 120; in bfq_init_queue()
6476 bfqd->bfq_large_burst_thresh = 8; in bfq_init_queue()
6477 bfqd->bfq_burst_interval = msecs_to_jiffies(180); in bfq_init_queue()
6479 bfqd->low_latency = true; in bfq_init_queue()
6484 bfqd->bfq_wr_coeff = 30; in bfq_init_queue()
6485 bfqd->bfq_wr_rt_max_time = msecs_to_jiffies(300); in bfq_init_queue()
6486 bfqd->bfq_wr_max_time = 0; in bfq_init_queue()
6487 bfqd->bfq_wr_min_idle_time = msecs_to_jiffies(2000); in bfq_init_queue()
6488 bfqd->bfq_wr_min_inter_arr_async = msecs_to_jiffies(500); in bfq_init_queue()
6489 bfqd->bfq_wr_max_softrt_rate = 7000; /* in bfq_init_queue()
6495 bfqd->wr_busy_queues = 0; in bfq_init_queue()
6501 bfqd->rate_dur_prod = ref_rate[blk_queue_nonrot(bfqd->queue)] * in bfq_init_queue()
6502 ref_wr_duration[blk_queue_nonrot(bfqd->queue)]; in bfq_init_queue()
6503 bfqd->peak_rate = ref_rate[blk_queue_nonrot(bfqd->queue)] * 2 / 3; in bfq_init_queue()
6505 spin_lock_init(&bfqd->lock); in bfq_init_queue()
6522 bfqd->root_group = bfq_create_group_hierarchy(bfqd, q->node); in bfq_init_queue()
6523 if (!bfqd->root_group) in bfq_init_queue()
6525 bfq_init_root_group(bfqd->root_group, bfqd); in bfq_init_queue()
6526 bfq_init_entity(&bfqd->oom_bfqq.entity, bfqd->root_group); in bfq_init_queue()
6532 kfree(bfqd); in bfq_init_queue()
6569 struct bfq_data *bfqd = e->elevator_data; \
6577 SHOW_FUNCTION(bfq_fifo_expire_sync_show, bfqd->bfq_fifo_expire[1], 2);
6578 SHOW_FUNCTION(bfq_fifo_expire_async_show, bfqd->bfq_fifo_expire[0], 2);
6579 SHOW_FUNCTION(bfq_back_seek_max_show, bfqd->bfq_back_max, 0);
6580 SHOW_FUNCTION(bfq_back_seek_penalty_show, bfqd->bfq_back_penalty, 0);
6581 SHOW_FUNCTION(bfq_slice_idle_show, bfqd->bfq_slice_idle, 2);
6582 SHOW_FUNCTION(bfq_max_budget_show, bfqd->bfq_user_max_budget, 0);
6583 SHOW_FUNCTION(bfq_timeout_sync_show, bfqd->bfq_timeout, 1);
6584 SHOW_FUNCTION(bfq_strict_guarantees_show, bfqd->strict_guarantees, 0);
6585 SHOW_FUNCTION(bfq_low_latency_show, bfqd->low_latency, 0);
6591 struct bfq_data *bfqd = e->elevator_data; \
6596 USEC_SHOW_FUNCTION(bfq_slice_idle_us_show, bfqd->bfq_slice_idle);
6603 struct bfq_data *bfqd = e->elevator_data; \
6622 STORE_FUNCTION(bfq_fifo_expire_sync_store, &bfqd->bfq_fifo_expire[1], 1,
6624 STORE_FUNCTION(bfq_fifo_expire_async_store, &bfqd->bfq_fifo_expire[0], 1,
6626 STORE_FUNCTION(bfq_back_seek_max_store, &bfqd->bfq_back_max, 0, INT_MAX, 0);
6627 STORE_FUNCTION(bfq_back_seek_penalty_store, &bfqd->bfq_back_penalty, 1,
6629 STORE_FUNCTION(bfq_slice_idle_store, &bfqd->bfq_slice_idle, 0, INT_MAX, 2);
6635 struct bfq_data *bfqd = e->elevator_data; \
6649 USEC_STORE_FUNCTION(bfq_slice_idle_us_store, &bfqd->bfq_slice_idle, 0,
6656 struct bfq_data *bfqd = e->elevator_data; in bfq_max_budget_store() local
6665 bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd); in bfq_max_budget_store()
6669 bfqd->bfq_max_budget = __data; in bfq_max_budget_store()
6672 bfqd->bfq_user_max_budget = __data; in bfq_max_budget_store()
6684 struct bfq_data *bfqd = e->elevator_data; in bfq_timeout_sync_store() local
6697 bfqd->bfq_timeout = msecs_to_jiffies(__data); in bfq_timeout_sync_store()
6698 if (bfqd->bfq_user_max_budget == 0) in bfq_timeout_sync_store()
6699 bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd); in bfq_timeout_sync_store()
6707 struct bfq_data *bfqd = e->elevator_data; in bfq_strict_guarantees_store() local
6717 if (!bfqd->strict_guarantees && __data == 1 in bfq_strict_guarantees_store()
6718 && bfqd->bfq_slice_idle < 8 * NSEC_PER_MSEC) in bfq_strict_guarantees_store()
6719 bfqd->bfq_slice_idle = 8 * NSEC_PER_MSEC; in bfq_strict_guarantees_store()
6721 bfqd->strict_guarantees = __data; in bfq_strict_guarantees_store()
6729 struct bfq_data *bfqd = e->elevator_data; in bfq_low_latency_store() local
6739 if (__data == 0 && bfqd->low_latency != 0) in bfq_low_latency_store()
6740 bfq_end_wr(bfqd); in bfq_low_latency_store()
6741 bfqd->low_latency = __data; in bfq_low_latency_store()