Lines Matching refs:cfqq
61 #define CFQQ_SEEKY(cfqq) (hweight32(cfqq->seek_history) > 32/8) argument
70 #define cfq_class_idle(cfqq) ((cfqq)->ioprio_class == IOPRIO_CLASS_IDLE) argument
71 #define cfq_class_rt(cfqq) ((cfqq)->ioprio_class == IOPRIO_CLASS_RT) argument
317 struct cfq_queue *cfqq[2]; member
403 static void cfq_put_queue(struct cfq_queue *cfqq);
435 static inline void cfq_mark_cfqq_##name(struct cfq_queue *cfqq) \
437 (cfqq)->flags |= (1 << CFQ_CFQQ_FLAG_##name); \
439 static inline void cfq_clear_cfqq_##name(struct cfq_queue *cfqq) \
441 (cfqq)->flags &= ~(1 << CFQ_CFQQ_FLAG_##name); \
443 static inline int cfq_cfqq_##name(const struct cfq_queue *cfqq) \
445 return ((cfqq)->flags & (1 << CFQ_CFQQ_FLAG_##name)) != 0; \
659 #define cfq_log_cfqq(cfqd, cfqq, fmt, args...) do { \ argument
661 cfqg_to_blkg((cfqq)->cfqg)->blkcg, \
662 "cfq%d%c%c " fmt, (cfqq)->pid, \
663 cfq_cfqq_sync((cfqq)) ? 'S' : 'A', \
664 cfqq_type((cfqq)) == SYNC_NOIDLE_WORKLOAD ? 'N' : ' ',\
786 #define cfq_log_cfqq(cfqd, cfqq, fmt, args...) \ argument
787 blk_add_trace_msg((cfqd)->queue, "cfq%d%c%c " fmt, (cfqq)->pid, \
788 cfq_cfqq_sync((cfqq)) ? 'S' : 'A', \
789 cfqq_type((cfqq)) == SYNC_NOIDLE_WORKLOAD ? 'N' : ' ',\
849 static inline enum wl_class_t cfqq_class(struct cfq_queue *cfqq) in cfqq_class() argument
851 if (cfq_class_idle(cfqq)) in cfqq_class()
853 if (cfq_class_rt(cfqq)) in cfqq_class()
859 static enum wl_type_t cfqq_type(struct cfq_queue *cfqq) in cfqq_type() argument
861 if (!cfq_cfqq_sync(cfqq)) in cfqq_type()
863 if (!cfq_cfqq_idle_window(cfqq)) in cfqq_type()
907 return cic->cfqq[is_sync]; in cic_to_cfqq()
910 static inline void cic_set_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq, in cic_set_cfqq() argument
913 cic->cfqq[is_sync] = cfqq; in cic_set_cfqq()
950 cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_prio_to_slice() argument
952 return cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio); in cfq_prio_to_slice()
1024 cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_scaled_cfqq_slice() argument
1026 u64 slice = cfq_prio_to_slice(cfqd, cfqq); in cfq_scaled_cfqq_slice()
1032 unsigned iq = cfq_group_get_avg_queues(cfqd, cfqq->cfqg, in cfq_scaled_cfqq_slice()
1033 cfq_class_rt(cfqq)); in cfq_scaled_cfqq_slice()
1036 u64 group_slice = cfq_group_slice(cfqd, cfqq->cfqg); in cfq_scaled_cfqq_slice()
1056 cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_set_prio_slice() argument
1058 u64 slice = cfq_scaled_cfqq_slice(cfqd, cfqq); in cfq_set_prio_slice()
1061 cfqq->slice_start = now; in cfq_set_prio_slice()
1062 cfqq->slice_end = now + slice; in cfq_set_prio_slice()
1063 cfqq->allocated_slice = slice; in cfq_set_prio_slice()
1064 cfq_log_cfqq(cfqd, cfqq, "set_slice=%llu", cfqq->slice_end - now); in cfq_set_prio_slice()
1072 static inline bool cfq_slice_used(struct cfq_queue *cfqq) in cfq_slice_used() argument
1074 if (cfq_cfqq_slice_new(cfqq)) in cfq_slice_used()
1076 if (ktime_get_ns() < cfqq->slice_end) in cfq_slice_used()
1201 cfq_find_next_rq(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_find_next_rq() argument
1216 rbnext = rb_first(&cfqq->sort_list); in cfq_find_next_rq()
1225 struct cfq_queue *cfqq) in cfq_slice_offset() argument
1230 return (cfqq->cfqg->nr_cfqq - 1) * (cfq_prio_slice(cfqd, 1, 0) - in cfq_slice_offset()
1231 cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio)); in cfq_slice_offset()
1428 static inline u64 cfq_cfqq_slice_usage(struct cfq_queue *cfqq, in cfq_cfqq_slice_usage() argument
1438 if (!cfqq->slice_start || cfqq->slice_start == now) { in cfq_cfqq_slice_usage()
1445 slice_used = max_t(u64, (now - cfqq->dispatch_start), in cfq_cfqq_slice_usage()
1448 slice_used = now - cfqq->slice_start; in cfq_cfqq_slice_usage()
1449 if (slice_used > cfqq->allocated_slice) { in cfq_cfqq_slice_usage()
1450 *unaccounted_time = slice_used - cfqq->allocated_slice; in cfq_cfqq_slice_usage()
1451 slice_used = cfqq->allocated_slice; in cfq_cfqq_slice_usage()
1453 if (cfqq->slice_start > cfqq->dispatch_start) in cfq_cfqq_slice_usage()
1454 *unaccounted_time += cfqq->slice_start - in cfq_cfqq_slice_usage()
1455 cfqq->dispatch_start; in cfq_cfqq_slice_usage()
1462 struct cfq_queue *cfqq) in cfq_group_served() argument
1472 used_sl = charge = cfq_cfqq_slice_usage(cfqq, &unaccounted_sl); in cfq_group_served()
1475 charge = cfqq->slice_dispatch; in cfq_group_served()
1476 else if (!cfq_cfqq_sync(cfqq) && !nr_sync) in cfq_group_served()
1477 charge = cfqq->allocated_slice; in cfq_group_served()
1500 cfq_log_cfqq(cfqq->cfqd, cfqq, in cfq_group_served()
1502 used_sl, cfqq->slice_dispatch, charge, in cfq_group_served()
1503 iops_mode(cfqd), cfqq->nr_sectors); in cfq_group_served()
1691 static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) in cfq_link_cfqq_cfqg() argument
1693 cfqq->cfqg = cfqg; in cfq_link_cfqq_cfqg()
2191 cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) { in cfq_link_cfqq_cfqg() argument
2192 cfqq->cfqg = cfqg; in cfq_link_cfqq_cfqg()
2202 static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_service_tree_add() argument
2213 st = st_for(cfqq->cfqg, cfqq_class(cfqq), cfqq_type(cfqq)); in cfq_service_tree_add()
2214 if (cfq_class_idle(cfqq)) { in cfq_service_tree_add()
2217 if (parent && parent != &cfqq->rb_node) { in cfq_service_tree_add()
2229 rb_key = cfq_slice_offset(cfqd, cfqq) + now; in cfq_service_tree_add()
2230 rb_key -= cfqq->slice_resid; in cfq_service_tree_add()
2231 cfqq->slice_resid = 0; in cfq_service_tree_add()
2238 if (!RB_EMPTY_NODE(&cfqq->rb_node)) { in cfq_service_tree_add()
2243 if (rb_key == cfqq->rb_key && cfqq->service_tree == st) in cfq_service_tree_add()
2246 cfq_rb_erase(&cfqq->rb_node, cfqq->service_tree); in cfq_service_tree_add()
2247 cfqq->service_tree = NULL; in cfq_service_tree_add()
2251 cfqq->service_tree = st; in cfq_service_tree_add()
2268 cfqq->rb_key = rb_key; in cfq_service_tree_add()
2269 rb_link_node(&cfqq->rb_node, parent, p); in cfq_service_tree_add()
2270 rb_insert_color_cached(&cfqq->rb_node, &st->rb, leftmost); in cfq_service_tree_add()
2274 cfq_group_notify_queue_add(cfqd, cfqq->cfqg); in cfq_service_tree_add()
2283 struct cfq_queue *cfqq = NULL; in cfq_prio_tree_lookup() local
2291 cfqq = rb_entry(parent, struct cfq_queue, p_node); in cfq_prio_tree_lookup()
2297 if (sector > blk_rq_pos(cfqq->next_rq)) in cfq_prio_tree_lookup()
2299 else if (sector < blk_rq_pos(cfqq->next_rq)) in cfq_prio_tree_lookup()
2304 cfqq = NULL; in cfq_prio_tree_lookup()
2310 return cfqq; in cfq_prio_tree_lookup()
2313 static void cfq_prio_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_prio_tree_add() argument
2318 if (cfqq->p_root) { in cfq_prio_tree_add()
2319 rb_erase(&cfqq->p_node, cfqq->p_root); in cfq_prio_tree_add()
2320 cfqq->p_root = NULL; in cfq_prio_tree_add()
2323 if (cfq_class_idle(cfqq)) in cfq_prio_tree_add()
2325 if (!cfqq->next_rq) in cfq_prio_tree_add()
2328 cfqq->p_root = &cfqd->prio_trees[cfqq->org_ioprio]; in cfq_prio_tree_add()
2329 __cfqq = cfq_prio_tree_lookup(cfqd, cfqq->p_root, in cfq_prio_tree_add()
2330 blk_rq_pos(cfqq->next_rq), &parent, &p); in cfq_prio_tree_add()
2332 rb_link_node(&cfqq->p_node, parent, p); in cfq_prio_tree_add()
2333 rb_insert_color(&cfqq->p_node, cfqq->p_root); in cfq_prio_tree_add()
2335 cfqq->p_root = NULL; in cfq_prio_tree_add()
2341 static void cfq_resort_rr_list(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_resort_rr_list() argument
2346 if (cfq_cfqq_on_rr(cfqq)) { in cfq_resort_rr_list()
2347 cfq_service_tree_add(cfqd, cfqq, 0); in cfq_resort_rr_list()
2348 cfq_prio_tree_add(cfqd, cfqq); in cfq_resort_rr_list()
2356 static void cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_add_cfqq_rr() argument
2358 cfq_log_cfqq(cfqd, cfqq, "add_to_rr"); in cfq_add_cfqq_rr()
2359 BUG_ON(cfq_cfqq_on_rr(cfqq)); in cfq_add_cfqq_rr()
2360 cfq_mark_cfqq_on_rr(cfqq); in cfq_add_cfqq_rr()
2362 if (cfq_cfqq_sync(cfqq)) in cfq_add_cfqq_rr()
2365 cfq_resort_rr_list(cfqd, cfqq); in cfq_add_cfqq_rr()
2372 static void cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_del_cfqq_rr() argument
2374 cfq_log_cfqq(cfqd, cfqq, "del_from_rr"); in cfq_del_cfqq_rr()
2375 BUG_ON(!cfq_cfqq_on_rr(cfqq)); in cfq_del_cfqq_rr()
2376 cfq_clear_cfqq_on_rr(cfqq); in cfq_del_cfqq_rr()
2378 if (!RB_EMPTY_NODE(&cfqq->rb_node)) { in cfq_del_cfqq_rr()
2379 cfq_rb_erase(&cfqq->rb_node, cfqq->service_tree); in cfq_del_cfqq_rr()
2380 cfqq->service_tree = NULL; in cfq_del_cfqq_rr()
2382 if (cfqq->p_root) { in cfq_del_cfqq_rr()
2383 rb_erase(&cfqq->p_node, cfqq->p_root); in cfq_del_cfqq_rr()
2384 cfqq->p_root = NULL; in cfq_del_cfqq_rr()
2387 cfq_group_notify_queue_del(cfqd, cfqq->cfqg); in cfq_del_cfqq_rr()
2390 if (cfq_cfqq_sync(cfqq)) in cfq_del_cfqq_rr()
2399 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_del_rq_rb() local
2402 BUG_ON(!cfqq->queued[sync]); in cfq_del_rq_rb()
2403 cfqq->queued[sync]--; in cfq_del_rq_rb()
2405 elv_rb_del(&cfqq->sort_list, rq); in cfq_del_rq_rb()
2407 if (cfq_cfqq_on_rr(cfqq) && RB_EMPTY_ROOT(&cfqq->sort_list)) { in cfq_del_rq_rb()
2413 if (cfqq->p_root) { in cfq_del_rq_rb()
2414 rb_erase(&cfqq->p_node, cfqq->p_root); in cfq_del_rq_rb()
2415 cfqq->p_root = NULL; in cfq_del_rq_rb()
2422 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_add_rq_rb() local
2423 struct cfq_data *cfqd = cfqq->cfqd; in cfq_add_rq_rb()
2426 cfqq->queued[rq_is_sync(rq)]++; in cfq_add_rq_rb()
2428 elv_rb_add(&cfqq->sort_list, rq); in cfq_add_rq_rb()
2430 if (!cfq_cfqq_on_rr(cfqq)) in cfq_add_rq_rb()
2431 cfq_add_cfqq_rr(cfqd, cfqq); in cfq_add_rq_rb()
2436 prev = cfqq->next_rq; in cfq_add_rq_rb()
2437 cfqq->next_rq = cfq_choose_req(cfqd, cfqq->next_rq, rq, cfqd->last_position); in cfq_add_rq_rb()
2442 if (prev != cfqq->next_rq) in cfq_add_rq_rb()
2443 cfq_prio_tree_add(cfqd, cfqq); in cfq_add_rq_rb()
2445 BUG_ON(!cfqq->next_rq); in cfq_add_rq_rb()
2448 static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq) in cfq_reposition_rq_rb() argument
2450 elv_rb_del(&cfqq->sort_list, rq); in cfq_reposition_rq_rb()
2451 cfqq->queued[rq_is_sync(rq)]--; in cfq_reposition_rq_rb()
2454 cfqg_stats_update_io_add(RQ_CFQG(rq), cfqq->cfqd->serving_group, in cfq_reposition_rq_rb()
2463 struct cfq_queue *cfqq; in cfq_find_rq_fmerge() local
2469 cfqq = cic_to_cfqq(cic, op_is_sync(bio->bi_opf)); in cfq_find_rq_fmerge()
2470 if (cfqq) in cfq_find_rq_fmerge()
2471 return elv_rb_find(&cfqq->sort_list, bio_end_sector(bio)); in cfq_find_rq_fmerge()
2499 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_remove_request() local
2501 if (cfqq->next_rq == rq) in cfq_remove_request()
2502 cfqq->next_rq = cfq_find_next_rq(cfqq->cfqd, cfqq, rq); in cfq_remove_request()
2507 cfqq->cfqd->rq_queued--; in cfq_remove_request()
2510 WARN_ON(!cfqq->prio_pending); in cfq_remove_request()
2511 cfqq->prio_pending--; in cfq_remove_request()
2534 struct cfq_queue *cfqq = RQ_CFQQ(req); in cfq_merged_request() local
2536 cfq_reposition_rq_rb(cfqq, req); in cfq_merged_request()
2550 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_merged_requests() local
2558 cfqq == RQ_CFQQ(next)) { in cfq_merged_requests()
2563 if (cfqq->next_rq == next) in cfq_merged_requests()
2564 cfqq->next_rq = rq; in cfq_merged_requests()
2568 cfqq = RQ_CFQQ(next); in cfq_merged_requests()
2574 if (cfq_cfqq_on_rr(cfqq) && RB_EMPTY_ROOT(&cfqq->sort_list) && in cfq_merged_requests()
2575 cfqq != cfqd->active_queue) in cfq_merged_requests()
2576 cfq_del_cfqq_rr(cfqd, cfqq); in cfq_merged_requests()
2585 struct cfq_queue *cfqq; in cfq_allow_bio_merge() local
2601 cfqq = cic_to_cfqq(cic, is_sync); in cfq_allow_bio_merge()
2602 return cfqq == RQ_CFQQ(rq); in cfq_allow_bio_merge()
2611 static inline void cfq_del_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_del_timer() argument
2614 cfqg_stats_update_idle_time(cfqq->cfqg); in cfq_del_timer()
2618 struct cfq_queue *cfqq) in __cfq_set_active_queue() argument
2620 if (cfqq) { in __cfq_set_active_queue()
2621 cfq_log_cfqq(cfqd, cfqq, "set_active wl_class:%d wl_type:%d", in __cfq_set_active_queue()
2623 cfqg_stats_update_avg_queue_size(cfqq->cfqg); in __cfq_set_active_queue()
2624 cfqq->slice_start = 0; in __cfq_set_active_queue()
2625 cfqq->dispatch_start = ktime_get_ns(); in __cfq_set_active_queue()
2626 cfqq->allocated_slice = 0; in __cfq_set_active_queue()
2627 cfqq->slice_end = 0; in __cfq_set_active_queue()
2628 cfqq->slice_dispatch = 0; in __cfq_set_active_queue()
2629 cfqq->nr_sectors = 0; in __cfq_set_active_queue()
2631 cfq_clear_cfqq_wait_request(cfqq); in __cfq_set_active_queue()
2632 cfq_clear_cfqq_must_dispatch(cfqq); in __cfq_set_active_queue()
2633 cfq_clear_cfqq_must_alloc_slice(cfqq); in __cfq_set_active_queue()
2634 cfq_clear_cfqq_fifo_expire(cfqq); in __cfq_set_active_queue()
2635 cfq_mark_cfqq_slice_new(cfqq); in __cfq_set_active_queue()
2637 cfq_del_timer(cfqd, cfqq); in __cfq_set_active_queue()
2640 cfqd->active_queue = cfqq; in __cfq_set_active_queue()
2647 __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq, in __cfq_slice_expired() argument
2650 cfq_log_cfqq(cfqd, cfqq, "slice expired t=%d", timed_out); in __cfq_slice_expired()
2652 if (cfq_cfqq_wait_request(cfqq)) in __cfq_slice_expired()
2653 cfq_del_timer(cfqd, cfqq); in __cfq_slice_expired()
2655 cfq_clear_cfqq_wait_request(cfqq); in __cfq_slice_expired()
2656 cfq_clear_cfqq_wait_busy(cfqq); in __cfq_slice_expired()
2664 if (cfq_cfqq_coop(cfqq) && CFQQ_SEEKY(cfqq)) in __cfq_slice_expired()
2665 cfq_mark_cfqq_split_coop(cfqq); in __cfq_slice_expired()
2671 if (cfq_cfqq_slice_new(cfqq)) in __cfq_slice_expired()
2672 cfqq->slice_resid = cfq_scaled_cfqq_slice(cfqd, cfqq); in __cfq_slice_expired()
2674 cfqq->slice_resid = cfqq->slice_end - ktime_get_ns(); in __cfq_slice_expired()
2675 cfq_log_cfqq(cfqd, cfqq, "resid=%lld", cfqq->slice_resid); in __cfq_slice_expired()
2678 cfq_group_served(cfqd, cfqq->cfqg, cfqq); in __cfq_slice_expired()
2680 if (cfq_cfqq_on_rr(cfqq) && RB_EMPTY_ROOT(&cfqq->sort_list)) in __cfq_slice_expired()
2681 cfq_del_cfqq_rr(cfqd, cfqq); in __cfq_slice_expired()
2683 cfq_resort_rr_list(cfqd, cfqq); in __cfq_slice_expired()
2685 if (cfqq == cfqd->active_queue) in __cfq_slice_expired()
2696 struct cfq_queue *cfqq = cfqd->active_queue; in cfq_slice_expired() local
2698 if (cfqq) in cfq_slice_expired()
2699 __cfq_slice_expired(cfqd, cfqq, timed_out); in cfq_slice_expired()
2725 struct cfq_queue *cfqq; in cfq_get_next_queue_forced() local
2737 cfqq = cfq_rb_first(st); in cfq_get_next_queue_forced()
2738 if (cfqq) in cfq_get_next_queue_forced()
2739 return cfqq; in cfq_get_next_queue_forced()
2748 struct cfq_queue *cfqq) in cfq_set_active_queue() argument
2750 if (!cfqq) in cfq_set_active_queue()
2751 cfqq = cfq_get_next_queue(cfqd); in cfq_set_active_queue()
2753 __cfq_set_active_queue(cfqd, cfqq); in cfq_set_active_queue()
2754 return cfqq; in cfq_set_active_queue()
2766 static inline int cfq_rq_close(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_rq_close() argument
2826 struct cfq_queue *cfqq; in cfq_close_cooperator() local
2846 cfqq = cfqq_close(cfqd, cur_cfqq); in cfq_close_cooperator()
2847 if (!cfqq) in cfq_close_cooperator()
2851 if (cur_cfqq->cfqg != cfqq->cfqg) in cfq_close_cooperator()
2857 if (!cfq_cfqq_sync(cfqq)) in cfq_close_cooperator()
2859 if (CFQQ_SEEKY(cfqq)) in cfq_close_cooperator()
2865 if (cfq_class_rt(cfqq) != cfq_class_rt(cur_cfqq)) in cfq_close_cooperator()
2868 return cfqq; in cfq_close_cooperator()
2875 static bool cfq_should_idle(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_should_idle() argument
2877 enum wl_class_t wl_class = cfqq_class(cfqq); in cfq_should_idle()
2878 struct cfq_rb_root *st = cfqq->service_tree; in cfq_should_idle()
2891 if (cfq_cfqq_idle_window(cfqq) && in cfq_should_idle()
2899 if (st->count == 1 && cfq_cfqq_sync(cfqq) && in cfq_should_idle()
2902 cfq_log_cfqq(cfqd, cfqq, "Not idling. st->count:%d", st->count); in cfq_should_idle()
2908 struct cfq_queue *cfqq = cfqd->active_queue; in cfq_arm_slice_timer() local
2909 struct cfq_rb_root *st = cfqq->service_tree; in cfq_arm_slice_timer()
2923 WARN_ON(!RB_EMPTY_ROOT(&cfqq->sort_list)); in cfq_arm_slice_timer()
2924 WARN_ON(cfq_cfqq_slice_new(cfqq)); in cfq_arm_slice_timer()
2929 if (!cfq_should_idle(cfqd, cfqq)) { in cfq_arm_slice_timer()
2940 if (cfqq->dispatched) in cfq_arm_slice_timer()
2956 (cfqq->slice_end - now < cic->ttime.ttime_mean)) { in cfq_arm_slice_timer()
2957 cfq_log_cfqq(cfqd, cfqq, "Not idling. think_time:%llu", in cfq_arm_slice_timer()
2967 (cfqq->cfqg->nr_cfqq > 1 || in cfq_arm_slice_timer()
2971 cfq_mark_cfqq_wait_request(cfqq); in cfq_arm_slice_timer()
2980 cfqg_stats_set_start_idle_time(cfqq->cfqg); in cfq_arm_slice_timer()
2981 cfq_log_cfqq(cfqd, cfqq, "arm_idle: %llu group_idle: %d", sl, in cfq_arm_slice_timer()
2991 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_dispatch_insert() local
2993 cfq_log_cfqq(cfqd, cfqq, "dispatch_insert"); in cfq_dispatch_insert()
2995 cfqq->next_rq = cfq_find_next_rq(cfqd, cfqq, rq); in cfq_dispatch_insert()
2997 cfqq->dispatched++; in cfq_dispatch_insert()
3001 cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]++; in cfq_dispatch_insert()
3002 cfqq->nr_sectors += blk_rq_sectors(rq); in cfq_dispatch_insert()
3008 static struct request *cfq_check_fifo(struct cfq_queue *cfqq) in cfq_check_fifo() argument
3012 if (cfq_cfqq_fifo_expire(cfqq)) in cfq_check_fifo()
3015 cfq_mark_cfqq_fifo_expire(cfqq); in cfq_check_fifo()
3017 if (list_empty(&cfqq->fifo)) in cfq_check_fifo()
3020 rq = rq_entry_fifo(cfqq->fifo.next); in cfq_check_fifo()
3028 cfq_prio_to_maxrq(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_prio_to_maxrq() argument
3032 WARN_ON(cfqq->ioprio >= IOPRIO_BE_NR); in cfq_prio_to_maxrq()
3034 return 2 * base_rq * (IOPRIO_BE_NR - cfqq->ioprio); in cfq_prio_to_maxrq()
3040 static int cfqq_process_refs(struct cfq_queue *cfqq) in cfqq_process_refs() argument
3044 io_refs = cfqq->allocated[READ] + cfqq->allocated[WRITE]; in cfqq_process_refs()
3045 process_refs = cfqq->ref - io_refs; in cfqq_process_refs()
3050 static void cfq_setup_merge(struct cfq_queue *cfqq, struct cfq_queue *new_cfqq) in cfq_setup_merge() argument
3066 if (__cfqq == cfqq) in cfq_setup_merge()
3071 process_refs = cfqq_process_refs(cfqq); in cfq_setup_merge()
3084 cfqq->new_cfqq = new_cfqq; in cfq_setup_merge()
3087 new_cfqq->new_cfqq = cfqq; in cfq_setup_merge()
3088 cfqq->ref += new_process_refs; in cfq_setup_merge()
3235 struct cfq_queue *cfqq, *new_cfqq = NULL; in cfq_select_queue() local
3238 cfqq = cfqd->active_queue; in cfq_select_queue()
3239 if (!cfqq) in cfq_select_queue()
3248 if (cfq_cfqq_wait_busy(cfqq) && !RB_EMPTY_ROOT(&cfqq->sort_list)) in cfq_select_queue()
3254 if (cfq_slice_used(cfqq) && !cfq_cfqq_must_dispatch(cfqq)) { in cfq_select_queue()
3264 if (cfqq->cfqg->nr_cfqq == 1 && RB_EMPTY_ROOT(&cfqq->sort_list) in cfq_select_queue()
3265 && cfqq->dispatched && cfq_should_idle(cfqd, cfqq)) { in cfq_select_queue()
3266 cfqq = NULL; in cfq_select_queue()
3276 if (!RB_EMPTY_ROOT(&cfqq->sort_list)) in cfq_select_queue()
3285 new_cfqq = cfq_close_cooperator(cfqd, cfqq); in cfq_select_queue()
3287 if (!cfqq->new_cfqq) in cfq_select_queue()
3288 cfq_setup_merge(cfqq, new_cfqq); in cfq_select_queue()
3298 cfqq = NULL; in cfq_select_queue()
3306 if (CFQQ_SEEKY(cfqq) && cfq_cfqq_idle_window(cfqq) && in cfq_select_queue()
3307 (cfq_cfqq_slice_new(cfqq) || in cfq_select_queue()
3308 (cfqq->slice_end - now > now - cfqq->slice_start))) { in cfq_select_queue()
3309 cfq_clear_cfqq_deep(cfqq); in cfq_select_queue()
3310 cfq_clear_cfqq_idle_window(cfqq); in cfq_select_queue()
3313 if (cfqq->dispatched && cfq_should_idle(cfqd, cfqq)) { in cfq_select_queue()
3314 cfqq = NULL; in cfq_select_queue()
3323 if (cfqd->cfq_group_idle && cfqq->cfqg->nr_cfqq == 1 && in cfq_select_queue()
3324 cfqq->cfqg->dispatched && in cfq_select_queue()
3325 !cfq_io_thinktime_big(cfqd, &cfqq->cfqg->ttime, true)) { in cfq_select_queue()
3326 cfqq = NULL; in cfq_select_queue()
3340 cfqq = cfq_set_active_queue(cfqd, new_cfqq); in cfq_select_queue()
3342 return cfqq; in cfq_select_queue()
3345 static int __cfq_forced_dispatch_cfqq(struct cfq_queue *cfqq) in __cfq_forced_dispatch_cfqq() argument
3349 while (cfqq->next_rq) { in __cfq_forced_dispatch_cfqq()
3350 cfq_dispatch_insert(cfqq->cfqd->queue, cfqq->next_rq); in __cfq_forced_dispatch_cfqq()
3354 BUG_ON(!list_empty(&cfqq->fifo)); in __cfq_forced_dispatch_cfqq()
3357 __cfq_slice_expired(cfqq->cfqd, cfqq, 0); in __cfq_forced_dispatch_cfqq()
3367 struct cfq_queue *cfqq; in cfq_forced_dispatch() local
3372 while ((cfqq = cfq_get_next_queue_forced(cfqd)) != NULL) { in cfq_forced_dispatch()
3373 __cfq_set_active_queue(cfqd, cfqq); in cfq_forced_dispatch()
3374 dispatched += __cfq_forced_dispatch_cfqq(cfqq); in cfq_forced_dispatch()
3384 struct cfq_queue *cfqq) in cfq_slice_used_soon() argument
3389 if (cfq_cfqq_slice_new(cfqq)) in cfq_slice_used_soon()
3391 if (now + cfqd->cfq_slice_idle * cfqq->dispatched > cfqq->slice_end) in cfq_slice_used_soon()
3397 static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_may_dispatch() argument
3401 if (cfq_cfqq_must_dispatch(cfqq)) in cfq_may_dispatch()
3407 if (cfq_should_idle(cfqd, cfqq) && cfqd->rq_in_flight[BLK_RW_ASYNC]) in cfq_may_dispatch()
3413 if (cfqd->rq_in_flight[BLK_RW_SYNC] && !cfq_cfqq_sync(cfqq)) in cfq_may_dispatch()
3417 if (cfq_class_idle(cfqq)) in cfq_may_dispatch()
3423 if (cfqq->dispatched >= max_dispatch) { in cfq_may_dispatch()
3428 if (cfq_class_idle(cfqq)) in cfq_may_dispatch()
3438 if (cfq_cfqq_sync(cfqq) && cfqd->busy_sync_queues == 1) in cfq_may_dispatch()
3444 if (cfqd->busy_queues > 1 && cfq_slice_used_soon(cfqd, cfqq) && in cfq_may_dispatch()
3468 if (!cfq_cfqq_sync(cfqq) && cfqd->cfq_latency) { in cfq_may_dispatch()
3473 if (!depth && !cfqq->dispatched) in cfq_may_dispatch()
3482 return cfqq->dispatched < max_dispatch; in cfq_may_dispatch()
3489 static bool cfq_dispatch_request(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_dispatch_request() argument
3493 BUG_ON(RB_EMPTY_ROOT(&cfqq->sort_list)); in cfq_dispatch_request()
3495 rq = cfq_check_fifo(cfqq); in cfq_dispatch_request()
3497 cfq_mark_cfqq_must_dispatch(cfqq); in cfq_dispatch_request()
3499 if (!cfq_may_dispatch(cfqd, cfqq)) in cfq_dispatch_request()
3506 rq = cfqq->next_rq; in cfq_dispatch_request()
3508 cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq); in cfq_dispatch_request()
3532 struct cfq_queue *cfqq; in cfq_dispatch_requests() local
3540 cfqq = cfq_select_queue(cfqd); in cfq_dispatch_requests()
3541 if (!cfqq) in cfq_dispatch_requests()
3547 if (!cfq_dispatch_request(cfqd, cfqq)) in cfq_dispatch_requests()
3550 cfqq->slice_dispatch++; in cfq_dispatch_requests()
3551 cfq_clear_cfqq_must_dispatch(cfqq); in cfq_dispatch_requests()
3557 if (cfqd->busy_queues > 1 && ((!cfq_cfqq_sync(cfqq) && in cfq_dispatch_requests()
3558 cfqq->slice_dispatch >= cfq_prio_to_maxrq(cfqd, cfqq)) || in cfq_dispatch_requests()
3559 cfq_class_idle(cfqq))) { in cfq_dispatch_requests()
3560 cfqq->slice_end = ktime_get_ns() + 1; in cfq_dispatch_requests()
3564 cfq_log_cfqq(cfqd, cfqq, "dispatched a request"); in cfq_dispatch_requests()
3575 static void cfq_put_queue(struct cfq_queue *cfqq) in cfq_put_queue() argument
3577 struct cfq_data *cfqd = cfqq->cfqd; in cfq_put_queue()
3580 BUG_ON(cfqq->ref <= 0); in cfq_put_queue()
3582 cfqq->ref--; in cfq_put_queue()
3583 if (cfqq->ref) in cfq_put_queue()
3586 cfq_log_cfqq(cfqd, cfqq, "put_queue"); in cfq_put_queue()
3587 BUG_ON(rb_first(&cfqq->sort_list)); in cfq_put_queue()
3588 BUG_ON(cfqq->allocated[READ] + cfqq->allocated[WRITE]); in cfq_put_queue()
3589 cfqg = cfqq->cfqg; in cfq_put_queue()
3591 if (unlikely(cfqd->active_queue == cfqq)) { in cfq_put_queue()
3592 __cfq_slice_expired(cfqd, cfqq, 0); in cfq_put_queue()
3596 BUG_ON(cfq_cfqq_on_rr(cfqq)); in cfq_put_queue()
3597 kmem_cache_free(cfq_pool, cfqq); in cfq_put_queue()
3601 static void cfq_put_cooperator(struct cfq_queue *cfqq) in cfq_put_cooperator() argument
3610 __cfqq = cfqq->new_cfqq; in cfq_put_cooperator()
3612 if (__cfqq == cfqq) { in cfq_put_cooperator()
3622 static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_exit_cfqq() argument
3624 if (unlikely(cfqq == cfqd->active_queue)) { in cfq_exit_cfqq()
3625 __cfq_slice_expired(cfqd, cfqq, 0); in cfq_exit_cfqq()
3629 cfq_put_cooperator(cfqq); in cfq_exit_cfqq()
3631 cfq_put_queue(cfqq); in cfq_exit_cfqq()
3657 static void cfq_init_prio_data(struct cfq_queue *cfqq, struct cfq_io_cq *cic) in cfq_init_prio_data() argument
3662 if (!cfq_cfqq_prio_changed(cfqq)) in cfq_init_prio_data()
3674 cfqq->ioprio = task_nice_ioprio(tsk); in cfq_init_prio_data()
3675 cfqq->ioprio_class = task_nice_ioclass(tsk); in cfq_init_prio_data()
3678 cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio); in cfq_init_prio_data()
3679 cfqq->ioprio_class = IOPRIO_CLASS_RT; in cfq_init_prio_data()
3682 cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio); in cfq_init_prio_data()
3683 cfqq->ioprio_class = IOPRIO_CLASS_BE; in cfq_init_prio_data()
3686 cfqq->ioprio_class = IOPRIO_CLASS_IDLE; in cfq_init_prio_data()
3687 cfqq->ioprio = 7; in cfq_init_prio_data()
3688 cfq_clear_cfqq_idle_window(cfqq); in cfq_init_prio_data()
3696 cfqq->org_ioprio = cfqq->ioprio; in cfq_init_prio_data()
3697 cfqq->org_ioprio_class = cfqq->ioprio_class; in cfq_init_prio_data()
3698 cfq_clear_cfqq_prio_changed(cfqq); in cfq_init_prio_data()
3705 struct cfq_queue *cfqq; in check_ioprio_changed() local
3714 cfqq = cic_to_cfqq(cic, false); in check_ioprio_changed()
3715 if (cfqq) { in check_ioprio_changed()
3716 cfq_put_queue(cfqq); in check_ioprio_changed()
3717 cfqq = cfq_get_queue(cfqd, BLK_RW_ASYNC, cic, bio); in check_ioprio_changed()
3718 cic_set_cfqq(cic, cfqq, false); in check_ioprio_changed()
3721 cfqq = cic_to_cfqq(cic, true); in check_ioprio_changed()
3722 if (cfqq) in check_ioprio_changed()
3723 cfq_mark_cfqq_prio_changed(cfqq); in check_ioprio_changed()
3728 static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_init_cfqq() argument
3731 RB_CLEAR_NODE(&cfqq->rb_node); in cfq_init_cfqq()
3732 RB_CLEAR_NODE(&cfqq->p_node); in cfq_init_cfqq()
3733 INIT_LIST_HEAD(&cfqq->fifo); in cfq_init_cfqq()
3735 cfqq->ref = 0; in cfq_init_cfqq()
3736 cfqq->cfqd = cfqd; in cfq_init_cfqq()
3738 cfq_mark_cfqq_prio_changed(cfqq); in cfq_init_cfqq()
3741 if (!cfq_class_idle(cfqq)) in cfq_init_cfqq()
3742 cfq_mark_cfqq_idle_window(cfqq); in cfq_init_cfqq()
3743 cfq_mark_cfqq_sync(cfqq); in cfq_init_cfqq()
3745 cfqq->pid = pid; in cfq_init_cfqq()
3752 struct cfq_queue *cfqq; in check_blkcg_changed() local
3770 cfqq = cic_to_cfqq(cic, false); in check_blkcg_changed()
3771 if (cfqq) { in check_blkcg_changed()
3772 cfq_log_cfqq(cfqd, cfqq, "changed cgroup"); in check_blkcg_changed()
3774 cfq_put_queue(cfqq); in check_blkcg_changed()
3777 cfqq = cic_to_cfqq(cic, true); in check_blkcg_changed()
3778 if (cfqq) { in check_blkcg_changed()
3779 cfq_log_cfqq(cfqd, cfqq, "changed cgroup"); in check_blkcg_changed()
3781 cfq_put_queue(cfqq); in check_blkcg_changed()
3817 struct cfq_queue *cfqq; in cfq_get_queue() local
3823 cfqq = &cfqd->oom_cfqq; in cfq_get_queue()
3834 cfqq = *async_cfqq; in cfq_get_queue()
3835 if (cfqq) in cfq_get_queue()
3839 cfqq = kmem_cache_alloc_node(cfq_pool, in cfq_get_queue()
3842 if (!cfqq) { in cfq_get_queue()
3843 cfqq = &cfqd->oom_cfqq; in cfq_get_queue()
3848 cfqq->ioprio_class = IOPRIO_CLASS_NONE; in cfq_get_queue()
3849 cfq_init_cfqq(cfqd, cfqq, current->pid, is_sync); in cfq_get_queue()
3850 cfq_init_prio_data(cfqq, cic); in cfq_get_queue()
3851 cfq_link_cfqq_cfqg(cfqq, cfqg); in cfq_get_queue()
3852 cfq_log_cfqq(cfqd, cfqq, "alloced"); in cfq_get_queue()
3856 cfqq->ref++; in cfq_get_queue()
3857 *async_cfqq = cfqq; in cfq_get_queue()
3860 cfqq->ref++; in cfq_get_queue()
3862 return cfqq; in cfq_get_queue()
3878 cfq_update_io_thinktime(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_update_io_thinktime() argument
3881 if (cfq_cfqq_sync(cfqq)) { in cfq_update_io_thinktime()
3883 __cfq_update_io_thinktime(&cfqq->service_tree->ttime, in cfq_update_io_thinktime()
3887 __cfq_update_io_thinktime(&cfqq->cfqg->ttime, cfqd->cfq_group_idle); in cfq_update_io_thinktime()
3892 cfq_update_io_seektime(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_update_io_seektime() argument
3897 if (cfqq->last_request_pos) { in cfq_update_io_seektime()
3898 if (cfqq->last_request_pos < blk_rq_pos(rq)) in cfq_update_io_seektime()
3899 sdist = blk_rq_pos(rq) - cfqq->last_request_pos; in cfq_update_io_seektime()
3901 sdist = cfqq->last_request_pos - blk_rq_pos(rq); in cfq_update_io_seektime()
3904 cfqq->seek_history <<= 1; in cfq_update_io_seektime()
3906 cfqq->seek_history |= (n_sec < CFQQ_SECT_THR_NONROT); in cfq_update_io_seektime()
3908 cfqq->seek_history |= (sdist > CFQQ_SEEK_THR); in cfq_update_io_seektime()
3922 cfq_update_idle_window(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_update_idle_window() argument
3930 if (!cfq_cfqq_sync(cfqq) || cfq_class_idle(cfqq)) in cfq_update_idle_window()
3933 enable_idle = old_idle = cfq_cfqq_idle_window(cfqq); in cfq_update_idle_window()
3935 if (cfqq->queued[0] + cfqq->queued[1] >= 4) in cfq_update_idle_window()
3936 cfq_mark_cfqq_deep(cfqq); in cfq_update_idle_window()
3938 if (cfqq->next_rq && req_noidle(cfqq->next_rq)) in cfq_update_idle_window()
3942 (!cfq_cfqq_deep(cfqq) && CFQQ_SEEKY(cfqq))) in cfq_update_idle_window()
3952 cfq_log_cfqq(cfqd, cfqq, "idle=%d", enable_idle); in cfq_update_idle_window()
3954 cfq_mark_cfqq_idle_window(cfqq); in cfq_update_idle_window()
3956 cfq_clear_cfqq_idle_window(cfqq); in cfq_update_idle_window()
3968 struct cfq_queue *cfqq; in cfq_should_preempt() local
3970 cfqq = cfqd->active_queue; in cfq_should_preempt()
3971 if (!cfqq) in cfq_should_preempt()
3977 if (cfq_class_idle(cfqq)) in cfq_should_preempt()
3983 if (cfq_class_rt(cfqq) && !cfq_class_rt(new_cfqq)) in cfq_should_preempt()
3990 if (rq_is_sync(rq) && !cfq_cfqq_sync(cfqq) && !cfq_cfqq_must_dispatch(cfqq)) in cfq_should_preempt()
3998 if (!cfqg_is_descendant(cfqq->cfqg, new_cfqq->cfqg)) in cfq_should_preempt()
4001 if (cfq_slice_used(cfqq)) in cfq_should_preempt()
4007 if (cfq_class_rt(new_cfqq) && !cfq_class_rt(cfqq)) in cfq_should_preempt()
4010 WARN_ON_ONCE(cfqq->ioprio_class != new_cfqq->ioprio_class); in cfq_should_preempt()
4014 RB_EMPTY_ROOT(&cfqq->sort_list)) in cfq_should_preempt()
4021 if ((rq->cmd_flags & REQ_PRIO) && !cfqq->prio_pending) in cfq_should_preempt()
4025 if (RB_EMPTY_ROOT(&cfqq->sort_list) && !cfq_should_idle(cfqd, cfqq)) in cfq_should_preempt()
4028 if (!cfqd->active_cic || !cfq_cfqq_wait_request(cfqq)) in cfq_should_preempt()
4035 if (cfq_rq_close(cfqd, cfqq, rq)) in cfq_should_preempt()
4045 static void cfq_preempt_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_preempt_queue() argument
4049 cfq_log_cfqq(cfqd, cfqq, "preempt"); in cfq_preempt_queue()
4056 if (old_type != cfqq_type(cfqq)) in cfq_preempt_queue()
4057 cfqq->cfqg->saved_wl_slice = 0; in cfq_preempt_queue()
4063 BUG_ON(!cfq_cfqq_on_rr(cfqq)); in cfq_preempt_queue()
4065 cfq_service_tree_add(cfqd, cfqq, 1); in cfq_preempt_queue()
4067 cfqq->slice_end = 0; in cfq_preempt_queue()
4068 cfq_mark_cfqq_slice_new(cfqq); in cfq_preempt_queue()
4076 cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq, in cfq_rq_enqueued() argument
4083 cfqq->prio_pending++; in cfq_rq_enqueued()
4085 cfq_update_io_thinktime(cfqd, cfqq, cic); in cfq_rq_enqueued()
4086 cfq_update_io_seektime(cfqd, cfqq, rq); in cfq_rq_enqueued()
4087 cfq_update_idle_window(cfqd, cfqq, cic); in cfq_rq_enqueued()
4089 cfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq); in cfq_rq_enqueued()
4091 if (cfqq == cfqd->active_queue) { in cfq_rq_enqueued()
4102 if (cfq_cfqq_wait_request(cfqq)) { in cfq_rq_enqueued()
4105 cfq_del_timer(cfqd, cfqq); in cfq_rq_enqueued()
4106 cfq_clear_cfqq_wait_request(cfqq); in cfq_rq_enqueued()
4109 cfqg_stats_update_idle_time(cfqq->cfqg); in cfq_rq_enqueued()
4110 cfq_mark_cfqq_must_dispatch(cfqq); in cfq_rq_enqueued()
4113 } else if (cfq_should_preempt(cfqd, cfqq, rq)) { in cfq_rq_enqueued()
4120 cfq_preempt_queue(cfqd, cfqq); in cfq_rq_enqueued()
4128 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_insert_request() local
4130 cfq_log_cfqq(cfqd, cfqq, "insert_request"); in cfq_insert_request()
4131 cfq_init_prio_data(cfqq, RQ_CIC(rq)); in cfq_insert_request()
4134 list_add_tail(&rq->queuelist, &cfqq->fifo); in cfq_insert_request()
4138 cfq_rq_enqueued(cfqd, cfqq, rq); in cfq_insert_request()
4147 struct cfq_queue *cfqq = cfqd->active_queue; in cfq_update_hw_tag() local
4164 if (cfqq && cfq_cfqq_idle_window(cfqq) && in cfq_update_hw_tag()
4165 cfqq->dispatched + cfqq->queued[0] + cfqq->queued[1] < in cfq_update_hw_tag()
4178 static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq) in cfq_should_wait_busy() argument
4184 if (!RB_EMPTY_ROOT(&cfqq->sort_list)) in cfq_should_wait_busy()
4188 if (cfqq->cfqg->nr_cfqq > 1) in cfq_should_wait_busy()
4192 if (cfq_io_thinktime_big(cfqd, &cfqq->cfqg->ttime, true)) in cfq_should_wait_busy()
4195 if (cfq_slice_used(cfqq)) in cfq_should_wait_busy()
4200 && (cfqq->slice_end - now < cic->ttime.ttime_mean)) in cfq_should_wait_busy()
4210 if (cfqq->slice_end - now <= jiffies_to_nsecs(1)) in cfq_should_wait_busy()
4218 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_completed_request() local
4219 struct cfq_data *cfqd = cfqq->cfqd; in cfq_completed_request()
4223 cfq_log_cfqq(cfqd, cfqq, "complete rqnoidle %d", req_noidle(rq)); in cfq_completed_request()
4228 WARN_ON(!cfqq->dispatched); in cfq_completed_request()
4230 cfqq->dispatched--; in cfq_completed_request()
4232 cfqg_stats_update_completion(cfqq->cfqg, rq->start_time_ns, in cfq_completed_request()
4235 cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--; in cfq_completed_request()
4242 if (cfq_cfqq_on_rr(cfqq)) in cfq_completed_request()
4243 st = cfqq->service_tree; in cfq_completed_request()
4245 st = st_for(cfqq->cfqg, cfqq_class(cfqq), in cfq_completed_request()
4246 cfqq_type(cfqq)); in cfq_completed_request()
4254 cfqq->cfqg->ttime.last_end_request = now; in cfq_completed_request()
4261 if (cfqd->active_queue == cfqq) { in cfq_completed_request()
4262 const bool cfqq_empty = RB_EMPTY_ROOT(&cfqq->sort_list); in cfq_completed_request()
4264 if (cfq_cfqq_slice_new(cfqq)) { in cfq_completed_request()
4265 cfq_set_prio_slice(cfqd, cfqq); in cfq_completed_request()
4266 cfq_clear_cfqq_slice_new(cfqq); in cfq_completed_request()
4273 if (cfq_should_wait_busy(cfqd, cfqq)) { in cfq_completed_request()
4277 cfqq->slice_end = now + extend_sl; in cfq_completed_request()
4278 cfq_mark_cfqq_wait_busy(cfqq); in cfq_completed_request()
4279 cfq_log_cfqq(cfqd, cfqq, "will busy wait"); in cfq_completed_request()
4290 if (cfq_slice_used(cfqq) || cfq_class_idle(cfqq)) in cfq_completed_request()
4293 !cfq_close_cooperator(cfqd, cfqq)) { in cfq_completed_request()
4302 static void cfqq_boost_on_prio(struct cfq_queue *cfqq, unsigned int op) in cfqq_boost_on_prio() argument
4310 cfqq->ioprio_class = cfqq->org_ioprio_class; in cfqq_boost_on_prio()
4311 cfqq->ioprio = cfqq->org_ioprio; in cfqq_boost_on_prio()
4313 if (cfq_class_idle(cfqq)) in cfqq_boost_on_prio()
4314 cfqq->ioprio_class = IOPRIO_CLASS_BE; in cfqq_boost_on_prio()
4315 if (cfqq->ioprio > IOPRIO_NORM) in cfqq_boost_on_prio()
4316 cfqq->ioprio = IOPRIO_NORM; in cfqq_boost_on_prio()
4320 static inline int __cfq_may_queue(struct cfq_queue *cfqq) in __cfq_may_queue() argument
4322 if (cfq_cfqq_wait_request(cfqq) && !cfq_cfqq_must_alloc_slice(cfqq)) { in __cfq_may_queue()
4323 cfq_mark_cfqq_must_alloc_slice(cfqq); in __cfq_may_queue()
4335 struct cfq_queue *cfqq; in cfq_may_queue() local
4347 cfqq = cic_to_cfqq(cic, op_is_sync(op)); in cfq_may_queue()
4348 if (cfqq) { in cfq_may_queue()
4349 cfq_init_prio_data(cfqq, cic); in cfq_may_queue()
4350 cfqq_boost_on_prio(cfqq, op); in cfq_may_queue()
4352 return __cfq_may_queue(cfqq); in cfq_may_queue()
4363 struct cfq_queue *cfqq = RQ_CFQQ(rq); in cfq_put_request() local
4365 if (cfqq) { in cfq_put_request()
4368 BUG_ON(!cfqq->allocated[rw]); in cfq_put_request()
4369 cfqq->allocated[rw]--; in cfq_put_request()
4376 cfq_put_queue(cfqq); in cfq_put_request()
4382 struct cfq_queue *cfqq) in cfq_merge_cfqqs() argument
4384 cfq_log_cfqq(cfqd, cfqq, "merging with queue %p", cfqq->new_cfqq); in cfq_merge_cfqqs()
4385 cic_set_cfqq(cic, cfqq->new_cfqq, 1); in cfq_merge_cfqqs()
4386 cfq_mark_cfqq_coop(cfqq->new_cfqq); in cfq_merge_cfqqs()
4387 cfq_put_queue(cfqq); in cfq_merge_cfqqs()
4396 split_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq) in split_cfqq() argument
4398 if (cfqq_process_refs(cfqq) == 1) { in split_cfqq()
4399 cfqq->pid = current->pid; in split_cfqq()
4400 cfq_clear_cfqq_coop(cfqq); in split_cfqq()
4401 cfq_clear_cfqq_split_coop(cfqq); in split_cfqq()
4402 return cfqq; in split_cfqq()
4407 cfq_put_cooperator(cfqq); in split_cfqq()
4409 cfq_put_queue(cfqq); in split_cfqq()
4423 struct cfq_queue *cfqq; in cfq_set_request() local
4430 cfqq = cic_to_cfqq(cic, is_sync); in cfq_set_request()
4431 if (!cfqq || cfqq == &cfqd->oom_cfqq) { in cfq_set_request()
4432 if (cfqq) in cfq_set_request()
4433 cfq_put_queue(cfqq); in cfq_set_request()
4434 cfqq = cfq_get_queue(cfqd, is_sync, cic, bio); in cfq_set_request()
4435 cic_set_cfqq(cic, cfqq, is_sync); in cfq_set_request()
4440 if (cfq_cfqq_coop(cfqq) && cfq_cfqq_split_coop(cfqq)) { in cfq_set_request()
4441 cfq_log_cfqq(cfqd, cfqq, "breaking apart cfqq"); in cfq_set_request()
4442 cfqq = split_cfqq(cic, cfqq); in cfq_set_request()
4443 if (!cfqq) in cfq_set_request()
4453 if (cfqq->new_cfqq) in cfq_set_request()
4454 cfqq = cfq_merge_cfqqs(cfqd, cic, cfqq); in cfq_set_request()
4457 cfqq->allocated[rw]++; in cfq_set_request()
4459 cfqq->ref++; in cfq_set_request()
4460 cfqg_get(cfqq->cfqg); in cfq_set_request()
4461 rq->elv.priv[0] = cfqq; in cfq_set_request()
4462 rq->elv.priv[1] = cfqq->cfqg; in cfq_set_request()
4486 struct cfq_queue *cfqq; in cfq_idle_slice_timer() local
4494 cfqq = cfqd->active_queue; in cfq_idle_slice_timer()
4495 if (cfqq) { in cfq_idle_slice_timer()
4501 if (cfq_cfqq_must_dispatch(cfqq)) in cfq_idle_slice_timer()
4507 if (cfq_slice_used(cfqq)) in cfq_idle_slice_timer()
4520 if (!RB_EMPTY_ROOT(&cfqq->sort_list)) in cfq_idle_slice_timer()
4526 cfq_clear_cfqq_deep(cfqq); in cfq_idle_slice_timer()