64 #define RQ_CFQQ(rq)		(struct cfq_queue *) ((rq)->elv.priv[0])
109 struct cfq_queue {  struct
159 	struct cfq_queue *new_cfqq;  argument
310 	struct cfq_queue *async_cfqq[2][IOPRIO_BE_NR];
311 	struct cfq_queue *async_idle_cfqq;
317 	struct cfq_queue	*cfqq[2];
375 	struct cfq_queue *active_queue;
397 	struct cfq_queue oom_cfqq;
403 static void cfq_put_queue(struct cfq_queue *cfqq);
435 static inline void cfq_mark_cfqq_##name(struct cfq_queue *cfqq)		\
439 static inline void cfq_clear_cfqq_##name(struct cfq_queue *cfqq)	\
443 static inline int cfq_cfqq_##name(const struct cfq_queue *cfqq)		\
849 static inline enum wl_class_t cfqq_class(struct cfq_queue *cfqq)  in cfqq_class()
859 static enum wl_type_t cfqq_type(struct cfq_queue *cfqq)  in cfqq_type()
888 static struct cfq_queue *cfq_get_queue(struct cfq_data *cfqd, bool is_sync,
905 static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_cq *cic, bool is_sync)  in cic_to_cfqq()
910 static inline void cic_set_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq,  in cic_set_cfqq()
950 cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_prio_to_slice()
1024 cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_scaled_cfqq_slice()
1056 cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_set_prio_slice()
1072 static inline bool cfq_slice_used(struct cfq_queue *cfqq)  in cfq_slice_used()
1172 static struct cfq_queue *cfq_rb_first(struct cfq_rb_root *root)  in cfq_rb_first()
1178 	return rb_entry(rb_first_cached(&root->rb), struct cfq_queue, rb_node);  in cfq_rb_first()
1201 cfq_find_next_rq(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_find_next_rq()
1225 			    struct cfq_queue *cfqq)  in cfq_slice_offset()
1428 static inline u64 cfq_cfqq_slice_usage(struct cfq_queue *cfqq,  in cfq_cfqq_slice_usage()
1462 				struct cfq_queue *cfqq)  in cfq_group_served()
1691 static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg)  in cfq_link_cfqq_cfqg()
2191 cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) {  in cfq_link_cfqq_cfqg()
2202 static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_service_tree_add()
2206 	struct cfq_queue *__cfqq;  in cfq_service_tree_add()
2218 			__cfqq = rb_entry(parent, struct cfq_queue, rb_node);  in cfq_service_tree_add()
2255 		__cfqq = rb_entry(parent, struct cfq_queue, rb_node);  in cfq_service_tree_add()
2277 static struct cfq_queue *
2283 	struct cfq_queue *cfqq = NULL;  in cfq_prio_tree_lookup()
2291 		cfqq = rb_entry(parent, struct cfq_queue, p_node);  in cfq_prio_tree_lookup()
2313 static void cfq_prio_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_prio_tree_add()
2316 	struct cfq_queue *__cfqq;  in cfq_prio_tree_add()
2341 static void cfq_resort_rr_list(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_resort_rr_list()
2356 static void cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_add_cfqq_rr()
2372 static void cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_del_cfqq_rr()
2399 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_del_rq_rb()
2422 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_add_rq_rb()
2448 static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq)  in cfq_reposition_rq_rb()
2463 	struct cfq_queue *cfqq;  in cfq_find_rq_fmerge()
2499 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_remove_request()
2534 		struct cfq_queue *cfqq = RQ_CFQQ(req);  in cfq_merged_request()
2550 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_merged_requests()
2585 	struct cfq_queue *cfqq;  in cfq_allow_bio_merge()
2611 static inline void cfq_del_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_del_timer()
2618 				   struct cfq_queue *cfqq)  in __cfq_set_active_queue()
2647 __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in __cfq_slice_expired()
2696 	struct cfq_queue *cfqq = cfqd->active_queue;  in cfq_slice_expired()
2706 static struct cfq_queue *cfq_get_next_queue(struct cfq_data *cfqd)  in cfq_get_next_queue()
2722 static struct cfq_queue *cfq_get_next_queue_forced(struct cfq_data *cfqd)  in cfq_get_next_queue_forced()
2725 	struct cfq_queue *cfqq;  in cfq_get_next_queue_forced()
2747 static struct cfq_queue *cfq_set_active_queue(struct cfq_data *cfqd,  in cfq_set_active_queue()
2748 					      struct cfq_queue *cfqq)  in cfq_set_active_queue()
2766 static inline int cfq_rq_close(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_rq_close()
2772 static struct cfq_queue *cfqq_close(struct cfq_data *cfqd,  in cfqq_close()
2773 				    struct cfq_queue *cur_cfqq)  in cfqq_close()
2777 	struct cfq_queue *__cfqq;  in cfqq_close()
2795 	__cfqq = rb_entry(parent, struct cfq_queue, p_node);  in cfqq_close()
2806 	__cfqq = rb_entry(node, struct cfq_queue, p_node);  in cfqq_close()
2823 static struct cfq_queue *cfq_close_cooperator(struct cfq_data *cfqd,  in cfq_close_cooperator()
2824 					      struct cfq_queue *cur_cfqq)  in cfq_close_cooperator()
2826 	struct cfq_queue *cfqq;  in cfq_close_cooperator()
2875 static bool cfq_should_idle(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_should_idle()
2908 	struct cfq_queue *cfqq = cfqd->active_queue;  in cfq_arm_slice_timer()
2991 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_dispatch_insert()
3008 static struct request *cfq_check_fifo(struct cfq_queue *cfqq)  in cfq_check_fifo()
3028 cfq_prio_to_maxrq(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_prio_to_maxrq()
3040 static int cfqq_process_refs(struct cfq_queue *cfqq)  in cfqq_process_refs()
3050 static void cfq_setup_merge(struct cfq_queue *cfqq, struct cfq_queue *new_cfqq)  in cfq_setup_merge()
3053 	struct cfq_queue *__cfqq;  in cfq_setup_merge()
3095 	struct cfq_queue *queue;  in cfq_choose_wl_type()
3233 static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)  in cfq_select_queue()
3235 	struct cfq_queue *cfqq, *new_cfqq = NULL;  in cfq_select_queue()
3345 static int __cfq_forced_dispatch_cfqq(struct cfq_queue *cfqq)  in __cfq_forced_dispatch_cfqq()
3367 	struct cfq_queue *cfqq;  in cfq_forced_dispatch()
3384 	struct cfq_queue *cfqq)  in cfq_slice_used_soon()
3397 static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_may_dispatch()
3489 static bool cfq_dispatch_request(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_dispatch_request()
3532 	struct cfq_queue *cfqq;  in cfq_dispatch_requests()
3575 static void cfq_put_queue(struct cfq_queue *cfqq)  in cfq_put_queue()
3601 static void cfq_put_cooperator(struct cfq_queue *cfqq)  in cfq_put_cooperator()
3603 	struct cfq_queue *__cfqq, *next;  in cfq_put_cooperator()
3622 static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_exit_cfqq()
3657 static void cfq_init_prio_data(struct cfq_queue *cfqq, struct cfq_io_cq *cic)  in cfq_init_prio_data()
3705 	struct cfq_queue *cfqq;  in check_ioprio_changed()
3728 static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_init_cfqq()
3752 	struct cfq_queue *cfqq;  in check_blkcg_changed()
3792 static struct cfq_queue **
3810 static struct cfq_queue *
3816 	struct cfq_queue **async_cfqq = NULL;  in cfq_get_queue()
3817 	struct cfq_queue *cfqq;  in cfq_get_queue()
3878 cfq_update_io_thinktime(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_update_io_thinktime()
3892 cfq_update_io_seektime(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_update_io_seektime()
3922 cfq_update_idle_window(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_update_idle_window()
3965 cfq_should_preempt(struct cfq_data *cfqd, struct cfq_queue *new_cfqq,  in cfq_should_preempt()
3968 	struct cfq_queue *cfqq;  in cfq_should_preempt()
4045 static void cfq_preempt_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_preempt_queue()
4076 cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,  in cfq_rq_enqueued()
4128 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_insert_request()
4147 	struct cfq_queue *cfqq = cfqd->active_queue;  in cfq_update_hw_tag()
4178 static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq)  in cfq_should_wait_busy()
4218 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_completed_request()
4302 static void cfqq_boost_on_prio(struct cfq_queue *cfqq, unsigned int op)  in cfqq_boost_on_prio()
4320 static inline int __cfq_may_queue(struct cfq_queue *cfqq)  in __cfq_may_queue()
4335 	struct cfq_queue *cfqq;  in cfq_may_queue()
4363 	struct cfq_queue *cfqq = RQ_CFQQ(rq);  in cfq_put_request()
4380 static struct cfq_queue *
4382 		struct cfq_queue *cfqq)  in cfq_merge_cfqqs()
4395 static struct cfq_queue *
4396 split_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq)  in split_cfqq()
4423 	struct cfq_queue *cfqq;  in cfq_set_request()
4486 	struct cfq_queue *cfqq;  in cfq_idle_slice_timer()
4877 	cfq_pool = KMEM_CACHE(cfq_queue, 0);  in cfq_init()