72 static inline struct blk_mq_hw_ctx *blk_qc_to_hctx(struct request_queue *q,  in blk_qc_to_hctx()
79 static inline struct request *blk_qc_to_rq(struct blk_mq_hw_ctx *hctx,  in blk_qc_to_rq()
100 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)  in blk_mq_hctx_has_pending()
110 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,  in blk_mq_hctx_mark_pending()
119 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,  in blk_mq_hctx_clear_pending()
316 	struct blk_mq_hw_ctx *hctx;  in blk_mq_wake_waiters()
345 	struct blk_mq_hw_ctx *hctx = data->hctx;  in blk_mq_rq_ctx_init()
681 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in __blk_mq_free_request()
698 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_mq_free_request()
1029 static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx,  in blk_mq_flush_tag_batch()
1048 	struct blk_mq_hw_ctx *cur_hctx = NULL;  in blk_mq_end_request_batch()
1583 	struct blk_mq_hw_ctx *hctx;  in blk_mq_timeout_work()
1623 	struct blk_mq_hw_ctx *hctx;
1630 	struct blk_mq_hw_ctx *hctx = flush_data->hctx;  in flush_busy_ctx()
1645 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)  in blk_mq_flush_busy_ctxs()
1657 	struct blk_mq_hw_ctx *hctx;
1665 	struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;  in dispatch_rq_from_ctx()
1681 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,  in blk_mq_dequeue_from_ctx()
1720 bool __blk_mq_get_driver_tag(struct blk_mq_hw_ctx *hctx, struct request *rq)  in __blk_mq_get_driver_tag()
1737 	struct blk_mq_hw_ctx *hctx;  in blk_mq_dispatch_wake()
1739 	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);  in blk_mq_dispatch_wake()
1761 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,  in blk_mq_mark_tag_wait()
1834 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)  in blk_mq_update_dispatch_busy()
1892 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_mq_prep_dispatch_rq()
1943 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,  in blk_mq_dispatch_rq_list()
2102 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)  in __blk_mq_run_hw_queue()
2114 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)  in blk_mq_first_mapped_cpu()
2129 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)  in blk_mq_hctx_next_cpu()
2178 static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async,  in __blk_mq_delay_run_hw_queue()
2202 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)  in blk_mq_delay_run_hw_queue()
2217 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)  in blk_mq_run_hw_queue()
2242 static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)  in blk_mq_get_sq_hctx()
2252 	struct blk_mq_hw_ctx *hctx = ctx->hctxs[HCTX_TYPE_DEFAULT];  in blk_mq_get_sq_hctx()
2266 	struct blk_mq_hw_ctx *hctx, *sq_hctx;  in blk_mq_run_hw_queues()
2294 	struct blk_mq_hw_ctx *hctx, *sq_hctx;  in blk_mq_delay_run_hw_queues()
2332 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)  in blk_mq_stop_hw_queue()
2351 	struct blk_mq_hw_ctx *hctx;  in blk_mq_stop_hw_queues()
2359 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)  in blk_mq_start_hw_queue()
2369 	struct blk_mq_hw_ctx *hctx;  in blk_mq_start_hw_queues()
2377 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)  in blk_mq_start_stopped_hw_queue()
2389 	struct blk_mq_hw_ctx *hctx;  in blk_mq_start_stopped_hw_queues()
2399 	struct blk_mq_hw_ctx *hctx;  in blk_mq_run_work_fn()
2401 	hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);  in blk_mq_run_work_fn()
2412 static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,  in __blk_mq_insert_req_list()
2429 void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,  in __blk_mq_insert_request()
2452 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_mq_request_bypass_insert()
2465 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,  in blk_mq_insert_requests()
2487 static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int *queued,  in blk_mq_commit_rqs()
2515 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,  in __blk_mq_issue_directly()
2548 static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,  in __blk_mq_try_issue_directly()
2603 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,  in blk_mq_try_issue_directly()
2622 	struct blk_mq_hw_ctx *hctx = NULL;  in blk_mq_plug_issue_direct()
2672 	struct blk_mq_hw_ctx *this_hctx = NULL;  in blk_mq_dispatch_plug_list()
2740 void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,  in blk_mq_try_issue_list_directly()
3361 	struct blk_mq_hw_ctx *hctx;
3375 static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)  in blk_mq_hctx_has_requests()
3388 		struct blk_mq_hw_ctx *hctx)  in blk_mq_last_cpu_in_hctx()
3399 	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,  in blk_mq_hctx_notify_offline()
3400 			struct blk_mq_hw_ctx, cpuhp_online);  in blk_mq_hctx_notify_offline()
3432 	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,  in blk_mq_hctx_notify_online()
3433 			struct blk_mq_hw_ctx, cpuhp_online);  in blk_mq_hctx_notify_online()
3447 	struct blk_mq_hw_ctx *hctx;  in blk_mq_hctx_notify_dead()
3452 	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);  in blk_mq_hctx_notify_dead()
3477 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)  in blk_mq_remove_cpuhp()
3518 		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)  in blk_mq_exit_hctx()
3546 	struct blk_mq_hw_ctx *hctx;  in blk_mq_exit_hw_queues()
3558 		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)  in blk_mq_init_hctx()
3593 static struct blk_mq_hw_ctx *
3597 	struct blk_mq_hw_ctx *hctx;  in blk_mq_alloc_hctx()
3600 	hctx = kzalloc_node(sizeof(struct blk_mq_hw_ctx), gfp, node);  in blk_mq_alloc_hctx()
3666 		struct blk_mq_hw_ctx *hctx;  in blk_mq_init_cpu_queues()
3746 	struct blk_mq_hw_ctx *hctx;  in blk_mq_map_swqueue()
3851 	struct blk_mq_hw_ctx *hctx;  in queue_set_hctx_shared()
3951 	struct blk_mq_hw_ctx *hctx, *next;  in blk_mq_release()
4059 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(  in blk_mq_alloc_and_init_hctx()
4063 	struct blk_mq_hw_ctx *hctx = NULL, *tmp;  in blk_mq_alloc_and_init_hctx()
4096 	struct blk_mq_hw_ctx *hctx;  in blk_mq_realloc_hw_ctxs()
4104 		struct blk_mq_hw_ctx *old_hctx = xa_load(&q->hctx_table, i);  in blk_mq_realloc_hw_ctxs()
4485 	struct blk_mq_hw_ctx *hctx;  in blk_mq_update_nr_requests()
4759 	struct blk_mq_hw_ctx *hctx = blk_qc_to_hctx(q, qc);  in blk_mq_poll_hybrid()
4826 	struct blk_mq_hw_ctx *hctx = blk_qc_to_hctx(q, cookie);  in blk_mq_poll_classic()
4871 		struct blk_mq_hw_ctx *hctx;  in blk_mq_cancel_work_sync()