48 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *,
50 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list);
51 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
121 extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);
160 	struct blk_mq_hw_ctx *hctx;  member
176 int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
189 						 struct blk_mq_hw_ctx *hctx)  in bt_wait_ptr()  argument
191 	if (!hctx)  in bt_wait_ptr()
193 	return sbq_wait_ptr(bt, &hctx->wait_index);  in bt_wait_ptr()
199 static inline void blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)  in blk_mq_tag_busy()  argument
201 	if (hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)  in blk_mq_tag_busy()
202 		__blk_mq_tag_busy(hctx);  in blk_mq_tag_busy()
205 static inline void blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)  in blk_mq_tag_idle()  argument
207 	if (hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)  in blk_mq_tag_idle()
208 		__blk_mq_tag_idle(hctx);  in blk_mq_tag_idle()
225 		return data->hctx->sched_tags;  in blk_mq_tags_from_data()
226 	return data->hctx->tags;  in blk_mq_tags_from_data()
229 static inline bool blk_mq_hctx_stopped(struct blk_mq_hw_ctx *hctx)  in blk_mq_hctx_stopped()  argument
231 	return test_bit(BLK_MQ_S_STOPPED, &hctx->state);  in blk_mq_hctx_stopped()
234 static inline bool blk_mq_hw_queue_mapped(struct blk_mq_hw_ctx *hctx)  in blk_mq_hw_queue_mapped()  argument
236 	return hctx->nr_ctx && hctx->tags;  in blk_mq_hw_queue_mapped()
274 static inline void __blk_mq_inc_active_requests(struct blk_mq_hw_ctx *hctx)  in __blk_mq_inc_active_requests()  argument
276 	if (blk_mq_is_shared_tags(hctx->flags))  in __blk_mq_inc_active_requests()
277 		atomic_inc(&hctx->queue->nr_active_requests_shared_tags);  in __blk_mq_inc_active_requests()
279 		atomic_inc(&hctx->nr_active);  in __blk_mq_inc_active_requests()
282 static inline void __blk_mq_sub_active_requests(struct blk_mq_hw_ctx *hctx,  in __blk_mq_sub_active_requests()  argument
285 	if (blk_mq_is_shared_tags(hctx->flags))  in __blk_mq_sub_active_requests()
286 		atomic_sub(val, &hctx->queue->nr_active_requests_shared_tags);  in __blk_mq_sub_active_requests()
288 		atomic_sub(val, &hctx->nr_active);  in __blk_mq_sub_active_requests()
291 static inline void __blk_mq_dec_active_requests(struct blk_mq_hw_ctx *hctx)  in __blk_mq_dec_active_requests()  argument
293 	__blk_mq_sub_active_requests(hctx, 1);  in __blk_mq_dec_active_requests()
296 static inline int __blk_mq_active_requests(struct blk_mq_hw_ctx *hctx)  in __blk_mq_active_requests()  argument
298 	if (blk_mq_is_shared_tags(hctx->flags))  in __blk_mq_active_requests()
299 		return atomic_read(&hctx->queue->nr_active_requests_shared_tags);  in __blk_mq_active_requests()
300 	return atomic_read(&hctx->nr_active);  in __blk_mq_active_requests()
302 static inline void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,  in __blk_mq_put_driver_tag()  argument
305 	blk_mq_put_tag(hctx->tags, rq->mq_ctx, rq->tag);  in __blk_mq_put_driver_tag()
310 		__blk_mq_dec_active_requests(hctx);  in __blk_mq_put_driver_tag()
322 bool __blk_mq_get_driver_tag(struct blk_mq_hw_ctx *hctx, struct request *rq);
326 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in blk_mq_get_driver_tag()  local
329 	    !(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {  in blk_mq_get_driver_tag()
330 		hctx->tags->rqs[rq->tag] = rq;  in blk_mq_get_driver_tag()
334 	return __blk_mq_get_driver_tag(hctx, rq);  in blk_mq_get_driver_tag()
391 static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,  in hctx_may_queue()  argument
396 	if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED))  in hctx_may_queue()
405 	if (blk_mq_is_shared_tags(hctx->flags)) {  in hctx_may_queue()
406 		struct request_queue *q = hctx->queue;  in hctx_may_queue()
411 		if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))  in hctx_may_queue()
415 	users = READ_ONCE(hctx->tags->active_queues);  in hctx_may_queue()
423 	return __blk_mq_active_requests(hctx) < depth;  in hctx_may_queue()