461 static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)  in kyber_init_hctx()  argument
463 	struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;  in kyber_init_hctx()
467 	khd = kmalloc_node(sizeof(*khd), GFP_KERNEL, hctx->numa_node);  in kyber_init_hctx()
471 	khd->kcqs = kmalloc_array_node(hctx->nr_ctx,  in kyber_init_hctx()
473 				       GFP_KERNEL, hctx->numa_node);  in kyber_init_hctx()
477 	for (i = 0; i < hctx->nr_ctx; i++)  in kyber_init_hctx()
481 		if (sbitmap_init_node(&khd->kcq_map[i], hctx->nr_ctx,  in kyber_init_hctx()
482 				      ilog2(8), GFP_KERNEL, hctx->numa_node)) {  in kyber_init_hctx()
496 		khd->domain_wait[i].wait.private = hctx;  in kyber_init_hctx()
504 	hctx->sched_data = khd;  in kyber_init_hctx()
505 	sbitmap_queue_min_shallow_depth(&hctx->sched_tags->bitmap_tags,  in kyber_init_hctx()
517 static void kyber_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)  in kyber_exit_hctx()  argument
519 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_exit_hctx()
525 	kfree(hctx->sched_data);  in kyber_exit_hctx()
565 static bool kyber_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio,  in kyber_bio_merge()  argument
568 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_bio_merge()
569 	struct blk_mq_ctx *ctx = blk_mq_get_ctx(hctx->queue);  in kyber_bio_merge()
570 	struct kyber_ctx_queue *kcq = &khd->kcqs[ctx->index_hw[hctx->type]];  in kyber_bio_merge()
576 	merged = blk_mq_bio_list_merge(hctx->queue, rq_list, bio, nr_segs);  in kyber_bio_merge()
587 static void kyber_insert_requests(struct blk_mq_hw_ctx *hctx,  in kyber_insert_requests()  argument
590 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_insert_requests()
595 		struct kyber_ctx_queue *kcq = &khd->kcqs[rq->mq_ctx->index_hw[hctx->type]];  in kyber_insert_requests()
604 				rq->mq_ctx->index_hw[hctx->type]);  in kyber_insert_requests()
694 	struct blk_mq_hw_ctx *hctx = READ_ONCE(wqe->private);  in kyber_domain_wake()  local
698 	blk_mq_run_hw_queue(hctx, true);  in kyber_domain_wake()
704 				  struct blk_mq_hw_ctx *hctx)  in kyber_get_domain_token()  argument
752 			  struct blk_mq_hw_ctx *hctx)  in kyber_dispatch_cur_domain()  argument
770 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
781 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
799 static struct request *kyber_dispatch_request(struct blk_mq_hw_ctx *hctx)  in kyber_dispatch_request()  argument
801 	struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;  in kyber_dispatch_request()
802 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_dispatch_request()
813 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
834 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
845 static bool kyber_has_work(struct blk_mq_hw_ctx *hctx)  in kyber_has_work()  argument
847 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_has_work()
909 	struct blk_mq_hw_ctx *hctx = m->private;			\
910 	struct kyber_hctx_data *khd = hctx->sched_data;			\
919 	struct blk_mq_hw_ctx *hctx = m->private;			\
920 	struct kyber_hctx_data *khd = hctx->sched_data;			\
928 	struct blk_mq_hw_ctx *hctx = m->private;			\
929 	struct kyber_hctx_data *khd = hctx->sched_data;			\
943 	struct blk_mq_hw_ctx *hctx = data;				\
944 	struct kyber_hctx_data *khd = hctx->sched_data;			\
967 	struct blk_mq_hw_ctx *hctx = data;  in kyber_cur_domain_show()  local
968 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_cur_domain_show()
976 	struct blk_mq_hw_ctx *hctx = data;  in kyber_batching_show()  local
977 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_batching_show()