406 static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)  in kyber_init_hctx()  argument
408 	struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;  in kyber_init_hctx()
412 	khd = kmalloc_node(sizeof(*khd), GFP_KERNEL, hctx->numa_node);  in kyber_init_hctx()
416 	khd->kcqs = kmalloc_array_node(hctx->nr_ctx,  in kyber_init_hctx()
418 				       GFP_KERNEL, hctx->numa_node);  in kyber_init_hctx()
422 	for (i = 0; i < hctx->nr_ctx; i++)  in kyber_init_hctx()
426 		if (sbitmap_init_node(&khd->kcq_map[i], hctx->nr_ctx,  in kyber_init_hctx()
427 				      ilog2(8), GFP_KERNEL, hctx->numa_node)) {  in kyber_init_hctx()
440 		khd->domain_wait[i].private = hctx;  in kyber_init_hctx()
448 	hctx->sched_data = khd;  in kyber_init_hctx()
449 	sbitmap_queue_min_shallow_depth(&hctx->sched_tags->bitmap_tags,  in kyber_init_hctx()
461 static void kyber_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)  in kyber_exit_hctx()  argument
463 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_exit_hctx()
469 	kfree(hctx->sched_data);  in kyber_exit_hctx()
509 static bool kyber_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio)  in kyber_bio_merge()  argument
511 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_bio_merge()
512 	struct blk_mq_ctx *ctx = blk_mq_get_ctx(hctx->queue);  in kyber_bio_merge()
519 	merged = blk_mq_bio_list_merge(hctx->queue, rq_list, bio);  in kyber_bio_merge()
531 static void kyber_insert_requests(struct blk_mq_hw_ctx *hctx,  in kyber_insert_requests()  argument
534 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_insert_requests()
635 	struct blk_mq_hw_ctx *hctx = READ_ONCE(wait->private);  in kyber_domain_wake()  local
638 	blk_mq_run_hw_queue(hctx, true);  in kyber_domain_wake()
644 				  struct blk_mq_hw_ctx *hctx)  in kyber_get_domain_token()  argument
692 			  struct blk_mq_hw_ctx *hctx)  in kyber_dispatch_cur_domain()  argument
710 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
718 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
733 static struct request *kyber_dispatch_request(struct blk_mq_hw_ctx *hctx)  in kyber_dispatch_request()  argument
735 	struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;  in kyber_dispatch_request()
736 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_dispatch_request()
747 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
768 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
779 static bool kyber_has_work(struct blk_mq_hw_ctx *hctx)  in kyber_has_work()  argument
781 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_has_work()
843 	struct blk_mq_hw_ctx *hctx = m->private;			\
844 	struct kyber_hctx_data *khd = hctx->sched_data;			\
853 	struct blk_mq_hw_ctx *hctx = m->private;			\
854 	struct kyber_hctx_data *khd = hctx->sched_data;			\
862 	struct blk_mq_hw_ctx *hctx = m->private;			\
863 	struct kyber_hctx_data *khd = hctx->sched_data;			\
877 	struct blk_mq_hw_ctx *hctx = data;				\
878 	struct kyber_hctx_data *khd = hctx->sched_data;			\
900 	struct blk_mq_hw_ctx *hctx = data;  in kyber_cur_domain_show()  local
901 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_cur_domain_show()
922 	struct blk_mq_hw_ctx *hctx = data;  in kyber_batching_show()  local
923 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_batching_show()