464 	struct kyber_hctx_data *khd;  in kyber_init_hctx()  local
467 	khd = kmalloc_node(sizeof(*khd), GFP_KERNEL, hctx->numa_node);  in kyber_init_hctx()
468 	if (!khd)  in kyber_init_hctx()
471 	khd->kcqs = kmalloc_array_node(hctx->nr_ctx,  in kyber_init_hctx()
474 	if (!khd->kcqs)  in kyber_init_hctx()
478 		kyber_ctx_queue_init(&khd->kcqs[i]);  in kyber_init_hctx()
481 		if (sbitmap_init_node(&khd->kcq_map[i], hctx->nr_ctx,  in kyber_init_hctx()
484 				sbitmap_free(&khd->kcq_map[i]);  in kyber_init_hctx()
489 	spin_lock_init(&khd->lock);  in kyber_init_hctx()
492 		INIT_LIST_HEAD(&khd->rqs[i]);  in kyber_init_hctx()
493 		khd->domain_wait[i].sbq = NULL;  in kyber_init_hctx()
494 		init_waitqueue_func_entry(&khd->domain_wait[i].wait,  in kyber_init_hctx()
496 		khd->domain_wait[i].wait.private = hctx;  in kyber_init_hctx()
497 		INIT_LIST_HEAD(&khd->domain_wait[i].wait.entry);  in kyber_init_hctx()
498 		atomic_set(&khd->wait_index[i], 0);  in kyber_init_hctx()
501 	khd->cur_domain = 0;  in kyber_init_hctx()
502 	khd->batching = 0;  in kyber_init_hctx()
504 	hctx->sched_data = khd;  in kyber_init_hctx()
511 	kfree(khd->kcqs);  in kyber_init_hctx()
513 	kfree(khd);  in kyber_init_hctx()
519 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_exit_hctx()  local
523 		sbitmap_free(&khd->kcq_map[i]);  in kyber_exit_hctx()
524 	kfree(khd->kcqs);  in kyber_exit_hctx()
568 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_bio_merge()  local
570 	struct kyber_ctx_queue *kcq = &khd->kcqs[ctx->index_hw[hctx->type]];  in kyber_bio_merge()
590 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_insert_requests()  local
595 		struct kyber_ctx_queue *kcq = &khd->kcqs[rq->mq_ctx->index_hw[hctx->type]];  in kyber_insert_requests()
603 		sbitmap_set_bit(&khd->kcq_map[sched_domain],  in kyber_insert_requests()
658 	struct kyber_hctx_data *khd;  member
666 	struct kyber_ctx_queue *kcq = &flush_data->khd->kcqs[bitnr];  in flush_busy_kcq()
677 static void kyber_flush_busy_kcqs(struct kyber_hctx_data *khd,  in kyber_flush_busy_kcqs()  argument
682 		.khd = khd,  in kyber_flush_busy_kcqs()
687 	sbitmap_for_each_set(&khd->kcq_map[sched_domain],  in kyber_flush_busy_kcqs()
703 				  struct kyber_hctx_data *khd,  in kyber_get_domain_token()  argument
706 	unsigned int sched_domain = khd->cur_domain;  in kyber_get_domain_token()
708 	struct sbq_wait *wait = &khd->domain_wait[sched_domain];  in kyber_get_domain_token()
721 				  &khd->wait_index[sched_domain]);  in kyber_get_domain_token()
722 		khd->domain_ws[sched_domain] = ws;  in kyber_get_domain_token()
740 		ws = khd->domain_ws[sched_domain];  in kyber_get_domain_token()
751 			  struct kyber_hctx_data *khd,  in kyber_dispatch_cur_domain()  argument
758 	rqs = &khd->rqs[khd->cur_domain];  in kyber_dispatch_cur_domain()
770 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
772 			khd->batching++;  in kyber_dispatch_cur_domain()
778 					      kyber_domain_names[khd->cur_domain]);  in kyber_dispatch_cur_domain()
780 	} else if (sbitmap_any_bit_set(&khd->kcq_map[khd->cur_domain])) {  in kyber_dispatch_cur_domain()
781 		nr = kyber_get_domain_token(kqd, khd, hctx);  in kyber_dispatch_cur_domain()
783 			kyber_flush_busy_kcqs(khd, khd->cur_domain, rqs);  in kyber_dispatch_cur_domain()
785 			khd->batching++;  in kyber_dispatch_cur_domain()
791 					      kyber_domain_names[khd->cur_domain]);  in kyber_dispatch_cur_domain()
802 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_dispatch_request()  local
806 	spin_lock(&khd->lock);  in kyber_dispatch_request()
812 	if (khd->batching < kyber_batch_size[khd->cur_domain]) {  in kyber_dispatch_request()
813 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
827 	khd->batching = 0;  in kyber_dispatch_request()
829 		if (khd->cur_domain == KYBER_NUM_DOMAINS - 1)  in kyber_dispatch_request()
830 			khd->cur_domain = 0;  in kyber_dispatch_request()
832 			khd->cur_domain++;  in kyber_dispatch_request()
834 		rq = kyber_dispatch_cur_domain(kqd, khd, hctx);  in kyber_dispatch_request()
841 	spin_unlock(&khd->lock);  in kyber_dispatch_request()
847 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_has_work()  local
851 		if (!list_empty_careful(&khd->rqs[i]) ||  in kyber_has_work()
852 		    sbitmap_any_bit_set(&khd->kcq_map[i]))  in kyber_has_work()
907 	__acquires(&khd->lock)						\
910 	struct kyber_hctx_data *khd = hctx->sched_data;			\
912 	spin_lock(&khd->lock);						\
913 	return seq_list_start(&khd->rqs[domain], *pos);			\
920 	struct kyber_hctx_data *khd = hctx->sched_data;			\
922 	return seq_list_next(v, &khd->rqs[domain], pos);		\
926 	__releases(&khd->lock)						\
929 	struct kyber_hctx_data *khd = hctx->sched_data;			\
931 	spin_unlock(&khd->lock);					\
944 	struct kyber_hctx_data *khd = hctx->sched_data;			\
945 	wait_queue_entry_t *wait = &khd->domain_wait[domain].wait;	\
968 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_cur_domain_show()  local
970 	seq_printf(m, "%s\n", kyber_domain_names[khd->cur_domain]);  in kyber_cur_domain_show()
977 	struct kyber_hctx_data *khd = hctx->sched_data;  in kyber_batching_show()  local
979 	seq_printf(m, "%u\n", khd->batching);  in kyber_batching_show()