68 deadline_rb_root(struct deadline_data *dd, struct request *rq)  in deadline_rb_root()  argument
70 	return &dd->sort_list[rq_data_dir(rq)];  in deadline_rb_root()
77 deadline_latter_request(struct request *rq)  in deadline_latter_request()  argument
79 	struct rb_node *node = rb_next(&rq->rb_node);  in deadline_latter_request()
88 deadline_add_rq_rb(struct deadline_data *dd, struct request *rq)  in deadline_add_rq_rb()  argument
90 	struct rb_root *root = deadline_rb_root(dd, rq);  in deadline_add_rq_rb()
92 	elv_rb_add(root, rq);  in deadline_add_rq_rb()
96 deadline_del_rq_rb(struct deadline_data *dd, struct request *rq)  in deadline_del_rq_rb()  argument
98 	const int data_dir = rq_data_dir(rq);  in deadline_del_rq_rb()
100 	if (dd->next_rq[data_dir] == rq)  in deadline_del_rq_rb()
101 		dd->next_rq[data_dir] = deadline_latter_request(rq);  in deadline_del_rq_rb()
103 	elv_rb_del(deadline_rb_root(dd, rq), rq);  in deadline_del_rq_rb()
109 static void deadline_remove_request(struct request_queue *q, struct request *rq)  in deadline_remove_request()  argument
113 	list_del_init(&rq->queuelist);  in deadline_remove_request()
118 	if (!RB_EMPTY_NODE(&rq->rb_node))  in deadline_remove_request()
119 		deadline_del_rq_rb(dd, rq);  in deadline_remove_request()
121 	elv_rqhash_del(q, rq);  in deadline_remove_request()
122 	if (q->last_merge == rq)  in deadline_remove_request()
165 deadline_move_request(struct deadline_data *dd, struct request *rq)  in deadline_move_request()  argument
167 	const int data_dir = rq_data_dir(rq);  in deadline_move_request()
171 	dd->next_rq[data_dir] = deadline_latter_request(rq);  in deadline_move_request()
176 	deadline_remove_request(rq->q, rq);  in deadline_move_request()
185 	struct request *rq = rq_entry_fifo(dd->fifo_list[ddir].next);  in deadline_check_fifo()  local
190 	if (time_after_eq(jiffies, (unsigned long)rq->fifo_time))  in deadline_check_fifo()
203 	struct request *rq;  in deadline_fifo_request()  local
212 	rq = rq_entry_fifo(dd->fifo_list[data_dir].next);  in deadline_fifo_request()
213 	if (data_dir == READ || !blk_queue_is_zoned(rq->q))  in deadline_fifo_request()
214 		return rq;  in deadline_fifo_request()
221 	list_for_each_entry(rq, &dd->fifo_list[WRITE], queuelist) {  in deadline_fifo_request()
222 		if (blk_req_can_dispatch_to_zone(rq))  in deadline_fifo_request()
225 	rq = NULL;  in deadline_fifo_request()
229 	return rq;  in deadline_fifo_request()
239 	struct request *rq;  in deadline_next_request()  local
245 	rq = dd->next_rq[data_dir];  in deadline_next_request()
246 	if (!rq)  in deadline_next_request()
249 	if (data_dir == READ || !blk_queue_is_zoned(rq->q))  in deadline_next_request()
250 		return rq;  in deadline_next_request()
257 	while (rq) {  in deadline_next_request()
258 		if (blk_req_can_dispatch_to_zone(rq))  in deadline_next_request()
260 		rq = deadline_latter_request(rq);  in deadline_next_request()
264 	return rq;  in deadline_next_request()
273 	struct request *rq, *next_rq;  in __dd_dispatch_request()  local
278 		rq = list_first_entry(&dd->dispatch, struct request, queuelist);  in __dd_dispatch_request()
279 		list_del_init(&rq->queuelist);  in __dd_dispatch_request()
289 	rq = deadline_next_request(dd, WRITE);  in __dd_dispatch_request()
290 	if (!rq)  in __dd_dispatch_request()
291 		rq = deadline_next_request(dd, READ);  in __dd_dispatch_request()
293 	if (rq && dd->batching < dd->fifo_batch)  in __dd_dispatch_request()
342 		rq = deadline_fifo_request(dd, data_dir);  in __dd_dispatch_request()
348 		rq = next_rq;  in __dd_dispatch_request()
355 	if (!rq)  in __dd_dispatch_request()
365 	deadline_move_request(dd, rq);  in __dd_dispatch_request()
370 	blk_req_zone_write_lock(rq);  in __dd_dispatch_request()
371 	rq->rq_flags |= RQF_STARTED;  in __dd_dispatch_request()
372 	return rq;  in __dd_dispatch_request()
384 	struct request *rq;  in dd_dispatch_request()  local
387 	rq = __dd_dispatch_request(dd);  in dd_dispatch_request()
390 	return rq;  in dd_dispatch_request()
439 static int dd_request_merge(struct request_queue *q, struct request **rq,  in dd_request_merge()  argument
454 			*rq = __rq;  in dd_request_merge()
483 static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,  in dd_insert_request()  argument
488 	const int data_dir = rq_data_dir(rq);  in dd_insert_request()
494 	blk_req_zone_write_unlock(rq);  in dd_insert_request()
496 	if (blk_mq_sched_try_insert_merge(q, rq))  in dd_insert_request()
499 	blk_mq_sched_request_inserted(rq);  in dd_insert_request()
501 	if (at_head || blk_rq_is_passthrough(rq)) {  in dd_insert_request()
503 			list_add(&rq->queuelist, &dd->dispatch);  in dd_insert_request()
505 			list_add_tail(&rq->queuelist, &dd->dispatch);  in dd_insert_request()
507 		deadline_add_rq_rb(dd, rq);  in dd_insert_request()
509 		if (rq_mergeable(rq)) {  in dd_insert_request()
510 			elv_rqhash_add(q, rq);  in dd_insert_request()
512 				q->last_merge = rq;  in dd_insert_request()
518 		rq->fifo_time = jiffies + dd->fifo_expire[data_dir];  in dd_insert_request()
519 		list_add_tail(&rq->queuelist, &dd->fifo_list[data_dir]);  in dd_insert_request()
531 		struct request *rq;  in dd_insert_requests()  local
533 		rq = list_first_entry(list, struct request, queuelist);  in dd_insert_requests()
534 		list_del_init(&rq->queuelist);  in dd_insert_requests()
535 		dd_insert_request(hctx, rq, at_head);  in dd_insert_requests()
544 static void dd_prepare_request(struct request *rq, struct bio *bio)  in dd_prepare_request()  argument
562 static void dd_finish_request(struct request *rq)  in dd_finish_request()  argument
564 	struct request_queue *q = rq->q;  in dd_finish_request()
571 		blk_req_zone_write_unlock(rq);  in dd_finish_request()
573 			blk_mq_sched_mark_restart_hctx(rq->mq_hctx);  in dd_finish_request()
698 	struct request *rq = dd->next_rq[ddir];				\
700 	if (rq)								\
701 		__blk_mq_debugfs_rq_show(m, rq);			\