60 	struct request_queue *q = rq->q;  in elv_iosched_allow_bio_merge()  local
61 	struct elevator_queue *e = q->elevator;  in elv_iosched_allow_bio_merge()
64 		return e->type->ops.mq.allow_merge(q, rq, bio);  in elv_iosched_allow_bio_merge()
66 		return e->type->ops.sq.elevator_allow_bio_merge_fn(q, rq, bio);  in elv_iosched_allow_bio_merge()
116 static struct elevator_type *elevator_get(struct request_queue *q,  in elevator_get()  argument
123 	e = elevator_find(name, q->mq_ops != NULL);  in elevator_get()
128 		e = elevator_find(name, q->mq_ops != NULL);  in elevator_get()
174 struct elevator_queue *elevator_alloc(struct request_queue *q,  in elevator_alloc()  argument
179 	eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);  in elevator_alloc()
207 int elevator_init(struct request_queue *q)  in elevator_init()  argument
216 	mutex_lock(&q->sysfs_lock);  in elevator_init()
217 	if (unlikely(q->elevator))  in elevator_init()
221 		e = elevator_get(q, chosen_elevator, false);  in elevator_init()
228 		e = elevator_get(q, CONFIG_DEFAULT_IOSCHED, false);  in elevator_init()
232 		e = elevator_get(q, "noop", false);  in elevator_init()
235 	err = e->ops.sq.elevator_init_fn(q, e);  in elevator_init()
239 	mutex_unlock(&q->sysfs_lock);  in elevator_init()
243 void elevator_exit(struct request_queue *q, struct elevator_queue *e)  in elevator_exit()  argument
247 		blk_mq_exit_sched(q, e);  in elevator_exit()
261 void elv_rqhash_del(struct request_queue *q, struct request *rq)  in elv_rqhash_del()  argument
268 void elv_rqhash_add(struct request_queue *q, struct request *rq)  in elv_rqhash_add()  argument
270 	struct elevator_queue *e = q->elevator;  in elv_rqhash_add()
278 void elv_rqhash_reposition(struct request_queue *q, struct request *rq)  in elv_rqhash_reposition()  argument
281 	elv_rqhash_add(q, rq);  in elv_rqhash_reposition()
284 struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)  in elv_rqhash_find()  argument
286 	struct elevator_queue *e = q->elevator;  in elv_rqhash_find()
363 void elv_dispatch_sort(struct request_queue *q, struct request *rq)  in elv_dispatch_sort()  argument
368 	if (q->last_merge == rq)  in elv_dispatch_sort()
369 		q->last_merge = NULL;  in elv_dispatch_sort()
371 	elv_rqhash_del(q, rq);  in elv_dispatch_sort()
373 	q->nr_sorted--;  in elv_dispatch_sort()
375 	boundary = q->end_sector;  in elv_dispatch_sort()
376 	list_for_each_prev(entry, &q->queue_head) {  in elv_dispatch_sort()
405 void elv_dispatch_add_tail(struct request_queue *q, struct request *rq)  in elv_dispatch_add_tail()  argument
407 	if (q->last_merge == rq)  in elv_dispatch_add_tail()
408 		q->last_merge = NULL;  in elv_dispatch_add_tail()
410 	elv_rqhash_del(q, rq);  in elv_dispatch_add_tail()
412 	q->nr_sorted--;  in elv_dispatch_add_tail()
414 	q->end_sector = rq_end_sector(rq);  in elv_dispatch_add_tail()
415 	q->boundary_rq = rq;  in elv_dispatch_add_tail()
416 	list_add_tail(&rq->queuelist, &q->queue_head);  in elv_dispatch_add_tail()
420 enum elv_merge elv_merge(struct request_queue *q, struct request **req,  in elv_merge()  argument
423 	struct elevator_queue *e = q->elevator;  in elv_merge()
432 	if (blk_queue_nomerges(q) || !bio_mergeable(bio))  in elv_merge()
438 	if (q->last_merge && elv_bio_merge_ok(q->last_merge, bio)) {  in elv_merge()
439 		enum elv_merge ret = blk_try_merge(q->last_merge, bio);  in elv_merge()
442 			*req = q->last_merge;  in elv_merge()
447 	if (blk_queue_noxmerges(q))  in elv_merge()
453 	__rq = elv_rqhash_find(q, bio->bi_iter.bi_sector);  in elv_merge()
460 		return e->type->ops.mq.request_merge(q, req, bio);  in elv_merge()
462 		return e->type->ops.sq.elevator_merge_fn(q, req, bio);  in elv_merge()
474 bool elv_attempt_insert_merge(struct request_queue *q, struct request *rq)  in elv_attempt_insert_merge()  argument
479 	if (blk_queue_nomerges(q))  in elv_attempt_insert_merge()
485 	if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq))  in elv_attempt_insert_merge()
488 	if (blk_queue_noxmerges(q))  in elv_attempt_insert_merge()
496 		__rq = elv_rqhash_find(q, blk_rq_pos(rq));  in elv_attempt_insert_merge()
497 		if (!__rq || !blk_attempt_req_merge(q, __rq, rq))  in elv_attempt_insert_merge()
508 void elv_merged_request(struct request_queue *q, struct request *rq,  in elv_merged_request()  argument
511 	struct elevator_queue *e = q->elevator;  in elv_merged_request()
514 		e->type->ops.mq.request_merged(q, rq, type);  in elv_merged_request()
516 		e->type->ops.sq.elevator_merged_fn(q, rq, type);  in elv_merged_request()
519 		elv_rqhash_reposition(q, rq);  in elv_merged_request()
521 	q->last_merge = rq;  in elv_merged_request()
524 void elv_merge_requests(struct request_queue *q, struct request *rq,  in elv_merge_requests()  argument
527 	struct elevator_queue *e = q->elevator;  in elv_merge_requests()
531 		e->type->ops.mq.requests_merged(q, rq, next);  in elv_merge_requests()
535 			e->type->ops.sq.elevator_merge_req_fn(q, rq, next);  in elv_merge_requests()
538 	elv_rqhash_reposition(q, rq);  in elv_merge_requests()
541 		elv_rqhash_del(q, next);  in elv_merge_requests()
542 		q->nr_sorted--;  in elv_merge_requests()
545 	q->last_merge = rq;  in elv_merge_requests()
548 void elv_bio_merged(struct request_queue *q, struct request *rq,  in elv_bio_merged()  argument
551 	struct elevator_queue *e = q->elevator;  in elv_bio_merged()
557 		e->type->ops.sq.elevator_bio_merged_fn(q, rq, bio);  in elv_bio_merged()
563 	if (rq->q->dev && !(rq->rq_flags & RQF_PM))  in blk_pm_requeue_request()
564 		rq->q->nr_pending--;  in blk_pm_requeue_request()
567 static void blk_pm_add_request(struct request_queue *q, struct request *rq)  in blk_pm_add_request()  argument
569 	if (q->dev && !(rq->rq_flags & RQF_PM) && q->nr_pending++ == 0 &&  in blk_pm_add_request()
570 	    (q->rpm_status == RPM_SUSPENDED || q->rpm_status == RPM_SUSPENDING))  in blk_pm_add_request()
571 		pm_request_resume(q->dev);  in blk_pm_add_request()
575 static inline void blk_pm_add_request(struct request_queue *q,  in blk_pm_add_request()  argument
581 void elv_requeue_request(struct request_queue *q, struct request *rq)  in elv_requeue_request()  argument
588 		q->in_flight[rq_is_sync(rq)]--;  in elv_requeue_request()
590 			elv_deactivate_rq(q, rq);  in elv_requeue_request()
597 	__elv_add_request(q, rq, ELEVATOR_INSERT_REQUEUE);  in elv_requeue_request()
600 void elv_drain_elevator(struct request_queue *q)  in elv_drain_elevator()  argument
602 	struct elevator_queue *e = q->elevator;  in elv_drain_elevator()
608 	lockdep_assert_held(q->queue_lock);  in elv_drain_elevator()
610 	while (e->type->ops.sq.elevator_dispatch_fn(q, 1))  in elv_drain_elevator()
612 	if (q->nr_sorted && !blk_queue_is_zoned(q) && printed++ < 10 ) {  in elv_drain_elevator()
615 		       q->elevator->type->elevator_name, q->nr_sorted);  in elv_drain_elevator()
619 void __elv_add_request(struct request_queue *q, struct request *rq, int where)  in __elv_add_request()  argument
621 	trace_block_rq_insert(q, rq);  in __elv_add_request()
623 	blk_pm_add_request(q, rq);  in __elv_add_request()
625 	rq->q = q;  in __elv_add_request()
630 			q->end_sector = rq_end_sector(rq);  in __elv_add_request()
631 			q->boundary_rq = rq;  in __elv_add_request()
642 		list_add(&rq->queuelist, &q->queue_head);  in __elv_add_request()
647 		elv_drain_elevator(q);  in __elv_add_request()
648 		list_add_tail(&rq->queuelist, &q->queue_head);  in __elv_add_request()
659 		__blk_run_queue(q);  in __elv_add_request()
668 		if (elv_attempt_insert_merge(q, rq))  in __elv_add_request()
674 		q->nr_sorted++;  in __elv_add_request()
676 			elv_rqhash_add(q, rq);  in __elv_add_request()
677 			if (!q->last_merge)  in __elv_add_request()
678 				q->last_merge = rq;  in __elv_add_request()
686 		q->elevator->type->ops.sq.elevator_add_req_fn(q, rq);  in __elv_add_request()
701 void elv_add_request(struct request_queue *q, struct request *rq, int where)  in elv_add_request()  argument
705 	spin_lock_irqsave(q->queue_lock, flags);  in elv_add_request()
706 	__elv_add_request(q, rq, where);  in elv_add_request()
707 	spin_unlock_irqrestore(q->queue_lock, flags);  in elv_add_request()
711 struct request *elv_latter_request(struct request_queue *q, struct request *rq)  in elv_latter_request()  argument
713 	struct elevator_queue *e = q->elevator;  in elv_latter_request()
716 		return e->type->ops.mq.next_request(q, rq);  in elv_latter_request()
718 		return e->type->ops.sq.elevator_latter_req_fn(q, rq);  in elv_latter_request()
723 struct request *elv_former_request(struct request_queue *q, struct request *rq)  in elv_former_request()  argument
725 	struct elevator_queue *e = q->elevator;  in elv_former_request()
728 		return e->type->ops.mq.former_request(q, rq);  in elv_former_request()
730 		return e->type->ops.sq.elevator_former_req_fn(q, rq);  in elv_former_request()
734 int elv_set_request(struct request_queue *q, struct request *rq,  in elv_set_request()  argument
737 	struct elevator_queue *e = q->elevator;  in elv_set_request()
743 		return e->type->ops.sq.elevator_set_req_fn(q, rq, bio, gfp_mask);  in elv_set_request()
747 void elv_put_request(struct request_queue *q, struct request *rq)  in elv_put_request()  argument
749 	struct elevator_queue *e = q->elevator;  in elv_put_request()
758 int elv_may_queue(struct request_queue *q, unsigned int op)  in elv_may_queue()  argument
760 	struct elevator_queue *e = q->elevator;  in elv_may_queue()
766 		return e->type->ops.sq.elevator_may_queue_fn(q, op);  in elv_may_queue()
771 void elv_completed_request(struct request_queue *q, struct request *rq)  in elv_completed_request()  argument
773 	struct elevator_queue *e = q->elevator;  in elv_completed_request()
782 		q->in_flight[rq_is_sync(rq)]--;  in elv_completed_request()
785 			e->type->ops.sq.elevator_completed_req_fn(q, rq);  in elv_completed_request()
836 int elv_register_queue(struct request_queue *q)  in elv_register_queue()  argument
838 	struct elevator_queue *e = q->elevator;  in elv_register_queue()
841 	lockdep_assert_held(&q->sysfs_lock);  in elv_register_queue()
843 	error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");  in elv_register_queue()
856 			e->type->ops.sq.elevator_registered_fn(q);  in elv_register_queue()
861 void elv_unregister_queue(struct request_queue *q)  in elv_unregister_queue()  argument
863 	lockdep_assert_held(&q->sysfs_lock);  in elv_unregister_queue()
865 	if (q) {  in elv_unregister_queue()
866 		struct elevator_queue *e = q->elevator;  in elv_unregister_queue()
872 		wbt_enable_default(q);  in elv_unregister_queue()
935 int elevator_switch_mq(struct request_queue *q,  in elevator_switch_mq()  argument
940 	lockdep_assert_held(&q->sysfs_lock);  in elevator_switch_mq()
942 	if (q->elevator) {  in elevator_switch_mq()
943 		if (q->elevator->registered)  in elevator_switch_mq()
944 			elv_unregister_queue(q);  in elevator_switch_mq()
945 		ioc_clear_queue(q);  in elevator_switch_mq()
946 		elevator_exit(q, q->elevator);  in elevator_switch_mq()
949 	ret = blk_mq_init_sched(q, new_e);  in elevator_switch_mq()
954 		ret = elv_register_queue(q);  in elevator_switch_mq()
956 			elevator_exit(q, q->elevator);  in elevator_switch_mq()
962 		blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);  in elevator_switch_mq()
964 		blk_add_trace_msg(q, "elv switch: none");  in elevator_switch_mq()
975 int elevator_init_mq(struct request_queue *q)  in elevator_init_mq()  argument
980 	if (q->nr_hw_queues != 1)  in elevator_init_mq()
987 	mutex_lock(&q->sysfs_lock);  in elevator_init_mq()
988 	if (unlikely(q->elevator))  in elevator_init_mq()
991 	e = elevator_get(q, "mq-deadline", false);  in elevator_init_mq()
995 	err = blk_mq_init_sched(q, e);  in elevator_init_mq()
999 	mutex_unlock(&q->sysfs_lock);  in elevator_init_mq()
1010 static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)  in elevator_switch()  argument
1012 	struct elevator_queue *old = q->elevator;  in elevator_switch()
1016 	lockdep_assert_held(&q->sysfs_lock);  in elevator_switch()
1018 	if (q->mq_ops) {  in elevator_switch()
1019 		blk_mq_freeze_queue(q);  in elevator_switch()
1020 		blk_mq_quiesce_queue(q);  in elevator_switch()
1022 		err = elevator_switch_mq(q, new_e);  in elevator_switch()
1024 		blk_mq_unquiesce_queue(q);  in elevator_switch()
1025 		blk_mq_unfreeze_queue(q);  in elevator_switch()
1040 		blk_queue_bypass_start(q);  in elevator_switch()
1044 			elv_unregister_queue(q);  in elevator_switch()
1046 		ioc_clear_queue(q);  in elevator_switch()
1050 	err = new_e->ops.sq.elevator_init_fn(q, new_e);  in elevator_switch()
1054 	err = elv_register_queue(q);  in elevator_switch()
1060 		elevator_exit(q, old);  in elevator_switch()
1061 		blk_queue_bypass_end(q);  in elevator_switch()
1064 	blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);  in elevator_switch()
1069 	elevator_exit(q, q->elevator);  in elevator_switch()
1073 		q->elevator = old;  in elevator_switch()
1074 		elv_register_queue(q);  in elevator_switch()
1075 		blk_queue_bypass_end(q);  in elevator_switch()
1084 static int __elevator_change(struct request_queue *q, const char *name)  in __elevator_change()  argument
1090 	if (!test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))  in __elevator_change()
1096 	if (q->mq_ops && !strncmp(name, "none", 4))  in __elevator_change()
1097 		return elevator_switch(q, NULL);  in __elevator_change()
1100 	e = elevator_get(q, strstrip(elevator_name), true);  in __elevator_change()
1104 	if (q->elevator && elevator_match(q->elevator->type, elevator_name)) {  in __elevator_change()
1109 	return elevator_switch(q, e);  in __elevator_change()
1112 static inline bool elv_support_iosched(struct request_queue *q)  in elv_support_iosched()  argument
1114 	if (q->mq_ops && q->tag_set && (q->tag_set->flags &  in elv_support_iosched()
1120 ssize_t elv_iosched_store(struct request_queue *q, const char *name,  in elv_iosched_store()  argument
1125 	if (!(q->mq_ops || q->request_fn) || !elv_support_iosched(q))  in elv_iosched_store()
1128 	ret = __elevator_change(q, name);  in elv_iosched_store()
1135 ssize_t elv_iosched_show(struct request_queue *q, char *name)  in elv_iosched_show()  argument
1137 	struct elevator_queue *e = q->elevator;  in elv_iosched_show()
1140 	bool uses_mq = q->mq_ops != NULL;  in elv_iosched_show()
1143 	if (!queue_is_rq_based(q))  in elv_iosched_show()
1146 	if (!q->elevator)  in elv_iosched_show()
1158 		if (__e->uses_mq && q->mq_ops && elv_support_iosched(q))  in elv_iosched_show()
1160 		else if (!__e->uses_mq && !q->mq_ops)  in elv_iosched_show()
1165 	if (q->mq_ops && q->elevator)  in elv_iosched_show()
1172 struct request *elv_rb_former_request(struct request_queue *q,  in elv_rb_former_request()  argument
1184 struct request *elv_rb_latter_request(struct request_queue *q,  in elv_rb_latter_request()  argument