61 	struct elevator_queue *e = q->elevator;  in elv_iosched_allow_bio_merge()  local
63 	if (e->uses_mq && e->type->ops.mq.allow_merge)  in elv_iosched_allow_bio_merge()
64 		return e->type->ops.mq.allow_merge(q, rq, bio);  in elv_iosched_allow_bio_merge()
65 	else if (!e->uses_mq && e->type->ops.sq.elevator_allow_bio_merge_fn)  in elv_iosched_allow_bio_merge()
66 		return e->type->ops.sq.elevator_allow_bio_merge_fn(q, rq, bio);  in elv_iosched_allow_bio_merge()
86 static bool elevator_match(const struct elevator_type *e, const char *name)  in elevator_match()  argument
88 	if (!strcmp(e->elevator_name, name))  in elevator_match()
90 	if (e->elevator_alias && !strcmp(e->elevator_alias, name))  in elevator_match()
101 	struct elevator_type *e;  in elevator_find()  local
103 	list_for_each_entry(e, &elv_list, list) {  in elevator_find()
104 		if (elevator_match(e, name) && (mq == e->uses_mq))  in elevator_find()
105 			return e;  in elevator_find()
111 static void elevator_put(struct elevator_type *e)  in elevator_put()  argument
113 	module_put(e->elevator_owner);  in elevator_put()
119 	struct elevator_type *e;  in elevator_get()  local
123 	e = elevator_find(name, q->mq_ops != NULL);  in elevator_get()
124 	if (!e && try_loading) {  in elevator_get()
128 		e = elevator_find(name, q->mq_ops != NULL);  in elevator_get()
131 	if (e && !try_module_get(e->elevator_owner))  in elevator_get()
132 		e = NULL;  in elevator_get()
135 	return e;  in elevator_get()
155 	struct elevator_type *e;  in load_default_elevator_module()  local
165 	e = elevator_find(chosen_elevator, false);  in load_default_elevator_module()
168 	if (!e)  in load_default_elevator_module()
175 				  struct elevator_type *e)  in elevator_alloc()  argument
183 	eq->type = e;  in elevator_alloc()
187 	eq->uses_mq = e->uses_mq;  in elevator_alloc()
195 	struct elevator_queue *e;  in elevator_release()  local
197 	e = container_of(kobj, struct elevator_queue, kobj);  in elevator_release()
198 	elevator_put(e->type);  in elevator_release()
199 	kfree(e);  in elevator_release()
209 	struct elevator_type *e = NULL;  in elevator_init()  local
221 		e = elevator_get(q, chosen_elevator, false);  in elevator_init()
222 		if (!e)  in elevator_init()
227 	if (!e)  in elevator_init()
228 		e = elevator_get(q, CONFIG_DEFAULT_IOSCHED, false);  in elevator_init()
229 	if (!e) {  in elevator_init()
232 		e = elevator_get(q, "noop", false);  in elevator_init()
235 	err = e->ops.sq.elevator_init_fn(q, e);  in elevator_init()
237 		elevator_put(e);  in elevator_init()
243 void elevator_exit(struct request_queue *q, struct elevator_queue *e)  in elevator_exit()  argument
245 	mutex_lock(&e->sysfs_lock);  in elevator_exit()
246 	if (e->uses_mq && e->type->ops.mq.exit_sched)  in elevator_exit()
247 		blk_mq_exit_sched(q, e);  in elevator_exit()
248 	else if (!e->uses_mq && e->type->ops.sq.elevator_exit_fn)  in elevator_exit()
249 		e->type->ops.sq.elevator_exit_fn(e);  in elevator_exit()
250 	mutex_unlock(&e->sysfs_lock);  in elevator_exit()
252 	kobject_put(&e->kobj);  in elevator_exit()
270 	struct elevator_queue *e = q->elevator;  in elv_rqhash_add()  local
273 	hash_add(e->hash, &rq->hash, rq_hash_key(rq));  in elv_rqhash_add()
286 	struct elevator_queue *e = q->elevator;  in elv_rqhash_find()  local
290 	hash_for_each_possible_safe(e->hash, rq, next, hash, offset) {  in elv_rqhash_find()
423 	struct elevator_queue *e = q->elevator;  in elv_merge()  local
459 	if (e->uses_mq && e->type->ops.mq.request_merge)  in elv_merge()
460 		return e->type->ops.mq.request_merge(q, req, bio);  in elv_merge()
461 	else if (!e->uses_mq && e->type->ops.sq.elevator_merge_fn)  in elv_merge()
462 		return e->type->ops.sq.elevator_merge_fn(q, req, bio);  in elv_merge()
511 	struct elevator_queue *e = q->elevator;  in elv_merged_request()  local
513 	if (e->uses_mq && e->type->ops.mq.request_merged)  in elv_merged_request()
514 		e->type->ops.mq.request_merged(q, rq, type);  in elv_merged_request()
515 	else if (!e->uses_mq && e->type->ops.sq.elevator_merged_fn)  in elv_merged_request()
516 		e->type->ops.sq.elevator_merged_fn(q, rq, type);  in elv_merged_request()
527 	struct elevator_queue *e = q->elevator;  in elv_merge_requests()  local
530 	if (e->uses_mq && e->type->ops.mq.requests_merged)  in elv_merge_requests()
531 		e->type->ops.mq.requests_merged(q, rq, next);  in elv_merge_requests()
532 	else if (e->type->ops.sq.elevator_merge_req_fn) {  in elv_merge_requests()
535 			e->type->ops.sq.elevator_merge_req_fn(q, rq, next);  in elv_merge_requests()
551 	struct elevator_queue *e = q->elevator;  in elv_bio_merged()  local
553 	if (WARN_ON_ONCE(e->uses_mq))  in elv_bio_merged()
556 	if (e->type->ops.sq.elevator_bio_merged_fn)  in elv_bio_merged()
557 		e->type->ops.sq.elevator_bio_merged_fn(q, rq, bio);  in elv_bio_merged()
602 	struct elevator_queue *e = q->elevator;  in elv_drain_elevator()  local
605 	if (WARN_ON_ONCE(e->uses_mq))  in elv_drain_elevator()
610 	while (e->type->ops.sq.elevator_dispatch_fn(q, 1))  in elv_drain_elevator()
713 	struct elevator_queue *e = q->elevator;  in elv_latter_request()  local
715 	if (e->uses_mq && e->type->ops.mq.next_request)  in elv_latter_request()
716 		return e->type->ops.mq.next_request(q, rq);  in elv_latter_request()
717 	else if (!e->uses_mq && e->type->ops.sq.elevator_latter_req_fn)  in elv_latter_request()
718 		return e->type->ops.sq.elevator_latter_req_fn(q, rq);  in elv_latter_request()
725 	struct elevator_queue *e = q->elevator;  in elv_former_request()  local
727 	if (e->uses_mq && e->type->ops.mq.former_request)  in elv_former_request()
728 		return e->type->ops.mq.former_request(q, rq);  in elv_former_request()
729 	if (!e->uses_mq && e->type->ops.sq.elevator_former_req_fn)  in elv_former_request()
730 		return e->type->ops.sq.elevator_former_req_fn(q, rq);  in elv_former_request()
737 	struct elevator_queue *e = q->elevator;  in elv_set_request()  local
739 	if (WARN_ON_ONCE(e->uses_mq))  in elv_set_request()
742 	if (e->type->ops.sq.elevator_set_req_fn)  in elv_set_request()
743 		return e->type->ops.sq.elevator_set_req_fn(q, rq, bio, gfp_mask);  in elv_set_request()
749 	struct elevator_queue *e = q->elevator;  in elv_put_request()  local
751 	if (WARN_ON_ONCE(e->uses_mq))  in elv_put_request()
754 	if (e->type->ops.sq.elevator_put_req_fn)  in elv_put_request()
755 		e->type->ops.sq.elevator_put_req_fn(rq);  in elv_put_request()
760 	struct elevator_queue *e = q->elevator;  in elv_may_queue()  local
762 	if (WARN_ON_ONCE(e->uses_mq))  in elv_may_queue()
765 	if (e->type->ops.sq.elevator_may_queue_fn)  in elv_may_queue()
766 		return e->type->ops.sq.elevator_may_queue_fn(q, op);  in elv_may_queue()
773 	struct elevator_queue *e = q->elevator;  in elv_completed_request()  local
775 	if (WARN_ON_ONCE(e->uses_mq))  in elv_completed_request()
784 		    e->type->ops.sq.elevator_completed_req_fn)  in elv_completed_request()
785 			e->type->ops.sq.elevator_completed_req_fn(q, rq);  in elv_completed_request()
795 	struct elevator_queue *e;  in elv_attr_show()  local
801 	e = container_of(kobj, struct elevator_queue, kobj);  in elv_attr_show()
802 	mutex_lock(&e->sysfs_lock);  in elv_attr_show()
803 	error = e->type ? entry->show(e, page) : -ENOENT;  in elv_attr_show()
804 	mutex_unlock(&e->sysfs_lock);  in elv_attr_show()
813 	struct elevator_queue *e;  in elv_attr_store()  local
819 	e = container_of(kobj, struct elevator_queue, kobj);  in elv_attr_store()
820 	mutex_lock(&e->sysfs_lock);  in elv_attr_store()
821 	error = e->type ? entry->store(e, page, length) : -ENOENT;  in elv_attr_store()
822 	mutex_unlock(&e->sysfs_lock);  in elv_attr_store()
838 	struct elevator_queue *e = q->elevator;  in elv_register_queue()  local
843 	error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");  in elv_register_queue()
845 		struct elv_fs_entry *attr = e->type->elevator_attrs;  in elv_register_queue()
848 				if (sysfs_create_file(&e->kobj, &attr->attr))  in elv_register_queue()
853 		kobject_uevent(&e->kobj, KOBJ_ADD);  in elv_register_queue()
854 		e->registered = 1;  in elv_register_queue()
855 		if (!e->uses_mq && e->type->ops.sq.elevator_registered_fn)  in elv_register_queue()
856 			e->type->ops.sq.elevator_registered_fn(q);  in elv_register_queue()
866 		struct elevator_queue *e = q->elevator;  in elv_unregister_queue()  local
868 		kobject_uevent(&e->kobj, KOBJ_REMOVE);  in elv_unregister_queue()
869 		kobject_del(&e->kobj);  in elv_unregister_queue()
870 		e->registered = 0;  in elv_unregister_queue()
876 int elv_register(struct elevator_type *e)  in elv_register()  argument
881 	if (e->icq_size) {  in elv_register()
882 		if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||  in elv_register()
883 		    WARN_ON(e->icq_align < __alignof__(struct io_cq)))  in elv_register()
886 		snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),  in elv_register()
887 			 "%s_io_cq", e->elevator_name);  in elv_register()
888 		e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,  in elv_register()
889 						 e->icq_align, 0, NULL);  in elv_register()
890 		if (!e->icq_cache)  in elv_register()
896 	if (elevator_find(e->elevator_name, e->uses_mq)) {  in elv_register()
898 		kmem_cache_destroy(e->icq_cache);  in elv_register()
901 	list_add_tail(&e->list, &elv_list);  in elv_register()
905 	if (elevator_match(e, chosen_elevator) ||  in elv_register()
907 			 elevator_match(e, CONFIG_DEFAULT_IOSCHED)))  in elv_register()
910 	printk(KERN_INFO "io scheduler %s registered%s\n", e->elevator_name,  in elv_register()
916 void elv_unregister(struct elevator_type *e)  in elv_unregister()  argument
920 	list_del_init(&e->list);  in elv_unregister()
927 	if (e->icq_cache) {  in elv_unregister()
929 		kmem_cache_destroy(e->icq_cache);  in elv_unregister()
930 		e->icq_cache = NULL;  in elv_unregister()
977 	struct elevator_type *e;  in elevator_init_mq()  local
991 	e = elevator_get(q, "mq-deadline", false);  in elevator_init_mq()
992 	if (!e)  in elevator_init_mq()
995 	err = blk_mq_init_sched(q, e);  in elevator_init_mq()
997 		elevator_put(e);  in elevator_init_mq()
1087 	struct elevator_type *e;  in __elevator_change()  local
1100 	e = elevator_get(q, strstrip(elevator_name), true);  in __elevator_change()
1101 	if (!e)  in __elevator_change()
1105 		elevator_put(e);  in __elevator_change()
1109 	return elevator_switch(q, e);  in __elevator_change()
1137 	struct elevator_queue *e = q->elevator;  in elv_iosched_show()  local
1149 		elv = e->type;  in elv_iosched_show()