63 	struct elevator_queue *e = q->elevator;  in elv_iosched_allow_bio_merge()  local
65 	if (e->type->ops.allow_merge)  in elv_iosched_allow_bio_merge()
66 		return e->type->ops.allow_merge(q, rq, bio);  in elv_iosched_allow_bio_merge()
94  * @e: Scheduler to test
98  * Return true if the elevator @e name matches @name and if @e provides all
101 static bool elevator_match(const struct elevator_type *e, const char *name,  in elevator_match()  argument
104 	if (!elv_support_features(e->elevator_features, required_features))  in elevator_match()
106 	if (!strcmp(e->elevator_name, name))  in elevator_match()
108 	if (e->elevator_alias && !strcmp(e->elevator_alias, name))  in elevator_match()
125 	struct elevator_type *e;  in elevator_find()  local
127 	list_for_each_entry(e, &elv_list, list) {  in elevator_find()
128 		if (elevator_match(e, name, required_features))  in elevator_find()
129 			return e;  in elevator_find()
135 static void elevator_put(struct elevator_type *e)  in elevator_put()  argument
137 	module_put(e->elevator_owner);  in elevator_put()
143 	struct elevator_type *e;  in elevator_get()  local
147 	e = elevator_find(name, q->required_elevator_features);  in elevator_get()
148 	if (!e && try_loading) {  in elevator_get()
152 		e = elevator_find(name, q->required_elevator_features);  in elevator_get()
155 	if (e && !try_module_get(e->elevator_owner))  in elevator_get()
156 		e = NULL;  in elevator_get()
159 	return e;  in elevator_get()
165 				  struct elevator_type *e)  in elevator_alloc()  argument
173 	eq->type = e;  in elevator_alloc()
184 	struct elevator_queue *e;  in elevator_release()  local
186 	e = container_of(kobj, struct elevator_queue, kobj);  in elevator_release()
187 	elevator_put(e->type);  in elevator_release()
188 	kfree(e);  in elevator_release()
191 void __elevator_exit(struct request_queue *q, struct elevator_queue *e)  in __elevator_exit()  argument
193 	mutex_lock(&e->sysfs_lock);  in __elevator_exit()
194 	blk_mq_exit_sched(q, e);  in __elevator_exit()
195 	mutex_unlock(&e->sysfs_lock);  in __elevator_exit()
197 	kobject_put(&e->kobj);  in __elevator_exit()
215 	struct elevator_queue *e = q->elevator;  in elv_rqhash_add()  local
218 	hash_add(e->hash, &rq->hash, rq_hash_key(rq));  in elv_rqhash_add()
231 	struct elevator_queue *e = q->elevator;  in elv_rqhash_find()  local
235 	hash_for_each_possible_safe(e->hash, rq, next, hash, offset) {  in elv_rqhash_find()
306 	struct elevator_queue *e = q->elevator;  in elv_merge()  local
342 	if (e->type->ops.request_merge)  in elv_merge()
343 		return e->type->ops.request_merge(q, req, bio);  in elv_merge()
392 	struct elevator_queue *e = q->elevator;  in elv_merged_request()  local
394 	if (e->type->ops.request_merged)  in elv_merged_request()
395 		e->type->ops.request_merged(q, rq, type);  in elv_merged_request()
406 	struct elevator_queue *e = q->elevator;  in elv_merge_requests()  local
408 	if (e->type->ops.requests_merged)  in elv_merge_requests()
409 		e->type->ops.requests_merged(q, rq, next);  in elv_merge_requests()
417 	struct elevator_queue *e = q->elevator;  in elv_latter_request()  local
419 	if (e->type->ops.next_request)  in elv_latter_request()
420 		return e->type->ops.next_request(q, rq);  in elv_latter_request()
427 	struct elevator_queue *e = q->elevator;  in elv_former_request()  local
429 	if (e->type->ops.former_request)  in elv_former_request()
430 		return e->type->ops.former_request(q, rq);  in elv_former_request()
441 	struct elevator_queue *e;  in elv_attr_show()  local
447 	e = container_of(kobj, struct elevator_queue, kobj);  in elv_attr_show()
448 	mutex_lock(&e->sysfs_lock);  in elv_attr_show()
449 	error = e->type ? entry->show(e, page) : -ENOENT;  in elv_attr_show()
450 	mutex_unlock(&e->sysfs_lock);  in elv_attr_show()
459 	struct elevator_queue *e;  in elv_attr_store()  local
465 	e = container_of(kobj, struct elevator_queue, kobj);  in elv_attr_store()
466 	mutex_lock(&e->sysfs_lock);  in elv_attr_store()
467 	error = e->type ? entry->store(e, page, length) : -ENOENT;  in elv_attr_store()
468 	mutex_unlock(&e->sysfs_lock);  in elv_attr_store()
484 	struct elevator_queue *e = q->elevator;  in elv_register_queue()  local
489 	error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");  in elv_register_queue()
491 		struct elv_fs_entry *attr = e->type->elevator_attrs;  in elv_register_queue()
494 				if (sysfs_create_file(&e->kobj, &attr->attr))  in elv_register_queue()
500 			kobject_uevent(&e->kobj, KOBJ_ADD);  in elv_register_queue()
502 		e->registered = 1;  in elv_register_queue()
512 		struct elevator_queue *e = q->elevator;  in elv_unregister_queue()  local
514 		kobject_uevent(&e->kobj, KOBJ_REMOVE);  in elv_unregister_queue()
515 		kobject_del(&e->kobj);  in elv_unregister_queue()
517 		e->registered = 0;  in elv_unregister_queue()
523 int elv_register(struct elevator_type *e)  in elv_register()  argument
526 	if (e->icq_size) {  in elv_register()
527 		if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||  in elv_register()
528 		    WARN_ON(e->icq_align < __alignof__(struct io_cq)))  in elv_register()
531 		snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),  in elv_register()
532 			 "%s_io_cq", e->elevator_name);  in elv_register()
533 		e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,  in elv_register()
534 						 e->icq_align, 0, NULL);  in elv_register()
535 		if (!e->icq_cache)  in elv_register()
541 	if (elevator_find(e->elevator_name, 0)) {  in elv_register()
543 		kmem_cache_destroy(e->icq_cache);  in elv_register()
546 	list_add_tail(&e->list, &elv_list);  in elv_register()
549 	printk(KERN_INFO "io scheduler %s registered\n", e->elevator_name);  in elv_register()
555 void elv_unregister(struct elevator_type *e)  in elv_unregister()  argument
559 	list_del_init(&e->list);  in elv_unregister()
566 	if (e->icq_cache) {  in elv_unregister()
568 		kmem_cache_destroy(e->icq_cache);  in elv_unregister()
569 		e->icq_cache = NULL;  in elv_unregister()
636 	struct elevator_type *e, *found = NULL;  in elevator_get_by_features()  local
640 	list_for_each_entry(e, &elv_list, list) {  in elevator_get_by_features()
641 		if (elv_support_features(e->elevator_features,  in elevator_get_by_features()
643 			found = e;  in elevator_get_by_features()
663 	struct elevator_type *e;  in elevator_init_mq()  local
675 		e = elevator_get_default(q);  in elevator_init_mq()
677 		e = elevator_get_by_features(q);  in elevator_init_mq()
678 	if (!e)  in elevator_init_mq()
684 	err = blk_mq_init_sched(q, e);  in elevator_init_mq()
691 			"falling back to \"none\"\n", e->elevator_name);  in elevator_init_mq()
692 		elevator_put(e);  in elevator_init_mq()
726 	struct elevator_type *e;  in __elevator_change()  local
742 	e = elevator_get(q, strstrip(elevator_name), true);  in __elevator_change()
743 	if (!e)  in __elevator_change()
748 		elevator_put(e);  in __elevator_change()
752 	return elevator_switch(q, e);  in __elevator_change()
772 	struct elevator_queue *e = q->elevator;  in elv_iosched_show()  local
783 		elv = e->type;  in elv_iosched_show()