63 	struct elevator_queue *e = q->elevator;  in elv_iosched_allow_bio_merge()  local
65 	if (e->type->ops.allow_merge)  in elv_iosched_allow_bio_merge()
66 		return e->type->ops.allow_merge(q, rq, bio);  in elv_iosched_allow_bio_merge()
101 static bool elevator_match(const struct elevator_type *e, const char *name,  in elevator_match()  argument
104 	if (!elv_support_features(e->elevator_features, required_features))  in elevator_match()
106 	if (!strcmp(e->elevator_name, name))  in elevator_match()
108 	if (e->elevator_alias && !strcmp(e->elevator_alias, name))  in elevator_match()
125 	struct elevator_type *e;  in elevator_find()  local
127 	list_for_each_entry(e, &elv_list, list) {  in elevator_find()
128 		if (elevator_match(e, name, required_features))  in elevator_find()
129 			return e;  in elevator_find()
135 static void elevator_put(struct elevator_type *e)  in elevator_put()  argument
137 	module_put(e->elevator_owner);  in elevator_put()
143 	struct elevator_type *e;  in elevator_get()  local
147 	e = elevator_find(name, q->required_elevator_features);  in elevator_get()
148 	if (!e && try_loading) {  in elevator_get()
152 		e = elevator_find(name, q->required_elevator_features);  in elevator_get()
155 	if (e && !try_module_get(e->elevator_owner))  in elevator_get()
156 		e = NULL;  in elevator_get()
159 	return e;  in elevator_get()
165 				  struct elevator_type *e)  in elevator_alloc()  argument
173 	eq->type = e;  in elevator_alloc()
184 	struct elevator_queue *e;  in elevator_release()  local
186 	e = container_of(kobj, struct elevator_queue, kobj);  in elevator_release()
187 	elevator_put(e->type);  in elevator_release()
188 	kfree(e);  in elevator_release()
193 	struct elevator_queue *e = q->elevator;  in elevator_exit()  local
198 	mutex_lock(&e->sysfs_lock);  in elevator_exit()
199 	blk_mq_exit_sched(q, e);  in elevator_exit()
200 	mutex_unlock(&e->sysfs_lock);  in elevator_exit()
202 	kobject_put(&e->kobj);  in elevator_exit()
220 	struct elevator_queue *e = q->elevator;  in elv_rqhash_add()  local
223 	hash_add(e->hash, &rq->hash, rq_hash_key(rq));  in elv_rqhash_add()
236 	struct elevator_queue *e = q->elevator;  in elv_rqhash_find()  local
240 	hash_for_each_possible_safe(e->hash, rq, next, hash, offset) {  in elv_rqhash_find()
311 	struct elevator_queue *e = q->elevator;  in elv_merge()  local
350 	if (e->type->ops.request_merge)  in elv_merge()
351 		return e->type->ops.request_merge(q, req, bio);  in elv_merge()
405 	struct elevator_queue *e = q->elevator;  in elv_merged_request()  local
407 	if (e->type->ops.request_merged)  in elv_merged_request()
408 		e->type->ops.request_merged(q, rq, type);  in elv_merged_request()
419 	struct elevator_queue *e = q->elevator;  in elv_merge_requests()  local
421 	if (e->type->ops.requests_merged)  in elv_merge_requests()
422 		e->type->ops.requests_merged(q, rq, next);  in elv_merge_requests()
430 	struct elevator_queue *e = q->elevator;  in elv_latter_request()  local
432 	if (e->type->ops.next_request)  in elv_latter_request()
433 		return e->type->ops.next_request(q, rq);  in elv_latter_request()
440 	struct elevator_queue *e = q->elevator;  in elv_former_request()  local
442 	if (e->type->ops.former_request)  in elv_former_request()
443 		return e->type->ops.former_request(q, rq);  in elv_former_request()
454 	struct elevator_queue *e;  in elv_attr_show()  local
460 	e = container_of(kobj, struct elevator_queue, kobj);  in elv_attr_show()
461 	mutex_lock(&e->sysfs_lock);  in elv_attr_show()
462 	error = e->type ? entry->show(e, page) : -ENOENT;  in elv_attr_show()
463 	mutex_unlock(&e->sysfs_lock);  in elv_attr_show()
472 	struct elevator_queue *e;  in elv_attr_store()  local
478 	e = container_of(kobj, struct elevator_queue, kobj);  in elv_attr_store()
479 	mutex_lock(&e->sysfs_lock);  in elv_attr_store()
480 	error = e->type ? entry->store(e, page, length) : -ENOENT;  in elv_attr_store()
481 	mutex_unlock(&e->sysfs_lock);  in elv_attr_store()
497 	struct elevator_queue *e = q->elevator;  in elv_register_queue()  local
502 	error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");  in elv_register_queue()
504 		struct elv_fs_entry *attr = e->type->elevator_attrs;  in elv_register_queue()
507 				if (sysfs_create_file(&e->kobj, &attr->attr))  in elv_register_queue()
513 			kobject_uevent(&e->kobj, KOBJ_ADD);  in elv_register_queue()
515 		e->registered = 1;  in elv_register_queue()
522 	struct elevator_queue *e = q->elevator;  in elv_unregister_queue()  local
526 	if (e && e->registered) {  in elv_unregister_queue()
527 		struct elevator_queue *e = q->elevator;  in elv_unregister_queue()  local
529 		kobject_uevent(&e->kobj, KOBJ_REMOVE);  in elv_unregister_queue()
530 		kobject_del(&e->kobj);  in elv_unregister_queue()
532 		e->registered = 0;  in elv_unregister_queue()
536 int elv_register(struct elevator_type *e)  in elv_register()  argument
539 	if (WARN_ON_ONCE(!e->ops.insert_requests || !e->ops.dispatch_request))  in elv_register()
543 	if (e->icq_size) {  in elv_register()
544 		if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||  in elv_register()
545 		    WARN_ON(e->icq_align < __alignof__(struct io_cq)))  in elv_register()
548 		snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),  in elv_register()
549 			 "%s_io_cq", e->elevator_name);  in elv_register()
550 		e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,  in elv_register()
551 						 e->icq_align, 0, NULL);  in elv_register()
552 		if (!e->icq_cache)  in elv_register()
558 	if (elevator_find(e->elevator_name, 0)) {  in elv_register()
560 		kmem_cache_destroy(e->icq_cache);  in elv_register()
563 	list_add_tail(&e->list, &elv_list);  in elv_register()
566 	printk(KERN_INFO "io scheduler %s registered\n", e->elevator_name);  in elv_register()
572 void elv_unregister(struct elevator_type *e)  in elv_unregister()  argument
576 	list_del_init(&e->list);  in elv_unregister()
583 	if (e->icq_cache) {  in elv_unregister()
585 		kmem_cache_destroy(e->icq_cache);  in elv_unregister()
586 		e->icq_cache = NULL;  in elv_unregister()
654 	struct elevator_type *e, *found = NULL;  in elevator_get_by_features()  local
658 	list_for_each_entry(e, &elv_list, list) {  in elevator_get_by_features()
659 		if (elv_support_features(e->elevator_features,  in elevator_get_by_features()
661 			found = e;  in elevator_get_by_features()
681 	struct elevator_type *e;  in elevator_init_mq()  local
693 		e = elevator_get_default(q);  in elevator_init_mq()
695 		e = elevator_get_by_features(q);  in elevator_init_mq()
696 	if (!e)  in elevator_init_mq()
709 	err = blk_mq_init_sched(q, e);  in elevator_init_mq()
715 			"falling back to \"none\"\n", e->elevator_name);  in elevator_init_mq()
716 		elevator_put(e);  in elevator_init_mq()
749 	struct elevator_type *e;  in __elevator_change()  local
765 	e = elevator_get(q, strstrip(elevator_name), true);  in __elevator_change()
766 	if (!e)  in __elevator_change()
771 		elevator_put(e);  in __elevator_change()
775 	return elevator_switch(q, e);  in __elevator_change()
795 	struct elevator_queue *e = q->elevator;  in elv_iosched_show()  local
806 		elv = e->type;  in elv_iosched_show()