62 	struct request_queue *q = rq->q;  in elv_iosched_allow_bio_merge()  local
63 	struct elevator_queue *e = q->elevator;  in elv_iosched_allow_bio_merge()
66 		return e->type->ops.allow_merge(q, rq, bio);  in elv_iosched_allow_bio_merge()
86 static inline bool elv_support_features(struct request_queue *q,  in elv_support_features()  argument
89 	return (q->required_elevator_features & e->elevator_features) ==  in elv_support_features()
90 		q->required_elevator_features;  in elv_support_features()
116 static struct elevator_type *elevator_find_get(struct request_queue *q,  in elevator_find_get()  argument
123 	if (e && (!elv_support_features(q, e) || !elevator_tryget(e)))  in elevator_find_get()
131 struct elevator_queue *elevator_alloc(struct request_queue *q,  in elevator_alloc()  argument
136 	eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);  in elevator_alloc()
159 void elevator_exit(struct request_queue *q)  in elevator_exit()  argument
161 	struct elevator_queue *e = q->elevator;  in elevator_exit()
163 	ioc_clear_queue(q);  in elevator_exit()
164 	blk_mq_sched_free_rqs(q);  in elevator_exit()
167 	blk_mq_exit_sched(q, e);  in elevator_exit()
179 void elv_rqhash_del(struct request_queue *q, struct request *rq)  in elv_rqhash_del()  argument
186 void elv_rqhash_add(struct request_queue *q, struct request *rq)  in elv_rqhash_add()  argument
188 	struct elevator_queue *e = q->elevator;  in elv_rqhash_add()
196 void elv_rqhash_reposition(struct request_queue *q, struct request *rq)  in elv_rqhash_reposition()  argument
199 	elv_rqhash_add(q, rq);  in elv_rqhash_reposition()
202 struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)  in elv_rqhash_find()  argument
204 	struct elevator_queue *e = q->elevator;  in elv_rqhash_find()
276 enum elv_merge elv_merge(struct request_queue *q, struct request **req,  in elv_merge()  argument
279 	struct elevator_queue *e = q->elevator;  in elv_merge()
288 	if (blk_queue_nomerges(q) || !bio_mergeable(bio))  in elv_merge()
294 	if (q->last_merge && elv_bio_merge_ok(q->last_merge, bio)) {  in elv_merge()
295 		enum elv_merge ret = blk_try_merge(q->last_merge, bio);  in elv_merge()
298 			*req = q->last_merge;  in elv_merge()
303 	if (blk_queue_noxmerges(q))  in elv_merge()
309 	__rq = elv_rqhash_find(q, bio->bi_iter.bi_sector);  in elv_merge()
319 		return e->type->ops.request_merge(q, req, bio);  in elv_merge()
332 bool elv_attempt_insert_merge(struct request_queue *q, struct request *rq,  in elv_attempt_insert_merge()  argument
338 	if (blk_queue_nomerges(q))  in elv_attempt_insert_merge()
344 	if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq)) {  in elv_attempt_insert_merge()
349 	if (blk_queue_noxmerges(q))  in elv_attempt_insert_merge()
357 		__rq = elv_rqhash_find(q, blk_rq_pos(rq));  in elv_attempt_insert_merge()
358 		if (!__rq || !blk_attempt_req_merge(q, __rq, rq))  in elv_attempt_insert_merge()
370 void elv_merged_request(struct request_queue *q, struct request *rq,  in elv_merged_request()  argument
373 	struct elevator_queue *e = q->elevator;  in elv_merged_request()
376 		e->type->ops.request_merged(q, rq, type);  in elv_merged_request()
379 		elv_rqhash_reposition(q, rq);  in elv_merged_request()
381 	q->last_merge = rq;  in elv_merged_request()
384 void elv_merge_requests(struct request_queue *q, struct request *rq,  in elv_merge_requests()  argument
387 	struct elevator_queue *e = q->elevator;  in elv_merge_requests()
390 		e->type->ops.requests_merged(q, rq, next);  in elv_merge_requests()
392 	elv_rqhash_reposition(q, rq);  in elv_merge_requests()
393 	q->last_merge = rq;  in elv_merge_requests()
396 struct request *elv_latter_request(struct request_queue *q, struct request *rq)  in elv_latter_request()  argument
398 	struct elevator_queue *e = q->elevator;  in elv_latter_request()
401 		return e->type->ops.next_request(q, rq);  in elv_latter_request()
406 struct request *elv_former_request(struct request_queue *q, struct request *rq)  in elv_former_request()  argument
408 	struct elevator_queue *e = q->elevator;  in elv_former_request()
411 		return e->type->ops.former_request(q, rq);  in elv_former_request()
463 int elv_register_queue(struct request_queue *q, bool uevent)  in elv_register_queue()  argument
465 	struct elevator_queue *e = q->elevator;  in elv_register_queue()
468 	lockdep_assert_held(&q->sysfs_lock);  in elv_register_queue()
470 	error = kobject_add(&e->kobj, &q->disk->queue_kobj, "iosched");  in elv_register_queue()
488 void elv_unregister_queue(struct request_queue *q)  in elv_unregister_queue()  argument
490 	struct elevator_queue *e = q->elevator;  in elv_unregister_queue()
492 	lockdep_assert_held(&q->sysfs_lock);  in elv_unregister_queue()
558 static inline bool elv_support_iosched(struct request_queue *q)  in elv_support_iosched()  argument
560 	if (!queue_is_mq(q) ||  in elv_support_iosched()
561 	    (q->tag_set && (q->tag_set->flags & BLK_MQ_F_NO_SCHED)))  in elv_support_iosched()
570 static struct elevator_type *elevator_get_default(struct request_queue *q)  in elevator_get_default()  argument
572 	if (q->tag_set && q->tag_set->flags & BLK_MQ_F_NO_SCHED_BY_DEFAULT)  in elevator_get_default()
575 	if (q->nr_hw_queues != 1 &&  in elevator_get_default()
576 	    !blk_mq_is_shared_tags(q->tag_set->flags))  in elevator_get_default()
579 	return elevator_find_get(q, "mq-deadline");  in elevator_get_default()
586 static struct elevator_type *elevator_get_by_features(struct request_queue *q)  in elevator_get_by_features()  argument
593 		if (elv_support_features(q, e)) {  in elevator_get_by_features()
612 void elevator_init_mq(struct request_queue *q)  in elevator_init_mq()  argument
617 	if (!elv_support_iosched(q))  in elevator_init_mq()
620 	WARN_ON_ONCE(blk_queue_registered(q));  in elevator_init_mq()
622 	if (unlikely(q->elevator))  in elevator_init_mq()
625 	if (!q->required_elevator_features)  in elevator_init_mq()
626 		e = elevator_get_default(q);  in elevator_init_mq()
628 		e = elevator_get_by_features(q);  in elevator_init_mq()
639 	blk_mq_freeze_queue(q);  in elevator_init_mq()
640 	blk_mq_cancel_work_sync(q);  in elevator_init_mq()
642 	err = blk_mq_init_sched(q, e);  in elevator_init_mq()
644 	blk_mq_unfreeze_queue(q);  in elevator_init_mq()
660 int elevator_switch(struct request_queue *q, struct elevator_type *new_e)  in elevator_switch()  argument
664 	lockdep_assert_held(&q->sysfs_lock);  in elevator_switch()
666 	blk_mq_freeze_queue(q);  in elevator_switch()
667 	blk_mq_quiesce_queue(q);  in elevator_switch()
669 	if (q->elevator) {  in elevator_switch()
670 		elv_unregister_queue(q);  in elevator_switch()
671 		elevator_exit(q);  in elevator_switch()
674 	ret = blk_mq_init_sched(q, new_e);  in elevator_switch()
678 	ret = elv_register_queue(q, true);  in elevator_switch()
680 		elevator_exit(q);  in elevator_switch()
683 	blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);  in elevator_switch()
686 	blk_mq_unquiesce_queue(q);  in elevator_switch()
687 	blk_mq_unfreeze_queue(q);  in elevator_switch()
697 void elevator_disable(struct request_queue *q)  in elevator_disable()  argument
699 	lockdep_assert_held(&q->sysfs_lock);  in elevator_disable()
701 	blk_mq_freeze_queue(q);  in elevator_disable()
702 	blk_mq_quiesce_queue(q);  in elevator_disable()
704 	elv_unregister_queue(q);  in elevator_disable()
705 	elevator_exit(q);  in elevator_disable()
706 	blk_queue_flag_clear(QUEUE_FLAG_SQ_SCHED, q);  in elevator_disable()
707 	q->elevator = NULL;  in elevator_disable()
708 	q->nr_requests = q->tag_set->queue_depth;  in elevator_disable()
709 	blk_add_trace_msg(q, "elv switch: none");  in elevator_disable()
711 	blk_mq_unquiesce_queue(q);  in elevator_disable()
712 	blk_mq_unfreeze_queue(q);  in elevator_disable()
718 static int elevator_change(struct request_queue *q, const char *elevator_name)  in elevator_change()  argument
724 	if (!blk_queue_registered(q))  in elevator_change()
728 		if (q->elevator)  in elevator_change()
729 			elevator_disable(q);  in elevator_change()
733 	if (q->elevator && elevator_match(q->elevator->type, elevator_name))  in elevator_change()
736 	e = elevator_find_get(q, elevator_name);  in elevator_change()
739 		e = elevator_find_get(q, elevator_name);  in elevator_change()
743 	ret = elevator_switch(q, e);  in elevator_change()
748 ssize_t elv_iosched_store(struct request_queue *q, const char *buf,  in elv_iosched_store()  argument
754 	if (!elv_support_iosched(q))  in elv_iosched_store()
758 	ret = elevator_change(q, strstrip(elevator_name));  in elv_iosched_store()
764 ssize_t elv_iosched_show(struct request_queue *q, char *name)  in elv_iosched_show()  argument
766 	struct elevator_queue *eq = q->elevator;  in elv_iosched_show()
770 	if (!elv_support_iosched(q))  in elv_iosched_show()
773 	if (!q->elevator) {  in elv_iosched_show()
784 		else if (elv_support_features(q, e))  in elv_iosched_show()
793 struct request *elv_rb_former_request(struct request_queue *q,  in elv_rb_former_request()  argument
805 struct request *elv_rb_latter_request(struct request_queue *q,  in elv_rb_latter_request()  argument