36 static inline bool rnbd_clt_get_sess(struct rnbd_clt_session *sess)  in rnbd_clt_get_sess()  argument
38 	return refcount_inc_not_zero(&sess->refcount);  in rnbd_clt_get_sess()
41 static void free_sess(struct rnbd_clt_session *sess);
43 static void rnbd_clt_put_sess(struct rnbd_clt_session *sess)  in rnbd_clt_put_sess()  argument
47 	if (refcount_dec_and_test(&sess->refcount))  in rnbd_clt_put_sess()
48 		free_sess(sess);  in rnbd_clt_put_sess()
61 	rnbd_clt_put_sess(dev->sess);  in rnbd_clt_put_dev()
161 rnbd_get_cpu_qlist(struct rnbd_clt_session *sess, int cpu)  in rnbd_get_cpu_qlist()  argument
166 	bit = find_next_bit(sess->cpu_queues_bm, nr_cpu_ids, cpu);  in rnbd_get_cpu_qlist()
168 		return per_cpu_ptr(sess->cpu_queues, bit);  in rnbd_get_cpu_qlist()
171 		bit = find_first_bit(sess->cpu_queues_bm, cpu);  in rnbd_get_cpu_qlist()
173 			return per_cpu_ptr(sess->cpu_queues, bit);  in rnbd_get_cpu_qlist()
199 static bool rnbd_rerun_if_needed(struct rnbd_clt_session *sess)  in rnbd_rerun_if_needed()  argument
211 	cpup = get_cpu_ptr(sess->cpu_rr);  in rnbd_rerun_if_needed()
212 	for (cpu_q = rnbd_get_cpu_qlist(sess, nxt_cpu(*cpup)); cpu_q;  in rnbd_rerun_if_needed()
213 	     cpu_q = rnbd_get_cpu_qlist(sess, nxt_cpu(cpu_q->cpu))) {  in rnbd_rerun_if_needed()
216 		if (!test_bit(cpu_q->cpu, sess->cpu_queues_bm))  in rnbd_rerun_if_needed()
228 			clear_bit(cpu_q->cpu, sess->cpu_queues_bm);  in rnbd_rerun_if_needed()
246 	put_cpu_ptr(sess->cpu_rr);  in rnbd_rerun_if_needed()
279 static void rnbd_rerun_all_if_idle(struct rnbd_clt_session *sess)  in rnbd_rerun_all_if_idle()  argument
284 		requeued = rnbd_rerun_if_needed(sess);  in rnbd_rerun_all_if_idle()
285 	} while (atomic_read(&sess->busy) == 0 && requeued);  in rnbd_rerun_all_if_idle()
288 static struct rtrs_permit *rnbd_get_permit(struct rnbd_clt_session *sess,  in rnbd_get_permit()  argument
294 	permit = rtrs_clt_get_permit(sess->rtrs, con_type, wait);  in rnbd_get_permit()
301 		atomic_inc(&sess->busy);  in rnbd_get_permit()
306 static void rnbd_put_permit(struct rnbd_clt_session *sess,  in rnbd_put_permit()  argument
309 	rtrs_clt_put_permit(sess->rtrs, permit);  in rnbd_put_permit()
310 	atomic_dec(&sess->busy);  in rnbd_put_permit()
315 	rnbd_rerun_all_if_idle(sess);  in rnbd_put_permit()
318 static struct rnbd_iu *rnbd_get_iu(struct rnbd_clt_session *sess,  in rnbd_get_iu()  argument
329 	permit = rnbd_get_permit(sess, con_type, wait);  in rnbd_get_iu()
349 		rnbd_put_permit(sess, permit);  in rnbd_get_iu()
357 static void rnbd_put_iu(struct rnbd_clt_session *sess, struct rnbd_iu *iu)  in rnbd_put_iu()  argument
361 		rnbd_put_permit(sess, iu->permit);  in rnbd_put_iu()
369 	struct rnbd_clt_session *sess	= dev->sess;  in rnbd_softirq_done_fn()  local
374 	rnbd_put_permit(sess, iu->permit);  in rnbd_softirq_done_fn()
440 	rnbd_put_iu(dev->sess, iu);  in msg_close_conf()
447 	struct rnbd_clt_session *sess = dev->sess;  in send_msg_close()  local
456 	iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);  in send_msg_close()
467 	err = send_usr_msg(sess->rtrs, WRITE, iu, &vec, 0, NULL, 0,  in send_msg_close()
471 		rnbd_put_iu(sess, iu);  in send_msg_close()
476 	rnbd_put_iu(sess, iu);  in send_msg_close()
511 	rnbd_put_iu(dev->sess, iu);  in msg_open_conf()
519 	struct rnbd_clt_session *sess = iu->sess;  in msg_sess_info_conf()  local
522 		sess->ver = min_t(u8, rsp->ver, RNBD_PROTO_VER_MAJOR);  in msg_sess_info_conf()
526 	rnbd_put_iu(sess, iu);  in msg_sess_info_conf()
527 	rnbd_clt_put_sess(sess);  in msg_sess_info_conf()
532 	struct rnbd_clt_session *sess = dev->sess;  in send_msg_open()  local
546 	iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);  in send_msg_open()
562 	err = send_usr_msg(sess->rtrs, READ, iu,  in send_msg_open()
567 		rnbd_put_iu(sess, iu);  in send_msg_open()
573 	rnbd_put_iu(sess, iu);  in send_msg_open()
577 static int send_msg_sess_info(struct rnbd_clt_session *sess, enum wait_type wait)  in send_msg_sess_info()  argument
592 	iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);  in send_msg_sess_info()
599 	iu->sess = sess;  in send_msg_sess_info()
605 	if (!rnbd_clt_get_sess(sess)) {  in send_msg_sess_info()
615 	err = send_usr_msg(sess->rtrs, READ, iu,  in send_msg_sess_info()
619 		rnbd_clt_put_sess(sess);  in send_msg_sess_info()
621 		rnbd_put_iu(sess, iu);  in send_msg_sess_info()
626 	rnbd_put_iu(sess, iu);  in send_msg_sess_info()
630 static void set_dev_states_to_disconnected(struct rnbd_clt_session *sess)  in set_dev_states_to_disconnected()  argument
635 	mutex_lock(&sess->lock);  in set_dev_states_to_disconnected()
636 	list_for_each_entry(dev, &sess->devs_list, list) {  in set_dev_states_to_disconnected()
647 	mutex_unlock(&sess->lock);  in set_dev_states_to_disconnected()
650 static void remap_devs(struct rnbd_clt_session *sess)  in remap_devs()  argument
667 	err = send_msg_sess_info(sess, RTRS_PERMIT_NOWAIT);  in remap_devs()
669 		pr_err("send_msg_sess_info(\"%s\"): %d\n", sess->sessname, err);  in remap_devs()
673 	err = rtrs_clt_query(sess->rtrs, &attrs);  in remap_devs()
675 		pr_err("rtrs_clt_query(\"%s\"): %d\n", sess->sessname, err);  in remap_devs()
678 	mutex_lock(&sess->lock);  in remap_devs()
679 	sess->max_io_size = attrs.max_io_size;  in remap_devs()
681 	list_for_each_entry(dev, &sess->devs_list, list) {  in remap_devs()
701 	mutex_unlock(&sess->lock);  in remap_devs()
706 	struct rnbd_clt_session *sess = priv;  in rnbd_clt_link_ev()  local
710 		set_dev_states_to_disconnected(sess);  in rnbd_clt_link_ev()
713 		remap_devs(sess);  in rnbd_clt_link_ev()
717 		       ev, sess->sessname);  in rnbd_clt_link_ev()
735 static void destroy_mq_tags(struct rnbd_clt_session *sess)  in destroy_mq_tags()  argument
737 	if (sess->tag_set.tags)  in destroy_mq_tags()
738 		blk_mq_free_tag_set(&sess->tag_set);  in destroy_mq_tags()
741 static inline void wake_up_rtrs_waiters(struct rnbd_clt_session *sess)  in wake_up_rtrs_waiters()  argument
743 	sess->rtrs_ready = true;  in wake_up_rtrs_waiters()
744 	wake_up_all(&sess->rtrs_waitq);  in wake_up_rtrs_waiters()
747 static void close_rtrs(struct rnbd_clt_session *sess)  in close_rtrs()  argument
751 	if (!IS_ERR_OR_NULL(sess->rtrs)) {  in close_rtrs()
752 		rtrs_clt_close(sess->rtrs);  in close_rtrs()
753 		sess->rtrs = NULL;  in close_rtrs()
754 		wake_up_rtrs_waiters(sess);  in close_rtrs()
758 static void free_sess(struct rnbd_clt_session *sess)  in free_sess()  argument
760 	WARN_ON(!list_empty(&sess->devs_list));  in free_sess()
764 	close_rtrs(sess);  in free_sess()
765 	destroy_mq_tags(sess);  in free_sess()
766 	if (!list_empty(&sess->list)) {  in free_sess()
768 		list_del(&sess->list);  in free_sess()
771 	free_percpu(sess->cpu_queues);  in free_sess()
772 	free_percpu(sess->cpu_rr);  in free_sess()
773 	mutex_destroy(&sess->lock);  in free_sess()
774 	kfree(sess);  in free_sess()
779 	struct rnbd_clt_session *sess;  in alloc_sess()  local
782 	sess = kzalloc_node(sizeof(*sess), GFP_KERNEL, NUMA_NO_NODE);  in alloc_sess()
783 	if (!sess)  in alloc_sess()
785 	strscpy(sess->sessname, sessname, sizeof(sess->sessname));  in alloc_sess()
786 	atomic_set(&sess->busy, 0);  in alloc_sess()
787 	mutex_init(&sess->lock);  in alloc_sess()
788 	INIT_LIST_HEAD(&sess->devs_list);  in alloc_sess()
789 	INIT_LIST_HEAD(&sess->list);  in alloc_sess()
790 	bitmap_zero(sess->cpu_queues_bm, num_possible_cpus());  in alloc_sess()
791 	init_waitqueue_head(&sess->rtrs_waitq);  in alloc_sess()
792 	refcount_set(&sess->refcount, 1);  in alloc_sess()
794 	sess->cpu_queues = alloc_percpu(struct rnbd_cpu_qlist);  in alloc_sess()
795 	if (!sess->cpu_queues) {  in alloc_sess()
799 	rnbd_init_cpu_qlists(sess->cpu_queues);  in alloc_sess()
806 	sess->cpu_rr = alloc_percpu(int);  in alloc_sess()
807 	if (!sess->cpu_rr) {  in alloc_sess()
812 		* per_cpu_ptr(sess->cpu_rr, cpu) = cpu;  in alloc_sess()
814 	return sess;  in alloc_sess()
817 	free_sess(sess);  in alloc_sess()
822 static int wait_for_rtrs_connection(struct rnbd_clt_session *sess)  in wait_for_rtrs_connection()  argument
824 	wait_event(sess->rtrs_waitq, sess->rtrs_ready);  in wait_for_rtrs_connection()
825 	if (IS_ERR_OR_NULL(sess->rtrs))  in wait_for_rtrs_connection()
831 static void wait_for_rtrs_disconnection(struct rnbd_clt_session *sess)  in wait_for_rtrs_disconnection()  argument
837 	prepare_to_wait(&sess->rtrs_waitq, &wait, TASK_UNINTERRUPTIBLE);  in wait_for_rtrs_disconnection()
838 	if (IS_ERR_OR_NULL(sess->rtrs)) {  in wait_for_rtrs_disconnection()
839 		finish_wait(&sess->rtrs_waitq, &wait);  in wait_for_rtrs_disconnection()
857 	struct rnbd_clt_session *sess, *sn;  in __find_and_get_sess()  local
861 	list_for_each_entry_safe(sess, sn, &sess_list, list) {  in __find_and_get_sess()
862 		if (strcmp(sessname, sess->sessname))  in __find_and_get_sess()
865 		if (sess->rtrs_ready && IS_ERR_OR_NULL(sess->rtrs))  in __find_and_get_sess()
871 		if (rnbd_clt_get_sess(sess)) {  in __find_and_get_sess()
876 			err = wait_for_rtrs_connection(sess);  in __find_and_get_sess()
878 				rnbd_clt_put_sess(sess);  in __find_and_get_sess()
885 			return sess;  in __find_and_get_sess()
891 		wait_for_rtrs_disconnection(sess);  in __find_and_get_sess()
906 	struct rnbd_clt_session *sess = NULL;  in find_or_create_sess()  local
909 	sess = __find_and_get_sess(sessname);  in find_or_create_sess()
910 	if (!sess) {  in find_or_create_sess()
911 		sess = alloc_sess(sessname);  in find_or_create_sess()
912 		if (IS_ERR(sess)) {  in find_or_create_sess()
914 			return sess;  in find_or_create_sess()
916 		list_add(&sess->list, &sess_list);  in find_or_create_sess()
921 	return sess;  in find_or_create_sess()
992 	struct rtrs_clt_sess *rtrs = dev->sess->rtrs;  in rnbd_client_xfer_request()
1054 	struct rnbd_clt_session *sess = dev->sess;  in rnbd_clt_dev_add_to_requeue()  local
1060 	cpu_q = get_cpu_ptr(sess->cpu_queues);  in rnbd_clt_dev_add_to_requeue()
1067 		need_set = !test_bit(cpu_q->cpu, sess->cpu_queues_bm);  in rnbd_clt_dev_add_to_requeue()
1069 			set_bit(cpu_q->cpu, sess->cpu_queues_bm);  in rnbd_clt_dev_add_to_requeue()
1075 		if (atomic_read(&sess->busy)) {  in rnbd_clt_dev_add_to_requeue()
1083 				clear_bit(cpu_q->cpu, sess->cpu_queues_bm);  in rnbd_clt_dev_add_to_requeue()
1090 	put_cpu_ptr(sess->cpu_queues);  in rnbd_clt_dev_add_to_requeue()
1123 	iu->permit = rnbd_get_permit(dev->sess, RTRS_IO_CON,  in rnbd_queue_rq()
1141 		rnbd_put_permit(dev->sess, iu->permit);  in rnbd_queue_rq()
1154 	rnbd_put_permit(dev->sess, iu->permit);  in rnbd_queue_rq()
1163 	return rtrs_clt_rdma_cq_direct(dev->sess->rtrs, hctx->queue_num);  in rnbd_rdma_poll()
1168 	struct rnbd_clt_session *sess = set->driver_data;  in rnbd_rdma_map_queues()  local
1178 	if (sess->nr_poll_queues) {  in rnbd_rdma_map_queues()
1180 		set->map[HCTX_TYPE_POLL].nr_queues = sess->nr_poll_queues;  in rnbd_rdma_map_queues()
1185 			sess->sessname,  in rnbd_rdma_map_queues()
1191 			sess->sessname,  in rnbd_rdma_map_queues()
1204 static int setup_mq_tags(struct rnbd_clt_session *sess)  in setup_mq_tags()  argument
1206 	struct blk_mq_tag_set *tag_set = &sess->tag_set;  in setup_mq_tags()
1210 	tag_set->queue_depth	= sess->queue_depth;  in setup_mq_tags()
1217 	tag_set->nr_maps        = sess->nr_poll_queues ? HCTX_MAX_TYPES : 2;  in setup_mq_tags()
1222 	tag_set->nr_hw_queues	= num_online_cpus() + sess->nr_poll_queues;  in setup_mq_tags()
1223 	tag_set->driver_data    = sess;  in setup_mq_tags()
1233 	struct rnbd_clt_session *sess;  in find_and_get_or_create_sess()  local
1239 	sess = find_or_create_sess(sessname, &first);  in find_and_get_or_create_sess()
1240 	if (sess == ERR_PTR(-ENOMEM)) {  in find_and_get_or_create_sess()
1242 	} else if ((nr_poll_queues && !first) ||  (!nr_poll_queues && sess->nr_poll_queues)) {  in find_and_get_or_create_sess()
1252 		return sess;  in find_and_get_or_create_sess()
1261 		.priv = sess,  in find_and_get_or_create_sess()
1267 	sess->rtrs = rtrs_clt_open(&rtrs_ops, sessname,  in find_and_get_or_create_sess()
1272 	if (IS_ERR(sess->rtrs)) {  in find_and_get_or_create_sess()
1273 		err = PTR_ERR(sess->rtrs);  in find_and_get_or_create_sess()
1277 	err = rtrs_clt_query(sess->rtrs, &attrs);  in find_and_get_or_create_sess()
1281 	sess->max_io_size = attrs.max_io_size;  in find_and_get_or_create_sess()
1282 	sess->queue_depth = attrs.queue_depth;  in find_and_get_or_create_sess()
1283 	sess->nr_poll_queues = nr_poll_queues;  in find_and_get_or_create_sess()
1284 	sess->max_segments = attrs.max_segments;  in find_and_get_or_create_sess()
1286 	err = setup_mq_tags(sess);  in find_and_get_or_create_sess()
1290 	err = send_msg_sess_info(sess, RTRS_PERMIT_WAIT);  in find_and_get_or_create_sess()
1294 	wake_up_rtrs_waiters(sess);  in find_and_get_or_create_sess()
1296 	return sess;  in find_and_get_or_create_sess()
1299 	close_rtrs(sess);  in find_and_get_or_create_sess()
1301 	rnbd_clt_put_sess(sess);  in find_and_get_or_create_sess()
1306 	wake_up_rtrs_waiters(sess);  in find_and_get_or_create_sess()
1340 				 dev->sess->max_io_size / SECTOR_SIZE);  in setup_request_queue()
1359 	blk_queue_max_segments(dev->queue, dev->sess->max_segments);  in setup_request_queue()
1360 	blk_queue_io_opt(dev->queue, dev->sess->max_io_size);  in setup_request_queue()
1409 	dev->gd = blk_mq_alloc_disk(&dev->sess->tag_set, dev);  in rnbd_client_setup_device()
1419 static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess,  in init_dev()  argument
1447 		       pathname, sess->sessname, ret);  in init_dev()
1458 	dev->sess		= sess;  in init_dev()
1469 	WARN_ON(!rnbd_clt_get_sess(sess));  in init_dev()
1482 	struct rnbd_clt_session *sess;  in __exists_dev()  local
1486 	list_for_each_entry(sess, &sess_list, list) {  in __exists_dev()
1487 		if (sessname && strncmp(sess->sessname, sessname,  in __exists_dev()
1488 					sizeof(sess->sessname)))  in __exists_dev()
1490 		mutex_lock(&sess->lock);  in __exists_dev()
1491 		list_for_each_entry(dev, &sess->devs_list, list) {  in __exists_dev()
1498 		mutex_unlock(&sess->lock);  in __exists_dev()
1520 	struct rnbd_clt_session *sess = dev->sess;  in insert_dev_if_not_exists_devpath()  local
1523 	found = __exists_dev(dev->pathname, sess->sessname);  in insert_dev_if_not_exists_devpath()
1525 		mutex_lock(&sess->lock);  in insert_dev_if_not_exists_devpath()
1526 		list_add_tail(&dev->list, &sess->devs_list);  in insert_dev_if_not_exists_devpath()
1527 		mutex_unlock(&sess->lock);  in insert_dev_if_not_exists_devpath()
1536 	struct rnbd_clt_session *sess = dev->sess;  in delete_dev()  local
1538 	mutex_lock(&sess->lock);  in delete_dev()
1540 	mutex_unlock(&sess->lock);  in delete_dev()
1550 	struct rnbd_clt_session *sess;  in rnbd_clt_map_device()  local
1564 	sess = find_and_get_or_create_sess(sessname, paths, path_cnt, port_nr, nr_poll_queues);  in rnbd_clt_map_device()
1565 	if (IS_ERR(sess))  in rnbd_clt_map_device()
1566 		return ERR_CAST(sess);  in rnbd_clt_map_device()
1568 	dev = init_dev(sess, access_mode, pathname, nr_poll_queues);  in rnbd_clt_map_device()
1571 		       pathname, sess->sessname, PTR_ERR(dev));  in rnbd_clt_map_device()
1586 	iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);  in rnbd_clt_map_device()
1601 	ret = send_usr_msg(sess->rtrs, READ, iu,  in rnbd_clt_map_device()
1606 		rnbd_put_iu(sess, iu);  in rnbd_clt_map_device()
1618 		 sess->sessname, pathname);  in rnbd_clt_map_device()
1637 		       sess->max_segments, sess->max_io_size / SECTOR_SIZE,  in rnbd_clt_map_device()
1643 	rnbd_put_iu(sess, iu);  in rnbd_clt_map_device()
1644 	rnbd_clt_put_sess(sess);  in rnbd_clt_map_device()
1652 	rnbd_put_iu(sess, iu);  in rnbd_clt_map_device()
1658 	rnbd_clt_put_sess(sess);  in rnbd_clt_map_device()
1685 	struct rnbd_clt_session *sess = dev->sess;  in rnbd_clt_unmap_device()  local
1710 	if (was_mapped && sess->rtrs)  in rnbd_clt_unmap_device()
1763 	struct rnbd_clt_session *sess, *sn;  in rnbd_destroy_sessions()  local
1784 	list_for_each_entry_safe(sess, sn, &sess_list, list) {  in rnbd_destroy_sessions()
1785 		if (!rnbd_clt_get_sess(sess))  in rnbd_destroy_sessions()
1787 		close_rtrs(sess);  in rnbd_destroy_sessions()
1788 		list_for_each_entry_safe(dev, tn, &sess->devs_list, list) {  in rnbd_destroy_sessions()
1798 		rnbd_clt_put_sess(sess);  in rnbd_destroy_sessions()