30 static struct svc_deferred_req *svc_deferred_dequeue(struct svc_xprt *xprt);
34 static void svc_delete_xprt(struct svc_xprt *xprt);
150 void svc_xprt_deferred_close(struct svc_xprt *xprt)  in svc_xprt_deferred_close()  argument
152 	if (!test_and_set_bit(XPT_CLOSE, &xprt->xpt_flags))  in svc_xprt_deferred_close()
153 		svc_xprt_enqueue(xprt);  in svc_xprt_deferred_close()
159 	struct svc_xprt *xprt =  in svc_xprt_free()  local
161 	struct module *owner = xprt->xpt_class->xcl_owner;  in svc_xprt_free()
162 	if (test_bit(XPT_CACHE_AUTH, &xprt->xpt_flags))  in svc_xprt_free()
163 		svcauth_unix_info_release(xprt);  in svc_xprt_free()
164 	put_cred(xprt->xpt_cred);  in svc_xprt_free()
165 	put_net_track(xprt->xpt_net, &xprt->ns_tracker);  in svc_xprt_free()
167 	if (xprt->xpt_bc_xprt)  in svc_xprt_free()
168 		xprt_put(xprt->xpt_bc_xprt);  in svc_xprt_free()
169 	if (xprt->xpt_bc_xps)  in svc_xprt_free()
170 		xprt_switch_put(xprt->xpt_bc_xps);  in svc_xprt_free()
171 	trace_svc_xprt_free(xprt);  in svc_xprt_free()
172 	xprt->xpt_ops->xpo_free(xprt);  in svc_xprt_free()
176 void svc_xprt_put(struct svc_xprt *xprt)  in svc_xprt_put()  argument
178 	kref_put(&xprt->xpt_ref, svc_xprt_free);  in svc_xprt_put()
187 		   struct svc_xprt *xprt, struct svc_serv *serv)  in svc_xprt_init()  argument
189 	memset(xprt, 0, sizeof(*xprt));  in svc_xprt_init()
190 	xprt->xpt_class = xcl;  in svc_xprt_init()
191 	xprt->xpt_ops = xcl->xcl_ops;  in svc_xprt_init()
192 	kref_init(&xprt->xpt_ref);  in svc_xprt_init()
193 	xprt->xpt_server = serv;  in svc_xprt_init()
194 	INIT_LIST_HEAD(&xprt->xpt_list);  in svc_xprt_init()
195 	INIT_LIST_HEAD(&xprt->xpt_ready);  in svc_xprt_init()
196 	INIT_LIST_HEAD(&xprt->xpt_deferred);  in svc_xprt_init()
197 	INIT_LIST_HEAD(&xprt->xpt_users);  in svc_xprt_init()
198 	mutex_init(&xprt->xpt_mutex);  in svc_xprt_init()
199 	spin_lock_init(&xprt->xpt_lock);  in svc_xprt_init()
200 	set_bit(XPT_BUSY, &xprt->xpt_flags);  in svc_xprt_init()
201 	xprt->xpt_net = get_net_track(net, &xprt->ns_tracker, GFP_ATOMIC);  in svc_xprt_init()
202 	strcpy(xprt->xpt_remotebuf, "uninitialized");  in svc_xprt_init()
225 	struct svc_xprt *xprt;  in __svc_xpo_create()  local
244 	xprt = xcl->xcl_ops->xpo_create(serv, net, sap, len, flags);  in __svc_xpo_create()
245 	if (IS_ERR(xprt))  in __svc_xpo_create()
247 					  xcl->xcl_name, sap, len, xprt);  in __svc_xpo_create()
248 	return xprt;  in __svc_xpo_create()
261 void svc_xprt_received(struct svc_xprt *xprt)  in svc_xprt_received()  argument
263 	if (!test_bit(XPT_BUSY, &xprt->xpt_flags)) {  in svc_xprt_received()
264 		WARN_ONCE(1, "xprt=0x%p already busy!", xprt);  in svc_xprt_received()
271 	svc_xprt_get(xprt);  in svc_xprt_received()
273 	clear_bit(XPT_BUSY, &xprt->xpt_flags);  in svc_xprt_received()
274 	svc_xprt_enqueue(xprt);  in svc_xprt_received()
275 	svc_xprt_put(xprt);  in svc_xprt_received()
357 void svc_xprt_copy_addrs(struct svc_rqst *rqstp, struct svc_xprt *xprt)  in svc_xprt_copy_addrs()  argument
359 	memcpy(&rqstp->rq_addr, &xprt->xpt_remote, xprt->xpt_remotelen);  in svc_xprt_copy_addrs()
360 	rqstp->rq_addrlen = xprt->xpt_remotelen;  in svc_xprt_copy_addrs()
366 	memcpy(&rqstp->rq_daddr, &xprt->xpt_local, xprt->xpt_locallen);  in svc_xprt_copy_addrs()
367 	rqstp->rq_daddrlen = xprt->xpt_locallen;  in svc_xprt_copy_addrs()
384 static bool svc_xprt_slots_in_range(struct svc_xprt *xprt)  in svc_xprt_slots_in_range()  argument
387 	int nrqsts = atomic_read(&xprt->xpt_nr_rqsts);  in svc_xprt_slots_in_range()
392 static bool svc_xprt_reserve_slot(struct svc_rqst *rqstp, struct svc_xprt *xprt)  in svc_xprt_reserve_slot()  argument
395 		if (!svc_xprt_slots_in_range(xprt))  in svc_xprt_reserve_slot()
397 		atomic_inc(&xprt->xpt_nr_rqsts);  in svc_xprt_reserve_slot()
405 	struct svc_xprt	*xprt = rqstp->rq_xprt;  in svc_xprt_release_slot()  local
407 		atomic_dec(&xprt->xpt_nr_rqsts);  in svc_xprt_release_slot()
409 		svc_xprt_enqueue(xprt);  in svc_xprt_release_slot()
413 static bool svc_xprt_ready(struct svc_xprt *xprt)  in svc_xprt_ready()  argument
426 	xpt_flags = READ_ONCE(xprt->xpt_flags);  in svc_xprt_ready()
433 		if (xprt->xpt_ops->xpo_has_wspace(xprt) &&  in svc_xprt_ready()
434 		    svc_xprt_slots_in_range(xprt))  in svc_xprt_ready()
436 		trace_svc_xprt_no_write_space(xprt);  in svc_xprt_ready()
447 void svc_xprt_enqueue(struct svc_xprt *xprt)  in svc_xprt_enqueue()  argument
452 	if (!svc_xprt_ready(xprt))  in svc_xprt_enqueue()
460 	if (test_and_set_bit(XPT_BUSY, &xprt->xpt_flags))  in svc_xprt_enqueue()
463 	pool = svc_pool_for_cpu(xprt->xpt_server);  in svc_xprt_enqueue()
468 	list_add_tail(&xprt->xpt_ready, &pool->sp_sockets);  in svc_xprt_enqueue()
486 	trace_svc_xprt_enqueue(xprt, rqstp);  in svc_xprt_enqueue()
495 	struct svc_xprt	*xprt = NULL;  in svc_xprt_dequeue()  local
502 		xprt = list_first_entry(&pool->sp_sockets,  in svc_xprt_dequeue()
504 		list_del_init(&xprt->xpt_ready);  in svc_xprt_dequeue()
505 		svc_xprt_get(xprt);  in svc_xprt_dequeue()
509 	return xprt;  in svc_xprt_dequeue()
524 	struct svc_xprt *xprt = rqstp->rq_xprt;  in svc_reserve()  local
528 	if (xprt && space < rqstp->rq_reserved) {  in svc_reserve()
529 		atomic_sub((rqstp->rq_reserved - space), &xprt->xpt_reserved);  in svc_reserve()
532 		svc_xprt_enqueue(xprt);  in svc_reserve()
539 	struct svc_xprt	*xprt = rqstp->rq_xprt;  in svc_xprt_release()  local
541 	xprt->xpt_ops->xpo_release_rqst(rqstp);  in svc_xprt_release()
565 	svc_xprt_put(xprt);  in svc_xprt_release()
639 		struct svc_xprt *xprt = NULL;  in svc_check_conn_limits()  local
651 			xprt = list_entry(serv->sv_tempsocks.prev,  in svc_check_conn_limits()
654 			set_bit(XPT_CLOSE, &xprt->xpt_flags);  in svc_check_conn_limits()
655 			svc_xprt_get(xprt);  in svc_check_conn_limits()
659 		if (xprt) {  in svc_check_conn_limits()
660 			svc_xprt_enqueue(xprt);  in svc_check_conn_limits()
661 			svc_xprt_put(xprt);  in svc_check_conn_limits()
805 static int svc_handle_xprt(struct svc_rqst *rqstp, struct svc_xprt *xprt)  in svc_handle_xprt()  argument
810 	if (test_bit(XPT_CLOSE, &xprt->xpt_flags)) {  in svc_handle_xprt()
811 		if (test_and_clear_bit(XPT_KILL_TEMP, &xprt->xpt_flags))  in svc_handle_xprt()
812 			xprt->xpt_ops->xpo_kill_temp_xprt(xprt);  in svc_handle_xprt()
813 		svc_delete_xprt(xprt);  in svc_handle_xprt()
817 	if (test_bit(XPT_LISTENER, &xprt->xpt_flags)) {  in svc_handle_xprt()
823 		__module_get(xprt->xpt_class->xcl_owner);  in svc_handle_xprt()
824 		svc_check_conn_limits(xprt->xpt_server);  in svc_handle_xprt()
825 		newxpt = xprt->xpt_ops->xpo_accept(xprt);  in svc_handle_xprt()
827 			newxpt->xpt_cred = get_cred(xprt->xpt_cred);  in svc_handle_xprt()
831 			module_put(xprt->xpt_class->xcl_owner);  in svc_handle_xprt()
833 		svc_xprt_received(xprt);  in svc_handle_xprt()
834 	} else if (svc_xprt_reserve_slot(rqstp, xprt)) {  in svc_handle_xprt()
837 			rqstp, rqstp->rq_pool->sp_id, xprt,  in svc_handle_xprt()
838 			kref_read(&xprt->xpt_ref));  in svc_handle_xprt()
839 		rqstp->rq_deferred = svc_deferred_dequeue(xprt);  in svc_handle_xprt()
843 			len = xprt->xpt_ops->xpo_recvfrom(rqstp);  in svc_handle_xprt()
846 		atomic_add(rqstp->rq_reserved, &xprt->xpt_reserved);  in svc_handle_xprt()
848 		svc_xprt_received(xprt);  in svc_handle_xprt()
861 	struct svc_xprt		*xprt = NULL;  in svc_recv()  local
875 	xprt = svc_get_next_xprt(rqstp, timeout);  in svc_recv()
876 	if (IS_ERR(xprt)) {  in svc_recv()
877 		err = PTR_ERR(xprt);  in svc_recv()
881 	len = svc_handle_xprt(rqstp, xprt);  in svc_recv()
889 	clear_bit(XPT_OLD, &xprt->xpt_flags);  in svc_recv()
891 	xprt->xpt_ops->xpo_secure_port(rqstp);  in svc_recv()
921 	struct svc_xprt	*xprt;  in svc_send()  local
925 	xprt = rqstp->rq_xprt;  in svc_send()
926 	if (!xprt)  in svc_send()
937 	len = xprt->xpt_ops->xpo_sendto(rqstp);  in svc_send()
955 	struct svc_xprt *xprt;  in svc_age_temp_xprts()  local
968 		xprt = list_entry(le, struct svc_xprt, xpt_list);  in svc_age_temp_xprts()
972 		if (!test_and_set_bit(XPT_OLD, &xprt->xpt_flags))  in svc_age_temp_xprts()
974 		if (kref_read(&xprt->xpt_ref) > 1 ||  in svc_age_temp_xprts()
975 		    test_bit(XPT_BUSY, &xprt->xpt_flags))  in svc_age_temp_xprts()
978 		set_bit(XPT_CLOSE, &xprt->xpt_flags);  in svc_age_temp_xprts()
979 		dprintk("queuing xprt %p for closing\n", xprt);  in svc_age_temp_xprts()
982 		svc_xprt_enqueue(xprt);  in svc_age_temp_xprts()
997 	struct svc_xprt *xprt;  in svc_age_temp_xprts_now()  local
1003 		xprt = list_entry(le, struct svc_xprt, xpt_list);  in svc_age_temp_xprts_now()
1005 				&xprt->xpt_local)) {  in svc_age_temp_xprts_now()
1006 			dprintk("svc_age_temp_xprts_now: found %p\n", xprt);  in svc_age_temp_xprts_now()
1015 		xprt = list_entry(le, struct svc_xprt, xpt_list);  in svc_age_temp_xprts_now()
1016 		set_bit(XPT_CLOSE, &xprt->xpt_flags);  in svc_age_temp_xprts_now()
1017 		set_bit(XPT_KILL_TEMP, &xprt->xpt_flags);  in svc_age_temp_xprts_now()
1019 				xprt);  in svc_age_temp_xprts_now()
1020 		svc_xprt_enqueue(xprt);  in svc_age_temp_xprts_now()
1025 static void call_xpt_users(struct svc_xprt *xprt)  in call_xpt_users()  argument
1029 	spin_lock(&xprt->xpt_lock);  in call_xpt_users()
1030 	while (!list_empty(&xprt->xpt_users)) {  in call_xpt_users()
1031 		u = list_first_entry(&xprt->xpt_users, struct svc_xpt_user, list);  in call_xpt_users()
1035 	spin_unlock(&xprt->xpt_lock);  in call_xpt_users()
1041 static void svc_delete_xprt(struct svc_xprt *xprt)  in svc_delete_xprt()  argument
1043 	struct svc_serv	*serv = xprt->xpt_server;  in svc_delete_xprt()
1046 	if (test_and_set_bit(XPT_DEAD, &xprt->xpt_flags))  in svc_delete_xprt()
1049 	trace_svc_xprt_detach(xprt);  in svc_delete_xprt()
1050 	xprt->xpt_ops->xpo_detach(xprt);  in svc_delete_xprt()
1051 	if (xprt->xpt_bc_xprt)  in svc_delete_xprt()
1052 		xprt->xpt_bc_xprt->ops->close(xprt->xpt_bc_xprt);  in svc_delete_xprt()
1055 	list_del_init(&xprt->xpt_list);  in svc_delete_xprt()
1056 	WARN_ON_ONCE(!list_empty(&xprt->xpt_ready));  in svc_delete_xprt()
1057 	if (test_bit(XPT_TEMP, &xprt->xpt_flags))  in svc_delete_xprt()
1061 	while ((dr = svc_deferred_dequeue(xprt)) != NULL)  in svc_delete_xprt()
1064 	call_xpt_users(xprt);  in svc_delete_xprt()
1065 	svc_xprt_put(xprt);  in svc_delete_xprt()
1073 void svc_xprt_close(struct svc_xprt *xprt)  in svc_xprt_close()  argument
1075 	trace_svc_xprt_close(xprt);  in svc_xprt_close()
1076 	set_bit(XPT_CLOSE, &xprt->xpt_flags);  in svc_xprt_close()
1077 	if (test_and_set_bit(XPT_BUSY, &xprt->xpt_flags))  in svc_xprt_close()
1086 	svc_delete_xprt(xprt);  in svc_xprt_close()
1092 	struct svc_xprt *xprt;  in svc_close_list()  local
1096 	list_for_each_entry(xprt, xprt_list, xpt_list) {  in svc_close_list()
1097 		if (xprt->xpt_net != net)  in svc_close_list()
1100 		set_bit(XPT_CLOSE, &xprt->xpt_flags);  in svc_close_list()
1101 		svc_xprt_enqueue(xprt);  in svc_close_list()
1110 	struct svc_xprt *xprt;  in svc_dequeue_net()  local
1118 		list_for_each_entry_safe(xprt, tmp, &pool->sp_sockets, xpt_ready) {  in svc_dequeue_net()
1119 			if (xprt->xpt_net != net)  in svc_dequeue_net()
1121 			list_del_init(&xprt->xpt_ready);  in svc_dequeue_net()
1123 			return xprt;  in svc_dequeue_net()
1132 	struct svc_xprt *xprt;  in svc_clean_up_xprts()  local
1134 	while ((xprt = svc_dequeue_net(serv, net))) {  in svc_clean_up_xprts()
1135 		set_bit(XPT_CLOSE, &xprt->xpt_flags);  in svc_clean_up_xprts()
1136 		svc_delete_xprt(xprt);  in svc_clean_up_xprts()
1177 	struct svc_xprt *xprt = dr->xprt;  in svc_revisit()  local
1179 	spin_lock(&xprt->xpt_lock);  in svc_revisit()
1180 	set_bit(XPT_DEFERRED, &xprt->xpt_flags);  in svc_revisit()
1181 	if (too_many || test_bit(XPT_DEAD, &xprt->xpt_flags)) {  in svc_revisit()
1182 		spin_unlock(&xprt->xpt_lock);  in svc_revisit()
1184 		svc_xprt_put(xprt);  in svc_revisit()
1188 	dr->xprt = NULL;  in svc_revisit()
1189 	list_add(&dr->handle.recent, &xprt->xpt_deferred);  in svc_revisit()
1190 	spin_unlock(&xprt->xpt_lock);  in svc_revisit()
1192 	svc_xprt_enqueue(xprt);  in svc_revisit()
1193 	svc_xprt_put(xprt);  in svc_revisit()
1240 	dr->xprt = rqstp->rq_xprt;  in svc_defer()
1275 static struct svc_deferred_req *svc_deferred_dequeue(struct svc_xprt *xprt)  in svc_deferred_dequeue()  argument
1279 	if (!test_bit(XPT_DEFERRED, &xprt->xpt_flags))  in svc_deferred_dequeue()
1281 	spin_lock(&xprt->xpt_lock);  in svc_deferred_dequeue()
1282 	if (!list_empty(&xprt->xpt_deferred)) {  in svc_deferred_dequeue()
1283 		dr = list_entry(xprt->xpt_deferred.next,  in svc_deferred_dequeue()
1288 		clear_bit(XPT_DEFERRED, &xprt->xpt_flags);  in svc_deferred_dequeue()
1289 	spin_unlock(&xprt->xpt_lock);  in svc_deferred_dequeue()
1313 	struct svc_xprt *xprt;  in svc_find_xprt()  local
1321 	list_for_each_entry(xprt, &serv->sv_permsocks, xpt_list) {  in svc_find_xprt()
1322 		if (xprt->xpt_net != net)  in svc_find_xprt()
1324 		if (strcmp(xprt->xpt_class->xcl_name, xcl_name))  in svc_find_xprt()
1326 		if (af != AF_UNSPEC && af != xprt->xpt_local.ss_family)  in svc_find_xprt()
1328 		if (port != 0 && port != svc_xprt_local_port(xprt))  in svc_find_xprt()
1330 		found = xprt;  in svc_find_xprt()
1331 		svc_xprt_get(xprt);  in svc_find_xprt()
1339 static int svc_one_xprt_name(const struct svc_xprt *xprt,  in svc_one_xprt_name()  argument
1345 			xprt->xpt_class->xcl_name,  in svc_one_xprt_name()
1346 			svc_xprt_local_port(xprt));  in svc_one_xprt_name()
1366 	struct svc_xprt *xprt;  in svc_xprt_names()  local
1378 	list_for_each_entry(xprt, &serv->sv_permsocks, xpt_list) {  in svc_xprt_names()
1379 		len = svc_one_xprt_name(xprt, pos, buflen - totlen);  in svc_xprt_names()