233 static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type,  in smc_llc_flow_parallel()  argument
239 	    flow_type != msg_type && !lgr->delayed_event) {  in smc_llc_flow_parallel()
240 		lgr->delayed_event = qentry;  in smc_llc_flow_parallel()
247 			     SMC_LGR_ID_SIZE, &lgr->id,  in smc_llc_flow_parallel()
248 			     lgr->net->net_cookie,  in smc_llc_flow_parallel()
250 			     flow_type, lgr->role);  in smc_llc_flow_parallel()
258 	struct smc_link_group *lgr = qentry->link->lgr;  in smc_llc_flow_start()  local
260 	spin_lock_bh(&lgr->llc_flow_lock);  in smc_llc_flow_start()
263 		smc_llc_flow_parallel(lgr, flow->type, qentry);  in smc_llc_flow_start()
264 		spin_unlock_bh(&lgr->llc_flow_lock);  in smc_llc_flow_start()
282 	spin_unlock_bh(&lgr->llc_flow_lock);  in smc_llc_flow_start()
287 int smc_llc_flow_initiate(struct smc_link_group *lgr,  in smc_llc_flow_initiate()  argument
299 	if (list_empty(&lgr->list))  in smc_llc_flow_initiate()
301 	spin_lock_bh(&lgr->llc_flow_lock);  in smc_llc_flow_initiate()
302 	if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&  in smc_llc_flow_initiate()
303 	    (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||  in smc_llc_flow_initiate()
304 	     lgr->llc_flow_rmt.type == allowed_remote)) {  in smc_llc_flow_initiate()
305 		lgr->llc_flow_lcl.type = type;  in smc_llc_flow_initiate()
306 		spin_unlock_bh(&lgr->llc_flow_lock);  in smc_llc_flow_initiate()
309 	spin_unlock_bh(&lgr->llc_flow_lock);  in smc_llc_flow_initiate()
310 	rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) ||  in smc_llc_flow_initiate()
311 				(lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&  in smc_llc_flow_initiate()
312 				 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||  in smc_llc_flow_initiate()
313 				  lgr->llc_flow_rmt.type == allowed_remote))),  in smc_llc_flow_initiate()
321 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)  in smc_llc_flow_stop()  argument
323 	spin_lock_bh(&lgr->llc_flow_lock);  in smc_llc_flow_stop()
326 	spin_unlock_bh(&lgr->llc_flow_lock);  in smc_llc_flow_stop()
327 	if (!list_empty(&lgr->list) && lgr->delayed_event &&  in smc_llc_flow_stop()
328 	    flow == &lgr->llc_flow_lcl)  in smc_llc_flow_stop()
329 		schedule_work(&lgr->llc_event_work);  in smc_llc_flow_stop()
331 		wake_up(&lgr->llc_flow_waiter);  in smc_llc_flow_stop()
337 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,  in smc_llc_wait()  argument
341 	struct smc_llc_flow *flow = &lgr->llc_flow_lcl;  in smc_llc_wait()
344 	wait_event_timeout(lgr->llc_msg_waiter,  in smc_llc_wait()
347 			    list_empty(&lgr->list)),  in smc_llc_wait()
350 	    (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {  in smc_llc_wait()
359 			smc_llc_flow_start(&lgr->llc_flow_lcl,  in smc_llc_wait()
365 			     SMC_LGR_ID_SIZE, &lgr->id, lgr->net->net_cookie,  in smc_llc_wait()
367 			     flow->type, lgr->role,  in smc_llc_wait()
436 				 struct smc_link_group *lgr, size_t len)  in smc_llc_init_msg_hdr()  argument
438 	if (lgr->smc_version == SMC_V2) {  in smc_llc_init_msg_hdr()
464 	smc_llc_init_msg_hdr(&confllc->hd, link->lgr, sizeof(*confllc));  in smc_llc_send_confirm_link()
500 	smc_llc_init_msg_hdr(&rkeyllc->hd, send_link->lgr, sizeof(*rkeyllc));  in smc_llc_send_confirm_rkey()
504 		link = &send_link->lgr->lnk[i];  in smc_llc_send_confirm_rkey()
545 	smc_llc_init_msg_hdr(&rkeyllc->hd, link->lgr, sizeof(*rkeyllc));  in smc_llc_send_delete_rkey()
556 static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr,  in _smc_llc_get_next_rmb()  argument
562 		buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst],  in _smc_llc_get_next_rmb()
572 static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr,  in smc_llc_get_next_rmb()  argument
578 	if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) {  in smc_llc_get_next_rmb()
580 		return _smc_llc_get_next_rmb(lgr, buf_lst);  in smc_llc_get_next_rmb()
586 static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr,  in smc_llc_get_first_rmb()  argument
590 	return smc_llc_get_next_rmb(lgr, buf_lst, NULL);  in smc_llc_get_first_rmb()
596 	struct smc_link_group *lgr = link->lgr;  in smc_llc_fill_ext_v2()  local
603 	ext->v2_direct = !lgr->uses_gateway;  in smc_llc_fill_ext_v2()
608 	mutex_lock(&lgr->rmbs_lock);  in smc_llc_fill_ext_v2()
609 	ext->num_rkeys = lgr->conns_num;  in smc_llc_fill_ext_v2()
612 	buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);  in smc_llc_fill_ext_v2()
621 		buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);  in smc_llc_fill_ext_v2()
623 			buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);  in smc_llc_fill_ext_v2()
627 	mutex_unlock(&lgr->rmbs_lock);  in smc_llc_fill_ext_v2()
644 	if (link->lgr->smc_version == SMC_V2) {  in smc_llc_send_add_link()
681 	smc_llc_init_msg_hdr(&addllc->hd, link->lgr, len);  in smc_llc_send_add_link()
683 	if (link->lgr->smc_version == SMC_V2)  in smc_llc_send_add_link()
711 	smc_llc_init_msg_hdr(&delllc->hd, link->lgr, sizeof(*delllc));  in smc_llc_send_delete_link()
744 	smc_llc_init_msg_hdr(&testllc->hd, link->lgr, sizeof(*testllc));  in smc_llc_send_test_link()
796 static int smc_llc_alloc_alt_link(struct smc_link_group *lgr,  in smc_llc_alloc_alt_link()  argument
801 	if (lgr->type == SMC_LGR_SYMMETRIC ||  in smc_llc_alloc_alt_link()
802 	    (lgr->type != SMC_LGR_SINGLE &&  in smc_llc_alloc_alt_link()
810 			if (lgr->lnk[i].state == SMC_LNK_UNUSED)  in smc_llc_alloc_alt_link()
814 			if (lgr->lnk[i].state == SMC_LNK_UNUSED)  in smc_llc_alloc_alt_link()
826 	struct smc_link_group *lgr = link->lgr;  in smc_llc_add_link_cont()  local
861 		*buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);  in smc_llc_add_link_cont()
863 			*buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);  in smc_llc_add_link_cont()
867 	if (lgr->role == SMC_CLNT)  in smc_llc_add_link_cont()
879 	struct smc_link_group *lgr = link->lgr;  in smc_llc_cli_rkey_exchange()  local
887 	mutex_lock(&lgr->rmbs_lock);  in smc_llc_cli_rkey_exchange()
888 	num_rkeys_send = lgr->conns_num;  in smc_llc_cli_rkey_exchange()
889 	buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);  in smc_llc_cli_rkey_exchange()
891 		qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME,  in smc_llc_cli_rkey_exchange()
901 			smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,  in smc_llc_cli_rkey_exchange()
907 		smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_cli_rkey_exchange()
914 	mutex_unlock(&lgr->rmbs_lock);  in smc_llc_cli_rkey_exchange()
924 	smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,  in smc_llc_cli_add_link_reject()
934 	struct smc_link_group *lgr = link->lgr;  in smc_llc_cli_conf_link()  local
939 	qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0);  in smc_llc_cli_conf_link()
950 		smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_cli_conf_link()
954 	smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_cli_conf_link()
981 		smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);  in smc_llc_cli_conf_link()
983 		smcr_lgr_set_type(lgr, lgr_new_t);  in smc_llc_cli_conf_link()
991 	struct smc_link_group *lgr = link->lgr;  in smc_llc_save_add_link_rkeys()  local
994 	ext = (struct smc_llc_msg_add_link_v2_ext *)((u8 *)lgr->wr_rx_buf_v2 +  in smc_llc_save_add_link_rkeys()
997 	mutex_lock(&lgr->rmbs_lock);  in smc_llc_save_add_link_rkeys()
999 		smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,  in smc_llc_save_add_link_rkeys()
1004 	mutex_unlock(&lgr->rmbs_lock);  in smc_llc_save_add_link_rkeys()
1022 	struct smc_link_group *lgr = smc_get_lgr(link);  in smc_llc_cli_add_link()  local
1036 	ini->vlan_id = lgr->vlan_id;  in smc_llc_cli_add_link()
1037 	if (lgr->smc_version == SMC_V2) {  in smc_llc_cli_add_link()
1039 		ini->smcrv2.saddr = lgr->saddr;  in smc_llc_cli_add_link()
1042 	smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);  in smc_llc_cli_add_link()
1044 	    (lgr->smc_version == SMC_V2 ||  in smc_llc_cli_add_link()
1050 	if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {  in smc_llc_cli_add_link()
1054 	} else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {  in smc_llc_cli_add_link()
1059 	lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);  in smc_llc_cli_add_link()
1062 	lnk_new = &lgr->lnk[lnk_idx];  in smc_llc_cli_add_link()
1063 	rc = smcr_link_init(lgr, lnk_new, lnk_idx, ini);  in smc_llc_cli_add_link()
1083 	if (lgr->smc_version == SMC_V2) {  in smc_llc_cli_add_link()
1116 	if (link->lgr->type == SMC_LGR_SYMMETRIC ||  in smc_llc_send_request_add_link()
1117 	    link->lgr->type == SMC_LGR_ASYMMETRIC_PEER)  in smc_llc_send_request_add_link()
1120 	smc_fill_gid_list(link->lgr, &gidlist, link->smcibdev, link->gid);  in smc_llc_send_request_add_link()
1135 	smc_llc_init_msg_hdr(&llc->hd, link->lgr, len);  in smc_llc_send_request_add_link()
1139 		link->lgr->llc_flow_lcl.type = SMC_LLC_FLOW_REQ_ADD_LINK;  in smc_llc_send_request_add_link()
1148 	struct smc_link_group *lgr = smc_get_lgr(link);  in smc_llc_cli_add_link_invite()  local
1151 	if (lgr->smc_version == SMC_V2) {  in smc_llc_cli_add_link_invite()
1156 	if (lgr->type == SMC_LGR_SYMMETRIC ||  in smc_llc_cli_add_link_invite()
1157 	    lgr->type == SMC_LGR_ASYMMETRIC_PEER)  in smc_llc_cli_add_link_invite()
1164 	ini->vlan_id = lgr->vlan_id;  in smc_llc_cli_add_link_invite()
1165 	smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);  in smc_llc_cli_add_link_invite()
1194 static void smc_llc_process_cli_add_link(struct smc_link_group *lgr)  in smc_llc_process_cli_add_link()  argument
1198 	qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);  in smc_llc_process_cli_add_link()
1200 	mutex_lock(&lgr->llc_conf_mutex);  in smc_llc_process_cli_add_link()
1205 	mutex_unlock(&lgr->llc_conf_mutex);  in smc_llc_process_cli_add_link()
1208 static int smc_llc_active_link_count(struct smc_link_group *lgr)  in smc_llc_active_link_count()  argument
1213 		if (!smc_link_active(&lgr->lnk[i]))  in smc_llc_active_link_count()
1221 static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr)  in smc_llc_find_asym_link()  argument
1231 			if (!smc_link_usable(&lgr->lnk[i]) ||  in smc_llc_find_asym_link()
1232 			    !smc_link_usable(&lgr->lnk[j]))  in smc_llc_find_asym_link()
1234 			if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid,  in smc_llc_find_asym_link()
1246 		if (!smc_link_usable(&lgr->lnk[k]))  in smc_llc_find_asym_link()
1249 		    !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid,  in smc_llc_find_asym_link()
1255 		    !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid,  in smc_llc_find_asym_link()
1262 	return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx];  in smc_llc_find_asym_link()
1265 static void smc_llc_delete_asym_link(struct smc_link_group *lgr)  in smc_llc_delete_asym_link()  argument
1271 	lnk_asym = smc_llc_find_asym_link(lgr);  in smc_llc_delete_asym_link()
1276 	lnk_new = smc_switch_conns(lgr, lnk_asym, false);  in smc_llc_delete_asym_link()
1281 	lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK;  in smc_llc_delete_asym_link()
1288 	qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME,  in smc_llc_delete_asym_link()
1294 	smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_delete_asym_link()
1303 	struct smc_link_group *lgr = link->lgr;  in smc_llc_srv_rkey_exchange()  local
1311 	mutex_lock(&lgr->rmbs_lock);  in smc_llc_srv_rkey_exchange()
1312 	num_rkeys_send = lgr->conns_num;  in smc_llc_srv_rkey_exchange()
1313 	buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);  in smc_llc_srv_rkey_exchange()
1317 		qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME,  in smc_llc_srv_rkey_exchange()
1327 			smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,  in smc_llc_srv_rkey_exchange()
1333 		smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_srv_rkey_exchange()
1336 	mutex_unlock(&lgr->rmbs_lock);  in smc_llc_srv_rkey_exchange()
1344 	struct smc_link_group *lgr = link->lgr;  in smc_llc_srv_conf_link()  local
1353 	qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 0);  in smc_llc_srv_conf_link()
1360 			smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_srv_conf_link()
1367 		smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);  in smc_llc_srv_conf_link()
1369 		smcr_lgr_set_type(lgr, lgr_new_t);  in smc_llc_srv_conf_link()
1370 	smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_srv_conf_link()
1377 	smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,  in smc_llc_send_req_add_link_response()
1387 	struct smc_link_group *lgr = link->lgr;  in smc_llc_srv_add_link()  local
1406 	ini->vlan_id = lgr->vlan_id;  in smc_llc_srv_add_link()
1407 	if (lgr->smc_version == SMC_V2) {  in smc_llc_srv_add_link()
1409 		ini->smcrv2.saddr = lgr->saddr;  in smc_llc_srv_add_link()
1417 	smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);  in smc_llc_srv_add_link()
1418 	if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {  in smc_llc_srv_add_link()
1422 	} else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {  in smc_llc_srv_add_link()
1427 	lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);  in smc_llc_srv_add_link()
1433 	rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, ini);  in smc_llc_srv_add_link()
1436 	link_new = &lgr->lnk[lnk_idx];  in smc_llc_srv_add_link()
1449 	qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK);  in smc_llc_srv_add_link()
1456 		smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_srv_add_link()
1460 	if (lgr->type == SMC_LGR_SINGLE &&  in smc_llc_srv_add_link()
1462 	     (lgr->smc_version == SMC_V2 ||  in smc_llc_srv_add_link()
1467 	smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_srv_add_link()
1475 	if (lgr->smc_version == SMC_V2) {  in smc_llc_srv_add_link()
1499 static void smc_llc_process_srv_add_link(struct smc_link_group *lgr)  in smc_llc_process_srv_add_link()  argument
1501 	struct smc_link *link = lgr->llc_flow_lcl.qentry->link;  in smc_llc_process_srv_add_link()
1505 	qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);  in smc_llc_process_srv_add_link()
1507 	mutex_lock(&lgr->llc_conf_mutex);  in smc_llc_process_srv_add_link()
1509 	if (!rc && lgr->type == SMC_LGR_SYMMETRIC) {  in smc_llc_process_srv_add_link()
1511 		smc_llc_delete_asym_link(lgr);  in smc_llc_process_srv_add_link()
1513 	mutex_unlock(&lgr->llc_conf_mutex);  in smc_llc_process_srv_add_link()
1523 	smc_llc_init_msg_hdr(&add_llc.hd, link->lgr, sizeof(add_llc));  in smc_llc_add_link_local()
1531 	struct smc_link_group *lgr = container_of(work, struct smc_link_group,  in smc_llc_add_link_work()  local
1534 	if (list_empty(&lgr->list)) {  in smc_llc_add_link_work()
1536 		smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_add_link_work()
1540 	if (lgr->role == SMC_CLNT)  in smc_llc_add_link_work()
1541 		smc_llc_process_cli_add_link(lgr);  in smc_llc_add_link_work()
1543 		smc_llc_process_srv_add_link(lgr);  in smc_llc_add_link_work()
1545 	if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_REQ_ADD_LINK)  in smc_llc_add_link_work()
1546 		smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);  in smc_llc_add_link_work()
1557 	smc_llc_init_msg_hdr(&del_llc.hd, link->lgr, sizeof(del_llc));  in smc_llc_srv_delete_link_local()
1564 static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr)  in smc_llc_process_cli_delete_link()  argument
1572 	qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);  in smc_llc_process_cli_delete_link()
1577 		smc_lgr_terminate_sched(lgr);  in smc_llc_process_cli_delete_link()
1580 	mutex_lock(&lgr->llc_conf_mutex);  in smc_llc_process_cli_delete_link()
1583 		if (lgr->lnk[lnk_idx].link_id != del_llc->link_num)  in smc_llc_process_cli_delete_link()
1585 		lnk_del = &lgr->lnk[lnk_idx];  in smc_llc_process_cli_delete_link()
1595 	lnk_asym = smc_llc_find_asym_link(lgr);  in smc_llc_process_cli_delete_link()
1601 		smc_switch_conns(lgr, lnk_del, false);  in smc_llc_process_cli_delete_link()
1604 	active_links = smc_llc_active_link_count(lgr);  in smc_llc_process_cli_delete_link()
1608 		smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);  in smc_llc_process_cli_delete_link()
1610 		smcr_lgr_set_type(lgr, SMC_LGR_NONE);  in smc_llc_process_cli_delete_link()
1611 		smc_lgr_terminate_sched(lgr);  in smc_llc_process_cli_delete_link()
1614 	mutex_unlock(&lgr->llc_conf_mutex);  in smc_llc_process_cli_delete_link()
1622 void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn)  in smc_llc_send_link_delete_all()  argument
1628 	smc_llc_init_msg_hdr(&delllc.hd, lgr, sizeof(delllc));  in smc_llc_send_link_delete_all()
1635 		if (!smc_link_sendable(&lgr->lnk[i]))  in smc_llc_send_link_delete_all()
1637 		if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc))  in smc_llc_send_link_delete_all()
1642 static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr)  in smc_llc_process_srv_delete_link()  argument
1650 	mutex_lock(&lgr->llc_conf_mutex);  in smc_llc_process_srv_delete_link()
1651 	qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);  in smc_llc_process_srv_delete_link()
1657 		smc_llc_send_link_delete_all(lgr, true, ntohl(  in smc_llc_process_srv_delete_link()
1659 		smc_lgr_terminate_sched(lgr);  in smc_llc_process_srv_delete_link()
1665 		if (lgr->lnk[i].link_id == del_llc->link_num) {  in smc_llc_process_srv_delete_link()
1666 			lnk_del = &lgr->lnk[i];  in smc_llc_process_srv_delete_link()
1674 		if (smc_switch_conns(lgr, lnk_del, false))  in smc_llc_process_srv_delete_link()
1677 	if (!list_empty(&lgr->list)) {  in smc_llc_process_srv_delete_link()
1685 			qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME,  in smc_llc_process_srv_delete_link()
1688 				smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_process_srv_delete_link()
1693 	active_links = smc_llc_active_link_count(lgr);  in smc_llc_process_srv_delete_link()
1695 		smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);  in smc_llc_process_srv_delete_link()
1697 		smcr_lgr_set_type(lgr, SMC_LGR_NONE);  in smc_llc_process_srv_delete_link()
1698 		smc_lgr_terminate_sched(lgr);  in smc_llc_process_srv_delete_link()
1701 	if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) {  in smc_llc_process_srv_delete_link()
1706 	mutex_unlock(&lgr->llc_conf_mutex);  in smc_llc_process_srv_delete_link()
1712 	struct smc_link_group *lgr = container_of(work, struct smc_link_group,  in smc_llc_delete_link_work()  local
1715 	if (list_empty(&lgr->list)) {  in smc_llc_delete_link_work()
1717 		smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_delete_link_work()
1721 	if (lgr->role == SMC_CLNT)  in smc_llc_delete_link_work()
1722 		smc_llc_process_cli_delete_link(lgr);  in smc_llc_delete_link_work()
1724 		smc_llc_process_srv_delete_link(lgr);  in smc_llc_delete_link_work()
1726 	smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);  in smc_llc_delete_link_work()
1730 static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr)  in smc_llc_rmt_conf_rkey()  argument
1739 	qentry = lgr->llc_flow_rmt.qentry;  in smc_llc_rmt_conf_rkey()
1754 		smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id,  in smc_llc_rmt_conf_rkey()
1764 	smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));  in smc_llc_rmt_conf_rkey()
1766 	smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);  in smc_llc_rmt_conf_rkey()
1770 static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr)  in smc_llc_rmt_delete_rkey()  argument
1778 	qentry = lgr->llc_flow_rmt.qentry;  in smc_llc_rmt_delete_rkey()
1782 	if (lgr->smc_version == SMC_V2) {  in smc_llc_rmt_delete_rkey()
1785 		memcpy(lgr->wr_rx_buf_v2, llc, sizeof(*llc));  in smc_llc_rmt_delete_rkey()
1786 		llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)lgr->wr_rx_buf_v2;  in smc_llc_rmt_delete_rkey()
1796 		smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));  in smc_llc_rmt_delete_rkey()
1816 	smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);  in smc_llc_rmt_delete_rkey()
1819 static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type)  in smc_llc_protocol_violation()  argument
1822 			    "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id,  in smc_llc_protocol_violation()
1823 			    lgr->net->net_cookie, type);  in smc_llc_protocol_violation()
1824 	smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL);  in smc_llc_protocol_violation()
1825 	smc_lgr_terminate_sched(lgr);  in smc_llc_protocol_violation()
1829 static void smc_llc_event_flush(struct smc_link_group *lgr)  in smc_llc_event_flush()  argument
1833 	spin_lock_bh(&lgr->llc_event_q_lock);  in smc_llc_event_flush()
1834 	list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {  in smc_llc_event_flush()
1838 	spin_unlock_bh(&lgr->llc_event_q_lock);  in smc_llc_event_flush()
1845 	struct smc_link_group *lgr = link->lgr;  in smc_llc_event_handler()  local
1856 		if (list_empty(&lgr->list))  in smc_llc_event_handler()
1858 		if (lgr->role == SMC_CLNT) {  in smc_llc_event_handler()
1860 				if (lgr->llc_flow_lcl.type ==  in smc_llc_event_handler()
1863 				if (smc_llc_flow_start(&lgr->llc_flow_lcl,  in smc_llc_event_handler()
1865 					schedule_work(&lgr->llc_add_link_work);  in smc_llc_event_handler()
1869 			if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&  in smc_llc_event_handler()
1870 			    !lgr->llc_flow_lcl.qentry) {  in smc_llc_event_handler()
1872 				smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,  in smc_llc_event_handler()
1874 				wake_up(&lgr->llc_msg_waiter);  in smc_llc_event_handler()
1877 			if (lgr->llc_flow_lcl.type ==  in smc_llc_event_handler()
1880 				lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;  in smc_llc_event_handler()
1881 				smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,  in smc_llc_event_handler()
1883 				schedule_work(&lgr->llc_add_link_work);  in smc_llc_event_handler()
1886 			if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {  in smc_llc_event_handler()
1887 				schedule_work(&lgr->llc_add_link_work);  in smc_llc_event_handler()
1889 		} else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {  in smc_llc_event_handler()
1891 			schedule_work(&lgr->llc_add_link_work);  in smc_llc_event_handler()
1896 		if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {  in smc_llc_event_handler()
1898 			smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);  in smc_llc_event_handler()
1899 			wake_up(&lgr->llc_msg_waiter);  in smc_llc_event_handler()
1904 		if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&  in smc_llc_event_handler()
1905 		    !lgr->llc_flow_lcl.qentry) {  in smc_llc_event_handler()
1907 			smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);  in smc_llc_event_handler()
1908 			wake_up(&lgr->llc_msg_waiter);  in smc_llc_event_handler()
1909 		} else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {  in smc_llc_event_handler()
1910 			schedule_work(&lgr->llc_del_link_work);  in smc_llc_event_handler()
1915 		if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {  in smc_llc_event_handler()
1917 			smc_llc_rmt_conf_rkey(lgr);  in smc_llc_event_handler()
1918 			smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);  in smc_llc_event_handler()
1928 		if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {  in smc_llc_event_handler()
1930 			smc_llc_rmt_delete_rkey(lgr);  in smc_llc_event_handler()
1931 			smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);  in smc_llc_event_handler()
1938 		if (lgr->role == SMC_CLNT &&  in smc_llc_event_handler()
1939 		    lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK &&  in smc_llc_event_handler()
1941 			smc_llc_flow_stop(link->lgr, &lgr->llc_flow_lcl);  in smc_llc_event_handler()
1942 		} else if (lgr->role == SMC_SERV) {  in smc_llc_event_handler()
1943 			if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {  in smc_llc_event_handler()
1945 				lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;  in smc_llc_event_handler()
1946 				schedule_work(&lgr->llc_add_link_work);  in smc_llc_event_handler()
1952 		smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type);  in smc_llc_event_handler()
1962 	struct smc_link_group *lgr = container_of(work, struct smc_link_group,  in smc_llc_event_work()  local
1966 	if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {  in smc_llc_event_work()
1967 		qentry = lgr->delayed_event;  in smc_llc_event_work()
1968 		lgr->delayed_event = NULL;  in smc_llc_event_work()
1976 	spin_lock_bh(&lgr->llc_event_q_lock);  in smc_llc_event_work()
1977 	if (!list_empty(&lgr->llc_event_q)) {  in smc_llc_event_work()
1978 		qentry = list_first_entry(&lgr->llc_event_q,  in smc_llc_event_work()
1981 		spin_unlock_bh(&lgr->llc_event_q_lock);  in smc_llc_event_work()
1985 	spin_unlock_bh(&lgr->llc_event_q_lock);  in smc_llc_event_work()
1992 	enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type;  in smc_llc_rx_response()
1993 	struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl;  in smc_llc_rx_response()
2020 		smc_llc_protocol_violation(link->lgr,  in smc_llc_rx_response()
2028 	smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry);  in smc_llc_rx_response()
2029 	wake_up(&link->lgr->llc_msg_waiter);  in smc_llc_rx_response()
2034 	struct smc_link_group *lgr = link->lgr;  in smc_llc_enqueue()  local
2053 	spin_lock_irqsave(&lgr->llc_event_q_lock, flags);  in smc_llc_enqueue()
2054 	list_add_tail(&qentry->list, &lgr->llc_event_q);  in smc_llc_enqueue()
2055 	spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);  in smc_llc_enqueue()
2056 	queue_work(system_highpri_wq, &lgr->llc_event_work);  in smc_llc_enqueue()
2112 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)  in smc_llc_lgr_init()  argument
2116 	INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);  in smc_llc_lgr_init()
2117 	INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work);  in smc_llc_lgr_init()
2118 	INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work);  in smc_llc_lgr_init()
2119 	INIT_LIST_HEAD(&lgr->llc_event_q);  in smc_llc_lgr_init()
2120 	spin_lock_init(&lgr->llc_event_q_lock);  in smc_llc_lgr_init()
2121 	spin_lock_init(&lgr->llc_flow_lock);  in smc_llc_lgr_init()
2122 	init_waitqueue_head(&lgr->llc_flow_waiter);  in smc_llc_lgr_init()
2123 	init_waitqueue_head(&lgr->llc_msg_waiter);  in smc_llc_lgr_init()
2124 	mutex_init(&lgr->llc_conf_mutex);  in smc_llc_lgr_init()
2125 	lgr->llc_testlink_time = READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time);  in smc_llc_lgr_init()
2129 void smc_llc_lgr_clear(struct smc_link_group *lgr)  in smc_llc_lgr_clear()  argument
2131 	smc_llc_event_flush(lgr);  in smc_llc_lgr_clear()
2132 	wake_up_all(&lgr->llc_flow_waiter);  in smc_llc_lgr_clear()
2133 	wake_up_all(&lgr->llc_msg_waiter);  in smc_llc_lgr_clear()
2134 	cancel_work_sync(&lgr->llc_event_work);  in smc_llc_lgr_clear()
2135 	cancel_work_sync(&lgr->llc_add_link_work);  in smc_llc_lgr_clear()
2136 	cancel_work_sync(&lgr->llc_del_link_work);  in smc_llc_lgr_clear()
2137 	if (lgr->delayed_event) {  in smc_llc_lgr_clear()
2138 		kfree(lgr->delayed_event);  in smc_llc_lgr_clear()
2139 		lgr->delayed_event = NULL;  in smc_llc_lgr_clear()
2154 			    SMC_LGR_ID_SIZE, &link->lgr->id,  in smc_llc_link_active()
2155 			    link->lgr->net->net_cookie,  in smc_llc_link_active()
2160 	if (link->lgr->llc_testlink_time) {  in smc_llc_link_active()
2161 		link->llc_testlink_time = link->lgr->llc_testlink_time;  in smc_llc_link_active()
2173 				    SMC_LGR_ID_SIZE, &link->lgr->id,  in smc_llc_link_clear()
2174 				    link->lgr->net->net_cookie,  in smc_llc_link_clear()
2186 	struct smc_link_group *lgr = send_link->lgr;  in smc_llc_do_confirm_rkey()  local
2194 	qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,  in smc_llc_do_confirm_rkey()
2200 		smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_do_confirm_rkey()
2205 int smc_llc_do_delete_rkey(struct smc_link_group *lgr,  in smc_llc_do_delete_rkey()  argument
2212 	send_link = smc_llc_usable_link(lgr);  in smc_llc_do_delete_rkey()
2221 	qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,  in smc_llc_do_delete_rkey()
2227 		smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);  in smc_llc_do_delete_rkey()
2235 	link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id);  in smc_llc_link_set_uid()