1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2004-2007 Intel Corporation.  All rights reserved.
4  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
5  * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.
6  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
7  * Copyright (c) 2019, Mellanox Technologies inc.  All rights reserved.
8  */
9 
10 #include <linux/completion.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/device.h>
13 #include <linux/module.h>
14 #include <linux/err.h>
15 #include <linux/idr.h>
16 #include <linux/interrupt.h>
17 #include <linux/random.h>
18 #include <linux/rbtree.h>
19 #include <linux/spinlock.h>
20 #include <linux/slab.h>
21 #include <linux/sysfs.h>
22 #include <linux/workqueue.h>
23 #include <linux/kdev_t.h>
24 #include <linux/etherdevice.h>
25 
26 #include <rdma/ib_cache.h>
27 #include <rdma/ib_cm.h>
28 #include <rdma/ib_sysfs.h>
29 #include "cm_msgs.h"
30 #include "core_priv.h"
31 #include "cm_trace.h"
32 
33 MODULE_AUTHOR("Sean Hefty");
34 MODULE_DESCRIPTION("InfiniBand CM");
35 MODULE_LICENSE("Dual BSD/GPL");
36 
37 static const char * const ibcm_rej_reason_strs[] = {
38 	[IB_CM_REJ_NO_QP]			= "no QP",
39 	[IB_CM_REJ_NO_EEC]			= "no EEC",
40 	[IB_CM_REJ_NO_RESOURCES]		= "no resources",
41 	[IB_CM_REJ_TIMEOUT]			= "timeout",
42 	[IB_CM_REJ_UNSUPPORTED]			= "unsupported",
43 	[IB_CM_REJ_INVALID_COMM_ID]		= "invalid comm ID",
44 	[IB_CM_REJ_INVALID_COMM_INSTANCE]	= "invalid comm instance",
45 	[IB_CM_REJ_INVALID_SERVICE_ID]		= "invalid service ID",
46 	[IB_CM_REJ_INVALID_TRANSPORT_TYPE]	= "invalid transport type",
47 	[IB_CM_REJ_STALE_CONN]			= "stale conn",
48 	[IB_CM_REJ_RDC_NOT_EXIST]		= "RDC not exist",
49 	[IB_CM_REJ_INVALID_GID]			= "invalid GID",
50 	[IB_CM_REJ_INVALID_LID]			= "invalid LID",
51 	[IB_CM_REJ_INVALID_SL]			= "invalid SL",
52 	[IB_CM_REJ_INVALID_TRAFFIC_CLASS]	= "invalid traffic class",
53 	[IB_CM_REJ_INVALID_HOP_LIMIT]		= "invalid hop limit",
54 	[IB_CM_REJ_INVALID_PACKET_RATE]		= "invalid packet rate",
55 	[IB_CM_REJ_INVALID_ALT_GID]		= "invalid alt GID",
56 	[IB_CM_REJ_INVALID_ALT_LID]		= "invalid alt LID",
57 	[IB_CM_REJ_INVALID_ALT_SL]		= "invalid alt SL",
58 	[IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS]	= "invalid alt traffic class",
59 	[IB_CM_REJ_INVALID_ALT_HOP_LIMIT]	= "invalid alt hop limit",
60 	[IB_CM_REJ_INVALID_ALT_PACKET_RATE]	= "invalid alt packet rate",
61 	[IB_CM_REJ_PORT_CM_REDIRECT]		= "port CM redirect",
62 	[IB_CM_REJ_PORT_REDIRECT]		= "port redirect",
63 	[IB_CM_REJ_INVALID_MTU]			= "invalid MTU",
64 	[IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES]	= "insufficient resp resources",
65 	[IB_CM_REJ_CONSUMER_DEFINED]		= "consumer defined",
66 	[IB_CM_REJ_INVALID_RNR_RETRY]		= "invalid RNR retry",
67 	[IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID]	= "duplicate local comm ID",
68 	[IB_CM_REJ_INVALID_CLASS_VERSION]	= "invalid class version",
69 	[IB_CM_REJ_INVALID_FLOW_LABEL]		= "invalid flow label",
70 	[IB_CM_REJ_INVALID_ALT_FLOW_LABEL]	= "invalid alt flow label",
71 	[IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED] =
72 		"vendor option is not supported",
73 };
74 
ibcm_reject_msg(int reason)75 const char *__attribute_const__ ibcm_reject_msg(int reason)
76 {
77 	size_t index = reason;
78 
79 	if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
80 	    ibcm_rej_reason_strs[index])
81 		return ibcm_rej_reason_strs[index];
82 	else
83 		return "unrecognized reason";
84 }
85 EXPORT_SYMBOL(ibcm_reject_msg);
86 
87 struct cm_id_private;
88 struct cm_work;
89 static int cm_add_one(struct ib_device *device);
90 static void cm_remove_one(struct ib_device *device, void *client_data);
91 static void cm_process_work(struct cm_id_private *cm_id_priv,
92 			    struct cm_work *work);
93 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
94 				   struct ib_cm_sidr_rep_param *param);
95 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
96 			       const void *private_data, u8 private_data_len);
97 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
98 			       void *private_data, u8 private_data_len);
99 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
100 			      enum ib_cm_rej_reason reason, void *ari,
101 			      u8 ari_length, const void *private_data,
102 			      u8 private_data_len);
103 
104 static struct ib_client cm_client = {
105 	.name   = "cm",
106 	.add    = cm_add_one,
107 	.remove = cm_remove_one
108 };
109 
110 static struct ib_cm {
111 	spinlock_t lock;
112 	struct list_head device_list;
113 	rwlock_t device_lock;
114 	struct rb_root listen_service_table;
115 	u64 listen_service_id;
116 	/* struct rb_root peer_service_table; todo: fix peer to peer */
117 	struct rb_root remote_qp_table;
118 	struct rb_root remote_id_table;
119 	struct rb_root remote_sidr_table;
120 	struct xarray local_id_table;
121 	u32 local_id_next;
122 	__be32 random_id_operand;
123 	struct list_head timewait_list;
124 	struct workqueue_struct *wq;
125 } cm;
126 
127 /* Counter indexes ordered by attribute ID */
128 enum {
129 	CM_REQ_COUNTER,
130 	CM_MRA_COUNTER,
131 	CM_REJ_COUNTER,
132 	CM_REP_COUNTER,
133 	CM_RTU_COUNTER,
134 	CM_DREQ_COUNTER,
135 	CM_DREP_COUNTER,
136 	CM_SIDR_REQ_COUNTER,
137 	CM_SIDR_REP_COUNTER,
138 	CM_LAP_COUNTER,
139 	CM_APR_COUNTER,
140 	CM_ATTR_COUNT,
141 	CM_ATTR_ID_OFFSET = 0x0010,
142 };
143 
144 enum {
145 	CM_XMIT,
146 	CM_XMIT_RETRIES,
147 	CM_RECV,
148 	CM_RECV_DUPLICATES,
149 	CM_COUNTER_GROUPS
150 };
151 
152 struct cm_counter_attribute {
153 	struct ib_port_attribute attr;
154 	unsigned short group;
155 	unsigned short index;
156 };
157 
158 struct cm_port {
159 	struct cm_device *cm_dev;
160 	struct ib_mad_agent *mad_agent;
161 	u32 port_num;
162 	atomic_long_t counters[CM_COUNTER_GROUPS][CM_ATTR_COUNT];
163 };
164 
165 struct cm_device {
166 	struct kref kref;
167 	struct list_head list;
168 	spinlock_t mad_agent_lock;
169 	struct ib_device *ib_device;
170 	u8 ack_delay;
171 	int going_down;
172 	struct cm_port *port[];
173 };
174 
175 struct cm_av {
176 	struct cm_port *port;
177 	struct rdma_ah_attr ah_attr;
178 	u16 dlid_datapath;
179 	u16 pkey_index;
180 	u8 timeout;
181 };
182 
183 struct cm_work {
184 	struct delayed_work work;
185 	struct list_head list;
186 	struct cm_port *port;
187 	struct ib_mad_recv_wc *mad_recv_wc;	/* Received MADs */
188 	__be32 local_id;			/* Established / timewait */
189 	__be32 remote_id;
190 	struct ib_cm_event cm_event;
191 	struct sa_path_rec path[];
192 };
193 
194 struct cm_timewait_info {
195 	struct cm_work work;
196 	struct list_head list;
197 	struct rb_node remote_qp_node;
198 	struct rb_node remote_id_node;
199 	__be64 remote_ca_guid;
200 	__be32 remote_qpn;
201 	u8 inserted_remote_qp;
202 	u8 inserted_remote_id;
203 };
204 
205 struct cm_id_private {
206 	struct ib_cm_id	id;
207 
208 	struct rb_node service_node;
209 	struct rb_node sidr_id_node;
210 	u32 sidr_slid;
211 	spinlock_t lock;	/* Do not acquire inside cm.lock */
212 	struct completion comp;
213 	refcount_t refcount;
214 	/* Number of clients sharing this ib_cm_id. Only valid for listeners.
215 	 * Protected by the cm.lock spinlock.
216 	 */
217 	int listen_sharecount;
218 	struct rcu_head rcu;
219 
220 	struct ib_mad_send_buf *msg;
221 	struct cm_timewait_info *timewait_info;
222 	/* todo: use alternate port on send failure */
223 	struct cm_av av;
224 	struct cm_av alt_av;
225 
226 	void *private_data;
227 	__be64 tid;
228 	__be32 local_qpn;
229 	__be32 remote_qpn;
230 	enum ib_qp_type qp_type;
231 	__be32 sq_psn;
232 	__be32 rq_psn;
233 	int timeout_ms;
234 	enum ib_mtu path_mtu;
235 	__be16 pkey;
236 	u8 private_data_len;
237 	u8 max_cm_retries;
238 	u8 responder_resources;
239 	u8 initiator_depth;
240 	u8 retry_count;
241 	u8 rnr_retry_count;
242 	u8 service_timeout;
243 	u8 target_ack_delay;
244 
245 	struct list_head work_list;
246 	atomic_t work_count;
247 
248 	struct rdma_ucm_ece ece;
249 };
250 
cm_dev_release(struct kref * kref)251 static void cm_dev_release(struct kref *kref)
252 {
253 	struct cm_device *cm_dev = container_of(kref, struct cm_device, kref);
254 	u32 i;
255 
256 	rdma_for_each_port(cm_dev->ib_device, i)
257 		kfree(cm_dev->port[i - 1]);
258 
259 	kfree(cm_dev);
260 }
261 
cm_device_put(struct cm_device * cm_dev)262 static void cm_device_put(struct cm_device *cm_dev)
263 {
264 	kref_put(&cm_dev->kref, cm_dev_release);
265 }
266 
267 static void cm_work_handler(struct work_struct *work);
268 
cm_deref_id(struct cm_id_private * cm_id_priv)269 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
270 {
271 	if (refcount_dec_and_test(&cm_id_priv->refcount))
272 		complete(&cm_id_priv->comp);
273 }
274 
cm_alloc_msg(struct cm_id_private * cm_id_priv)275 static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv)
276 {
277 	struct ib_mad_agent *mad_agent;
278 	struct ib_mad_send_buf *m;
279 	struct ib_ah *ah;
280 
281 	lockdep_assert_held(&cm_id_priv->lock);
282 
283 	if (!cm_id_priv->av.port)
284 		return ERR_PTR(-EINVAL);
285 
286 	spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
287 	mad_agent = cm_id_priv->av.port->mad_agent;
288 	if (!mad_agent) {
289 		m = ERR_PTR(-EINVAL);
290 		goto out;
291 	}
292 
293 	ah = rdma_create_ah(mad_agent->qp->pd, &cm_id_priv->av.ah_attr, 0);
294 	if (IS_ERR(ah)) {
295 		m = ERR_CAST(ah);
296 		goto out;
297 	}
298 
299 	m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
300 			       cm_id_priv->av.pkey_index,
301 			       0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
302 			       GFP_ATOMIC,
303 			       IB_MGMT_BASE_VERSION);
304 	if (IS_ERR(m)) {
305 		rdma_destroy_ah(ah, 0);
306 		goto out;
307 	}
308 
309 	/* Timeout set by caller if response is expected. */
310 	m->ah = ah;
311 	m->retries = cm_id_priv->max_cm_retries;
312 
313 	refcount_inc(&cm_id_priv->refcount);
314 	m->context[0] = cm_id_priv;
315 
316 out:
317 	spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
318 	return m;
319 }
320 
cm_free_msg(struct ib_mad_send_buf * msg)321 static void cm_free_msg(struct ib_mad_send_buf *msg)
322 {
323 	struct cm_id_private *cm_id_priv = msg->context[0];
324 
325 	if (msg->ah)
326 		rdma_destroy_ah(msg->ah, 0);
327 	cm_deref_id(cm_id_priv);
328 	ib_free_send_mad(msg);
329 }
330 
331 static struct ib_mad_send_buf *
cm_alloc_priv_msg(struct cm_id_private * cm_id_priv)332 cm_alloc_priv_msg(struct cm_id_private *cm_id_priv)
333 {
334 	struct ib_mad_send_buf *msg;
335 
336 	lockdep_assert_held(&cm_id_priv->lock);
337 
338 	msg = cm_alloc_msg(cm_id_priv);
339 	if (IS_ERR(msg))
340 		return msg;
341 	cm_id_priv->msg = msg;
342 	return msg;
343 }
344 
cm_free_priv_msg(struct ib_mad_send_buf * msg)345 static void cm_free_priv_msg(struct ib_mad_send_buf *msg)
346 {
347 	struct cm_id_private *cm_id_priv = msg->context[0];
348 
349 	lockdep_assert_held(&cm_id_priv->lock);
350 
351 	if (!WARN_ON(cm_id_priv->msg != msg))
352 		cm_id_priv->msg = NULL;
353 
354 	if (msg->ah)
355 		rdma_destroy_ah(msg->ah, 0);
356 	cm_deref_id(cm_id_priv);
357 	ib_free_send_mad(msg);
358 }
359 
cm_alloc_response_msg_no_ah(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc)360 static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port,
361 							   struct ib_mad_recv_wc *mad_recv_wc)
362 {
363 	return ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
364 				  0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
365 				  GFP_ATOMIC,
366 				  IB_MGMT_BASE_VERSION);
367 }
368 
cm_create_response_msg_ah(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,struct ib_mad_send_buf * msg)369 static int cm_create_response_msg_ah(struct cm_port *port,
370 				     struct ib_mad_recv_wc *mad_recv_wc,
371 				     struct ib_mad_send_buf *msg)
372 {
373 	struct ib_ah *ah;
374 
375 	ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
376 				  mad_recv_wc->recv_buf.grh, port->port_num);
377 	if (IS_ERR(ah))
378 		return PTR_ERR(ah);
379 
380 	msg->ah = ah;
381 	return 0;
382 }
383 
cm_alloc_response_msg(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,struct ib_mad_send_buf ** msg)384 static int cm_alloc_response_msg(struct cm_port *port,
385 				 struct ib_mad_recv_wc *mad_recv_wc,
386 				 struct ib_mad_send_buf **msg)
387 {
388 	struct ib_mad_send_buf *m;
389 	int ret;
390 
391 	m = cm_alloc_response_msg_no_ah(port, mad_recv_wc);
392 	if (IS_ERR(m))
393 		return PTR_ERR(m);
394 
395 	ret = cm_create_response_msg_ah(port, mad_recv_wc, m);
396 	if (ret) {
397 		ib_free_send_mad(m);
398 		return ret;
399 	}
400 
401 	*msg = m;
402 	return 0;
403 }
404 
cm_free_response_msg(struct ib_mad_send_buf * msg)405 static void cm_free_response_msg(struct ib_mad_send_buf *msg)
406 {
407 	if (msg->ah)
408 		rdma_destroy_ah(msg->ah, 0);
409 	ib_free_send_mad(msg);
410 }
411 
cm_copy_private_data(const void * private_data,u8 private_data_len)412 static void *cm_copy_private_data(const void *private_data, u8 private_data_len)
413 {
414 	void *data;
415 
416 	if (!private_data || !private_data_len)
417 		return NULL;
418 
419 	data = kmemdup(private_data, private_data_len, GFP_KERNEL);
420 	if (!data)
421 		return ERR_PTR(-ENOMEM);
422 
423 	return data;
424 }
425 
cm_set_private_data(struct cm_id_private * cm_id_priv,void * private_data,u8 private_data_len)426 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
427 				 void *private_data, u8 private_data_len)
428 {
429 	if (cm_id_priv->private_data && cm_id_priv->private_data_len)
430 		kfree(cm_id_priv->private_data);
431 
432 	cm_id_priv->private_data = private_data;
433 	cm_id_priv->private_data_len = private_data_len;
434 }
435 
cm_set_av_port(struct cm_av * av,struct cm_port * port)436 static void cm_set_av_port(struct cm_av *av, struct cm_port *port)
437 {
438 	struct cm_port *old_port = av->port;
439 
440 	if (old_port == port)
441 		return;
442 
443 	av->port = port;
444 	if (old_port)
445 		cm_device_put(old_port->cm_dev);
446 	if (port)
447 		kref_get(&port->cm_dev->kref);
448 }
449 
cm_init_av_for_lap(struct cm_port * port,struct ib_wc * wc,struct rdma_ah_attr * ah_attr,struct cm_av * av)450 static void cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
451 			       struct rdma_ah_attr *ah_attr, struct cm_av *av)
452 {
453 	cm_set_av_port(av, port);
454 	av->pkey_index = wc->pkey_index;
455 	rdma_move_ah_attr(&av->ah_attr, ah_attr);
456 }
457 
cm_init_av_for_response(struct cm_port * port,struct ib_wc * wc,struct ib_grh * grh,struct cm_av * av)458 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
459 				   struct ib_grh *grh, struct cm_av *av)
460 {
461 	cm_set_av_port(av, port);
462 	av->pkey_index = wc->pkey_index;
463 	return ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
464 				       port->port_num, wc,
465 				       grh, &av->ah_attr);
466 }
467 
468 static struct cm_port *
get_cm_port_from_path(struct sa_path_rec * path,const struct ib_gid_attr * attr)469 get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)
470 {
471 	struct cm_device *cm_dev;
472 	struct cm_port *port = NULL;
473 	unsigned long flags;
474 
475 	if (attr) {
476 		read_lock_irqsave(&cm.device_lock, flags);
477 		list_for_each_entry(cm_dev, &cm.device_list, list) {
478 			if (cm_dev->ib_device == attr->device) {
479 				port = cm_dev->port[attr->port_num - 1];
480 				break;
481 			}
482 		}
483 		read_unlock_irqrestore(&cm.device_lock, flags);
484 	} else {
485 		/* SGID attribute can be NULL in following
486 		 * conditions.
487 		 * (a) Alternative path
488 		 * (b) IB link layer without GRH
489 		 * (c) LAP send messages
490 		 */
491 		read_lock_irqsave(&cm.device_lock, flags);
492 		list_for_each_entry(cm_dev, &cm.device_list, list) {
493 			attr = rdma_find_gid(cm_dev->ib_device,
494 					     &path->sgid,
495 					     sa_conv_pathrec_to_gid_type(path),
496 					     NULL);
497 			if (!IS_ERR(attr)) {
498 				port = cm_dev->port[attr->port_num - 1];
499 				break;
500 			}
501 		}
502 		read_unlock_irqrestore(&cm.device_lock, flags);
503 		if (port)
504 			rdma_put_gid_attr(attr);
505 	}
506 	return port;
507 }
508 
cm_init_av_by_path(struct sa_path_rec * path,const struct ib_gid_attr * sgid_attr,struct cm_av * av)509 static int cm_init_av_by_path(struct sa_path_rec *path,
510 			      const struct ib_gid_attr *sgid_attr,
511 			      struct cm_av *av)
512 {
513 	struct rdma_ah_attr new_ah_attr;
514 	struct cm_device *cm_dev;
515 	struct cm_port *port;
516 	int ret;
517 
518 	port = get_cm_port_from_path(path, sgid_attr);
519 	if (!port)
520 		return -EINVAL;
521 	cm_dev = port->cm_dev;
522 
523 	ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
524 				  be16_to_cpu(path->pkey), &av->pkey_index);
525 	if (ret)
526 		return ret;
527 
528 	cm_set_av_port(av, port);
529 
530 	/*
531 	 * av->ah_attr might be initialized based on wc or during
532 	 * request processing time which might have reference to sgid_attr.
533 	 * So initialize a new ah_attr on stack.
534 	 * If initialization fails, old ah_attr is used for sending any
535 	 * responses. If initialization is successful, than new ah_attr
536 	 * is used by overwriting the old one. So that right ah_attr
537 	 * can be used to return an error response.
538 	 */
539 	ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
540 					&new_ah_attr, sgid_attr);
541 	if (ret)
542 		return ret;
543 
544 	av->timeout = path->packet_life_time + 1;
545 	rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
546 	return 0;
547 }
548 
549 /* Move av created by cm_init_av_by_path(), so av.dgid is not moved */
cm_move_av_from_path(struct cm_av * dest,struct cm_av * src)550 static void cm_move_av_from_path(struct cm_av *dest, struct cm_av *src)
551 {
552 	cm_set_av_port(dest, src->port);
553 	cm_set_av_port(src, NULL);
554 	dest->pkey_index = src->pkey_index;
555 	rdma_move_ah_attr(&dest->ah_attr, &src->ah_attr);
556 	dest->timeout = src->timeout;
557 }
558 
cm_destroy_av(struct cm_av * av)559 static void cm_destroy_av(struct cm_av *av)
560 {
561 	rdma_destroy_ah_attr(&av->ah_attr);
562 	cm_set_av_port(av, NULL);
563 }
564 
cm_local_id(__be32 local_id)565 static u32 cm_local_id(__be32 local_id)
566 {
567 	return (__force u32) (local_id ^ cm.random_id_operand);
568 }
569 
cm_acquire_id(__be32 local_id,__be32 remote_id)570 static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id)
571 {
572 	struct cm_id_private *cm_id_priv;
573 
574 	rcu_read_lock();
575 	cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id));
576 	if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id ||
577 	    !refcount_inc_not_zero(&cm_id_priv->refcount))
578 		cm_id_priv = NULL;
579 	rcu_read_unlock();
580 
581 	return cm_id_priv;
582 }
583 
584 /*
585  * Trivial helpers to strip endian annotation and compare; the
586  * endianness doesn't actually matter since we just need a stable
587  * order for the RB tree.
588  */
be32_lt(__be32 a,__be32 b)589 static int be32_lt(__be32 a, __be32 b)
590 {
591 	return (__force u32) a < (__force u32) b;
592 }
593 
be32_gt(__be32 a,__be32 b)594 static int be32_gt(__be32 a, __be32 b)
595 {
596 	return (__force u32) a > (__force u32) b;
597 }
598 
be64_lt(__be64 a,__be64 b)599 static int be64_lt(__be64 a, __be64 b)
600 {
601 	return (__force u64) a < (__force u64) b;
602 }
603 
be64_gt(__be64 a,__be64 b)604 static int be64_gt(__be64 a, __be64 b)
605 {
606 	return (__force u64) a > (__force u64) b;
607 }
608 
609 /*
610  * Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv
611  * if the new ID was inserted, NULL if it could not be inserted due to a
612  * collision, or the existing cm_id_priv ready for shared usage.
613  */
cm_insert_listen(struct cm_id_private * cm_id_priv,ib_cm_handler shared_handler)614 static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv,
615 					      ib_cm_handler shared_handler)
616 {
617 	struct rb_node **link = &cm.listen_service_table.rb_node;
618 	struct rb_node *parent = NULL;
619 	struct cm_id_private *cur_cm_id_priv;
620 	__be64 service_id = cm_id_priv->id.service_id;
621 	unsigned long flags;
622 
623 	spin_lock_irqsave(&cm.lock, flags);
624 	while (*link) {
625 		parent = *link;
626 		cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
627 					  service_node);
628 
629 		if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
630 			link = &(*link)->rb_left;
631 		else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
632 			link = &(*link)->rb_right;
633 		else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
634 			link = &(*link)->rb_left;
635 		else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
636 			link = &(*link)->rb_right;
637 		else {
638 			/*
639 			 * Sharing an ib_cm_id with different handlers is not
640 			 * supported
641 			 */
642 			if (cur_cm_id_priv->id.cm_handler != shared_handler ||
643 			    cur_cm_id_priv->id.context ||
644 			    WARN_ON(!cur_cm_id_priv->id.cm_handler)) {
645 				spin_unlock_irqrestore(&cm.lock, flags);
646 				return NULL;
647 			}
648 			refcount_inc(&cur_cm_id_priv->refcount);
649 			cur_cm_id_priv->listen_sharecount++;
650 			spin_unlock_irqrestore(&cm.lock, flags);
651 			return cur_cm_id_priv;
652 		}
653 	}
654 	cm_id_priv->listen_sharecount++;
655 	rb_link_node(&cm_id_priv->service_node, parent, link);
656 	rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
657 	spin_unlock_irqrestore(&cm.lock, flags);
658 	return cm_id_priv;
659 }
660 
cm_find_listen(struct ib_device * device,__be64 service_id)661 static struct cm_id_private *cm_find_listen(struct ib_device *device,
662 					    __be64 service_id)
663 {
664 	struct rb_node *node = cm.listen_service_table.rb_node;
665 	struct cm_id_private *cm_id_priv;
666 
667 	while (node) {
668 		cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
669 
670 		if (device < cm_id_priv->id.device)
671 			node = node->rb_left;
672 		else if (device > cm_id_priv->id.device)
673 			node = node->rb_right;
674 		else if (be64_lt(service_id, cm_id_priv->id.service_id))
675 			node = node->rb_left;
676 		else if (be64_gt(service_id, cm_id_priv->id.service_id))
677 			node = node->rb_right;
678 		else {
679 			refcount_inc(&cm_id_priv->refcount);
680 			return cm_id_priv;
681 		}
682 	}
683 	return NULL;
684 }
685 
686 static struct cm_timewait_info *
cm_insert_remote_id(struct cm_timewait_info * timewait_info)687 cm_insert_remote_id(struct cm_timewait_info *timewait_info)
688 {
689 	struct rb_node **link = &cm.remote_id_table.rb_node;
690 	struct rb_node *parent = NULL;
691 	struct cm_timewait_info *cur_timewait_info;
692 	__be64 remote_ca_guid = timewait_info->remote_ca_guid;
693 	__be32 remote_id = timewait_info->work.remote_id;
694 
695 	while (*link) {
696 		parent = *link;
697 		cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
698 					     remote_id_node);
699 		if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
700 			link = &(*link)->rb_left;
701 		else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
702 			link = &(*link)->rb_right;
703 		else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
704 			link = &(*link)->rb_left;
705 		else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
706 			link = &(*link)->rb_right;
707 		else
708 			return cur_timewait_info;
709 	}
710 	timewait_info->inserted_remote_id = 1;
711 	rb_link_node(&timewait_info->remote_id_node, parent, link);
712 	rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
713 	return NULL;
714 }
715 
cm_find_remote_id(__be64 remote_ca_guid,__be32 remote_id)716 static struct cm_id_private *cm_find_remote_id(__be64 remote_ca_guid,
717 					       __be32 remote_id)
718 {
719 	struct rb_node *node = cm.remote_id_table.rb_node;
720 	struct cm_timewait_info *timewait_info;
721 	struct cm_id_private *res = NULL;
722 
723 	spin_lock_irq(&cm.lock);
724 	while (node) {
725 		timewait_info = rb_entry(node, struct cm_timewait_info,
726 					 remote_id_node);
727 		if (be32_lt(remote_id, timewait_info->work.remote_id))
728 			node = node->rb_left;
729 		else if (be32_gt(remote_id, timewait_info->work.remote_id))
730 			node = node->rb_right;
731 		else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
732 			node = node->rb_left;
733 		else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
734 			node = node->rb_right;
735 		else {
736 			res = cm_acquire_id(timewait_info->work.local_id,
737 					     timewait_info->work.remote_id);
738 			break;
739 		}
740 	}
741 	spin_unlock_irq(&cm.lock);
742 	return res;
743 }
744 
745 static struct cm_timewait_info *
cm_insert_remote_qpn(struct cm_timewait_info * timewait_info)746 cm_insert_remote_qpn(struct cm_timewait_info *timewait_info)
747 {
748 	struct rb_node **link = &cm.remote_qp_table.rb_node;
749 	struct rb_node *parent = NULL;
750 	struct cm_timewait_info *cur_timewait_info;
751 	__be64 remote_ca_guid = timewait_info->remote_ca_guid;
752 	__be32 remote_qpn = timewait_info->remote_qpn;
753 
754 	while (*link) {
755 		parent = *link;
756 		cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
757 					     remote_qp_node);
758 		if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
759 			link = &(*link)->rb_left;
760 		else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
761 			link = &(*link)->rb_right;
762 		else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
763 			link = &(*link)->rb_left;
764 		else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
765 			link = &(*link)->rb_right;
766 		else
767 			return cur_timewait_info;
768 	}
769 	timewait_info->inserted_remote_qp = 1;
770 	rb_link_node(&timewait_info->remote_qp_node, parent, link);
771 	rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
772 	return NULL;
773 }
774 
775 static struct cm_id_private *
cm_insert_remote_sidr(struct cm_id_private * cm_id_priv)776 cm_insert_remote_sidr(struct cm_id_private *cm_id_priv)
777 {
778 	struct rb_node **link = &cm.remote_sidr_table.rb_node;
779 	struct rb_node *parent = NULL;
780 	struct cm_id_private *cur_cm_id_priv;
781 	__be32 remote_id = cm_id_priv->id.remote_id;
782 
783 	while (*link) {
784 		parent = *link;
785 		cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
786 					  sidr_id_node);
787 		if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
788 			link = &(*link)->rb_left;
789 		else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
790 			link = &(*link)->rb_right;
791 		else {
792 			if (cur_cm_id_priv->sidr_slid < cm_id_priv->sidr_slid)
793 				link = &(*link)->rb_left;
794 			else if (cur_cm_id_priv->sidr_slid > cm_id_priv->sidr_slid)
795 				link = &(*link)->rb_right;
796 			else
797 				return cur_cm_id_priv;
798 		}
799 	}
800 	rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
801 	rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
802 	return NULL;
803 }
804 
cm_alloc_id_priv(struct ib_device * device,ib_cm_handler cm_handler,void * context)805 static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device,
806 					      ib_cm_handler cm_handler,
807 					      void *context)
808 {
809 	struct cm_id_private *cm_id_priv;
810 	u32 id;
811 	int ret;
812 
813 	cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
814 	if (!cm_id_priv)
815 		return ERR_PTR(-ENOMEM);
816 
817 	cm_id_priv->id.state = IB_CM_IDLE;
818 	cm_id_priv->id.device = device;
819 	cm_id_priv->id.cm_handler = cm_handler;
820 	cm_id_priv->id.context = context;
821 	cm_id_priv->id.remote_cm_qpn = 1;
822 
823 	RB_CLEAR_NODE(&cm_id_priv->service_node);
824 	RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
825 	spin_lock_init(&cm_id_priv->lock);
826 	init_completion(&cm_id_priv->comp);
827 	INIT_LIST_HEAD(&cm_id_priv->work_list);
828 	atomic_set(&cm_id_priv->work_count, -1);
829 	refcount_set(&cm_id_priv->refcount, 1);
830 
831 	ret = xa_alloc_cyclic(&cm.local_id_table, &id, NULL, xa_limit_32b,
832 			      &cm.local_id_next, GFP_KERNEL);
833 	if (ret < 0)
834 		goto error;
835 	cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
836 
837 	return cm_id_priv;
838 
839 error:
840 	kfree(cm_id_priv);
841 	return ERR_PTR(ret);
842 }
843 
844 /*
845  * Make the ID visible to the MAD handlers and other threads that use the
846  * xarray.
847  */
cm_finalize_id(struct cm_id_private * cm_id_priv)848 static void cm_finalize_id(struct cm_id_private *cm_id_priv)
849 {
850 	xa_store(&cm.local_id_table, cm_local_id(cm_id_priv->id.local_id),
851 		 cm_id_priv, GFP_ATOMIC);
852 }
853 
ib_create_cm_id(struct ib_device * device,ib_cm_handler cm_handler,void * context)854 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
855 				 ib_cm_handler cm_handler,
856 				 void *context)
857 {
858 	struct cm_id_private *cm_id_priv;
859 
860 	cm_id_priv = cm_alloc_id_priv(device, cm_handler, context);
861 	if (IS_ERR(cm_id_priv))
862 		return ERR_CAST(cm_id_priv);
863 
864 	cm_finalize_id(cm_id_priv);
865 	return &cm_id_priv->id;
866 }
867 EXPORT_SYMBOL(ib_create_cm_id);
868 
cm_dequeue_work(struct cm_id_private * cm_id_priv)869 static struct cm_work *cm_dequeue_work(struct cm_id_private *cm_id_priv)
870 {
871 	struct cm_work *work;
872 
873 	if (list_empty(&cm_id_priv->work_list))
874 		return NULL;
875 
876 	work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
877 	list_del(&work->list);
878 	return work;
879 }
880 
cm_free_work(struct cm_work * work)881 static void cm_free_work(struct cm_work *work)
882 {
883 	if (work->mad_recv_wc)
884 		ib_free_recv_mad(work->mad_recv_wc);
885 	kfree(work);
886 }
887 
cm_queue_work_unlock(struct cm_id_private * cm_id_priv,struct cm_work * work)888 static void cm_queue_work_unlock(struct cm_id_private *cm_id_priv,
889 				 struct cm_work *work)
890 	__releases(&cm_id_priv->lock)
891 {
892 	bool immediate;
893 
894 	/*
895 	 * To deliver the event to the user callback we have the drop the
896 	 * spinlock, however, we need to ensure that the user callback is single
897 	 * threaded and receives events in the temporal order. If there are
898 	 * already events being processed then thread new events onto a list,
899 	 * the thread currently processing will pick them up.
900 	 */
901 	immediate = atomic_inc_and_test(&cm_id_priv->work_count);
902 	if (!immediate) {
903 		list_add_tail(&work->list, &cm_id_priv->work_list);
904 		/*
905 		 * This routine always consumes incoming reference. Once queued
906 		 * to the work_list then a reference is held by the thread
907 		 * currently running cm_process_work() and this reference is not
908 		 * needed.
909 		 */
910 		cm_deref_id(cm_id_priv);
911 	}
912 	spin_unlock_irq(&cm_id_priv->lock);
913 
914 	if (immediate)
915 		cm_process_work(cm_id_priv, work);
916 }
917 
cm_convert_to_ms(int iba_time)918 static inline int cm_convert_to_ms(int iba_time)
919 {
920 	/* approximate conversion to ms from 4.096us x 2^iba_time */
921 	return 1 << max(iba_time - 8, 0);
922 }
923 
924 /*
925  * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
926  * Because of how ack_timeout is stored, adding one doubles the timeout.
927  * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
928  * increment it (round up) only if the other is within 50%.
929  */
cm_ack_timeout(u8 ca_ack_delay,u8 packet_life_time)930 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
931 {
932 	int ack_timeout = packet_life_time + 1;
933 
934 	if (ack_timeout >= ca_ack_delay)
935 		ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
936 	else
937 		ack_timeout = ca_ack_delay +
938 			      (ack_timeout >= (ca_ack_delay - 1));
939 
940 	return min(31, ack_timeout);
941 }
942 
cm_remove_remote(struct cm_id_private * cm_id_priv)943 static void cm_remove_remote(struct cm_id_private *cm_id_priv)
944 {
945 	struct cm_timewait_info *timewait_info = cm_id_priv->timewait_info;
946 
947 	if (timewait_info->inserted_remote_id) {
948 		rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
949 		timewait_info->inserted_remote_id = 0;
950 	}
951 
952 	if (timewait_info->inserted_remote_qp) {
953 		rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
954 		timewait_info->inserted_remote_qp = 0;
955 	}
956 }
957 
cm_create_timewait_info(__be32 local_id)958 static struct cm_timewait_info *cm_create_timewait_info(__be32 local_id)
959 {
960 	struct cm_timewait_info *timewait_info;
961 
962 	timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
963 	if (!timewait_info)
964 		return ERR_PTR(-ENOMEM);
965 
966 	timewait_info->work.local_id = local_id;
967 	INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
968 	timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
969 	return timewait_info;
970 }
971 
cm_enter_timewait(struct cm_id_private * cm_id_priv)972 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
973 {
974 	int wait_time;
975 	unsigned long flags;
976 	struct cm_device *cm_dev;
977 
978 	lockdep_assert_held(&cm_id_priv->lock);
979 
980 	cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client);
981 	if (!cm_dev)
982 		return;
983 
984 	spin_lock_irqsave(&cm.lock, flags);
985 	cm_remove_remote(cm_id_priv);
986 	list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
987 	spin_unlock_irqrestore(&cm.lock, flags);
988 
989 	/*
990 	 * The cm_id could be destroyed by the user before we exit timewait.
991 	 * To protect against this, we search for the cm_id after exiting
992 	 * timewait before notifying the user that we've exited timewait.
993 	 */
994 	cm_id_priv->id.state = IB_CM_TIMEWAIT;
995 	wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
996 
997 	/* Check if the device started its remove_one */
998 	spin_lock_irqsave(&cm.lock, flags);
999 	if (!cm_dev->going_down)
1000 		queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
1001 				   msecs_to_jiffies(wait_time));
1002 	spin_unlock_irqrestore(&cm.lock, flags);
1003 
1004 	/*
1005 	 * The timewait_info is converted into a work and gets freed during
1006 	 * cm_free_work() in cm_timewait_handler().
1007 	 */
1008 	BUILD_BUG_ON(offsetof(struct cm_timewait_info, work) != 0);
1009 	cm_id_priv->timewait_info = NULL;
1010 }
1011 
cm_reset_to_idle(struct cm_id_private * cm_id_priv)1012 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
1013 {
1014 	unsigned long flags;
1015 
1016 	lockdep_assert_held(&cm_id_priv->lock);
1017 
1018 	cm_id_priv->id.state = IB_CM_IDLE;
1019 	if (cm_id_priv->timewait_info) {
1020 		spin_lock_irqsave(&cm.lock, flags);
1021 		cm_remove_remote(cm_id_priv);
1022 		spin_unlock_irqrestore(&cm.lock, flags);
1023 		kfree(cm_id_priv->timewait_info);
1024 		cm_id_priv->timewait_info = NULL;
1025 	}
1026 }
1027 
cm_destroy_id(struct ib_cm_id * cm_id,int err)1028 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
1029 {
1030 	struct cm_id_private *cm_id_priv;
1031 	struct cm_work *work;
1032 
1033 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1034 	spin_lock_irq(&cm_id_priv->lock);
1035 retest:
1036 	switch (cm_id->state) {
1037 	case IB_CM_LISTEN:
1038 		spin_lock(&cm.lock);
1039 		if (--cm_id_priv->listen_sharecount > 0) {
1040 			/* The id is still shared. */
1041 			WARN_ON(refcount_read(&cm_id_priv->refcount) == 1);
1042 			spin_unlock(&cm.lock);
1043 			spin_unlock_irq(&cm_id_priv->lock);
1044 			cm_deref_id(cm_id_priv);
1045 			return;
1046 		}
1047 		cm_id->state = IB_CM_IDLE;
1048 		rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
1049 		RB_CLEAR_NODE(&cm_id_priv->service_node);
1050 		spin_unlock(&cm.lock);
1051 		break;
1052 	case IB_CM_SIDR_REQ_SENT:
1053 		cm_id->state = IB_CM_IDLE;
1054 		ib_cancel_mad(cm_id_priv->msg);
1055 		break;
1056 	case IB_CM_SIDR_REQ_RCVD:
1057 		cm_send_sidr_rep_locked(cm_id_priv,
1058 					&(struct ib_cm_sidr_rep_param){
1059 						.status = IB_SIDR_REJECT });
1060 		/* cm_send_sidr_rep_locked will not move to IDLE if it fails */
1061 		cm_id->state = IB_CM_IDLE;
1062 		break;
1063 	case IB_CM_REQ_SENT:
1064 	case IB_CM_MRA_REQ_RCVD:
1065 		ib_cancel_mad(cm_id_priv->msg);
1066 		cm_send_rej_locked(cm_id_priv, IB_CM_REJ_TIMEOUT,
1067 				   &cm_id_priv->id.device->node_guid,
1068 				   sizeof(cm_id_priv->id.device->node_guid),
1069 				   NULL, 0);
1070 		break;
1071 	case IB_CM_REQ_RCVD:
1072 		if (err == -ENOMEM) {
1073 			/* Do not reject to allow future retries. */
1074 			cm_reset_to_idle(cm_id_priv);
1075 		} else {
1076 			cm_send_rej_locked(cm_id_priv,
1077 					   IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1078 					   NULL, 0);
1079 		}
1080 		break;
1081 	case IB_CM_REP_SENT:
1082 	case IB_CM_MRA_REP_RCVD:
1083 		ib_cancel_mad(cm_id_priv->msg);
1084 		cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1085 				   0, NULL, 0);
1086 		goto retest;
1087 	case IB_CM_MRA_REQ_SENT:
1088 	case IB_CM_REP_RCVD:
1089 	case IB_CM_MRA_REP_SENT:
1090 		cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1091 				   0, NULL, 0);
1092 		break;
1093 	case IB_CM_ESTABLISHED:
1094 		if (cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
1095 			cm_id->state = IB_CM_IDLE;
1096 			break;
1097 		}
1098 		cm_send_dreq_locked(cm_id_priv, NULL, 0);
1099 		goto retest;
1100 	case IB_CM_DREQ_SENT:
1101 		ib_cancel_mad(cm_id_priv->msg);
1102 		cm_enter_timewait(cm_id_priv);
1103 		goto retest;
1104 	case IB_CM_DREQ_RCVD:
1105 		cm_send_drep_locked(cm_id_priv, NULL, 0);
1106 		WARN_ON(cm_id->state != IB_CM_TIMEWAIT);
1107 		goto retest;
1108 	case IB_CM_TIMEWAIT:
1109 		/*
1110 		 * The cm_acquire_id in cm_timewait_handler will stop working
1111 		 * once we do xa_erase below, so just move to idle here for
1112 		 * consistency.
1113 		 */
1114 		cm_id->state = IB_CM_IDLE;
1115 		break;
1116 	case IB_CM_IDLE:
1117 		break;
1118 	}
1119 	WARN_ON(cm_id->state != IB_CM_IDLE);
1120 
1121 	spin_lock(&cm.lock);
1122 	/* Required for cleanup paths related cm_req_handler() */
1123 	if (cm_id_priv->timewait_info) {
1124 		cm_remove_remote(cm_id_priv);
1125 		kfree(cm_id_priv->timewait_info);
1126 		cm_id_priv->timewait_info = NULL;
1127 	}
1128 
1129 	WARN_ON(cm_id_priv->listen_sharecount);
1130 	WARN_ON(!RB_EMPTY_NODE(&cm_id_priv->service_node));
1131 	if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
1132 		rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
1133 	spin_unlock(&cm.lock);
1134 	spin_unlock_irq(&cm_id_priv->lock);
1135 
1136 	xa_erase(&cm.local_id_table, cm_local_id(cm_id->local_id));
1137 	cm_deref_id(cm_id_priv);
1138 	wait_for_completion(&cm_id_priv->comp);
1139 	while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1140 		cm_free_work(work);
1141 
1142 	cm_destroy_av(&cm_id_priv->av);
1143 	cm_destroy_av(&cm_id_priv->alt_av);
1144 	kfree(cm_id_priv->private_data);
1145 	kfree_rcu(cm_id_priv, rcu);
1146 }
1147 
ib_destroy_cm_id(struct ib_cm_id * cm_id)1148 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1149 {
1150 	cm_destroy_id(cm_id, 0);
1151 }
1152 EXPORT_SYMBOL(ib_destroy_cm_id);
1153 
cm_init_listen(struct cm_id_private * cm_id_priv,__be64 service_id)1154 static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id)
1155 {
1156 	if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1157 	    (service_id != IB_CM_ASSIGN_SERVICE_ID))
1158 		return -EINVAL;
1159 
1160 	if (service_id == IB_CM_ASSIGN_SERVICE_ID)
1161 		cm_id_priv->id.service_id = cpu_to_be64(cm.listen_service_id++);
1162 	else
1163 		cm_id_priv->id.service_id = service_id;
1164 
1165 	return 0;
1166 }
1167 
1168 /**
1169  * ib_cm_listen - Initiates listening on the specified service ID for
1170  *   connection and service ID resolution requests.
1171  * @cm_id: Connection identifier associated with the listen request.
1172  * @service_id: Service identifier matched against incoming connection
1173  *   and service ID resolution requests.  The service ID should be specified
1174  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1175  *   assign a service ID to the caller.
1176  */
ib_cm_listen(struct ib_cm_id * cm_id,__be64 service_id)1177 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id)
1178 {
1179 	struct cm_id_private *cm_id_priv =
1180 		container_of(cm_id, struct cm_id_private, id);
1181 	unsigned long flags;
1182 	int ret;
1183 
1184 	spin_lock_irqsave(&cm_id_priv->lock, flags);
1185 	if (cm_id_priv->id.state != IB_CM_IDLE) {
1186 		ret = -EINVAL;
1187 		goto out;
1188 	}
1189 
1190 	ret = cm_init_listen(cm_id_priv, service_id);
1191 	if (ret)
1192 		goto out;
1193 
1194 	if (!cm_insert_listen(cm_id_priv, NULL)) {
1195 		ret = -EBUSY;
1196 		goto out;
1197 	}
1198 
1199 	cm_id_priv->id.state = IB_CM_LISTEN;
1200 	ret = 0;
1201 
1202 out:
1203 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1204 	return ret;
1205 }
1206 EXPORT_SYMBOL(ib_cm_listen);
1207 
1208 /**
1209  * ib_cm_insert_listen - Create a new listening ib_cm_id and listen on
1210  *			 the given service ID.
1211  *
1212  * If there's an existing ID listening on that same device and service ID,
1213  * return it.
1214  *
1215  * @device: Device associated with the cm_id.  All related communication will
1216  * be associated with the specified device.
1217  * @cm_handler: Callback invoked to notify the user of CM events.
1218  * @service_id: Service identifier matched against incoming connection
1219  *   and service ID resolution requests.  The service ID should be specified
1220  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1221  *   assign a service ID to the caller.
1222  *
1223  * Callers should call ib_destroy_cm_id when done with the listener ID.
1224  */
ib_cm_insert_listen(struct ib_device * device,ib_cm_handler cm_handler,__be64 service_id)1225 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1226 				     ib_cm_handler cm_handler,
1227 				     __be64 service_id)
1228 {
1229 	struct cm_id_private *listen_id_priv;
1230 	struct cm_id_private *cm_id_priv;
1231 	int err = 0;
1232 
1233 	/* Create an ID in advance, since the creation may sleep */
1234 	cm_id_priv = cm_alloc_id_priv(device, cm_handler, NULL);
1235 	if (IS_ERR(cm_id_priv))
1236 		return ERR_CAST(cm_id_priv);
1237 
1238 	err = cm_init_listen(cm_id_priv, service_id);
1239 	if (err) {
1240 		ib_destroy_cm_id(&cm_id_priv->id);
1241 		return ERR_PTR(err);
1242 	}
1243 
1244 	spin_lock_irq(&cm_id_priv->lock);
1245 	listen_id_priv = cm_insert_listen(cm_id_priv, cm_handler);
1246 	if (listen_id_priv != cm_id_priv) {
1247 		spin_unlock_irq(&cm_id_priv->lock);
1248 		ib_destroy_cm_id(&cm_id_priv->id);
1249 		if (!listen_id_priv)
1250 			return ERR_PTR(-EINVAL);
1251 		return &listen_id_priv->id;
1252 	}
1253 	cm_id_priv->id.state = IB_CM_LISTEN;
1254 	spin_unlock_irq(&cm_id_priv->lock);
1255 
1256 	/*
1257 	 * A listen ID does not need to be in the xarray since it does not
1258 	 * receive mads, is not placed in the remote_id or remote_qpn rbtree,
1259 	 * and does not enter timewait.
1260 	 */
1261 
1262 	return &cm_id_priv->id;
1263 }
1264 EXPORT_SYMBOL(ib_cm_insert_listen);
1265 
cm_form_tid(struct cm_id_private * cm_id_priv)1266 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv)
1267 {
1268 	u64 hi_tid = 0, low_tid;
1269 
1270 	lockdep_assert_held(&cm_id_priv->lock);
1271 
1272 	low_tid = (u64)cm_id_priv->id.local_id;
1273 	if (!cm_id_priv->av.port)
1274 		return cpu_to_be64(low_tid);
1275 
1276 	spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1277 	if (cm_id_priv->av.port->mad_agent)
1278 		hi_tid = ((u64)cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1279 	spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1280 	return cpu_to_be64(hi_tid | low_tid);
1281 }
1282 
cm_format_mad_hdr(struct ib_mad_hdr * hdr,__be16 attr_id,__be64 tid)1283 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1284 			      __be16 attr_id, __be64 tid)
1285 {
1286 	hdr->base_version  = IB_MGMT_BASE_VERSION;
1287 	hdr->mgmt_class	   = IB_MGMT_CLASS_CM;
1288 	hdr->class_version = IB_CM_CLASS_VERSION;
1289 	hdr->method	   = IB_MGMT_METHOD_SEND;
1290 	hdr->attr_id	   = attr_id;
1291 	hdr->tid	   = tid;
1292 }
1293 
cm_format_mad_ece_hdr(struct ib_mad_hdr * hdr,__be16 attr_id,__be64 tid,u32 attr_mod)1294 static void cm_format_mad_ece_hdr(struct ib_mad_hdr *hdr, __be16 attr_id,
1295 				  __be64 tid, u32 attr_mod)
1296 {
1297 	cm_format_mad_hdr(hdr, attr_id, tid);
1298 	hdr->attr_mod = cpu_to_be32(attr_mod);
1299 }
1300 
cm_format_req(struct cm_req_msg * req_msg,struct cm_id_private * cm_id_priv,struct ib_cm_req_param * param)1301 static void cm_format_req(struct cm_req_msg *req_msg,
1302 			  struct cm_id_private *cm_id_priv,
1303 			  struct ib_cm_req_param *param)
1304 {
1305 	struct sa_path_rec *pri_path = param->primary_path;
1306 	struct sa_path_rec *alt_path = param->alternate_path;
1307 	bool pri_ext = false;
1308 	__be16 lid;
1309 
1310 	if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
1311 		pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
1312 					      pri_path->opa.slid);
1313 
1314 	cm_format_mad_ece_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1315 			      cm_form_tid(cm_id_priv), param->ece.attr_mod);
1316 
1317 	IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg,
1318 		be32_to_cpu(cm_id_priv->id.local_id));
1319 	IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id));
1320 	IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg,
1321 		be64_to_cpu(cm_id_priv->id.device->node_guid));
1322 	IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num);
1323 	IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth);
1324 	IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg,
1325 		param->remote_cm_response_timeout);
1326 	cm_req_set_qp_type(req_msg, param->qp_type);
1327 	IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control);
1328 	IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn);
1329 	IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg,
1330 		param->local_cm_response_timeout);
1331 	IBA_SET(CM_REQ_PARTITION_KEY, req_msg,
1332 		be16_to_cpu(param->primary_path->pkey));
1333 	IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg,
1334 		param->primary_path->mtu);
1335 	IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries);
1336 
1337 	if (param->qp_type != IB_QPT_XRC_INI) {
1338 		IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg,
1339 			param->responder_resources);
1340 		IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count);
1341 		IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg,
1342 			param->rnr_retry_count);
1343 		IBA_SET(CM_REQ_SRQ, req_msg, param->srq);
1344 	}
1345 
1346 	*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) =
1347 		pri_path->sgid;
1348 	*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) =
1349 		pri_path->dgid;
1350 	if (pri_ext) {
1351 		IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)
1352 			->global.interface_id =
1353 			OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
1354 		IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)
1355 			->global.interface_id =
1356 			OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
1357 	}
1358 	if (pri_path->hop_limit <= 1) {
1359 		IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1360 			be16_to_cpu(pri_ext ? 0 :
1361 					      htons(ntohl(sa_path_get_slid(
1362 						      pri_path)))));
1363 		IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1364 			be16_to_cpu(pri_ext ? 0 :
1365 					      htons(ntohl(sa_path_get_dlid(
1366 						      pri_path)))));
1367 	} else {
1368 
1369 		if (param->primary_path_inbound) {
1370 			lid = param->primary_path_inbound->ib.dlid;
1371 			IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1372 				be16_to_cpu(lid));
1373 		} else
1374 			IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1375 				be16_to_cpu(IB_LID_PERMISSIVE));
1376 
1377 		/* Work-around until there's a way to obtain remote LID info */
1378 		IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1379 			be16_to_cpu(IB_LID_PERMISSIVE));
1380 	}
1381 	IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg,
1382 		be32_to_cpu(pri_path->flow_label));
1383 	IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate);
1384 	IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class);
1385 	IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit);
1386 	IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl);
1387 	IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg,
1388 		(pri_path->hop_limit <= 1));
1389 	IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg,
1390 		cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1391 			       pri_path->packet_life_time));
1392 
1393 	if (alt_path) {
1394 		bool alt_ext = false;
1395 
1396 		if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
1397 			alt_ext = opa_is_extended_lid(alt_path->opa.dlid,
1398 						      alt_path->opa.slid);
1399 
1400 		*IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) =
1401 			alt_path->sgid;
1402 		*IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) =
1403 			alt_path->dgid;
1404 		if (alt_ext) {
1405 			IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1406 					req_msg)
1407 				->global.interface_id =
1408 				OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
1409 			IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID,
1410 					req_msg)
1411 				->global.interface_id =
1412 				OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
1413 		}
1414 		if (alt_path->hop_limit <= 1) {
1415 			IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1416 				be16_to_cpu(
1417 					alt_ext ? 0 :
1418 						  htons(ntohl(sa_path_get_slid(
1419 							  alt_path)))));
1420 			IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1421 				be16_to_cpu(
1422 					alt_ext ? 0 :
1423 						  htons(ntohl(sa_path_get_dlid(
1424 							  alt_path)))));
1425 		} else {
1426 			IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1427 				be16_to_cpu(IB_LID_PERMISSIVE));
1428 			IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1429 				be16_to_cpu(IB_LID_PERMISSIVE));
1430 		}
1431 		IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg,
1432 			be32_to_cpu(alt_path->flow_label));
1433 		IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate);
1434 		IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg,
1435 			alt_path->traffic_class);
1436 		IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg,
1437 			alt_path->hop_limit);
1438 		IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl);
1439 		IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg,
1440 			(alt_path->hop_limit <= 1));
1441 		IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg,
1442 			cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1443 				       alt_path->packet_life_time));
1444 	}
1445 	IBA_SET(CM_REQ_VENDOR_ID, req_msg, param->ece.vendor_id);
1446 
1447 	if (param->private_data && param->private_data_len)
1448 		IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data,
1449 			    param->private_data_len);
1450 }
1451 
cm_validate_req_param(struct ib_cm_req_param * param)1452 static int cm_validate_req_param(struct ib_cm_req_param *param)
1453 {
1454 	if (!param->primary_path)
1455 		return -EINVAL;
1456 
1457 	if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1458 	    param->qp_type != IB_QPT_XRC_INI)
1459 		return -EINVAL;
1460 
1461 	if (param->private_data &&
1462 	    param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1463 		return -EINVAL;
1464 
1465 	if (param->alternate_path &&
1466 	    (param->alternate_path->pkey != param->primary_path->pkey ||
1467 	     param->alternate_path->mtu != param->primary_path->mtu))
1468 		return -EINVAL;
1469 
1470 	return 0;
1471 }
1472 
ib_send_cm_req(struct ib_cm_id * cm_id,struct ib_cm_req_param * param)1473 int ib_send_cm_req(struct ib_cm_id *cm_id,
1474 		   struct ib_cm_req_param *param)
1475 {
1476 	struct cm_av av = {}, alt_av = {};
1477 	struct cm_id_private *cm_id_priv;
1478 	struct ib_mad_send_buf *msg;
1479 	struct cm_req_msg *req_msg;
1480 	unsigned long flags;
1481 	int ret;
1482 
1483 	ret = cm_validate_req_param(param);
1484 	if (ret)
1485 		return ret;
1486 
1487 	/* Verify that we're not in timewait. */
1488 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1489 	spin_lock_irqsave(&cm_id_priv->lock, flags);
1490 	if (cm_id->state != IB_CM_IDLE || WARN_ON(cm_id_priv->timewait_info)) {
1491 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1492 		return -EINVAL;
1493 	}
1494 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1495 
1496 	cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1497 							    id.local_id);
1498 	if (IS_ERR(cm_id_priv->timewait_info)) {
1499 		ret = PTR_ERR(cm_id_priv->timewait_info);
1500 		cm_id_priv->timewait_info = NULL;
1501 		return ret;
1502 	}
1503 
1504 	ret = cm_init_av_by_path(param->primary_path,
1505 				 param->ppath_sgid_attr, &av);
1506 	if (ret)
1507 		return ret;
1508 	if (param->alternate_path) {
1509 		ret = cm_init_av_by_path(param->alternate_path, NULL,
1510 					 &alt_av);
1511 		if (ret) {
1512 			cm_destroy_av(&av);
1513 			return ret;
1514 		}
1515 	}
1516 	cm_id->service_id = param->service_id;
1517 	cm_id_priv->timeout_ms = cm_convert_to_ms(
1518 				    param->primary_path->packet_life_time) * 2 +
1519 				 cm_convert_to_ms(
1520 				    param->remote_cm_response_timeout);
1521 	cm_id_priv->max_cm_retries = param->max_cm_retries;
1522 	cm_id_priv->initiator_depth = param->initiator_depth;
1523 	cm_id_priv->responder_resources = param->responder_resources;
1524 	cm_id_priv->retry_count = param->retry_count;
1525 	cm_id_priv->path_mtu = param->primary_path->mtu;
1526 	cm_id_priv->pkey = param->primary_path->pkey;
1527 	cm_id_priv->qp_type = param->qp_type;
1528 
1529 	spin_lock_irqsave(&cm_id_priv->lock, flags);
1530 
1531 	cm_move_av_from_path(&cm_id_priv->av, &av);
1532 	if (param->primary_path_outbound)
1533 		cm_id_priv->av.dlid_datapath =
1534 			be16_to_cpu(param->primary_path_outbound->ib.dlid);
1535 
1536 	if (param->alternate_path)
1537 		cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
1538 
1539 	msg = cm_alloc_priv_msg(cm_id_priv);
1540 	if (IS_ERR(msg)) {
1541 		ret = PTR_ERR(msg);
1542 		goto out_unlock;
1543 	}
1544 
1545 	req_msg = (struct cm_req_msg *)msg->mad;
1546 	cm_format_req(req_msg, cm_id_priv, param);
1547 	cm_id_priv->tid = req_msg->hdr.tid;
1548 	msg->timeout_ms = cm_id_priv->timeout_ms;
1549 	msg->context[1] = (void *)(unsigned long)IB_CM_REQ_SENT;
1550 
1551 	cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
1552 	cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
1553 
1554 	trace_icm_send_req(&cm_id_priv->id);
1555 	ret = ib_post_send_mad(msg, NULL);
1556 	if (ret)
1557 		goto out_free;
1558 	BUG_ON(cm_id->state != IB_CM_IDLE);
1559 	cm_id->state = IB_CM_REQ_SENT;
1560 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1561 	return 0;
1562 out_free:
1563 	cm_free_priv_msg(msg);
1564 out_unlock:
1565 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1566 	return ret;
1567 }
1568 EXPORT_SYMBOL(ib_send_cm_req);
1569 
cm_issue_rej(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,enum ib_cm_rej_reason reason,enum cm_msg_response msg_rejected,void * ari,u8 ari_length)1570 static int cm_issue_rej(struct cm_port *port,
1571 			struct ib_mad_recv_wc *mad_recv_wc,
1572 			enum ib_cm_rej_reason reason,
1573 			enum cm_msg_response msg_rejected,
1574 			void *ari, u8 ari_length)
1575 {
1576 	struct ib_mad_send_buf *msg = NULL;
1577 	struct cm_rej_msg *rej_msg, *rcv_msg;
1578 	int ret;
1579 
1580 	ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
1581 	if (ret)
1582 		return ret;
1583 
1584 	/* We just need common CM header information.  Cast to any message. */
1585 	rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1586 	rej_msg = (struct cm_rej_msg *) msg->mad;
1587 
1588 	cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1589 	IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1590 		IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg));
1591 	IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1592 		IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1593 	IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected);
1594 	IBA_SET(CM_REJ_REASON, rej_msg, reason);
1595 
1596 	if (ari && ari_length) {
1597 		IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1598 		IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1599 	}
1600 
1601 	trace_icm_issue_rej(
1602 		IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg),
1603 		IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1604 	ret = ib_post_send_mad(msg, NULL);
1605 	if (ret)
1606 		cm_free_response_msg(msg);
1607 
1608 	return ret;
1609 }
1610 
cm_req_has_alt_path(struct cm_req_msg * req_msg)1611 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
1612 {
1613 	return ((cpu_to_be16(
1614 			IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) ||
1615 		(ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1616 					       req_msg))));
1617 }
1618 
cm_path_set_rec_type(struct ib_device * ib_device,u32 port_num,struct sa_path_rec * path,union ib_gid * gid)1619 static void cm_path_set_rec_type(struct ib_device *ib_device, u32 port_num,
1620 				 struct sa_path_rec *path, union ib_gid *gid)
1621 {
1622 	if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
1623 		path->rec_type = SA_PATH_REC_TYPE_OPA;
1624 	else
1625 		path->rec_type = SA_PATH_REC_TYPE_IB;
1626 }
1627 
cm_format_path_lid_from_req(struct cm_req_msg * req_msg,struct sa_path_rec * primary_path,struct sa_path_rec * alt_path,struct ib_wc * wc)1628 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
1629 					struct sa_path_rec *primary_path,
1630 					struct sa_path_rec *alt_path,
1631 					struct ib_wc *wc)
1632 {
1633 	u32 lid;
1634 
1635 	if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1636 		sa_path_set_dlid(primary_path, wc->slid);
1637 		sa_path_set_slid(primary_path,
1638 				 IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
1639 					 req_msg));
1640 	} else {
1641 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1642 			CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg));
1643 		sa_path_set_dlid(primary_path, lid);
1644 
1645 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1646 			CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg));
1647 		sa_path_set_slid(primary_path, lid);
1648 	}
1649 
1650 	if (!cm_req_has_alt_path(req_msg))
1651 		return;
1652 
1653 	if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1654 		sa_path_set_dlid(alt_path,
1655 				 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
1656 					 req_msg));
1657 		sa_path_set_slid(alt_path,
1658 				 IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
1659 					 req_msg));
1660 	} else {
1661 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1662 			CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg));
1663 		sa_path_set_dlid(alt_path, lid);
1664 
1665 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1666 			CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg));
1667 		sa_path_set_slid(alt_path, lid);
1668 	}
1669 }
1670 
cm_format_paths_from_req(struct cm_req_msg * req_msg,struct sa_path_rec * primary_path,struct sa_path_rec * alt_path,struct ib_wc * wc)1671 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1672 				     struct sa_path_rec *primary_path,
1673 				     struct sa_path_rec *alt_path,
1674 				     struct ib_wc *wc)
1675 {
1676 	primary_path->dgid =
1677 		*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg);
1678 	primary_path->sgid =
1679 		*IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg);
1680 	primary_path->flow_label =
1681 		cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg));
1682 	primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg);
1683 	primary_path->traffic_class =
1684 		IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg);
1685 	primary_path->reversible = 1;
1686 	primary_path->pkey =
1687 		cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1688 	primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg);
1689 	primary_path->mtu_selector = IB_SA_EQ;
1690 	primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1691 	primary_path->rate_selector = IB_SA_EQ;
1692 	primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg);
1693 	primary_path->packet_life_time_selector = IB_SA_EQ;
1694 	primary_path->packet_life_time =
1695 		IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg);
1696 	primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1697 	primary_path->service_id =
1698 		cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1699 	if (sa_path_is_roce(primary_path))
1700 		primary_path->roce.route_resolved = false;
1701 
1702 	if (cm_req_has_alt_path(req_msg)) {
1703 		alt_path->dgid = *IBA_GET_MEM_PTR(
1704 			CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg);
1705 		alt_path->sgid = *IBA_GET_MEM_PTR(
1706 			CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg);
1707 		alt_path->flow_label = cpu_to_be32(
1708 			IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg));
1709 		alt_path->hop_limit =
1710 			IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg);
1711 		alt_path->traffic_class =
1712 			IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg);
1713 		alt_path->reversible = 1;
1714 		alt_path->pkey =
1715 			cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1716 		alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg);
1717 		alt_path->mtu_selector = IB_SA_EQ;
1718 		alt_path->mtu =
1719 			IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1720 		alt_path->rate_selector = IB_SA_EQ;
1721 		alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg);
1722 		alt_path->packet_life_time_selector = IB_SA_EQ;
1723 		alt_path->packet_life_time =
1724 			IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg);
1725 		alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1726 		alt_path->service_id =
1727 			cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1728 
1729 		if (sa_path_is_roce(alt_path))
1730 			alt_path->roce.route_resolved = false;
1731 	}
1732 	cm_format_path_lid_from_req(req_msg, primary_path, alt_path, wc);
1733 }
1734 
cm_get_bth_pkey(struct cm_work * work)1735 static u16 cm_get_bth_pkey(struct cm_work *work)
1736 {
1737 	struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1738 	u32 port_num = work->port->port_num;
1739 	u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1740 	u16 pkey;
1741 	int ret;
1742 
1743 	ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey);
1744 	if (ret) {
1745 		dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %u, pkey index %u). %d\n",
1746 				     port_num, pkey_index, ret);
1747 		return 0;
1748 	}
1749 
1750 	return pkey;
1751 }
1752 
1753 /**
1754  * cm_opa_to_ib_sgid - Convert OPA SGID to IB SGID
1755  * ULPs (such as IPoIB) do not understand OPA GIDs and will
1756  * reject them as the local_gid will not match the sgid. Therefore,
1757  * change the pathrec's SGID to an IB SGID.
1758  *
1759  * @work: Work completion
1760  * @path: Path record
1761  */
cm_opa_to_ib_sgid(struct cm_work * work,struct sa_path_rec * path)1762 static void cm_opa_to_ib_sgid(struct cm_work *work,
1763 			      struct sa_path_rec *path)
1764 {
1765 	struct ib_device *dev = work->port->cm_dev->ib_device;
1766 	u32 port_num = work->port->port_num;
1767 
1768 	if (rdma_cap_opa_ah(dev, port_num) &&
1769 	    (ib_is_opa_gid(&path->sgid))) {
1770 		union ib_gid sgid;
1771 
1772 		if (rdma_query_gid(dev, port_num, 0, &sgid)) {
1773 			dev_warn(&dev->dev,
1774 				 "Error updating sgid in CM request\n");
1775 			return;
1776 		}
1777 
1778 		path->sgid = sgid;
1779 	}
1780 }
1781 
cm_format_req_event(struct cm_work * work,struct cm_id_private * cm_id_priv,struct ib_cm_id * listen_id)1782 static void cm_format_req_event(struct cm_work *work,
1783 				struct cm_id_private *cm_id_priv,
1784 				struct ib_cm_id *listen_id)
1785 {
1786 	struct cm_req_msg *req_msg;
1787 	struct ib_cm_req_event_param *param;
1788 
1789 	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1790 	param = &work->cm_event.param.req_rcvd;
1791 	param->listen_id = listen_id;
1792 	param->bth_pkey = cm_get_bth_pkey(work);
1793 	param->port = cm_id_priv->av.port->port_num;
1794 	param->primary_path = &work->path[0];
1795 	cm_opa_to_ib_sgid(work, param->primary_path);
1796 	if (cm_req_has_alt_path(req_msg)) {
1797 		param->alternate_path = &work->path[1];
1798 		cm_opa_to_ib_sgid(work, param->alternate_path);
1799 	} else {
1800 		param->alternate_path = NULL;
1801 	}
1802 	param->remote_ca_guid =
1803 		cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
1804 	param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg);
1805 	param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg);
1806 	param->qp_type = cm_req_get_qp_type(req_msg);
1807 	param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg);
1808 	param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
1809 	param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
1810 	param->local_cm_response_timeout =
1811 		IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg);
1812 	param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg);
1813 	param->remote_cm_response_timeout =
1814 		IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg);
1815 	param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
1816 	param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
1817 	param->srq = IBA_GET(CM_REQ_SRQ, req_msg);
1818 	param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
1819 	param->ece.vendor_id = IBA_GET(CM_REQ_VENDOR_ID, req_msg);
1820 	param->ece.attr_mod = be32_to_cpu(req_msg->hdr.attr_mod);
1821 
1822 	work->cm_event.private_data =
1823 		IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg);
1824 }
1825 
cm_process_work(struct cm_id_private * cm_id_priv,struct cm_work * work)1826 static void cm_process_work(struct cm_id_private *cm_id_priv,
1827 			    struct cm_work *work)
1828 {
1829 	int ret;
1830 
1831 	/* We will typically only have the current event to report. */
1832 	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1833 	cm_free_work(work);
1834 
1835 	while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1836 		spin_lock_irq(&cm_id_priv->lock);
1837 		work = cm_dequeue_work(cm_id_priv);
1838 		spin_unlock_irq(&cm_id_priv->lock);
1839 		if (!work)
1840 			return;
1841 
1842 		ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1843 						&work->cm_event);
1844 		cm_free_work(work);
1845 	}
1846 	cm_deref_id(cm_id_priv);
1847 	if (ret)
1848 		cm_destroy_id(&cm_id_priv->id, ret);
1849 }
1850 
cm_format_mra(struct cm_mra_msg * mra_msg,struct cm_id_private * cm_id_priv,enum cm_msg_response msg_mraed,u8 service_timeout,const void * private_data,u8 private_data_len)1851 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1852 			  struct cm_id_private *cm_id_priv,
1853 			  enum cm_msg_response msg_mraed, u8 service_timeout,
1854 			  const void *private_data, u8 private_data_len)
1855 {
1856 	cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1857 	IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed);
1858 	IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg,
1859 		be32_to_cpu(cm_id_priv->id.local_id));
1860 	IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg,
1861 		be32_to_cpu(cm_id_priv->id.remote_id));
1862 	IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout);
1863 
1864 	if (private_data && private_data_len)
1865 		IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data,
1866 			    private_data_len);
1867 }
1868 
cm_format_rej(struct cm_rej_msg * rej_msg,struct cm_id_private * cm_id_priv,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len,enum ib_cm_state state)1869 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1870 			  struct cm_id_private *cm_id_priv,
1871 			  enum ib_cm_rej_reason reason, void *ari,
1872 			  u8 ari_length, const void *private_data,
1873 			  u8 private_data_len, enum ib_cm_state state)
1874 {
1875 	lockdep_assert_held(&cm_id_priv->lock);
1876 
1877 	cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1878 	IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1879 		be32_to_cpu(cm_id_priv->id.remote_id));
1880 
1881 	switch (state) {
1882 	case IB_CM_REQ_RCVD:
1883 		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0));
1884 		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1885 		break;
1886 	case IB_CM_MRA_REQ_SENT:
1887 		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1888 			be32_to_cpu(cm_id_priv->id.local_id));
1889 		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1890 		break;
1891 	case IB_CM_REP_RCVD:
1892 	case IB_CM_MRA_REP_SENT:
1893 		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1894 			be32_to_cpu(cm_id_priv->id.local_id));
1895 		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP);
1896 		break;
1897 	default:
1898 		IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1899 			be32_to_cpu(cm_id_priv->id.local_id));
1900 		IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg,
1901 			CM_MSG_RESPONSE_OTHER);
1902 		break;
1903 	}
1904 
1905 	IBA_SET(CM_REJ_REASON, rej_msg, reason);
1906 	if (ari && ari_length) {
1907 		IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1908 		IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1909 	}
1910 
1911 	if (private_data && private_data_len)
1912 		IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data,
1913 			    private_data_len);
1914 }
1915 
cm_dup_req_handler(struct cm_work * work,struct cm_id_private * cm_id_priv)1916 static void cm_dup_req_handler(struct cm_work *work,
1917 			       struct cm_id_private *cm_id_priv)
1918 {
1919 	struct ib_mad_send_buf *msg = NULL;
1920 	int ret;
1921 
1922 	atomic_long_inc(
1923 		&work->port->counters[CM_RECV_DUPLICATES][CM_REQ_COUNTER]);
1924 
1925 	/* Quick state check to discard duplicate REQs. */
1926 	spin_lock_irq(&cm_id_priv->lock);
1927 	if (cm_id_priv->id.state == IB_CM_REQ_RCVD) {
1928 		spin_unlock_irq(&cm_id_priv->lock);
1929 		return;
1930 	}
1931 	spin_unlock_irq(&cm_id_priv->lock);
1932 
1933 	ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
1934 	if (ret)
1935 		return;
1936 
1937 	spin_lock_irq(&cm_id_priv->lock);
1938 	switch (cm_id_priv->id.state) {
1939 	case IB_CM_MRA_REQ_SENT:
1940 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1941 			      CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
1942 			      cm_id_priv->private_data,
1943 			      cm_id_priv->private_data_len);
1944 		break;
1945 	case IB_CM_TIMEWAIT:
1946 		cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv,
1947 			      IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0,
1948 			      IB_CM_TIMEWAIT);
1949 		break;
1950 	default:
1951 		goto unlock;
1952 	}
1953 	spin_unlock_irq(&cm_id_priv->lock);
1954 
1955 	trace_icm_send_dup_req(&cm_id_priv->id);
1956 	ret = ib_post_send_mad(msg, NULL);
1957 	if (ret)
1958 		goto free;
1959 	return;
1960 
1961 unlock:	spin_unlock_irq(&cm_id_priv->lock);
1962 free:	cm_free_response_msg(msg);
1963 }
1964 
cm_match_req(struct cm_work * work,struct cm_id_private * cm_id_priv)1965 static struct cm_id_private *cm_match_req(struct cm_work *work,
1966 					  struct cm_id_private *cm_id_priv)
1967 {
1968 	struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1969 	struct cm_timewait_info *timewait_info;
1970 	struct cm_req_msg *req_msg;
1971 
1972 	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1973 
1974 	/* Check for possible duplicate REQ. */
1975 	spin_lock_irq(&cm.lock);
1976 	timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
1977 	if (timewait_info) {
1978 		cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
1979 					   timewait_info->work.remote_id);
1980 		spin_unlock_irq(&cm.lock);
1981 		if (cur_cm_id_priv) {
1982 			cm_dup_req_handler(work, cur_cm_id_priv);
1983 			cm_deref_id(cur_cm_id_priv);
1984 		}
1985 		return NULL;
1986 	}
1987 
1988 	/* Check for stale connections. */
1989 	timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
1990 	if (timewait_info) {
1991 		cm_remove_remote(cm_id_priv);
1992 		cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
1993 					   timewait_info->work.remote_id);
1994 
1995 		spin_unlock_irq(&cm.lock);
1996 		cm_issue_rej(work->port, work->mad_recv_wc,
1997 			     IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
1998 			     NULL, 0);
1999 		if (cur_cm_id_priv) {
2000 			ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2001 			cm_deref_id(cur_cm_id_priv);
2002 		}
2003 		return NULL;
2004 	}
2005 
2006 	/* Find matching listen request. */
2007 	listen_cm_id_priv = cm_find_listen(
2008 		cm_id_priv->id.device,
2009 		cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)));
2010 	if (!listen_cm_id_priv) {
2011 		cm_remove_remote(cm_id_priv);
2012 		spin_unlock_irq(&cm.lock);
2013 		cm_issue_rej(work->port, work->mad_recv_wc,
2014 			     IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
2015 			     NULL, 0);
2016 		return NULL;
2017 	}
2018 	spin_unlock_irq(&cm.lock);
2019 	return listen_cm_id_priv;
2020 }
2021 
2022 /*
2023  * Work-around for inter-subnet connections.  If the LIDs are permissive,
2024  * we need to override the LID/SL data in the REQ with the LID information
2025  * in the work completion.
2026  */
cm_process_routed_req(struct cm_req_msg * req_msg,struct ib_wc * wc)2027 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
2028 {
2029 	if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) {
2030 		if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
2031 					req_msg)) == IB_LID_PERMISSIVE) {
2032 			IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
2033 				be16_to_cpu(ib_lid_be16(wc->slid)));
2034 			IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl);
2035 		}
2036 
2037 		if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
2038 					req_msg)) == IB_LID_PERMISSIVE)
2039 			IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
2040 				wc->dlid_path_bits);
2041 	}
2042 
2043 	if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) {
2044 		if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
2045 					req_msg)) == IB_LID_PERMISSIVE) {
2046 			IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
2047 				be16_to_cpu(ib_lid_be16(wc->slid)));
2048 			IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl);
2049 		}
2050 
2051 		if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
2052 					req_msg)) == IB_LID_PERMISSIVE)
2053 			IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
2054 				wc->dlid_path_bits);
2055 	}
2056 }
2057 
cm_req_handler(struct cm_work * work)2058 static int cm_req_handler(struct cm_work *work)
2059 {
2060 	struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
2061 	struct cm_req_msg *req_msg;
2062 	const struct ib_global_route *grh;
2063 	const struct ib_gid_attr *gid_attr;
2064 	int ret;
2065 
2066 	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
2067 
2068 	cm_id_priv =
2069 		cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
2070 	if (IS_ERR(cm_id_priv))
2071 		return PTR_ERR(cm_id_priv);
2072 
2073 	cm_id_priv->id.remote_id =
2074 		cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg));
2075 	cm_id_priv->id.service_id =
2076 		cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
2077 	cm_id_priv->tid = req_msg->hdr.tid;
2078 	cm_id_priv->timeout_ms = cm_convert_to_ms(
2079 		IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg));
2080 	cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg);
2081 	cm_id_priv->remote_qpn =
2082 		cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
2083 	cm_id_priv->initiator_depth =
2084 		IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
2085 	cm_id_priv->responder_resources =
2086 		IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
2087 	cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
2088 	cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
2089 	cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
2090 	cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
2091 	cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
2092 	cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
2093 
2094 	ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
2095 				      work->mad_recv_wc->recv_buf.grh,
2096 				      &cm_id_priv->av);
2097 	if (ret)
2098 		goto destroy;
2099 	cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
2100 							    id.local_id);
2101 	if (IS_ERR(cm_id_priv->timewait_info)) {
2102 		ret = PTR_ERR(cm_id_priv->timewait_info);
2103 		cm_id_priv->timewait_info = NULL;
2104 		goto destroy;
2105 	}
2106 	cm_id_priv->timewait_info->work.remote_id = cm_id_priv->id.remote_id;
2107 	cm_id_priv->timewait_info->remote_ca_guid =
2108 		cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
2109 	cm_id_priv->timewait_info->remote_qpn = cm_id_priv->remote_qpn;
2110 
2111 	/*
2112 	 * Note that the ID pointer is not in the xarray at this point,
2113 	 * so this set is only visible to the local thread.
2114 	 */
2115 	cm_id_priv->id.state = IB_CM_REQ_RCVD;
2116 
2117 	listen_cm_id_priv = cm_match_req(work, cm_id_priv);
2118 	if (!listen_cm_id_priv) {
2119 		trace_icm_no_listener_err(&cm_id_priv->id);
2120 		cm_id_priv->id.state = IB_CM_IDLE;
2121 		ret = -EINVAL;
2122 		goto destroy;
2123 	}
2124 
2125 	memset(&work->path[0], 0, sizeof(work->path[0]));
2126 	if (cm_req_has_alt_path(req_msg))
2127 		memset(&work->path[1], 0, sizeof(work->path[1]));
2128 	grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
2129 	gid_attr = grh->sgid_attr;
2130 
2131 	if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) {
2132 		work->path[0].rec_type =
2133 			sa_conv_gid_to_pathrec_type(gid_attr->gid_type);
2134 	} else {
2135 		cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
2136 		cm_path_set_rec_type(
2137 			work->port->cm_dev->ib_device, work->port->port_num,
2138 			&work->path[0],
2139 			IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID,
2140 					req_msg));
2141 	}
2142 	if (cm_req_has_alt_path(req_msg))
2143 		work->path[1].rec_type = work->path[0].rec_type;
2144 	cm_format_paths_from_req(req_msg, &work->path[0],
2145 				 &work->path[1], work->mad_recv_wc->wc);
2146 	if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
2147 		sa_path_set_dmac(&work->path[0],
2148 				 cm_id_priv->av.ah_attr.roce.dmac);
2149 	work->path[0].hop_limit = grh->hop_limit;
2150 
2151 	/* This destroy call is needed to pair with cm_init_av_for_response */
2152 	cm_destroy_av(&cm_id_priv->av);
2153 	ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av);
2154 	if (ret) {
2155 		int err;
2156 
2157 		err = rdma_query_gid(work->port->cm_dev->ib_device,
2158 				     work->port->port_num, 0,
2159 				     &work->path[0].sgid);
2160 		if (err)
2161 			ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2162 				       NULL, 0, NULL, 0);
2163 		else
2164 			ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2165 				       &work->path[0].sgid,
2166 				       sizeof(work->path[0].sgid),
2167 				       NULL, 0);
2168 		goto rejected;
2169 	}
2170 	if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_IB)
2171 		cm_id_priv->av.dlid_datapath =
2172 			IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg);
2173 
2174 	if (cm_req_has_alt_path(req_msg)) {
2175 		ret = cm_init_av_by_path(&work->path[1], NULL,
2176 					 &cm_id_priv->alt_av);
2177 		if (ret) {
2178 			ib_send_cm_rej(&cm_id_priv->id,
2179 				       IB_CM_REJ_INVALID_ALT_GID,
2180 				       &work->path[0].sgid,
2181 				       sizeof(work->path[0].sgid), NULL, 0);
2182 			goto rejected;
2183 		}
2184 	}
2185 
2186 	cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
2187 	cm_id_priv->id.context = listen_cm_id_priv->id.context;
2188 	cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
2189 
2190 	/* Now MAD handlers can see the new ID */
2191 	spin_lock_irq(&cm_id_priv->lock);
2192 	cm_finalize_id(cm_id_priv);
2193 
2194 	/* Refcount belongs to the event, pairs with cm_process_work() */
2195 	refcount_inc(&cm_id_priv->refcount);
2196 	cm_queue_work_unlock(cm_id_priv, work);
2197 	/*
2198 	 * Since this ID was just created and was not made visible to other MAD
2199 	 * handlers until the cm_finalize_id() above we know that the
2200 	 * cm_process_work() will deliver the event and the listen_cm_id
2201 	 * embedded in the event can be derefed here.
2202 	 */
2203 	cm_deref_id(listen_cm_id_priv);
2204 	return 0;
2205 
2206 rejected:
2207 	cm_deref_id(listen_cm_id_priv);
2208 destroy:
2209 	ib_destroy_cm_id(&cm_id_priv->id);
2210 	return ret;
2211 }
2212 
cm_format_rep(struct cm_rep_msg * rep_msg,struct cm_id_private * cm_id_priv,struct ib_cm_rep_param * param)2213 static void cm_format_rep(struct cm_rep_msg *rep_msg,
2214 			  struct cm_id_private *cm_id_priv,
2215 			  struct ib_cm_rep_param *param)
2216 {
2217 	cm_format_mad_ece_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid,
2218 			      param->ece.attr_mod);
2219 	IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg,
2220 		be32_to_cpu(cm_id_priv->id.local_id));
2221 	IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg,
2222 		be32_to_cpu(cm_id_priv->id.remote_id));
2223 	IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn);
2224 	IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg,
2225 		param->responder_resources);
2226 	IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg,
2227 		cm_id_priv->av.port->cm_dev->ack_delay);
2228 	IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted);
2229 	IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count);
2230 	IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg,
2231 		be64_to_cpu(cm_id_priv->id.device->node_guid));
2232 
2233 	if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
2234 		IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg,
2235 			param->initiator_depth);
2236 		IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg,
2237 			param->flow_control);
2238 		IBA_SET(CM_REP_SRQ, rep_msg, param->srq);
2239 		IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num);
2240 	} else {
2241 		IBA_SET(CM_REP_SRQ, rep_msg, 1);
2242 		IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num);
2243 	}
2244 
2245 	IBA_SET(CM_REP_VENDOR_ID_L, rep_msg, param->ece.vendor_id);
2246 	IBA_SET(CM_REP_VENDOR_ID_M, rep_msg, param->ece.vendor_id >> 8);
2247 	IBA_SET(CM_REP_VENDOR_ID_H, rep_msg, param->ece.vendor_id >> 16);
2248 
2249 	if (param->private_data && param->private_data_len)
2250 		IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data,
2251 			    param->private_data_len);
2252 }
2253 
ib_send_cm_rep(struct ib_cm_id * cm_id,struct ib_cm_rep_param * param)2254 int ib_send_cm_rep(struct ib_cm_id *cm_id,
2255 		   struct ib_cm_rep_param *param)
2256 {
2257 	struct cm_id_private *cm_id_priv;
2258 	struct ib_mad_send_buf *msg;
2259 	struct cm_rep_msg *rep_msg;
2260 	unsigned long flags;
2261 	int ret;
2262 
2263 	if (param->private_data &&
2264 	    param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
2265 		return -EINVAL;
2266 
2267 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2268 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2269 	if (cm_id->state != IB_CM_REQ_RCVD &&
2270 	    cm_id->state != IB_CM_MRA_REQ_SENT) {
2271 		trace_icm_send_rep_err(cm_id_priv->id.local_id, cm_id->state);
2272 		ret = -EINVAL;
2273 		goto out;
2274 	}
2275 
2276 	msg = cm_alloc_priv_msg(cm_id_priv);
2277 	if (IS_ERR(msg)) {
2278 		ret = PTR_ERR(msg);
2279 		goto out;
2280 	}
2281 
2282 	rep_msg = (struct cm_rep_msg *) msg->mad;
2283 	cm_format_rep(rep_msg, cm_id_priv, param);
2284 	msg->timeout_ms = cm_id_priv->timeout_ms;
2285 	msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
2286 
2287 	trace_icm_send_rep(cm_id);
2288 	ret = ib_post_send_mad(msg, NULL);
2289 	if (ret)
2290 		goto out_free;
2291 
2292 	cm_id->state = IB_CM_REP_SENT;
2293 	cm_id_priv->initiator_depth = param->initiator_depth;
2294 	cm_id_priv->responder_resources = param->responder_resources;
2295 	cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2296 	WARN_ONCE(param->qp_num & 0xFF000000,
2297 		  "IBTA declares QPN to be 24 bits, but it is 0x%X\n",
2298 		  param->qp_num);
2299 	cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
2300 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2301 	return 0;
2302 
2303 out_free:
2304 	cm_free_priv_msg(msg);
2305 out:
2306 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2307 	return ret;
2308 }
2309 EXPORT_SYMBOL(ib_send_cm_rep);
2310 
cm_format_rtu(struct cm_rtu_msg * rtu_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2311 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
2312 			  struct cm_id_private *cm_id_priv,
2313 			  const void *private_data,
2314 			  u8 private_data_len)
2315 {
2316 	cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
2317 	IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg,
2318 		be32_to_cpu(cm_id_priv->id.local_id));
2319 	IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg,
2320 		be32_to_cpu(cm_id_priv->id.remote_id));
2321 
2322 	if (private_data && private_data_len)
2323 		IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data,
2324 			    private_data_len);
2325 }
2326 
ib_send_cm_rtu(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2327 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
2328 		   const void *private_data,
2329 		   u8 private_data_len)
2330 {
2331 	struct cm_id_private *cm_id_priv;
2332 	struct ib_mad_send_buf *msg;
2333 	unsigned long flags;
2334 	void *data;
2335 	int ret;
2336 
2337 	if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
2338 		return -EINVAL;
2339 
2340 	data = cm_copy_private_data(private_data, private_data_len);
2341 	if (IS_ERR(data))
2342 		return PTR_ERR(data);
2343 
2344 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2345 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2346 	if (cm_id->state != IB_CM_REP_RCVD &&
2347 	    cm_id->state != IB_CM_MRA_REP_SENT) {
2348 		trace_icm_send_cm_rtu_err(cm_id);
2349 		ret = -EINVAL;
2350 		goto error;
2351 	}
2352 
2353 	msg = cm_alloc_msg(cm_id_priv);
2354 	if (IS_ERR(msg)) {
2355 		ret = PTR_ERR(msg);
2356 		goto error;
2357 	}
2358 
2359 	cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2360 		      private_data, private_data_len);
2361 
2362 	trace_icm_send_rtu(cm_id);
2363 	ret = ib_post_send_mad(msg, NULL);
2364 	if (ret) {
2365 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2366 		cm_free_msg(msg);
2367 		kfree(data);
2368 		return ret;
2369 	}
2370 
2371 	cm_id->state = IB_CM_ESTABLISHED;
2372 	cm_set_private_data(cm_id_priv, data, private_data_len);
2373 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2374 	return 0;
2375 
2376 error:	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2377 	kfree(data);
2378 	return ret;
2379 }
2380 EXPORT_SYMBOL(ib_send_cm_rtu);
2381 
cm_format_rep_event(struct cm_work * work,enum ib_qp_type qp_type)2382 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
2383 {
2384 	struct cm_rep_msg *rep_msg;
2385 	struct ib_cm_rep_event_param *param;
2386 
2387 	rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2388 	param = &work->cm_event.param.rep_rcvd;
2389 	param->remote_ca_guid =
2390 		cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2391 	param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg);
2392 	param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2393 	param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg);
2394 	param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2395 	param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2396 	param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2397 	param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg);
2398 	param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg);
2399 	param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2400 	param->srq = IBA_GET(CM_REP_SRQ, rep_msg);
2401 	param->ece.vendor_id = IBA_GET(CM_REP_VENDOR_ID_H, rep_msg) << 16;
2402 	param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_M, rep_msg) << 8;
2403 	param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_L, rep_msg);
2404 	param->ece.attr_mod = be32_to_cpu(rep_msg->hdr.attr_mod);
2405 
2406 	work->cm_event.private_data =
2407 		IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg);
2408 }
2409 
cm_dup_rep_handler(struct cm_work * work)2410 static void cm_dup_rep_handler(struct cm_work *work)
2411 {
2412 	struct cm_id_private *cm_id_priv;
2413 	struct cm_rep_msg *rep_msg;
2414 	struct ib_mad_send_buf *msg = NULL;
2415 	int ret;
2416 
2417 	rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2418 	cm_id_priv = cm_acquire_id(
2419 		cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)),
2420 		cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)));
2421 	if (!cm_id_priv)
2422 		return;
2423 
2424 	atomic_long_inc(
2425 		&work->port->counters[CM_RECV_DUPLICATES][CM_REP_COUNTER]);
2426 	ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
2427 	if (ret)
2428 		goto deref;
2429 
2430 	spin_lock_irq(&cm_id_priv->lock);
2431 	if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2432 		cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2433 			      cm_id_priv->private_data,
2434 			      cm_id_priv->private_data_len);
2435 	else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2436 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2437 			      CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout,
2438 			      cm_id_priv->private_data,
2439 			      cm_id_priv->private_data_len);
2440 	else
2441 		goto unlock;
2442 	spin_unlock_irq(&cm_id_priv->lock);
2443 
2444 	trace_icm_send_dup_rep(&cm_id_priv->id);
2445 	ret = ib_post_send_mad(msg, NULL);
2446 	if (ret)
2447 		goto free;
2448 	goto deref;
2449 
2450 unlock:	spin_unlock_irq(&cm_id_priv->lock);
2451 free:	cm_free_response_msg(msg);
2452 deref:	cm_deref_id(cm_id_priv);
2453 }
2454 
cm_rep_handler(struct cm_work * work)2455 static int cm_rep_handler(struct cm_work *work)
2456 {
2457 	struct cm_id_private *cm_id_priv;
2458 	struct cm_rep_msg *rep_msg;
2459 	int ret;
2460 	struct cm_id_private *cur_cm_id_priv;
2461 	struct cm_timewait_info *timewait_info;
2462 
2463 	rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2464 	cm_id_priv = cm_acquire_id(
2465 		cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0);
2466 	if (!cm_id_priv) {
2467 		cm_dup_rep_handler(work);
2468 		trace_icm_remote_no_priv_err(
2469 			 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2470 		return -EINVAL;
2471 	}
2472 
2473 	cm_format_rep_event(work, cm_id_priv->qp_type);
2474 
2475 	spin_lock_irq(&cm_id_priv->lock);
2476 	switch (cm_id_priv->id.state) {
2477 	case IB_CM_REQ_SENT:
2478 	case IB_CM_MRA_REQ_RCVD:
2479 		break;
2480 	default:
2481 		ret = -EINVAL;
2482 		trace_icm_rep_unknown_err(
2483 			IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2484 			IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg),
2485 			cm_id_priv->id.state);
2486 		spin_unlock_irq(&cm_id_priv->lock);
2487 		goto error;
2488 	}
2489 
2490 	cm_id_priv->timewait_info->work.remote_id =
2491 		cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2492 	cm_id_priv->timewait_info->remote_ca_guid =
2493 		cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2494 	cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2495 
2496 	spin_lock(&cm.lock);
2497 	/* Check for duplicate REP. */
2498 	if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
2499 		spin_unlock(&cm.lock);
2500 		spin_unlock_irq(&cm_id_priv->lock);
2501 		ret = -EINVAL;
2502 		trace_icm_insert_failed_err(
2503 			 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2504 		goto error;
2505 	}
2506 	/* Check for a stale connection. */
2507 	timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2508 	if (timewait_info) {
2509 		cm_remove_remote(cm_id_priv);
2510 		cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2511 					   timewait_info->work.remote_id);
2512 
2513 		spin_unlock(&cm.lock);
2514 		spin_unlock_irq(&cm_id_priv->lock);
2515 		cm_issue_rej(work->port, work->mad_recv_wc,
2516 			     IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
2517 			     NULL, 0);
2518 		ret = -EINVAL;
2519 		trace_icm_staleconn_err(
2520 			IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2521 			IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2522 
2523 		if (cur_cm_id_priv) {
2524 			ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2525 			cm_deref_id(cur_cm_id_priv);
2526 		}
2527 
2528 		goto error;
2529 	}
2530 	spin_unlock(&cm.lock);
2531 
2532 	cm_id_priv->id.state = IB_CM_REP_RCVD;
2533 	cm_id_priv->id.remote_id =
2534 		cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2535 	cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2536 	cm_id_priv->initiator_depth =
2537 		IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2538 	cm_id_priv->responder_resources =
2539 		IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2540 	cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2541 	cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2542 	cm_id_priv->target_ack_delay =
2543 		IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2544 	cm_id_priv->av.timeout =
2545 			cm_ack_timeout(cm_id_priv->target_ack_delay,
2546 				       cm_id_priv->av.timeout - 1);
2547 	cm_id_priv->alt_av.timeout =
2548 			cm_ack_timeout(cm_id_priv->target_ack_delay,
2549 				       cm_id_priv->alt_av.timeout - 1);
2550 
2551 	ib_cancel_mad(cm_id_priv->msg);
2552 	cm_queue_work_unlock(cm_id_priv, work);
2553 	return 0;
2554 
2555 error:
2556 	cm_deref_id(cm_id_priv);
2557 	return ret;
2558 }
2559 
cm_establish_handler(struct cm_work * work)2560 static int cm_establish_handler(struct cm_work *work)
2561 {
2562 	struct cm_id_private *cm_id_priv;
2563 
2564 	/* See comment in cm_establish about lookup. */
2565 	cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
2566 	if (!cm_id_priv)
2567 		return -EINVAL;
2568 
2569 	spin_lock_irq(&cm_id_priv->lock);
2570 	if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2571 		spin_unlock_irq(&cm_id_priv->lock);
2572 		goto out;
2573 	}
2574 
2575 	ib_cancel_mad(cm_id_priv->msg);
2576 	cm_queue_work_unlock(cm_id_priv, work);
2577 	return 0;
2578 out:
2579 	cm_deref_id(cm_id_priv);
2580 	return -EINVAL;
2581 }
2582 
cm_rtu_handler(struct cm_work * work)2583 static int cm_rtu_handler(struct cm_work *work)
2584 {
2585 	struct cm_id_private *cm_id_priv;
2586 	struct cm_rtu_msg *rtu_msg;
2587 
2588 	rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2589 	cm_id_priv = cm_acquire_id(
2590 		cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)),
2591 		cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg)));
2592 	if (!cm_id_priv)
2593 		return -EINVAL;
2594 
2595 	work->cm_event.private_data =
2596 		IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg);
2597 
2598 	spin_lock_irq(&cm_id_priv->lock);
2599 	if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2600 	    cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2601 		spin_unlock_irq(&cm_id_priv->lock);
2602 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2603 						     [CM_RTU_COUNTER]);
2604 		goto out;
2605 	}
2606 	cm_id_priv->id.state = IB_CM_ESTABLISHED;
2607 
2608 	ib_cancel_mad(cm_id_priv->msg);
2609 	cm_queue_work_unlock(cm_id_priv, work);
2610 	return 0;
2611 out:
2612 	cm_deref_id(cm_id_priv);
2613 	return -EINVAL;
2614 }
2615 
cm_format_dreq(struct cm_dreq_msg * dreq_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2616 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2617 			  struct cm_id_private *cm_id_priv,
2618 			  const void *private_data,
2619 			  u8 private_data_len)
2620 {
2621 	cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
2622 			  cm_form_tid(cm_id_priv));
2623 	IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg,
2624 		be32_to_cpu(cm_id_priv->id.local_id));
2625 	IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg,
2626 		be32_to_cpu(cm_id_priv->id.remote_id));
2627 	IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg,
2628 		be32_to_cpu(cm_id_priv->remote_qpn));
2629 
2630 	if (private_data && private_data_len)
2631 		IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data,
2632 			    private_data_len);
2633 }
2634 
cm_send_dreq_locked(struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2635 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
2636 			       const void *private_data, u8 private_data_len)
2637 {
2638 	struct ib_mad_send_buf *msg;
2639 	int ret;
2640 
2641 	lockdep_assert_held(&cm_id_priv->lock);
2642 
2643 	if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2644 		return -EINVAL;
2645 
2646 	if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2647 		trace_icm_dreq_skipped(&cm_id_priv->id);
2648 		return -EINVAL;
2649 	}
2650 
2651 	if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2652 	    cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2653 		ib_cancel_mad(cm_id_priv->msg);
2654 
2655 	msg = cm_alloc_priv_msg(cm_id_priv);
2656 	if (IS_ERR(msg)) {
2657 		cm_enter_timewait(cm_id_priv);
2658 		return PTR_ERR(msg);
2659 	}
2660 
2661 	cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2662 		       private_data, private_data_len);
2663 	msg->timeout_ms = cm_id_priv->timeout_ms;
2664 	msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
2665 
2666 	trace_icm_send_dreq(&cm_id_priv->id);
2667 	ret = ib_post_send_mad(msg, NULL);
2668 	if (ret) {
2669 		cm_enter_timewait(cm_id_priv);
2670 		cm_free_priv_msg(msg);
2671 		return ret;
2672 	}
2673 
2674 	cm_id_priv->id.state = IB_CM_DREQ_SENT;
2675 	return 0;
2676 }
2677 
ib_send_cm_dreq(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2678 int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data,
2679 		    u8 private_data_len)
2680 {
2681 	struct cm_id_private *cm_id_priv =
2682 		container_of(cm_id, struct cm_id_private, id);
2683 	unsigned long flags;
2684 	int ret;
2685 
2686 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2687 	ret = cm_send_dreq_locked(cm_id_priv, private_data, private_data_len);
2688 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2689 	return ret;
2690 }
2691 EXPORT_SYMBOL(ib_send_cm_dreq);
2692 
cm_format_drep(struct cm_drep_msg * drep_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2693 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2694 			  struct cm_id_private *cm_id_priv,
2695 			  const void *private_data,
2696 			  u8 private_data_len)
2697 {
2698 	cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2699 	IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2700 		be32_to_cpu(cm_id_priv->id.local_id));
2701 	IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2702 		be32_to_cpu(cm_id_priv->id.remote_id));
2703 
2704 	if (private_data && private_data_len)
2705 		IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data,
2706 			    private_data_len);
2707 }
2708 
cm_send_drep_locked(struct cm_id_private * cm_id_priv,void * private_data,u8 private_data_len)2709 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
2710 			       void *private_data, u8 private_data_len)
2711 {
2712 	struct ib_mad_send_buf *msg;
2713 	int ret;
2714 
2715 	lockdep_assert_held(&cm_id_priv->lock);
2716 
2717 	if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2718 		return -EINVAL;
2719 
2720 	if (cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2721 		trace_icm_send_drep_err(&cm_id_priv->id);
2722 		kfree(private_data);
2723 		return -EINVAL;
2724 	}
2725 
2726 	cm_set_private_data(cm_id_priv, private_data, private_data_len);
2727 	cm_enter_timewait(cm_id_priv);
2728 
2729 	msg = cm_alloc_msg(cm_id_priv);
2730 	if (IS_ERR(msg))
2731 		return PTR_ERR(msg);
2732 
2733 	cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2734 		       private_data, private_data_len);
2735 
2736 	trace_icm_send_drep(&cm_id_priv->id);
2737 	ret = ib_post_send_mad(msg, NULL);
2738 	if (ret) {
2739 		cm_free_msg(msg);
2740 		return ret;
2741 	}
2742 	return 0;
2743 }
2744 
ib_send_cm_drep(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2745 int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data,
2746 		    u8 private_data_len)
2747 {
2748 	struct cm_id_private *cm_id_priv =
2749 		container_of(cm_id, struct cm_id_private, id);
2750 	unsigned long flags;
2751 	void *data;
2752 	int ret;
2753 
2754 	data = cm_copy_private_data(private_data, private_data_len);
2755 	if (IS_ERR(data))
2756 		return PTR_ERR(data);
2757 
2758 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2759 	ret = cm_send_drep_locked(cm_id_priv, data, private_data_len);
2760 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2761 	return ret;
2762 }
2763 EXPORT_SYMBOL(ib_send_cm_drep);
2764 
cm_issue_drep(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc)2765 static int cm_issue_drep(struct cm_port *port,
2766 			 struct ib_mad_recv_wc *mad_recv_wc)
2767 {
2768 	struct ib_mad_send_buf *msg = NULL;
2769 	struct cm_dreq_msg *dreq_msg;
2770 	struct cm_drep_msg *drep_msg;
2771 	int ret;
2772 
2773 	ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
2774 	if (ret)
2775 		return ret;
2776 
2777 	dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2778 	drep_msg = (struct cm_drep_msg *) msg->mad;
2779 
2780 	cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2781 	IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2782 		IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg));
2783 	IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2784 		IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2785 
2786 	trace_icm_issue_drep(
2787 		IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2788 		IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2789 	ret = ib_post_send_mad(msg, NULL);
2790 	if (ret)
2791 		cm_free_response_msg(msg);
2792 
2793 	return ret;
2794 }
2795 
cm_dreq_handler(struct cm_work * work)2796 static int cm_dreq_handler(struct cm_work *work)
2797 {
2798 	struct cm_id_private *cm_id_priv;
2799 	struct cm_dreq_msg *dreq_msg;
2800 	struct ib_mad_send_buf *msg = NULL;
2801 
2802 	dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2803 	cm_id_priv = cm_acquire_id(
2804 		cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)),
2805 		cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)));
2806 	if (!cm_id_priv) {
2807 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2808 						     [CM_DREQ_COUNTER]);
2809 		cm_issue_drep(work->port, work->mad_recv_wc);
2810 		trace_icm_no_priv_err(
2811 			IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2812 			IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2813 		return -EINVAL;
2814 	}
2815 
2816 	work->cm_event.private_data =
2817 		IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg);
2818 
2819 	spin_lock_irq(&cm_id_priv->lock);
2820 	if (cm_id_priv->local_qpn !=
2821 	    cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg)))
2822 		goto unlock;
2823 
2824 	switch (cm_id_priv->id.state) {
2825 	case IB_CM_REP_SENT:
2826 	case IB_CM_DREQ_SENT:
2827 	case IB_CM_MRA_REP_RCVD:
2828 		ib_cancel_mad(cm_id_priv->msg);
2829 		break;
2830 	case IB_CM_ESTABLISHED:
2831 		if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2832 		    cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2833 			ib_cancel_mad(cm_id_priv->msg);
2834 		break;
2835 	case IB_CM_TIMEWAIT:
2836 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2837 						     [CM_DREQ_COUNTER]);
2838 		msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
2839 		if (IS_ERR(msg))
2840 			goto unlock;
2841 
2842 		cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2843 			       cm_id_priv->private_data,
2844 			       cm_id_priv->private_data_len);
2845 		spin_unlock_irq(&cm_id_priv->lock);
2846 
2847 		if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
2848 		    ib_post_send_mad(msg, NULL))
2849 			cm_free_response_msg(msg);
2850 		goto deref;
2851 	case IB_CM_DREQ_RCVD:
2852 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2853 						     [CM_DREQ_COUNTER]);
2854 		goto unlock;
2855 	default:
2856 		trace_icm_dreq_unknown_err(&cm_id_priv->id);
2857 		goto unlock;
2858 	}
2859 	cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2860 	cm_id_priv->tid = dreq_msg->hdr.tid;
2861 	cm_queue_work_unlock(cm_id_priv, work);
2862 	return 0;
2863 
2864 unlock:	spin_unlock_irq(&cm_id_priv->lock);
2865 deref:	cm_deref_id(cm_id_priv);
2866 	return -EINVAL;
2867 }
2868 
cm_drep_handler(struct cm_work * work)2869 static int cm_drep_handler(struct cm_work *work)
2870 {
2871 	struct cm_id_private *cm_id_priv;
2872 	struct cm_drep_msg *drep_msg;
2873 
2874 	drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2875 	cm_id_priv = cm_acquire_id(
2876 		cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)),
2877 		cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg)));
2878 	if (!cm_id_priv)
2879 		return -EINVAL;
2880 
2881 	work->cm_event.private_data =
2882 		IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg);
2883 
2884 	spin_lock_irq(&cm_id_priv->lock);
2885 	if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2886 	    cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2887 		spin_unlock_irq(&cm_id_priv->lock);
2888 		goto out;
2889 	}
2890 	cm_enter_timewait(cm_id_priv);
2891 
2892 	ib_cancel_mad(cm_id_priv->msg);
2893 	cm_queue_work_unlock(cm_id_priv, work);
2894 	return 0;
2895 out:
2896 	cm_deref_id(cm_id_priv);
2897 	return -EINVAL;
2898 }
2899 
cm_send_rej_locked(struct cm_id_private * cm_id_priv,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len)2900 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
2901 			      enum ib_cm_rej_reason reason, void *ari,
2902 			      u8 ari_length, const void *private_data,
2903 			      u8 private_data_len)
2904 {
2905 	enum ib_cm_state state = cm_id_priv->id.state;
2906 	struct ib_mad_send_buf *msg;
2907 	int ret;
2908 
2909 	lockdep_assert_held(&cm_id_priv->lock);
2910 
2911 	if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2912 	    (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2913 		return -EINVAL;
2914 
2915 	switch (state) {
2916 	case IB_CM_REQ_SENT:
2917 	case IB_CM_MRA_REQ_RCVD:
2918 	case IB_CM_REQ_RCVD:
2919 	case IB_CM_MRA_REQ_SENT:
2920 	case IB_CM_REP_RCVD:
2921 	case IB_CM_MRA_REP_SENT:
2922 		cm_reset_to_idle(cm_id_priv);
2923 		msg = cm_alloc_msg(cm_id_priv);
2924 		if (IS_ERR(msg))
2925 			return PTR_ERR(msg);
2926 		cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2927 			      ari, ari_length, private_data, private_data_len,
2928 			      state);
2929 		break;
2930 	case IB_CM_REP_SENT:
2931 	case IB_CM_MRA_REP_RCVD:
2932 		cm_enter_timewait(cm_id_priv);
2933 		msg = cm_alloc_msg(cm_id_priv);
2934 		if (IS_ERR(msg))
2935 			return PTR_ERR(msg);
2936 		cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2937 			      ari, ari_length, private_data, private_data_len,
2938 			      state);
2939 		break;
2940 	default:
2941 		trace_icm_send_unknown_rej_err(&cm_id_priv->id);
2942 		return -EINVAL;
2943 	}
2944 
2945 	trace_icm_send_rej(&cm_id_priv->id, reason);
2946 	ret = ib_post_send_mad(msg, NULL);
2947 	if (ret) {
2948 		cm_free_msg(msg);
2949 		return ret;
2950 	}
2951 
2952 	return 0;
2953 }
2954 
ib_send_cm_rej(struct ib_cm_id * cm_id,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len)2955 int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason,
2956 		   void *ari, u8 ari_length, const void *private_data,
2957 		   u8 private_data_len)
2958 {
2959 	struct cm_id_private *cm_id_priv =
2960 		container_of(cm_id, struct cm_id_private, id);
2961 	unsigned long flags;
2962 	int ret;
2963 
2964 	spin_lock_irqsave(&cm_id_priv->lock, flags);
2965 	ret = cm_send_rej_locked(cm_id_priv, reason, ari, ari_length,
2966 				 private_data, private_data_len);
2967 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2968 	return ret;
2969 }
2970 EXPORT_SYMBOL(ib_send_cm_rej);
2971 
cm_format_rej_event(struct cm_work * work)2972 static void cm_format_rej_event(struct cm_work *work)
2973 {
2974 	struct cm_rej_msg *rej_msg;
2975 	struct ib_cm_rej_event_param *param;
2976 
2977 	rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2978 	param = &work->cm_event.param.rej_rcvd;
2979 	param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg);
2980 	param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg);
2981 	param->reason = IBA_GET(CM_REJ_REASON, rej_msg);
2982 	work->cm_event.private_data =
2983 		IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg);
2984 }
2985 
cm_acquire_rejected_id(struct cm_rej_msg * rej_msg)2986 static struct cm_id_private *cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
2987 {
2988 	struct cm_id_private *cm_id_priv;
2989 	__be32 remote_id;
2990 
2991 	remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg));
2992 
2993 	if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) {
2994 		cm_id_priv = cm_find_remote_id(
2995 			*((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)),
2996 			remote_id);
2997 	} else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) ==
2998 		   CM_MSG_RESPONSE_REQ)
2999 		cm_id_priv = cm_acquire_id(
3000 			cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3001 			0);
3002 	else
3003 		cm_id_priv = cm_acquire_id(
3004 			cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3005 			remote_id);
3006 
3007 	return cm_id_priv;
3008 }
3009 
cm_rej_handler(struct cm_work * work)3010 static int cm_rej_handler(struct cm_work *work)
3011 {
3012 	struct cm_id_private *cm_id_priv;
3013 	struct cm_rej_msg *rej_msg;
3014 
3015 	rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3016 	cm_id_priv = cm_acquire_rejected_id(rej_msg);
3017 	if (!cm_id_priv)
3018 		return -EINVAL;
3019 
3020 	cm_format_rej_event(work);
3021 
3022 	spin_lock_irq(&cm_id_priv->lock);
3023 	switch (cm_id_priv->id.state) {
3024 	case IB_CM_REQ_SENT:
3025 	case IB_CM_MRA_REQ_RCVD:
3026 	case IB_CM_REP_SENT:
3027 	case IB_CM_MRA_REP_RCVD:
3028 		ib_cancel_mad(cm_id_priv->msg);
3029 		fallthrough;
3030 	case IB_CM_REQ_RCVD:
3031 	case IB_CM_MRA_REQ_SENT:
3032 		if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN)
3033 			cm_enter_timewait(cm_id_priv);
3034 		else
3035 			cm_reset_to_idle(cm_id_priv);
3036 		break;
3037 	case IB_CM_DREQ_SENT:
3038 		ib_cancel_mad(cm_id_priv->msg);
3039 		fallthrough;
3040 	case IB_CM_REP_RCVD:
3041 	case IB_CM_MRA_REP_SENT:
3042 		cm_enter_timewait(cm_id_priv);
3043 		break;
3044 	case IB_CM_ESTABLISHED:
3045 		if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
3046 		    cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
3047 			if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
3048 				ib_cancel_mad(cm_id_priv->msg);
3049 			cm_enter_timewait(cm_id_priv);
3050 			break;
3051 		}
3052 		fallthrough;
3053 	default:
3054 		trace_icm_rej_unknown_err(&cm_id_priv->id);
3055 		spin_unlock_irq(&cm_id_priv->lock);
3056 		goto out;
3057 	}
3058 
3059 	cm_queue_work_unlock(cm_id_priv, work);
3060 	return 0;
3061 out:
3062 	cm_deref_id(cm_id_priv);
3063 	return -EINVAL;
3064 }
3065 
ib_send_cm_mra(struct ib_cm_id * cm_id,u8 service_timeout,const void * private_data,u8 private_data_len)3066 int ib_send_cm_mra(struct ib_cm_id *cm_id,
3067 		   u8 service_timeout,
3068 		   const void *private_data,
3069 		   u8 private_data_len)
3070 {
3071 	struct cm_id_private *cm_id_priv;
3072 	struct ib_mad_send_buf *msg;
3073 	enum ib_cm_state cm_state;
3074 	enum ib_cm_lap_state lap_state;
3075 	enum cm_msg_response msg_response;
3076 	void *data;
3077 	unsigned long flags;
3078 	int ret;
3079 
3080 	if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE)
3081 		return -EINVAL;
3082 
3083 	data = cm_copy_private_data(private_data, private_data_len);
3084 	if (IS_ERR(data))
3085 		return PTR_ERR(data);
3086 
3087 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3088 
3089 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3090 	switch (cm_id_priv->id.state) {
3091 	case IB_CM_REQ_RCVD:
3092 		cm_state = IB_CM_MRA_REQ_SENT;
3093 		lap_state = cm_id->lap_state;
3094 		msg_response = CM_MSG_RESPONSE_REQ;
3095 		break;
3096 	case IB_CM_REP_RCVD:
3097 		cm_state = IB_CM_MRA_REP_SENT;
3098 		lap_state = cm_id->lap_state;
3099 		msg_response = CM_MSG_RESPONSE_REP;
3100 		break;
3101 	case IB_CM_ESTABLISHED:
3102 		if (cm_id->lap_state == IB_CM_LAP_RCVD) {
3103 			cm_state = cm_id->state;
3104 			lap_state = IB_CM_MRA_LAP_SENT;
3105 			msg_response = CM_MSG_RESPONSE_OTHER;
3106 			break;
3107 		}
3108 		fallthrough;
3109 	default:
3110 		trace_icm_send_mra_unknown_err(&cm_id_priv->id);
3111 		ret = -EINVAL;
3112 		goto error_unlock;
3113 	}
3114 
3115 	if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
3116 		msg = cm_alloc_msg(cm_id_priv);
3117 		if (IS_ERR(msg)) {
3118 			ret = PTR_ERR(msg);
3119 			goto error_unlock;
3120 		}
3121 
3122 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3123 			      msg_response, service_timeout,
3124 			      private_data, private_data_len);
3125 		trace_icm_send_mra(cm_id);
3126 		ret = ib_post_send_mad(msg, NULL);
3127 		if (ret)
3128 			goto error_free_msg;
3129 	}
3130 
3131 	cm_id->state = cm_state;
3132 	cm_id->lap_state = lap_state;
3133 	cm_id_priv->service_timeout = service_timeout;
3134 	cm_set_private_data(cm_id_priv, data, private_data_len);
3135 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3136 	return 0;
3137 
3138 error_free_msg:
3139 	cm_free_msg(msg);
3140 error_unlock:
3141 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3142 	kfree(data);
3143 	return ret;
3144 }
3145 EXPORT_SYMBOL(ib_send_cm_mra);
3146 
cm_acquire_mraed_id(struct cm_mra_msg * mra_msg)3147 static struct cm_id_private *cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
3148 {
3149 	switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) {
3150 	case CM_MSG_RESPONSE_REQ:
3151 		return cm_acquire_id(
3152 			cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3153 			0);
3154 	case CM_MSG_RESPONSE_REP:
3155 	case CM_MSG_RESPONSE_OTHER:
3156 		return cm_acquire_id(
3157 			cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3158 			cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg)));
3159 	default:
3160 		return NULL;
3161 	}
3162 }
3163 
cm_mra_handler(struct cm_work * work)3164 static int cm_mra_handler(struct cm_work *work)
3165 {
3166 	struct cm_id_private *cm_id_priv;
3167 	struct cm_mra_msg *mra_msg;
3168 	int timeout;
3169 
3170 	mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
3171 	cm_id_priv = cm_acquire_mraed_id(mra_msg);
3172 	if (!cm_id_priv)
3173 		return -EINVAL;
3174 
3175 	work->cm_event.private_data =
3176 		IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg);
3177 	work->cm_event.param.mra_rcvd.service_timeout =
3178 		IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg);
3179 	timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) +
3180 		  cm_convert_to_ms(cm_id_priv->av.timeout);
3181 
3182 	spin_lock_irq(&cm_id_priv->lock);
3183 	switch (cm_id_priv->id.state) {
3184 	case IB_CM_REQ_SENT:
3185 		if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3186 			    CM_MSG_RESPONSE_REQ ||
3187 		    ib_modify_mad(cm_id_priv->msg, timeout))
3188 			goto out;
3189 		cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
3190 		break;
3191 	case IB_CM_REP_SENT:
3192 		if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3193 			    CM_MSG_RESPONSE_REP ||
3194 		    ib_modify_mad(cm_id_priv->msg, timeout))
3195 			goto out;
3196 		cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
3197 		break;
3198 	case IB_CM_ESTABLISHED:
3199 		if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3200 			    CM_MSG_RESPONSE_OTHER ||
3201 		    cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
3202 		    ib_modify_mad(cm_id_priv->msg, timeout)) {
3203 			if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
3204 				atomic_long_inc(
3205 					&work->port->counters[CM_RECV_DUPLICATES]
3206 							     [CM_MRA_COUNTER]);
3207 			goto out;
3208 		}
3209 		cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
3210 		break;
3211 	case IB_CM_MRA_REQ_RCVD:
3212 	case IB_CM_MRA_REP_RCVD:
3213 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3214 						     [CM_MRA_COUNTER]);
3215 		fallthrough;
3216 	default:
3217 		trace_icm_mra_unknown_err(&cm_id_priv->id);
3218 		goto out;
3219 	}
3220 
3221 	cm_id_priv->msg->context[1] = (void *) (unsigned long)
3222 				      cm_id_priv->id.state;
3223 	cm_queue_work_unlock(cm_id_priv, work);
3224 	return 0;
3225 out:
3226 	spin_unlock_irq(&cm_id_priv->lock);
3227 	cm_deref_id(cm_id_priv);
3228 	return -EINVAL;
3229 }
3230 
cm_format_path_lid_from_lap(struct cm_lap_msg * lap_msg,struct sa_path_rec * path)3231 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
3232 					struct sa_path_rec *path)
3233 {
3234 	u32 lid;
3235 
3236 	if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
3237 		sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID,
3238 					       lap_msg));
3239 		sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID,
3240 					       lap_msg));
3241 	} else {
3242 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3243 			CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg));
3244 		sa_path_set_dlid(path, lid);
3245 
3246 		lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3247 			CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg));
3248 		sa_path_set_slid(path, lid);
3249 	}
3250 }
3251 
cm_format_path_from_lap(struct cm_id_private * cm_id_priv,struct sa_path_rec * path,struct cm_lap_msg * lap_msg)3252 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
3253 				    struct sa_path_rec *path,
3254 				    struct cm_lap_msg *lap_msg)
3255 {
3256 	path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg);
3257 	path->sgid =
3258 		*IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg);
3259 	path->flow_label =
3260 		cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg));
3261 	path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg);
3262 	path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg);
3263 	path->reversible = 1;
3264 	path->pkey = cm_id_priv->pkey;
3265 	path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg);
3266 	path->mtu_selector = IB_SA_EQ;
3267 	path->mtu = cm_id_priv->path_mtu;
3268 	path->rate_selector = IB_SA_EQ;
3269 	path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg);
3270 	path->packet_life_time_selector = IB_SA_EQ;
3271 	path->packet_life_time =
3272 		IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg);
3273 	path->packet_life_time -= (path->packet_life_time > 0);
3274 	cm_format_path_lid_from_lap(lap_msg, path);
3275 }
3276 
cm_lap_handler(struct cm_work * work)3277 static int cm_lap_handler(struct cm_work *work)
3278 {
3279 	struct cm_id_private *cm_id_priv;
3280 	struct cm_lap_msg *lap_msg;
3281 	struct ib_cm_lap_event_param *param;
3282 	struct ib_mad_send_buf *msg = NULL;
3283 	struct rdma_ah_attr ah_attr;
3284 	struct cm_av alt_av = {};
3285 	int ret;
3286 
3287 	/* Currently Alternate path messages are not supported for
3288 	 * RoCE link layer.
3289 	 */
3290 	if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3291 			       work->port->port_num))
3292 		return -EINVAL;
3293 
3294 	/* todo: verify LAP request and send reject APR if invalid. */
3295 	lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
3296 	cm_id_priv = cm_acquire_id(
3297 		cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)),
3298 		cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg)));
3299 	if (!cm_id_priv)
3300 		return -EINVAL;
3301 
3302 	param = &work->cm_event.param.lap_rcvd;
3303 	memset(&work->path[0], 0, sizeof(work->path[1]));
3304 	cm_path_set_rec_type(work->port->cm_dev->ib_device,
3305 			     work->port->port_num, &work->path[0],
3306 			     IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID,
3307 					     lap_msg));
3308 	param->alternate_path = &work->path[0];
3309 	cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
3310 	work->cm_event.private_data =
3311 		IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg);
3312 
3313 	ret = ib_init_ah_attr_from_wc(work->port->cm_dev->ib_device,
3314 				      work->port->port_num,
3315 				      work->mad_recv_wc->wc,
3316 				      work->mad_recv_wc->recv_buf.grh,
3317 				      &ah_attr);
3318 	if (ret)
3319 		goto deref;
3320 
3321 	ret = cm_init_av_by_path(param->alternate_path, NULL, &alt_av);
3322 	if (ret) {
3323 		rdma_destroy_ah_attr(&ah_attr);
3324 		goto deref;
3325 	}
3326 
3327 	spin_lock_irq(&cm_id_priv->lock);
3328 	cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
3329 			   &ah_attr, &cm_id_priv->av);
3330 	cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
3331 
3332 	if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
3333 		goto unlock;
3334 
3335 	switch (cm_id_priv->id.lap_state) {
3336 	case IB_CM_LAP_UNINIT:
3337 	case IB_CM_LAP_IDLE:
3338 		break;
3339 	case IB_CM_MRA_LAP_SENT:
3340 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3341 						     [CM_LAP_COUNTER]);
3342 		msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
3343 		if (IS_ERR(msg))
3344 			goto unlock;
3345 
3346 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3347 			      CM_MSG_RESPONSE_OTHER,
3348 			      cm_id_priv->service_timeout,
3349 			      cm_id_priv->private_data,
3350 			      cm_id_priv->private_data_len);
3351 		spin_unlock_irq(&cm_id_priv->lock);
3352 
3353 		if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
3354 		    ib_post_send_mad(msg, NULL))
3355 			cm_free_response_msg(msg);
3356 		goto deref;
3357 	case IB_CM_LAP_RCVD:
3358 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3359 						     [CM_LAP_COUNTER]);
3360 		goto unlock;
3361 	default:
3362 		goto unlock;
3363 	}
3364 
3365 	cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
3366 	cm_id_priv->tid = lap_msg->hdr.tid;
3367 	cm_queue_work_unlock(cm_id_priv, work);
3368 	return 0;
3369 
3370 unlock:	spin_unlock_irq(&cm_id_priv->lock);
3371 deref:	cm_deref_id(cm_id_priv);
3372 	return -EINVAL;
3373 }
3374 
cm_apr_handler(struct cm_work * work)3375 static int cm_apr_handler(struct cm_work *work)
3376 {
3377 	struct cm_id_private *cm_id_priv;
3378 	struct cm_apr_msg *apr_msg;
3379 
3380 	/* Currently Alternate path messages are not supported for
3381 	 * RoCE link layer.
3382 	 */
3383 	if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3384 			       work->port->port_num))
3385 		return -EINVAL;
3386 
3387 	apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3388 	cm_id_priv = cm_acquire_id(
3389 		cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)),
3390 		cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg)));
3391 	if (!cm_id_priv)
3392 		return -EINVAL; /* Unmatched reply. */
3393 
3394 	work->cm_event.param.apr_rcvd.ap_status =
3395 		IBA_GET(CM_APR_AR_STATUS, apr_msg);
3396 	work->cm_event.param.apr_rcvd.apr_info =
3397 		IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg);
3398 	work->cm_event.param.apr_rcvd.info_len =
3399 		IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg);
3400 	work->cm_event.private_data =
3401 		IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg);
3402 
3403 	spin_lock_irq(&cm_id_priv->lock);
3404 	if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3405 	    (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3406 	     cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3407 		spin_unlock_irq(&cm_id_priv->lock);
3408 		goto out;
3409 	}
3410 	cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3411 	ib_cancel_mad(cm_id_priv->msg);
3412 	cm_queue_work_unlock(cm_id_priv, work);
3413 	return 0;
3414 out:
3415 	cm_deref_id(cm_id_priv);
3416 	return -EINVAL;
3417 }
3418 
cm_timewait_handler(struct cm_work * work)3419 static int cm_timewait_handler(struct cm_work *work)
3420 {
3421 	struct cm_timewait_info *timewait_info;
3422 	struct cm_id_private *cm_id_priv;
3423 
3424 	timewait_info = container_of(work, struct cm_timewait_info, work);
3425 	spin_lock_irq(&cm.lock);
3426 	list_del(&timewait_info->list);
3427 	spin_unlock_irq(&cm.lock);
3428 
3429 	cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
3430 				   timewait_info->work.remote_id);
3431 	if (!cm_id_priv)
3432 		return -EINVAL;
3433 
3434 	spin_lock_irq(&cm_id_priv->lock);
3435 	if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3436 	    cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3437 		spin_unlock_irq(&cm_id_priv->lock);
3438 		goto out;
3439 	}
3440 	cm_id_priv->id.state = IB_CM_IDLE;
3441 	cm_queue_work_unlock(cm_id_priv, work);
3442 	return 0;
3443 out:
3444 	cm_deref_id(cm_id_priv);
3445 	return -EINVAL;
3446 }
3447 
cm_format_sidr_req(struct cm_sidr_req_msg * sidr_req_msg,struct cm_id_private * cm_id_priv,struct ib_cm_sidr_req_param * param)3448 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3449 			       struct cm_id_private *cm_id_priv,
3450 			       struct ib_cm_sidr_req_param *param)
3451 {
3452 	cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3453 			  cm_form_tid(cm_id_priv));
3454 	IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg,
3455 		be32_to_cpu(cm_id_priv->id.local_id));
3456 	IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg,
3457 		be16_to_cpu(param->path->pkey));
3458 	IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg,
3459 		be64_to_cpu(param->service_id));
3460 
3461 	if (param->private_data && param->private_data_len)
3462 		IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg,
3463 			    param->private_data, param->private_data_len);
3464 }
3465 
ib_send_cm_sidr_req(struct ib_cm_id * cm_id,struct ib_cm_sidr_req_param * param)3466 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3467 			struct ib_cm_sidr_req_param *param)
3468 {
3469 	struct cm_id_private *cm_id_priv;
3470 	struct ib_mad_send_buf *msg;
3471 	struct cm_av av = {};
3472 	unsigned long flags;
3473 	int ret;
3474 
3475 	if (!param->path || (param->private_data &&
3476 	     param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3477 		return -EINVAL;
3478 
3479 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3480 	ret = cm_init_av_by_path(param->path, param->sgid_attr, &av);
3481 	if (ret)
3482 		return ret;
3483 
3484 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3485 	cm_move_av_from_path(&cm_id_priv->av, &av);
3486 	cm_id->service_id = param->service_id;
3487 	cm_id_priv->timeout_ms = param->timeout_ms;
3488 	cm_id_priv->max_cm_retries = param->max_cm_retries;
3489 	if (cm_id->state != IB_CM_IDLE) {
3490 		ret = -EINVAL;
3491 		goto out_unlock;
3492 	}
3493 
3494 	msg = cm_alloc_priv_msg(cm_id_priv);
3495 	if (IS_ERR(msg)) {
3496 		ret = PTR_ERR(msg);
3497 		goto out_unlock;
3498 	}
3499 
3500 	cm_format_sidr_req((struct cm_sidr_req_msg *)msg->mad, cm_id_priv,
3501 			   param);
3502 	msg->timeout_ms = cm_id_priv->timeout_ms;
3503 	msg->context[1] = (void *)(unsigned long)IB_CM_SIDR_REQ_SENT;
3504 
3505 	trace_icm_send_sidr_req(&cm_id_priv->id);
3506 	ret = ib_post_send_mad(msg, NULL);
3507 	if (ret)
3508 		goto out_free;
3509 	cm_id->state = IB_CM_SIDR_REQ_SENT;
3510 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3511 	return 0;
3512 out_free:
3513 	cm_free_priv_msg(msg);
3514 out_unlock:
3515 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3516 	return ret;
3517 }
3518 EXPORT_SYMBOL(ib_send_cm_sidr_req);
3519 
cm_format_sidr_req_event(struct cm_work * work,const struct cm_id_private * rx_cm_id,struct ib_cm_id * listen_id)3520 static void cm_format_sidr_req_event(struct cm_work *work,
3521 				     const struct cm_id_private *rx_cm_id,
3522 				     struct ib_cm_id *listen_id)
3523 {
3524 	struct cm_sidr_req_msg *sidr_req_msg;
3525 	struct ib_cm_sidr_req_event_param *param;
3526 
3527 	sidr_req_msg = (struct cm_sidr_req_msg *)
3528 				work->mad_recv_wc->recv_buf.mad;
3529 	param = &work->cm_event.param.sidr_req_rcvd;
3530 	param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg);
3531 	param->listen_id = listen_id;
3532 	param->service_id =
3533 		cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3534 	param->bth_pkey = cm_get_bth_pkey(work);
3535 	param->port = work->port->port_num;
3536 	param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr;
3537 	work->cm_event.private_data =
3538 		IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg);
3539 }
3540 
cm_sidr_req_handler(struct cm_work * work)3541 static int cm_sidr_req_handler(struct cm_work *work)
3542 {
3543 	struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
3544 	struct cm_sidr_req_msg *sidr_req_msg;
3545 	struct ib_wc *wc;
3546 	int ret;
3547 
3548 	cm_id_priv =
3549 		cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
3550 	if (IS_ERR(cm_id_priv))
3551 		return PTR_ERR(cm_id_priv);
3552 
3553 	/* Record SGID/SLID and request ID for lookup. */
3554 	sidr_req_msg = (struct cm_sidr_req_msg *)
3555 				work->mad_recv_wc->recv_buf.mad;
3556 
3557 	cm_id_priv->id.remote_id =
3558 		cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg));
3559 	cm_id_priv->id.service_id =
3560 		cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3561 	cm_id_priv->tid = sidr_req_msg->hdr.tid;
3562 
3563 	wc = work->mad_recv_wc->wc;
3564 	cm_id_priv->sidr_slid = wc->slid;
3565 	ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
3566 				      work->mad_recv_wc->recv_buf.grh,
3567 				      &cm_id_priv->av);
3568 	if (ret)
3569 		goto out;
3570 
3571 	spin_lock_irq(&cm.lock);
3572 	listen_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3573 	if (listen_cm_id_priv) {
3574 		spin_unlock_irq(&cm.lock);
3575 		atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3576 						     [CM_SIDR_REQ_COUNTER]);
3577 		goto out; /* Duplicate message. */
3578 	}
3579 	cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3580 	listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
3581 					   cm_id_priv->id.service_id);
3582 	if (!listen_cm_id_priv) {
3583 		spin_unlock_irq(&cm.lock);
3584 		ib_send_cm_sidr_rep(&cm_id_priv->id,
3585 				    &(struct ib_cm_sidr_rep_param){
3586 					    .status = IB_SIDR_UNSUPPORTED });
3587 		goto out; /* No match. */
3588 	}
3589 	spin_unlock_irq(&cm.lock);
3590 
3591 	cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
3592 	cm_id_priv->id.context = listen_cm_id_priv->id.context;
3593 
3594 	/*
3595 	 * A SIDR ID does not need to be in the xarray since it does not receive
3596 	 * mads, is not placed in the remote_id or remote_qpn rbtree, and does
3597 	 * not enter timewait.
3598 	 */
3599 
3600 	cm_format_sidr_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
3601 	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
3602 	cm_free_work(work);
3603 	/*
3604 	 * A pointer to the listen_cm_id is held in the event, so this deref
3605 	 * must be after the event is delivered above.
3606 	 */
3607 	cm_deref_id(listen_cm_id_priv);
3608 	if (ret)
3609 		cm_destroy_id(&cm_id_priv->id, ret);
3610 	return 0;
3611 out:
3612 	ib_destroy_cm_id(&cm_id_priv->id);
3613 	return -EINVAL;
3614 }
3615 
cm_format_sidr_rep(struct cm_sidr_rep_msg * sidr_rep_msg,struct cm_id_private * cm_id_priv,struct ib_cm_sidr_rep_param * param)3616 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3617 			       struct cm_id_private *cm_id_priv,
3618 			       struct ib_cm_sidr_rep_param *param)
3619 {
3620 	cm_format_mad_ece_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3621 			      cm_id_priv->tid, param->ece.attr_mod);
3622 	IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg,
3623 		be32_to_cpu(cm_id_priv->id.remote_id));
3624 	IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status);
3625 	IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num);
3626 	IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg,
3627 		be64_to_cpu(cm_id_priv->id.service_id));
3628 	IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey);
3629 	IBA_SET(CM_SIDR_REP_VENDOR_ID_L, sidr_rep_msg,
3630 		param->ece.vendor_id & 0xFF);
3631 	IBA_SET(CM_SIDR_REP_VENDOR_ID_H, sidr_rep_msg,
3632 		(param->ece.vendor_id >> 8) & 0xFF);
3633 
3634 	if (param->info && param->info_length)
3635 		IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg,
3636 			    param->info, param->info_length);
3637 
3638 	if (param->private_data && param->private_data_len)
3639 		IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg,
3640 			    param->private_data, param->private_data_len);
3641 }
3642 
cm_send_sidr_rep_locked(struct cm_id_private * cm_id_priv,struct ib_cm_sidr_rep_param * param)3643 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
3644 				   struct ib_cm_sidr_rep_param *param)
3645 {
3646 	struct ib_mad_send_buf *msg;
3647 	unsigned long flags;
3648 	int ret;
3649 
3650 	lockdep_assert_held(&cm_id_priv->lock);
3651 
3652 	if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3653 	    (param->private_data &&
3654 	     param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3655 		return -EINVAL;
3656 
3657 	if (cm_id_priv->id.state != IB_CM_SIDR_REQ_RCVD)
3658 		return -EINVAL;
3659 
3660 	msg = cm_alloc_msg(cm_id_priv);
3661 	if (IS_ERR(msg))
3662 		return PTR_ERR(msg);
3663 
3664 	cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3665 			   param);
3666 	trace_icm_send_sidr_rep(&cm_id_priv->id);
3667 	ret = ib_post_send_mad(msg, NULL);
3668 	if (ret) {
3669 		cm_free_msg(msg);
3670 		return ret;
3671 	}
3672 	cm_id_priv->id.state = IB_CM_IDLE;
3673 	spin_lock_irqsave(&cm.lock, flags);
3674 	if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3675 		rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3676 		RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3677 	}
3678 	spin_unlock_irqrestore(&cm.lock, flags);
3679 	return 0;
3680 }
3681 
ib_send_cm_sidr_rep(struct ib_cm_id * cm_id,struct ib_cm_sidr_rep_param * param)3682 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3683 			struct ib_cm_sidr_rep_param *param)
3684 {
3685 	struct cm_id_private *cm_id_priv =
3686 		container_of(cm_id, struct cm_id_private, id);
3687 	unsigned long flags;
3688 	int ret;
3689 
3690 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3691 	ret = cm_send_sidr_rep_locked(cm_id_priv, param);
3692 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3693 	return ret;
3694 }
3695 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3696 
cm_format_sidr_rep_event(struct cm_work * work,const struct cm_id_private * cm_id_priv)3697 static void cm_format_sidr_rep_event(struct cm_work *work,
3698 				     const struct cm_id_private *cm_id_priv)
3699 {
3700 	struct cm_sidr_rep_msg *sidr_rep_msg;
3701 	struct ib_cm_sidr_rep_event_param *param;
3702 
3703 	sidr_rep_msg = (struct cm_sidr_rep_msg *)
3704 				work->mad_recv_wc->recv_buf.mad;
3705 	param = &work->cm_event.param.sidr_rep_rcvd;
3706 	param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg);
3707 	param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg);
3708 	param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg);
3709 	param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION,
3710 				      sidr_rep_msg);
3711 	param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH,
3712 				  sidr_rep_msg);
3713 	param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
3714 	work->cm_event.private_data =
3715 		IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg);
3716 }
3717 
cm_sidr_rep_handler(struct cm_work * work)3718 static int cm_sidr_rep_handler(struct cm_work *work)
3719 {
3720 	struct cm_sidr_rep_msg *sidr_rep_msg;
3721 	struct cm_id_private *cm_id_priv;
3722 
3723 	sidr_rep_msg = (struct cm_sidr_rep_msg *)
3724 				work->mad_recv_wc->recv_buf.mad;
3725 	cm_id_priv = cm_acquire_id(
3726 		cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0);
3727 	if (!cm_id_priv)
3728 		return -EINVAL; /* Unmatched reply. */
3729 
3730 	spin_lock_irq(&cm_id_priv->lock);
3731 	if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3732 		spin_unlock_irq(&cm_id_priv->lock);
3733 		goto out;
3734 	}
3735 	cm_id_priv->id.state = IB_CM_IDLE;
3736 	ib_cancel_mad(cm_id_priv->msg);
3737 	spin_unlock_irq(&cm_id_priv->lock);
3738 
3739 	cm_format_sidr_rep_event(work, cm_id_priv);
3740 	cm_process_work(cm_id_priv, work);
3741 	return 0;
3742 out:
3743 	cm_deref_id(cm_id_priv);
3744 	return -EINVAL;
3745 }
3746 
cm_process_send_error(struct cm_id_private * cm_id_priv,struct ib_mad_send_buf * msg,enum ib_cm_state state,enum ib_wc_status wc_status)3747 static void cm_process_send_error(struct cm_id_private *cm_id_priv,
3748 				  struct ib_mad_send_buf *msg,
3749 				  enum ib_cm_state state,
3750 				  enum ib_wc_status wc_status)
3751 {
3752 	struct ib_cm_event cm_event = {};
3753 	int ret;
3754 
3755 	/* Discard old sends or ones without a response. */
3756 	spin_lock_irq(&cm_id_priv->lock);
3757 	if (msg != cm_id_priv->msg) {
3758 		spin_unlock_irq(&cm_id_priv->lock);
3759 		cm_free_msg(msg);
3760 		return;
3761 	}
3762 	cm_free_priv_msg(msg);
3763 
3764 	if (state != cm_id_priv->id.state || wc_status == IB_WC_SUCCESS ||
3765 	    wc_status == IB_WC_WR_FLUSH_ERR)
3766 		goto out_unlock;
3767 
3768 	trace_icm_mad_send_err(state, wc_status);
3769 	switch (state) {
3770 	case IB_CM_REQ_SENT:
3771 	case IB_CM_MRA_REQ_RCVD:
3772 		cm_reset_to_idle(cm_id_priv);
3773 		cm_event.event = IB_CM_REQ_ERROR;
3774 		break;
3775 	case IB_CM_REP_SENT:
3776 	case IB_CM_MRA_REP_RCVD:
3777 		cm_reset_to_idle(cm_id_priv);
3778 		cm_event.event = IB_CM_REP_ERROR;
3779 		break;
3780 	case IB_CM_DREQ_SENT:
3781 		cm_enter_timewait(cm_id_priv);
3782 		cm_event.event = IB_CM_DREQ_ERROR;
3783 		break;
3784 	case IB_CM_SIDR_REQ_SENT:
3785 		cm_id_priv->id.state = IB_CM_IDLE;
3786 		cm_event.event = IB_CM_SIDR_REQ_ERROR;
3787 		break;
3788 	default:
3789 		goto out_unlock;
3790 	}
3791 	spin_unlock_irq(&cm_id_priv->lock);
3792 	cm_event.param.send_status = wc_status;
3793 
3794 	/* No other events can occur on the cm_id at this point. */
3795 	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3796 	if (ret)
3797 		ib_destroy_cm_id(&cm_id_priv->id);
3798 	return;
3799 out_unlock:
3800 	spin_unlock_irq(&cm_id_priv->lock);
3801 }
3802 
cm_send_handler(struct ib_mad_agent * mad_agent,struct ib_mad_send_wc * mad_send_wc)3803 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3804 			    struct ib_mad_send_wc *mad_send_wc)
3805 {
3806 	struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3807 	struct cm_id_private *cm_id_priv = msg->context[0];
3808 	enum ib_cm_state state =
3809 		(enum ib_cm_state)(unsigned long)msg->context[1];
3810 	struct cm_port *port;
3811 	u16 attr_index;
3812 
3813 	port = mad_agent->context;
3814 	attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3815 				  msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3816 
3817 	/*
3818 	 * If the send was in response to a received message (context[0] is not
3819 	 * set to a cm_id), and is not a REJ, then it is a send that was
3820 	 * manually retried.
3821 	 */
3822 	if (!cm_id_priv && (attr_index != CM_REJ_COUNTER))
3823 		msg->retries = 1;
3824 
3825 	atomic_long_add(1 + msg->retries, &port->counters[CM_XMIT][attr_index]);
3826 	if (msg->retries)
3827 		atomic_long_add(msg->retries,
3828 				&port->counters[CM_XMIT_RETRIES][attr_index]);
3829 
3830 	if (cm_id_priv)
3831 		cm_process_send_error(cm_id_priv, msg, state,
3832 				      mad_send_wc->status);
3833 	else
3834 		cm_free_response_msg(msg);
3835 }
3836 
cm_work_handler(struct work_struct * _work)3837 static void cm_work_handler(struct work_struct *_work)
3838 {
3839 	struct cm_work *work = container_of(_work, struct cm_work, work.work);
3840 	int ret;
3841 
3842 	switch (work->cm_event.event) {
3843 	case IB_CM_REQ_RECEIVED:
3844 		ret = cm_req_handler(work);
3845 		break;
3846 	case IB_CM_MRA_RECEIVED:
3847 		ret = cm_mra_handler(work);
3848 		break;
3849 	case IB_CM_REJ_RECEIVED:
3850 		ret = cm_rej_handler(work);
3851 		break;
3852 	case IB_CM_REP_RECEIVED:
3853 		ret = cm_rep_handler(work);
3854 		break;
3855 	case IB_CM_RTU_RECEIVED:
3856 		ret = cm_rtu_handler(work);
3857 		break;
3858 	case IB_CM_USER_ESTABLISHED:
3859 		ret = cm_establish_handler(work);
3860 		break;
3861 	case IB_CM_DREQ_RECEIVED:
3862 		ret = cm_dreq_handler(work);
3863 		break;
3864 	case IB_CM_DREP_RECEIVED:
3865 		ret = cm_drep_handler(work);
3866 		break;
3867 	case IB_CM_SIDR_REQ_RECEIVED:
3868 		ret = cm_sidr_req_handler(work);
3869 		break;
3870 	case IB_CM_SIDR_REP_RECEIVED:
3871 		ret = cm_sidr_rep_handler(work);
3872 		break;
3873 	case IB_CM_LAP_RECEIVED:
3874 		ret = cm_lap_handler(work);
3875 		break;
3876 	case IB_CM_APR_RECEIVED:
3877 		ret = cm_apr_handler(work);
3878 		break;
3879 	case IB_CM_TIMEWAIT_EXIT:
3880 		ret = cm_timewait_handler(work);
3881 		break;
3882 	default:
3883 		trace_icm_handler_err(work->cm_event.event);
3884 		ret = -EINVAL;
3885 		break;
3886 	}
3887 	if (ret)
3888 		cm_free_work(work);
3889 }
3890 
cm_establish(struct ib_cm_id * cm_id)3891 static int cm_establish(struct ib_cm_id *cm_id)
3892 {
3893 	struct cm_id_private *cm_id_priv;
3894 	struct cm_work *work;
3895 	unsigned long flags;
3896 	int ret = 0;
3897 	struct cm_device *cm_dev;
3898 
3899 	cm_dev = ib_get_client_data(cm_id->device, &cm_client);
3900 	if (!cm_dev)
3901 		return -ENODEV;
3902 
3903 	work = kmalloc(sizeof *work, GFP_ATOMIC);
3904 	if (!work)
3905 		return -ENOMEM;
3906 
3907 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3908 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3909 	switch (cm_id->state) {
3910 	case IB_CM_REP_SENT:
3911 	case IB_CM_MRA_REP_RCVD:
3912 		cm_id->state = IB_CM_ESTABLISHED;
3913 		break;
3914 	case IB_CM_ESTABLISHED:
3915 		ret = -EISCONN;
3916 		break;
3917 	default:
3918 		trace_icm_establish_err(cm_id);
3919 		ret = -EINVAL;
3920 		break;
3921 	}
3922 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3923 
3924 	if (ret) {
3925 		kfree(work);
3926 		goto out;
3927 	}
3928 
3929 	/*
3930 	 * The CM worker thread may try to destroy the cm_id before it
3931 	 * can execute this work item.  To prevent potential deadlock,
3932 	 * we need to find the cm_id once we're in the context of the
3933 	 * worker thread, rather than holding a reference on it.
3934 	 */
3935 	INIT_DELAYED_WORK(&work->work, cm_work_handler);
3936 	work->local_id = cm_id->local_id;
3937 	work->remote_id = cm_id->remote_id;
3938 	work->mad_recv_wc = NULL;
3939 	work->cm_event.event = IB_CM_USER_ESTABLISHED;
3940 
3941 	/* Check if the device started its remove_one */
3942 	spin_lock_irqsave(&cm.lock, flags);
3943 	if (!cm_dev->going_down) {
3944 		queue_delayed_work(cm.wq, &work->work, 0);
3945 	} else {
3946 		kfree(work);
3947 		ret = -ENODEV;
3948 	}
3949 	spin_unlock_irqrestore(&cm.lock, flags);
3950 
3951 out:
3952 	return ret;
3953 }
3954 
cm_migrate(struct ib_cm_id * cm_id)3955 static int cm_migrate(struct ib_cm_id *cm_id)
3956 {
3957 	struct cm_id_private *cm_id_priv;
3958 	unsigned long flags;
3959 	int ret = 0;
3960 
3961 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3962 	spin_lock_irqsave(&cm_id_priv->lock, flags);
3963 	if (cm_id->state == IB_CM_ESTABLISHED &&
3964 	    (cm_id->lap_state == IB_CM_LAP_UNINIT ||
3965 	     cm_id->lap_state == IB_CM_LAP_IDLE)) {
3966 		cm_id->lap_state = IB_CM_LAP_IDLE;
3967 		cm_id_priv->av = cm_id_priv->alt_av;
3968 	} else
3969 		ret = -EINVAL;
3970 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3971 
3972 	return ret;
3973 }
3974 
ib_cm_notify(struct ib_cm_id * cm_id,enum ib_event_type event)3975 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3976 {
3977 	int ret;
3978 
3979 	switch (event) {
3980 	case IB_EVENT_COMM_EST:
3981 		ret = cm_establish(cm_id);
3982 		break;
3983 	case IB_EVENT_PATH_MIG:
3984 		ret = cm_migrate(cm_id);
3985 		break;
3986 	default:
3987 		ret = -EINVAL;
3988 	}
3989 	return ret;
3990 }
3991 EXPORT_SYMBOL(ib_cm_notify);
3992 
cm_recv_handler(struct ib_mad_agent * mad_agent,struct ib_mad_send_buf * send_buf,struct ib_mad_recv_wc * mad_recv_wc)3993 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
3994 			    struct ib_mad_send_buf *send_buf,
3995 			    struct ib_mad_recv_wc *mad_recv_wc)
3996 {
3997 	struct cm_port *port = mad_agent->context;
3998 	struct cm_work *work;
3999 	enum ib_cm_event_type event;
4000 	bool alt_path = false;
4001 	u16 attr_id;
4002 	int paths = 0;
4003 	int going_down = 0;
4004 
4005 	switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
4006 	case CM_REQ_ATTR_ID:
4007 		alt_path = cm_req_has_alt_path((struct cm_req_msg *)
4008 						mad_recv_wc->recv_buf.mad);
4009 		paths = 1 + (alt_path != 0);
4010 		event = IB_CM_REQ_RECEIVED;
4011 		break;
4012 	case CM_MRA_ATTR_ID:
4013 		event = IB_CM_MRA_RECEIVED;
4014 		break;
4015 	case CM_REJ_ATTR_ID:
4016 		event = IB_CM_REJ_RECEIVED;
4017 		break;
4018 	case CM_REP_ATTR_ID:
4019 		event = IB_CM_REP_RECEIVED;
4020 		break;
4021 	case CM_RTU_ATTR_ID:
4022 		event = IB_CM_RTU_RECEIVED;
4023 		break;
4024 	case CM_DREQ_ATTR_ID:
4025 		event = IB_CM_DREQ_RECEIVED;
4026 		break;
4027 	case CM_DREP_ATTR_ID:
4028 		event = IB_CM_DREP_RECEIVED;
4029 		break;
4030 	case CM_SIDR_REQ_ATTR_ID:
4031 		event = IB_CM_SIDR_REQ_RECEIVED;
4032 		break;
4033 	case CM_SIDR_REP_ATTR_ID:
4034 		event = IB_CM_SIDR_REP_RECEIVED;
4035 		break;
4036 	case CM_LAP_ATTR_ID:
4037 		paths = 1;
4038 		event = IB_CM_LAP_RECEIVED;
4039 		break;
4040 	case CM_APR_ATTR_ID:
4041 		event = IB_CM_APR_RECEIVED;
4042 		break;
4043 	default:
4044 		ib_free_recv_mad(mad_recv_wc);
4045 		return;
4046 	}
4047 
4048 	attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
4049 	atomic_long_inc(&port->counters[CM_RECV][attr_id - CM_ATTR_ID_OFFSET]);
4050 
4051 	work = kmalloc(struct_size(work, path, paths), GFP_KERNEL);
4052 	if (!work) {
4053 		ib_free_recv_mad(mad_recv_wc);
4054 		return;
4055 	}
4056 
4057 	INIT_DELAYED_WORK(&work->work, cm_work_handler);
4058 	work->cm_event.event = event;
4059 	work->mad_recv_wc = mad_recv_wc;
4060 	work->port = port;
4061 
4062 	/* Check if the device started its remove_one */
4063 	spin_lock_irq(&cm.lock);
4064 	if (!port->cm_dev->going_down)
4065 		queue_delayed_work(cm.wq, &work->work, 0);
4066 	else
4067 		going_down = 1;
4068 	spin_unlock_irq(&cm.lock);
4069 
4070 	if (going_down) {
4071 		kfree(work);
4072 		ib_free_recv_mad(mad_recv_wc);
4073 	}
4074 }
4075 
cm_init_qp_init_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4076 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
4077 				struct ib_qp_attr *qp_attr,
4078 				int *qp_attr_mask)
4079 {
4080 	unsigned long flags;
4081 	int ret;
4082 
4083 	spin_lock_irqsave(&cm_id_priv->lock, flags);
4084 	switch (cm_id_priv->id.state) {
4085 	case IB_CM_REQ_SENT:
4086 	case IB_CM_MRA_REQ_RCVD:
4087 	case IB_CM_REQ_RCVD:
4088 	case IB_CM_MRA_REQ_SENT:
4089 	case IB_CM_REP_RCVD:
4090 	case IB_CM_MRA_REP_SENT:
4091 	case IB_CM_REP_SENT:
4092 	case IB_CM_MRA_REP_RCVD:
4093 	case IB_CM_ESTABLISHED:
4094 		*qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
4095 				IB_QP_PKEY_INDEX | IB_QP_PORT;
4096 		qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4097 		if (cm_id_priv->responder_resources)
4098 			qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
4099 						    IB_ACCESS_REMOTE_ATOMIC;
4100 		qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4101 		if (cm_id_priv->av.port)
4102 			qp_attr->port_num = cm_id_priv->av.port->port_num;
4103 		ret = 0;
4104 		break;
4105 	default:
4106 		trace_icm_qp_init_err(&cm_id_priv->id);
4107 		ret = -EINVAL;
4108 		break;
4109 	}
4110 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4111 	return ret;
4112 }
4113 
cm_init_qp_rtr_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4114 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
4115 			       struct ib_qp_attr *qp_attr,
4116 			       int *qp_attr_mask)
4117 {
4118 	unsigned long flags;
4119 	int ret;
4120 
4121 	spin_lock_irqsave(&cm_id_priv->lock, flags);
4122 	switch (cm_id_priv->id.state) {
4123 	case IB_CM_REQ_RCVD:
4124 	case IB_CM_MRA_REQ_SENT:
4125 	case IB_CM_REP_RCVD:
4126 	case IB_CM_MRA_REP_SENT:
4127 	case IB_CM_REP_SENT:
4128 	case IB_CM_MRA_REP_RCVD:
4129 	case IB_CM_ESTABLISHED:
4130 		*qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
4131 				IB_QP_DEST_QPN | IB_QP_RQ_PSN;
4132 		qp_attr->ah_attr = cm_id_priv->av.ah_attr;
4133 		if ((qp_attr->ah_attr.type == RDMA_AH_ATTR_TYPE_IB) &&
4134 		    cm_id_priv->av.dlid_datapath &&
4135 		    (cm_id_priv->av.dlid_datapath != 0xffff))
4136 			qp_attr->ah_attr.ib.dlid = cm_id_priv->av.dlid_datapath;
4137 		qp_attr->path_mtu = cm_id_priv->path_mtu;
4138 		qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
4139 		qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
4140 		if (cm_id_priv->qp_type == IB_QPT_RC ||
4141 		    cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
4142 			*qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
4143 					 IB_QP_MIN_RNR_TIMER;
4144 			qp_attr->max_dest_rd_atomic =
4145 					cm_id_priv->responder_resources;
4146 			qp_attr->min_rnr_timer = 0;
4147 		}
4148 		if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr) &&
4149 		    cm_id_priv->alt_av.port) {
4150 			*qp_attr_mask |= IB_QP_ALT_PATH;
4151 			qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4152 			qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4153 			qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4154 			qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4155 		}
4156 		ret = 0;
4157 		break;
4158 	default:
4159 		trace_icm_qp_rtr_err(&cm_id_priv->id);
4160 		ret = -EINVAL;
4161 		break;
4162 	}
4163 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4164 	return ret;
4165 }
4166 
cm_init_qp_rts_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4167 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
4168 			       struct ib_qp_attr *qp_attr,
4169 			       int *qp_attr_mask)
4170 {
4171 	unsigned long flags;
4172 	int ret;
4173 
4174 	spin_lock_irqsave(&cm_id_priv->lock, flags);
4175 	switch (cm_id_priv->id.state) {
4176 	/* Allow transition to RTS before sending REP */
4177 	case IB_CM_REQ_RCVD:
4178 	case IB_CM_MRA_REQ_SENT:
4179 
4180 	case IB_CM_REP_RCVD:
4181 	case IB_CM_MRA_REP_SENT:
4182 	case IB_CM_REP_SENT:
4183 	case IB_CM_MRA_REP_RCVD:
4184 	case IB_CM_ESTABLISHED:
4185 		if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
4186 			*qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
4187 			qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
4188 			switch (cm_id_priv->qp_type) {
4189 			case IB_QPT_RC:
4190 			case IB_QPT_XRC_INI:
4191 				*qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4192 						 IB_QP_MAX_QP_RD_ATOMIC;
4193 				qp_attr->retry_cnt = cm_id_priv->retry_count;
4194 				qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
4195 				qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
4196 				fallthrough;
4197 			case IB_QPT_XRC_TGT:
4198 				*qp_attr_mask |= IB_QP_TIMEOUT;
4199 				qp_attr->timeout = cm_id_priv->av.timeout;
4200 				break;
4201 			default:
4202 				break;
4203 			}
4204 			if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4205 				*qp_attr_mask |= IB_QP_PATH_MIG_STATE;
4206 				qp_attr->path_mig_state = IB_MIG_REARM;
4207 			}
4208 		} else {
4209 			*qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
4210 			if (cm_id_priv->alt_av.port)
4211 				qp_attr->alt_port_num =
4212 					cm_id_priv->alt_av.port->port_num;
4213 			qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4214 			qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4215 			qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4216 			qp_attr->path_mig_state = IB_MIG_REARM;
4217 		}
4218 		ret = 0;
4219 		break;
4220 	default:
4221 		trace_icm_qp_rts_err(&cm_id_priv->id);
4222 		ret = -EINVAL;
4223 		break;
4224 	}
4225 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4226 	return ret;
4227 }
4228 
ib_cm_init_qp_attr(struct ib_cm_id * cm_id,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4229 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
4230 		       struct ib_qp_attr *qp_attr,
4231 		       int *qp_attr_mask)
4232 {
4233 	struct cm_id_private *cm_id_priv;
4234 	int ret;
4235 
4236 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4237 	switch (qp_attr->qp_state) {
4238 	case IB_QPS_INIT:
4239 		ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
4240 		break;
4241 	case IB_QPS_RTR:
4242 		ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
4243 		break;
4244 	case IB_QPS_RTS:
4245 		ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
4246 		break;
4247 	default:
4248 		ret = -EINVAL;
4249 		break;
4250 	}
4251 	return ret;
4252 }
4253 EXPORT_SYMBOL(ib_cm_init_qp_attr);
4254 
cm_show_counter(struct ib_device * ibdev,u32 port_num,struct ib_port_attribute * attr,char * buf)4255 static ssize_t cm_show_counter(struct ib_device *ibdev, u32 port_num,
4256 			       struct ib_port_attribute *attr, char *buf)
4257 {
4258 	struct cm_counter_attribute *cm_attr =
4259 		container_of(attr, struct cm_counter_attribute, attr);
4260 	struct cm_device *cm_dev = ib_get_client_data(ibdev, &cm_client);
4261 
4262 	if (WARN_ON(!cm_dev))
4263 		return -EINVAL;
4264 
4265 	return sysfs_emit(
4266 		buf, "%ld\n",
4267 		atomic_long_read(
4268 			&cm_dev->port[port_num - 1]
4269 				 ->counters[cm_attr->group][cm_attr->index]));
4270 }
4271 
4272 #define CM_COUNTER_ATTR(_name, _group, _index)                                 \
4273 	{                                                                      \
4274 		.attr = __ATTR(_name, 0444, cm_show_counter, NULL),            \
4275 		.group = _group, .index = _index                               \
4276 	}
4277 
4278 #define CM_COUNTER_GROUP(_group, _name)                                        \
4279 	static struct cm_counter_attribute cm_counter_attr_##_group[] = {      \
4280 		CM_COUNTER_ATTR(req, _group, CM_REQ_COUNTER),                  \
4281 		CM_COUNTER_ATTR(mra, _group, CM_MRA_COUNTER),                  \
4282 		CM_COUNTER_ATTR(rej, _group, CM_REJ_COUNTER),                  \
4283 		CM_COUNTER_ATTR(rep, _group, CM_REP_COUNTER),                  \
4284 		CM_COUNTER_ATTR(rtu, _group, CM_RTU_COUNTER),                  \
4285 		CM_COUNTER_ATTR(dreq, _group, CM_DREQ_COUNTER),                \
4286 		CM_COUNTER_ATTR(drep, _group, CM_DREP_COUNTER),                \
4287 		CM_COUNTER_ATTR(sidr_req, _group, CM_SIDR_REQ_COUNTER),        \
4288 		CM_COUNTER_ATTR(sidr_rep, _group, CM_SIDR_REP_COUNTER),        \
4289 		CM_COUNTER_ATTR(lap, _group, CM_LAP_COUNTER),                  \
4290 		CM_COUNTER_ATTR(apr, _group, CM_APR_COUNTER),                  \
4291 	};                                                                     \
4292 	static struct attribute *cm_counter_attrs_##_group[] = {               \
4293 		&cm_counter_attr_##_group[0].attr.attr,                        \
4294 		&cm_counter_attr_##_group[1].attr.attr,                        \
4295 		&cm_counter_attr_##_group[2].attr.attr,                        \
4296 		&cm_counter_attr_##_group[3].attr.attr,                        \
4297 		&cm_counter_attr_##_group[4].attr.attr,                        \
4298 		&cm_counter_attr_##_group[5].attr.attr,                        \
4299 		&cm_counter_attr_##_group[6].attr.attr,                        \
4300 		&cm_counter_attr_##_group[7].attr.attr,                        \
4301 		&cm_counter_attr_##_group[8].attr.attr,                        \
4302 		&cm_counter_attr_##_group[9].attr.attr,                        \
4303 		&cm_counter_attr_##_group[10].attr.attr,                       \
4304 		NULL,                                                          \
4305 	};                                                                     \
4306 	static const struct attribute_group cm_counter_group_##_group = {      \
4307 		.name = _name,                                                 \
4308 		.attrs = cm_counter_attrs_##_group,                            \
4309 	};
4310 
4311 CM_COUNTER_GROUP(CM_XMIT, "cm_tx_msgs")
4312 CM_COUNTER_GROUP(CM_XMIT_RETRIES, "cm_tx_retries")
4313 CM_COUNTER_GROUP(CM_RECV, "cm_rx_msgs")
4314 CM_COUNTER_GROUP(CM_RECV_DUPLICATES, "cm_rx_duplicates")
4315 
4316 static const struct attribute_group *cm_counter_groups[] = {
4317 	&cm_counter_group_CM_XMIT,
4318 	&cm_counter_group_CM_XMIT_RETRIES,
4319 	&cm_counter_group_CM_RECV,
4320 	&cm_counter_group_CM_RECV_DUPLICATES,
4321 	NULL,
4322 };
4323 
cm_add_one(struct ib_device * ib_device)4324 static int cm_add_one(struct ib_device *ib_device)
4325 {
4326 	struct cm_device *cm_dev;
4327 	struct cm_port *port;
4328 	struct ib_mad_reg_req reg_req = {
4329 		.mgmt_class = IB_MGMT_CLASS_CM,
4330 		.mgmt_class_version = IB_CM_CLASS_VERSION,
4331 	};
4332 	struct ib_port_modify port_modify = {
4333 		.set_port_cap_mask = IB_PORT_CM_SUP
4334 	};
4335 	unsigned long flags;
4336 	int ret;
4337 	int count = 0;
4338 	u32 i;
4339 
4340 	cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
4341 			 GFP_KERNEL);
4342 	if (!cm_dev)
4343 		return -ENOMEM;
4344 
4345 	kref_init(&cm_dev->kref);
4346 	spin_lock_init(&cm_dev->mad_agent_lock);
4347 	cm_dev->ib_device = ib_device;
4348 	cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4349 	cm_dev->going_down = 0;
4350 
4351 	ib_set_client_data(ib_device, &cm_client, cm_dev);
4352 
4353 	set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
4354 	rdma_for_each_port (ib_device, i) {
4355 		if (!rdma_cap_ib_cm(ib_device, i))
4356 			continue;
4357 
4358 		port = kzalloc(sizeof *port, GFP_KERNEL);
4359 		if (!port) {
4360 			ret = -ENOMEM;
4361 			goto error1;
4362 		}
4363 
4364 		cm_dev->port[i-1] = port;
4365 		port->cm_dev = cm_dev;
4366 		port->port_num = i;
4367 
4368 		ret = ib_port_register_client_groups(ib_device, i,
4369 						     cm_counter_groups);
4370 		if (ret)
4371 			goto error1;
4372 
4373 		port->mad_agent = ib_register_mad_agent(ib_device, i,
4374 							IB_QPT_GSI,
4375 							&reg_req,
4376 							0,
4377 							cm_send_handler,
4378 							cm_recv_handler,
4379 							port,
4380 							0);
4381 		if (IS_ERR(port->mad_agent)) {
4382 			ret = PTR_ERR(port->mad_agent);
4383 			goto error2;
4384 		}
4385 
4386 		ret = ib_modify_port(ib_device, i, 0, &port_modify);
4387 		if (ret)
4388 			goto error3;
4389 
4390 		count++;
4391 	}
4392 
4393 	if (!count) {
4394 		ret = -EOPNOTSUPP;
4395 		goto free;
4396 	}
4397 
4398 	write_lock_irqsave(&cm.device_lock, flags);
4399 	list_add_tail(&cm_dev->list, &cm.device_list);
4400 	write_unlock_irqrestore(&cm.device_lock, flags);
4401 	return 0;
4402 
4403 error3:
4404 	ib_unregister_mad_agent(port->mad_agent);
4405 error2:
4406 	ib_port_unregister_client_groups(ib_device, i, cm_counter_groups);
4407 error1:
4408 	port_modify.set_port_cap_mask = 0;
4409 	port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4410 	while (--i) {
4411 		if (!rdma_cap_ib_cm(ib_device, i))
4412 			continue;
4413 
4414 		port = cm_dev->port[i-1];
4415 		ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4416 		ib_unregister_mad_agent(port->mad_agent);
4417 		ib_port_unregister_client_groups(ib_device, i,
4418 						 cm_counter_groups);
4419 	}
4420 free:
4421 	cm_device_put(cm_dev);
4422 	return ret;
4423 }
4424 
cm_remove_one(struct ib_device * ib_device,void * client_data)4425 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4426 {
4427 	struct cm_device *cm_dev = client_data;
4428 	struct cm_port *port;
4429 	struct ib_port_modify port_modify = {
4430 		.clr_port_cap_mask = IB_PORT_CM_SUP
4431 	};
4432 	unsigned long flags;
4433 	u32 i;
4434 
4435 	write_lock_irqsave(&cm.device_lock, flags);
4436 	list_del(&cm_dev->list);
4437 	write_unlock_irqrestore(&cm.device_lock, flags);
4438 
4439 	spin_lock_irq(&cm.lock);
4440 	cm_dev->going_down = 1;
4441 	spin_unlock_irq(&cm.lock);
4442 
4443 	rdma_for_each_port (ib_device, i) {
4444 		struct ib_mad_agent *mad_agent;
4445 
4446 		if (!rdma_cap_ib_cm(ib_device, i))
4447 			continue;
4448 
4449 		port = cm_dev->port[i-1];
4450 		mad_agent = port->mad_agent;
4451 		ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4452 		/*
4453 		 * We flush the queue here after the going_down set, this
4454 		 * verify that no new works will be queued in the recv handler,
4455 		 * after that we can call the unregister_mad_agent
4456 		 */
4457 		flush_workqueue(cm.wq);
4458 		/*
4459 		 * The above ensures no call paths from the work are running,
4460 		 * the remaining paths all take the mad_agent_lock.
4461 		 */
4462 		spin_lock(&cm_dev->mad_agent_lock);
4463 		port->mad_agent = NULL;
4464 		spin_unlock(&cm_dev->mad_agent_lock);
4465 		ib_unregister_mad_agent(mad_agent);
4466 		ib_port_unregister_client_groups(ib_device, i,
4467 						 cm_counter_groups);
4468 	}
4469 
4470 	cm_device_put(cm_dev);
4471 }
4472 
ib_cm_init(void)4473 static int __init ib_cm_init(void)
4474 {
4475 	int ret;
4476 
4477 	INIT_LIST_HEAD(&cm.device_list);
4478 	rwlock_init(&cm.device_lock);
4479 	spin_lock_init(&cm.lock);
4480 	cm.listen_service_table = RB_ROOT;
4481 	cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4482 	cm.remote_id_table = RB_ROOT;
4483 	cm.remote_qp_table = RB_ROOT;
4484 	cm.remote_sidr_table = RB_ROOT;
4485 	xa_init_flags(&cm.local_id_table, XA_FLAGS_ALLOC);
4486 	get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
4487 	INIT_LIST_HEAD(&cm.timewait_list);
4488 
4489 	cm.wq = alloc_workqueue("ib_cm", 0, 1);
4490 	if (!cm.wq) {
4491 		ret = -ENOMEM;
4492 		goto error2;
4493 	}
4494 
4495 	ret = ib_register_client(&cm_client);
4496 	if (ret)
4497 		goto error3;
4498 
4499 	return 0;
4500 error3:
4501 	destroy_workqueue(cm.wq);
4502 error2:
4503 	return ret;
4504 }
4505 
ib_cm_cleanup(void)4506 static void __exit ib_cm_cleanup(void)
4507 {
4508 	struct cm_timewait_info *timewait_info, *tmp;
4509 
4510 	spin_lock_irq(&cm.lock);
4511 	list_for_each_entry(timewait_info, &cm.timewait_list, list)
4512 		cancel_delayed_work(&timewait_info->work.work);
4513 	spin_unlock_irq(&cm.lock);
4514 
4515 	ib_unregister_client(&cm_client);
4516 	destroy_workqueue(cm.wq);
4517 
4518 	list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4519 		list_del(&timewait_info->list);
4520 		kfree(timewait_info);
4521 	}
4522 
4523 	WARN_ON(!xa_empty(&cm.local_id_table));
4524 }
4525 
4526 module_init(ib_cm_init);
4527 module_exit(ib_cm_cleanup);
4528