1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7 #ifndef RXE_VERBS_H
8 #define RXE_VERBS_H
9
10 #include <linux/interrupt.h>
11 #include <linux/workqueue.h>
12 #include <rdma/rdma_user_rxe.h>
13 #include "rxe_pool.h"
14 #include "rxe_task.h"
15 #include "rxe_hw_counters.h"
16
pkey_match(u16 key1,u16 key2)17 static inline int pkey_match(u16 key1, u16 key2)
18 {
19 return (((key1 & 0x7fff) != 0) &&
20 ((key1 & 0x7fff) == (key2 & 0x7fff)) &&
21 ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0;
22 }
23
24 /* Return >0 if psn_a > psn_b
25 * 0 if psn_a == psn_b
26 * <0 if psn_a < psn_b
27 */
psn_compare(u32 psn_a,u32 psn_b)28 static inline int psn_compare(u32 psn_a, u32 psn_b)
29 {
30 s32 diff;
31
32 diff = (psn_a - psn_b) << 8;
33 return diff;
34 }
35
36 struct rxe_ucontext {
37 struct ib_ucontext ibuc;
38 struct rxe_pool_elem elem;
39 };
40
41 struct rxe_pd {
42 struct ib_pd ibpd;
43 struct rxe_pool_elem elem;
44 };
45
46 struct rxe_ah {
47 struct ib_ah ibah;
48 struct rxe_pool_elem elem;
49 struct rxe_av av;
50 bool is_user;
51 int ah_num;
52 };
53
54 struct rxe_cqe {
55 union {
56 struct ib_wc ibwc;
57 struct ib_uverbs_wc uibwc;
58 };
59 };
60
61 struct rxe_cq {
62 struct ib_cq ibcq;
63 struct rxe_pool_elem elem;
64 struct rxe_queue *queue;
65 spinlock_t cq_lock;
66 u8 notify;
67 bool is_dying;
68 bool is_user;
69 struct tasklet_struct comp_task;
70 atomic_t num_wq;
71 };
72
73 enum wqe_state {
74 wqe_state_posted,
75 wqe_state_processing,
76 wqe_state_pending,
77 wqe_state_done,
78 wqe_state_error,
79 };
80
81 struct rxe_sq {
82 int max_wr;
83 int max_sge;
84 int max_inline;
85 spinlock_t sq_lock; /* guard queue */
86 struct rxe_queue *queue;
87 };
88
89 struct rxe_rq {
90 int max_wr;
91 int max_sge;
92 spinlock_t producer_lock; /* guard queue producer */
93 spinlock_t consumer_lock; /* guard queue consumer */
94 struct rxe_queue *queue;
95 };
96
97 struct rxe_srq {
98 struct ib_srq ibsrq;
99 struct rxe_pool_elem elem;
100 struct rxe_pd *pd;
101 struct rxe_rq rq;
102 u32 srq_num;
103
104 int limit;
105 int error;
106 };
107
108 enum rxe_qp_state {
109 QP_STATE_RESET,
110 QP_STATE_INIT,
111 QP_STATE_READY,
112 QP_STATE_DRAIN, /* req only */
113 QP_STATE_DRAINED, /* req only */
114 QP_STATE_ERROR
115 };
116
117 struct rxe_req_info {
118 enum rxe_qp_state state;
119 int wqe_index;
120 u32 psn;
121 int opcode;
122 atomic_t rd_atomic;
123 int wait_fence;
124 int need_rd_atomic;
125 int wait_psn;
126 int need_retry;
127 int wait_for_rnr_timer;
128 int noack_pkts;
129 struct rxe_task task;
130 };
131
132 struct rxe_comp_info {
133 u32 psn;
134 int opcode;
135 int timeout;
136 int timeout_retry;
137 int started_retry;
138 u32 retry_cnt;
139 u32 rnr_retry;
140 struct rxe_task task;
141 };
142
143 enum rdatm_res_state {
144 rdatm_res_state_next,
145 rdatm_res_state_new,
146 rdatm_res_state_replay,
147 };
148
149 struct resp_res {
150 int type;
151 int replay;
152 u32 first_psn;
153 u32 last_psn;
154 u32 cur_psn;
155 enum rdatm_res_state state;
156
157 union {
158 struct {
159 struct sk_buff *skb;
160 } atomic;
161 struct {
162 u64 va_org;
163 u32 rkey;
164 u32 length;
165 u64 va;
166 u32 resid;
167 } read;
168 };
169 };
170
171 struct rxe_resp_info {
172 enum rxe_qp_state state;
173 u32 msn;
174 u32 psn;
175 u32 ack_psn;
176 int opcode;
177 int drop_msg;
178 int goto_error;
179 int sent_psn_nak;
180 enum ib_wc_status status;
181 u8 aeth_syndrome;
182
183 /* Receive only */
184 struct rxe_recv_wqe *wqe;
185
186 /* RDMA read / atomic only */
187 u64 va;
188 u64 offset;
189 struct rxe_mr *mr;
190 u32 resid;
191 u32 rkey;
192 u32 length;
193 u64 atomic_orig;
194
195 /* SRQ only */
196 struct {
197 struct rxe_recv_wqe wqe;
198 struct ib_sge sge[RXE_MAX_SGE];
199 } srq_wqe;
200
201 /* Responder resources. It's a circular list where the oldest
202 * resource is dropped first.
203 */
204 struct resp_res *resources;
205 unsigned int res_head;
206 unsigned int res_tail;
207 struct resp_res *res;
208 struct rxe_task task;
209 };
210
211 struct rxe_qp {
212 struct ib_qp ibqp;
213 struct rxe_pool_elem elem;
214 struct ib_qp_attr attr;
215 unsigned int valid;
216 unsigned int mtu;
217 bool is_user;
218
219 struct rxe_pd *pd;
220 struct rxe_srq *srq;
221 struct rxe_cq *scq;
222 struct rxe_cq *rcq;
223
224 enum ib_sig_type sq_sig_type;
225
226 struct rxe_sq sq;
227 struct rxe_rq rq;
228
229 struct socket *sk;
230 u32 dst_cookie;
231 u16 src_port;
232
233 struct rxe_av pri_av;
234 struct rxe_av alt_av;
235
236 atomic_t mcg_num;
237
238 struct sk_buff_head req_pkts;
239 struct sk_buff_head resp_pkts;
240
241 struct rxe_req_info req;
242 struct rxe_comp_info comp;
243 struct rxe_resp_info resp;
244
245 atomic_t ssn;
246 atomic_t skb_out;
247 int need_req_skb;
248
249 /* Timer for retranmitting packet when ACKs have been lost. RC
250 * only. The requester sets it when it is not already
251 * started. The responder resets it whenever an ack is
252 * received.
253 */
254 struct timer_list retrans_timer;
255 u64 qp_timeout_jiffies;
256
257 /* Timer for handling RNR NAKS. */
258 struct timer_list rnr_nak_timer;
259
260 spinlock_t state_lock; /* guard requester and completer */
261
262 struct execute_work cleanup_work;
263 };
264
265 enum rxe_mr_state {
266 RXE_MR_STATE_INVALID,
267 RXE_MR_STATE_FREE,
268 RXE_MR_STATE_VALID,
269 };
270
271 enum rxe_mr_copy_dir {
272 RXE_TO_MR_OBJ,
273 RXE_FROM_MR_OBJ,
274 };
275
276 enum rxe_mr_lookup_type {
277 RXE_LOOKUP_LOCAL,
278 RXE_LOOKUP_REMOTE,
279 };
280
281 #define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf))
282
283 struct rxe_phys_buf {
284 u64 addr;
285 u64 size;
286 };
287
288 struct rxe_map {
289 struct rxe_phys_buf buf[RXE_BUF_PER_MAP];
290 };
291
rkey_is_mw(u32 rkey)292 static inline int rkey_is_mw(u32 rkey)
293 {
294 u32 index = rkey >> 8;
295
296 return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX);
297 }
298
299 struct rxe_mr {
300 struct rxe_pool_elem elem;
301 struct ib_mr ibmr;
302
303 struct ib_umem *umem;
304
305 u32 lkey;
306 u32 rkey;
307 enum rxe_mr_state state;
308 enum ib_mr_type type;
309 u64 va;
310 u64 iova;
311 size_t length;
312 u32 offset;
313 int access;
314
315 int page_shift;
316 int page_mask;
317 int map_shift;
318 int map_mask;
319
320 u32 num_buf;
321 u32 nbuf;
322
323 u32 max_buf;
324 u32 num_map;
325
326 atomic_t num_mw;
327
328 struct rxe_map **map;
329 };
330
331 enum rxe_mw_state {
332 RXE_MW_STATE_INVALID = RXE_MR_STATE_INVALID,
333 RXE_MW_STATE_FREE = RXE_MR_STATE_FREE,
334 RXE_MW_STATE_VALID = RXE_MR_STATE_VALID,
335 };
336
337 struct rxe_mw {
338 struct ib_mw ibmw;
339 struct rxe_pool_elem elem;
340 spinlock_t lock;
341 enum rxe_mw_state state;
342 struct rxe_qp *qp; /* Type 2 only */
343 struct rxe_mr *mr;
344 u32 rkey;
345 int access;
346 u64 addr;
347 u64 length;
348 };
349
350 struct rxe_mcg {
351 struct rb_node node;
352 struct kref ref_cnt;
353 struct rxe_dev *rxe;
354 struct list_head qp_list;
355 union ib_gid mgid;
356 atomic_t qp_num;
357 u32 qkey;
358 u16 pkey;
359 };
360
361 struct rxe_mca {
362 struct list_head qp_list;
363 struct rxe_qp *qp;
364 };
365
366 struct rxe_port {
367 struct ib_port_attr attr;
368 __be64 port_guid;
369 __be64 subnet_prefix;
370 spinlock_t port_lock; /* guard port */
371 unsigned int mtu_cap;
372 /* special QPs */
373 u32 qp_gsi_index;
374 };
375
376 struct rxe_dev {
377 struct ib_device ib_dev;
378 struct ib_device_attr attr;
379 int max_ucontext;
380 int max_inline_data;
381 struct mutex usdev_lock;
382
383 struct net_device *ndev;
384
385 struct rxe_pool uc_pool;
386 struct rxe_pool pd_pool;
387 struct rxe_pool ah_pool;
388 struct rxe_pool srq_pool;
389 struct rxe_pool qp_pool;
390 struct rxe_pool cq_pool;
391 struct rxe_pool mr_pool;
392 struct rxe_pool mw_pool;
393
394 /* multicast support */
395 spinlock_t mcg_lock;
396 struct rb_root mcg_tree;
397 atomic_t mcg_num;
398 atomic_t mcg_attach;
399
400 spinlock_t pending_lock; /* guard pending_mmaps */
401 struct list_head pending_mmaps;
402
403 spinlock_t mmap_offset_lock; /* guard mmap_offset */
404 u64 mmap_offset;
405
406 atomic64_t stats_counters[RXE_NUM_OF_COUNTERS];
407
408 struct rxe_port port;
409 struct crypto_shash *tfm;
410 };
411
rxe_counter_inc(struct rxe_dev * rxe,enum rxe_counters index)412 static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
413 {
414 atomic64_inc(&rxe->stats_counters[index]);
415 }
416
to_rdev(struct ib_device * dev)417 static inline struct rxe_dev *to_rdev(struct ib_device *dev)
418 {
419 return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
420 }
421
to_ruc(struct ib_ucontext * uc)422 static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc)
423 {
424 return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL;
425 }
426
to_rpd(struct ib_pd * pd)427 static inline struct rxe_pd *to_rpd(struct ib_pd *pd)
428 {
429 return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL;
430 }
431
to_rah(struct ib_ah * ah)432 static inline struct rxe_ah *to_rah(struct ib_ah *ah)
433 {
434 return ah ? container_of(ah, struct rxe_ah, ibah) : NULL;
435 }
436
to_rsrq(struct ib_srq * srq)437 static inline struct rxe_srq *to_rsrq(struct ib_srq *srq)
438 {
439 return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL;
440 }
441
to_rqp(struct ib_qp * qp)442 static inline struct rxe_qp *to_rqp(struct ib_qp *qp)
443 {
444 return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL;
445 }
446
to_rcq(struct ib_cq * cq)447 static inline struct rxe_cq *to_rcq(struct ib_cq *cq)
448 {
449 return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL;
450 }
451
to_rmr(struct ib_mr * mr)452 static inline struct rxe_mr *to_rmr(struct ib_mr *mr)
453 {
454 return mr ? container_of(mr, struct rxe_mr, ibmr) : NULL;
455 }
456
to_rmw(struct ib_mw * mw)457 static inline struct rxe_mw *to_rmw(struct ib_mw *mw)
458 {
459 return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL;
460 }
461
rxe_ah_pd(struct rxe_ah * ah)462 static inline struct rxe_pd *rxe_ah_pd(struct rxe_ah *ah)
463 {
464 return to_rpd(ah->ibah.pd);
465 }
466
mr_pd(struct rxe_mr * mr)467 static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)
468 {
469 return to_rpd(mr->ibmr.pd);
470 }
471
rxe_mw_pd(struct rxe_mw * mw)472 static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)
473 {
474 return to_rpd(mw->ibmw.pd);
475 }
476
477 int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
478
479 #endif /* RXE_VERBS_H */
480