1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2016 Chelsio Communications, Inc.
4 */
5
6 #define DRV_NAME "cxgbit"
7 #define DRV_VERSION "1.0.0-ko"
8 #define pr_fmt(fmt) DRV_NAME ": " fmt
9
10 #include "cxgbit.h"
11
12 #ifdef CONFIG_CHELSIO_T4_DCB
13 #include <net/dcbevent.h>
14 #include "cxgb4_dcb.h"
15 #endif
16
17 LIST_HEAD(cdev_list_head);
18 /* cdev list lock */
19 DEFINE_MUTEX(cdev_list_lock);
20
_cxgbit_free_cdev(struct kref * kref)21 void _cxgbit_free_cdev(struct kref *kref)
22 {
23 struct cxgbit_device *cdev;
24
25 cdev = container_of(kref, struct cxgbit_device, kref);
26
27 cxgbi_ppm_release(cdev2ppm(cdev));
28 kfree(cdev);
29 }
30
cxgbit_set_mdsl(struct cxgbit_device * cdev)31 static void cxgbit_set_mdsl(struct cxgbit_device *cdev)
32 {
33 struct cxgb4_lld_info *lldi = &cdev->lldi;
34 u32 mdsl;
35
36 #define CXGBIT_T5_MAX_PDU_LEN 16224
37 #define CXGBIT_PDU_NONPAYLOAD_LEN 312 /* 48(BHS) + 256(AHS) + 8(Digest) */
38 if (is_t5(lldi->adapter_type)) {
39 mdsl = min_t(u32, lldi->iscsi_iolen - CXGBIT_PDU_NONPAYLOAD_LEN,
40 CXGBIT_T5_MAX_PDU_LEN - CXGBIT_PDU_NONPAYLOAD_LEN);
41 } else {
42 mdsl = lldi->iscsi_iolen - CXGBIT_PDU_NONPAYLOAD_LEN;
43 mdsl = min(mdsl, 16384U);
44 }
45
46 mdsl = round_down(mdsl, 4);
47 mdsl = min_t(u32, mdsl, 4 * PAGE_SIZE);
48 mdsl = min_t(u32, mdsl, (MAX_SKB_FRAGS - 1) * PAGE_SIZE);
49
50 cdev->mdsl = mdsl;
51 }
52
cxgbit_uld_add(const struct cxgb4_lld_info * lldi)53 static void *cxgbit_uld_add(const struct cxgb4_lld_info *lldi)
54 {
55 struct cxgbit_device *cdev;
56
57 if (is_t4(lldi->adapter_type))
58 return ERR_PTR(-ENODEV);
59
60 cdev = kzalloc(sizeof(*cdev), GFP_KERNEL);
61 if (!cdev)
62 return ERR_PTR(-ENOMEM);
63
64 kref_init(&cdev->kref);
65 spin_lock_init(&cdev->np_lock);
66
67 cdev->lldi = *lldi;
68
69 cxgbit_set_mdsl(cdev);
70
71 if (cxgbit_ddp_init(cdev) < 0) {
72 kfree(cdev);
73 return ERR_PTR(-EINVAL);
74 }
75
76 if (!test_bit(CDEV_DDP_ENABLE, &cdev->flags))
77 pr_info("cdev %s ddp init failed\n",
78 pci_name(lldi->pdev));
79
80 if (lldi->fw_vers >= 0x10d2b00)
81 set_bit(CDEV_ISO_ENABLE, &cdev->flags);
82
83 spin_lock_init(&cdev->cskq.lock);
84 INIT_LIST_HEAD(&cdev->cskq.list);
85
86 mutex_lock(&cdev_list_lock);
87 list_add_tail(&cdev->list, &cdev_list_head);
88 mutex_unlock(&cdev_list_lock);
89
90 pr_info("cdev %s added for iSCSI target transport\n",
91 pci_name(lldi->pdev));
92
93 return cdev;
94 }
95
cxgbit_close_conn(struct cxgbit_device * cdev)96 static void cxgbit_close_conn(struct cxgbit_device *cdev)
97 {
98 struct cxgbit_sock *csk;
99 struct sk_buff *skb;
100 bool wakeup_thread = false;
101
102 spin_lock_bh(&cdev->cskq.lock);
103 list_for_each_entry(csk, &cdev->cskq.list, list) {
104 skb = alloc_skb(0, GFP_ATOMIC);
105 if (!skb)
106 continue;
107
108 spin_lock_bh(&csk->rxq.lock);
109 __skb_queue_tail(&csk->rxq, skb);
110 if (skb_queue_len(&csk->rxq) == 1)
111 wakeup_thread = true;
112 spin_unlock_bh(&csk->rxq.lock);
113
114 if (wakeup_thread) {
115 wake_up(&csk->waitq);
116 wakeup_thread = false;
117 }
118 }
119 spin_unlock_bh(&cdev->cskq.lock);
120 }
121
cxgbit_detach_cdev(struct cxgbit_device * cdev)122 static void cxgbit_detach_cdev(struct cxgbit_device *cdev)
123 {
124 bool free_cdev = false;
125
126 spin_lock_bh(&cdev->cskq.lock);
127 if (list_empty(&cdev->cskq.list))
128 free_cdev = true;
129 spin_unlock_bh(&cdev->cskq.lock);
130
131 if (free_cdev) {
132 mutex_lock(&cdev_list_lock);
133 list_del(&cdev->list);
134 mutex_unlock(&cdev_list_lock);
135
136 cxgbit_put_cdev(cdev);
137 } else {
138 cxgbit_close_conn(cdev);
139 }
140 }
141
cxgbit_uld_state_change(void * handle,enum cxgb4_state state)142 static int cxgbit_uld_state_change(void *handle, enum cxgb4_state state)
143 {
144 struct cxgbit_device *cdev = handle;
145
146 switch (state) {
147 case CXGB4_STATE_UP:
148 set_bit(CDEV_STATE_UP, &cdev->flags);
149 pr_info("cdev %s state UP.\n", pci_name(cdev->lldi.pdev));
150 break;
151 case CXGB4_STATE_START_RECOVERY:
152 clear_bit(CDEV_STATE_UP, &cdev->flags);
153 cxgbit_close_conn(cdev);
154 pr_info("cdev %s state RECOVERY.\n", pci_name(cdev->lldi.pdev));
155 break;
156 case CXGB4_STATE_DOWN:
157 pr_info("cdev %s state DOWN.\n", pci_name(cdev->lldi.pdev));
158 break;
159 case CXGB4_STATE_DETACH:
160 clear_bit(CDEV_STATE_UP, &cdev->flags);
161 pr_info("cdev %s state DETACH.\n", pci_name(cdev->lldi.pdev));
162 cxgbit_detach_cdev(cdev);
163 break;
164 default:
165 pr_info("cdev %s unknown state %d.\n",
166 pci_name(cdev->lldi.pdev), state);
167 break;
168 }
169 return 0;
170 }
171
172 static void
cxgbit_process_ddpvld(struct cxgbit_sock * csk,struct cxgbit_lro_pdu_cb * pdu_cb,u32 ddpvld)173 cxgbit_process_ddpvld(struct cxgbit_sock *csk, struct cxgbit_lro_pdu_cb *pdu_cb,
174 u32 ddpvld)
175 {
176
177 if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_HCRC_SHIFT)) {
178 pr_info("tid 0x%x, status 0x%x, hcrc bad.\n", csk->tid, ddpvld);
179 pdu_cb->flags |= PDUCBF_RX_HCRC_ERR;
180 }
181
182 if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_DCRC_SHIFT)) {
183 pr_info("tid 0x%x, status 0x%x, dcrc bad.\n", csk->tid, ddpvld);
184 pdu_cb->flags |= PDUCBF_RX_DCRC_ERR;
185 }
186
187 if (ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_PAD_SHIFT))
188 pr_info("tid 0x%x, status 0x%x, pad bad.\n", csk->tid, ddpvld);
189
190 if ((ddpvld & (1 << CPL_RX_ISCSI_DDP_STATUS_DDP_SHIFT)) &&
191 (!(pdu_cb->flags & PDUCBF_RX_DATA))) {
192 pdu_cb->flags |= PDUCBF_RX_DATA_DDPD;
193 }
194 }
195
196 static void
cxgbit_lro_add_packet_rsp(struct sk_buff * skb,u8 op,const __be64 * rsp)197 cxgbit_lro_add_packet_rsp(struct sk_buff *skb, u8 op, const __be64 *rsp)
198 {
199 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
200 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb,
201 lro_cb->pdu_idx);
202 struct cpl_rx_iscsi_ddp *cpl = (struct cpl_rx_iscsi_ddp *)(rsp + 1);
203
204 cxgbit_process_ddpvld(lro_cb->csk, pdu_cb, be32_to_cpu(cpl->ddpvld));
205
206 pdu_cb->flags |= PDUCBF_RX_STATUS;
207 pdu_cb->ddigest = ntohl(cpl->ulp_crc);
208 pdu_cb->pdulen = ntohs(cpl->len);
209
210 if (pdu_cb->flags & PDUCBF_RX_HDR)
211 pdu_cb->complete = true;
212
213 lro_cb->pdu_totallen += pdu_cb->pdulen;
214 lro_cb->complete = true;
215 lro_cb->pdu_idx++;
216 }
217
218 static void
cxgbit_copy_frags(struct sk_buff * skb,const struct pkt_gl * gl,unsigned int offset)219 cxgbit_copy_frags(struct sk_buff *skb, const struct pkt_gl *gl,
220 unsigned int offset)
221 {
222 u8 skb_frag_idx = skb_shinfo(skb)->nr_frags;
223 u8 i;
224
225 /* usually there's just one frag */
226 __skb_fill_page_desc(skb, skb_frag_idx, gl->frags[0].page,
227 gl->frags[0].offset + offset,
228 gl->frags[0].size - offset);
229 for (i = 1; i < gl->nfrags; i++)
230 __skb_fill_page_desc(skb, skb_frag_idx + i,
231 gl->frags[i].page,
232 gl->frags[i].offset,
233 gl->frags[i].size);
234
235 skb_shinfo(skb)->nr_frags += gl->nfrags;
236
237 /* get a reference to the last page, we don't own it */
238 get_page(gl->frags[gl->nfrags - 1].page);
239 }
240
241 static void
cxgbit_lro_add_packet_gl(struct sk_buff * skb,u8 op,const struct pkt_gl * gl)242 cxgbit_lro_add_packet_gl(struct sk_buff *skb, u8 op, const struct pkt_gl *gl)
243 {
244 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
245 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb,
246 lro_cb->pdu_idx);
247 u32 len, offset;
248
249 if (op == CPL_ISCSI_HDR) {
250 struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)gl->va;
251
252 offset = sizeof(struct cpl_iscsi_hdr);
253 pdu_cb->flags |= PDUCBF_RX_HDR;
254 pdu_cb->seq = ntohl(cpl->seq);
255 len = ntohs(cpl->len);
256 pdu_cb->hdr = gl->va + offset;
257 pdu_cb->hlen = len;
258 pdu_cb->hfrag_idx = skb_shinfo(skb)->nr_frags;
259
260 if (unlikely(gl->nfrags > 1))
261 cxgbit_skcb_flags(skb) = 0;
262
263 lro_cb->complete = false;
264 } else if (op == CPL_ISCSI_DATA) {
265 struct cpl_iscsi_data *cpl = (struct cpl_iscsi_data *)gl->va;
266
267 offset = sizeof(struct cpl_iscsi_data);
268 pdu_cb->flags |= PDUCBF_RX_DATA;
269 len = ntohs(cpl->len);
270 pdu_cb->dlen = len;
271 pdu_cb->doffset = lro_cb->offset;
272 pdu_cb->nr_dfrags = gl->nfrags;
273 pdu_cb->dfrag_idx = skb_shinfo(skb)->nr_frags;
274 lro_cb->complete = false;
275 } else {
276 struct cpl_rx_iscsi_cmp *cpl;
277
278 cpl = (struct cpl_rx_iscsi_cmp *)gl->va;
279 offset = sizeof(struct cpl_rx_iscsi_cmp);
280 pdu_cb->flags |= (PDUCBF_RX_HDR | PDUCBF_RX_STATUS);
281 len = be16_to_cpu(cpl->len);
282 pdu_cb->hdr = gl->va + offset;
283 pdu_cb->hlen = len;
284 pdu_cb->hfrag_idx = skb_shinfo(skb)->nr_frags;
285 pdu_cb->ddigest = be32_to_cpu(cpl->ulp_crc);
286 pdu_cb->pdulen = ntohs(cpl->len);
287
288 if (unlikely(gl->nfrags > 1))
289 cxgbit_skcb_flags(skb) = 0;
290
291 cxgbit_process_ddpvld(lro_cb->csk, pdu_cb,
292 be32_to_cpu(cpl->ddpvld));
293
294 if (pdu_cb->flags & PDUCBF_RX_DATA_DDPD) {
295 pdu_cb->flags |= PDUCBF_RX_DDP_CMP;
296 pdu_cb->complete = true;
297 } else if (pdu_cb->flags & PDUCBF_RX_DATA) {
298 pdu_cb->complete = true;
299 }
300
301 lro_cb->pdu_totallen += pdu_cb->hlen + pdu_cb->dlen;
302 lro_cb->complete = true;
303 lro_cb->pdu_idx++;
304 }
305
306 cxgbit_copy_frags(skb, gl, offset);
307
308 pdu_cb->frags += gl->nfrags;
309 lro_cb->offset += len;
310 skb->len += len;
311 skb->data_len += len;
312 skb->truesize += len;
313 }
314
315 static struct sk_buff *
cxgbit_lro_init_skb(struct cxgbit_sock * csk,u8 op,const struct pkt_gl * gl,const __be64 * rsp,struct napi_struct * napi)316 cxgbit_lro_init_skb(struct cxgbit_sock *csk, u8 op, const struct pkt_gl *gl,
317 const __be64 *rsp, struct napi_struct *napi)
318 {
319 struct sk_buff *skb;
320 struct cxgbit_lro_cb *lro_cb;
321
322 skb = napi_alloc_skb(napi, LRO_SKB_MAX_HEADROOM);
323
324 if (unlikely(!skb))
325 return NULL;
326
327 memset(skb->data, 0, LRO_SKB_MAX_HEADROOM);
328
329 cxgbit_skcb_flags(skb) |= SKCBF_RX_LRO;
330
331 lro_cb = cxgbit_skb_lro_cb(skb);
332
333 cxgbit_get_csk(csk);
334
335 lro_cb->csk = csk;
336
337 return skb;
338 }
339
cxgbit_queue_lro_skb(struct cxgbit_sock * csk,struct sk_buff * skb)340 static void cxgbit_queue_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
341 {
342 bool wakeup_thread = false;
343
344 spin_lock(&csk->rxq.lock);
345 __skb_queue_tail(&csk->rxq, skb);
346 if (skb_queue_len(&csk->rxq) == 1)
347 wakeup_thread = true;
348 spin_unlock(&csk->rxq.lock);
349
350 if (wakeup_thread)
351 wake_up(&csk->waitq);
352 }
353
cxgbit_lro_flush(struct t4_lro_mgr * lro_mgr,struct sk_buff * skb)354 static void cxgbit_lro_flush(struct t4_lro_mgr *lro_mgr, struct sk_buff *skb)
355 {
356 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
357 struct cxgbit_sock *csk = lro_cb->csk;
358
359 csk->lro_skb = NULL;
360
361 __skb_unlink(skb, &lro_mgr->lroq);
362 cxgbit_queue_lro_skb(csk, skb);
363
364 cxgbit_put_csk(csk);
365
366 lro_mgr->lro_pkts++;
367 lro_mgr->lro_session_cnt--;
368 }
369
cxgbit_uld_lro_flush(struct t4_lro_mgr * lro_mgr)370 static void cxgbit_uld_lro_flush(struct t4_lro_mgr *lro_mgr)
371 {
372 struct sk_buff *skb;
373
374 while ((skb = skb_peek(&lro_mgr->lroq)))
375 cxgbit_lro_flush(lro_mgr, skb);
376 }
377
378 static int
cxgbit_lro_receive(struct cxgbit_sock * csk,u8 op,const __be64 * rsp,const struct pkt_gl * gl,struct t4_lro_mgr * lro_mgr,struct napi_struct * napi)379 cxgbit_lro_receive(struct cxgbit_sock *csk, u8 op, const __be64 *rsp,
380 const struct pkt_gl *gl, struct t4_lro_mgr *lro_mgr,
381 struct napi_struct *napi)
382 {
383 struct sk_buff *skb;
384 struct cxgbit_lro_cb *lro_cb;
385
386 if (!csk) {
387 pr_err("%s: csk NULL, op 0x%x.\n", __func__, op);
388 goto out;
389 }
390
391 if (csk->lro_skb)
392 goto add_packet;
393
394 start_lro:
395 if (lro_mgr->lro_session_cnt >= MAX_LRO_SESSIONS) {
396 cxgbit_uld_lro_flush(lro_mgr);
397 goto start_lro;
398 }
399
400 skb = cxgbit_lro_init_skb(csk, op, gl, rsp, napi);
401 if (unlikely(!skb))
402 goto out;
403
404 csk->lro_skb = skb;
405
406 __skb_queue_tail(&lro_mgr->lroq, skb);
407 lro_mgr->lro_session_cnt++;
408
409 add_packet:
410 skb = csk->lro_skb;
411 lro_cb = cxgbit_skb_lro_cb(skb);
412
413 if ((gl && (((skb_shinfo(skb)->nr_frags + gl->nfrags) >
414 MAX_SKB_FRAGS) || (lro_cb->pdu_totallen >= LRO_FLUSH_LEN_MAX))) ||
415 (lro_cb->pdu_idx >= MAX_SKB_FRAGS)) {
416 cxgbit_lro_flush(lro_mgr, skb);
417 goto start_lro;
418 }
419
420 if (gl)
421 cxgbit_lro_add_packet_gl(skb, op, gl);
422 else
423 cxgbit_lro_add_packet_rsp(skb, op, rsp);
424
425 lro_mgr->lro_merged++;
426
427 return 0;
428
429 out:
430 return -1;
431 }
432
433 static int
cxgbit_uld_lro_rx_handler(void * hndl,const __be64 * rsp,const struct pkt_gl * gl,struct t4_lro_mgr * lro_mgr,struct napi_struct * napi)434 cxgbit_uld_lro_rx_handler(void *hndl, const __be64 *rsp,
435 const struct pkt_gl *gl, struct t4_lro_mgr *lro_mgr,
436 struct napi_struct *napi)
437 {
438 struct cxgbit_device *cdev = hndl;
439 struct cxgb4_lld_info *lldi = &cdev->lldi;
440 struct cpl_tx_data *rpl = NULL;
441 struct cxgbit_sock *csk = NULL;
442 unsigned int tid = 0;
443 struct sk_buff *skb;
444 unsigned int op = *(u8 *)rsp;
445 bool lro_flush = true;
446
447 switch (op) {
448 case CPL_ISCSI_HDR:
449 case CPL_ISCSI_DATA:
450 case CPL_RX_ISCSI_CMP:
451 case CPL_RX_ISCSI_DDP:
452 case CPL_FW4_ACK:
453 lro_flush = false;
454 fallthrough;
455 case CPL_ABORT_RPL_RSS:
456 case CPL_PASS_ESTABLISH:
457 case CPL_PEER_CLOSE:
458 case CPL_CLOSE_CON_RPL:
459 case CPL_ABORT_REQ_RSS:
460 case CPL_SET_TCB_RPL:
461 case CPL_RX_DATA:
462 rpl = gl ? (struct cpl_tx_data *)gl->va :
463 (struct cpl_tx_data *)(rsp + 1);
464 tid = GET_TID(rpl);
465 csk = lookup_tid(lldi->tids, tid);
466 break;
467 default:
468 break;
469 }
470
471 if (csk && csk->lro_skb && lro_flush)
472 cxgbit_lro_flush(lro_mgr, csk->lro_skb);
473
474 if (!gl) {
475 unsigned int len;
476
477 if (op == CPL_RX_ISCSI_DDP) {
478 if (!cxgbit_lro_receive(csk, op, rsp, NULL, lro_mgr,
479 napi))
480 return 0;
481 }
482
483 len = 64 - sizeof(struct rsp_ctrl) - 8;
484 skb = napi_alloc_skb(napi, len);
485 if (!skb)
486 goto nomem;
487 __skb_put(skb, len);
488 skb_copy_to_linear_data(skb, &rsp[1], len);
489 } else {
490 if (unlikely(op != *(u8 *)gl->va)) {
491 pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n",
492 gl->va, be64_to_cpu(*rsp),
493 get_unaligned_be64(gl->va),
494 gl->tot_len);
495 return 0;
496 }
497
498 if ((op == CPL_ISCSI_HDR) || (op == CPL_ISCSI_DATA) ||
499 (op == CPL_RX_ISCSI_CMP)) {
500 if (!cxgbit_lro_receive(csk, op, rsp, gl, lro_mgr,
501 napi))
502 return 0;
503 }
504
505 #define RX_PULL_LEN 128
506 skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
507 if (unlikely(!skb))
508 goto nomem;
509 }
510
511 rpl = (struct cpl_tx_data *)skb->data;
512 op = rpl->ot.opcode;
513 cxgbit_skcb_rx_opcode(skb) = op;
514
515 pr_debug("cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n",
516 cdev, op, rpl->ot.opcode_tid,
517 ntohl(rpl->ot.opcode_tid), skb);
518
519 if (op < NUM_CPL_CMDS && cxgbit_cplhandlers[op]) {
520 cxgbit_cplhandlers[op](cdev, skb);
521 } else {
522 pr_err("No handler for opcode 0x%x.\n", op);
523 __kfree_skb(skb);
524 }
525 return 0;
526 nomem:
527 pr_err("%s OOM bailing out.\n", __func__);
528 return 1;
529 }
530
531 #ifdef CONFIG_CHELSIO_T4_DCB
532 struct cxgbit_dcb_work {
533 struct dcb_app_type dcb_app;
534 struct work_struct work;
535 };
536
537 static void
cxgbit_update_dcb_priority(struct cxgbit_device * cdev,u8 port_id,u8 dcb_priority,u16 port_num)538 cxgbit_update_dcb_priority(struct cxgbit_device *cdev, u8 port_id,
539 u8 dcb_priority, u16 port_num)
540 {
541 struct cxgbit_sock *csk;
542 struct sk_buff *skb;
543 u16 local_port;
544 bool wakeup_thread = false;
545
546 spin_lock_bh(&cdev->cskq.lock);
547 list_for_each_entry(csk, &cdev->cskq.list, list) {
548 if (csk->port_id != port_id)
549 continue;
550
551 if (csk->com.local_addr.ss_family == AF_INET6) {
552 struct sockaddr_in6 *sock_in6;
553
554 sock_in6 = (struct sockaddr_in6 *)&csk->com.local_addr;
555 local_port = ntohs(sock_in6->sin6_port);
556 } else {
557 struct sockaddr_in *sock_in;
558
559 sock_in = (struct sockaddr_in *)&csk->com.local_addr;
560 local_port = ntohs(sock_in->sin_port);
561 }
562
563 if (local_port != port_num)
564 continue;
565
566 if (csk->dcb_priority == dcb_priority)
567 continue;
568
569 skb = alloc_skb(0, GFP_ATOMIC);
570 if (!skb)
571 continue;
572
573 spin_lock(&csk->rxq.lock);
574 __skb_queue_tail(&csk->rxq, skb);
575 if (skb_queue_len(&csk->rxq) == 1)
576 wakeup_thread = true;
577 spin_unlock(&csk->rxq.lock);
578
579 if (wakeup_thread) {
580 wake_up(&csk->waitq);
581 wakeup_thread = false;
582 }
583 }
584 spin_unlock_bh(&cdev->cskq.lock);
585 }
586
cxgbit_dcb_workfn(struct work_struct * work)587 static void cxgbit_dcb_workfn(struct work_struct *work)
588 {
589 struct cxgbit_dcb_work *dcb_work;
590 struct net_device *ndev;
591 struct cxgbit_device *cdev = NULL;
592 struct dcb_app_type *iscsi_app;
593 u8 priority, port_id = 0xff;
594
595 dcb_work = container_of(work, struct cxgbit_dcb_work, work);
596 iscsi_app = &dcb_work->dcb_app;
597
598 if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_IEEE) {
599 if ((iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_STREAM) &&
600 (iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_ANY))
601 goto out;
602
603 priority = iscsi_app->app.priority;
604
605 } else if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_CEE) {
606 if (iscsi_app->app.selector != DCB_APP_IDTYPE_PORTNUM)
607 goto out;
608
609 if (!iscsi_app->app.priority)
610 goto out;
611
612 priority = ffs(iscsi_app->app.priority) - 1;
613 } else {
614 goto out;
615 }
616
617 pr_debug("priority for ifid %d is %u\n",
618 iscsi_app->ifindex, priority);
619
620 ndev = dev_get_by_index(&init_net, iscsi_app->ifindex);
621
622 if (!ndev)
623 goto out;
624
625 mutex_lock(&cdev_list_lock);
626 cdev = cxgbit_find_device(ndev, &port_id);
627
628 dev_put(ndev);
629
630 if (!cdev) {
631 mutex_unlock(&cdev_list_lock);
632 goto out;
633 }
634
635 cxgbit_update_dcb_priority(cdev, port_id, priority,
636 iscsi_app->app.protocol);
637 mutex_unlock(&cdev_list_lock);
638 out:
639 kfree(dcb_work);
640 }
641
642 static int
cxgbit_dcbevent_notify(struct notifier_block * nb,unsigned long action,void * data)643 cxgbit_dcbevent_notify(struct notifier_block *nb, unsigned long action,
644 void *data)
645 {
646 struct cxgbit_dcb_work *dcb_work;
647 struct dcb_app_type *dcb_app = data;
648
649 dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC);
650 if (!dcb_work)
651 return NOTIFY_DONE;
652
653 dcb_work->dcb_app = *dcb_app;
654 INIT_WORK(&dcb_work->work, cxgbit_dcb_workfn);
655 schedule_work(&dcb_work->work);
656 return NOTIFY_OK;
657 }
658 #endif
659
cxgbit_get_sup_prot_ops(struct iscsit_conn * conn)660 static enum target_prot_op cxgbit_get_sup_prot_ops(struct iscsit_conn *conn)
661 {
662 return TARGET_PROT_NORMAL;
663 }
664
665 static struct iscsit_transport cxgbit_transport = {
666 .name = DRV_NAME,
667 .transport_type = ISCSI_CXGBIT,
668 .rdma_shutdown = false,
669 .priv_size = sizeof(struct cxgbit_cmd),
670 .owner = THIS_MODULE,
671 .iscsit_setup_np = cxgbit_setup_np,
672 .iscsit_accept_np = cxgbit_accept_np,
673 .iscsit_free_np = cxgbit_free_np,
674 .iscsit_free_conn = cxgbit_free_conn,
675 .iscsit_get_login_rx = cxgbit_get_login_rx,
676 .iscsit_put_login_tx = cxgbit_put_login_tx,
677 .iscsit_immediate_queue = iscsit_immediate_queue,
678 .iscsit_response_queue = iscsit_response_queue,
679 .iscsit_get_dataout = iscsit_build_r2ts_for_cmd,
680 .iscsit_queue_data_in = iscsit_queue_rsp,
681 .iscsit_queue_status = iscsit_queue_rsp,
682 .iscsit_xmit_pdu = cxgbit_xmit_pdu,
683 .iscsit_get_r2t_ttt = cxgbit_get_r2t_ttt,
684 .iscsit_get_rx_pdu = cxgbit_get_rx_pdu,
685 .iscsit_validate_params = cxgbit_validate_params,
686 .iscsit_unmap_cmd = cxgbit_unmap_cmd,
687 .iscsit_aborted_task = iscsit_aborted_task,
688 .iscsit_get_sup_prot_ops = cxgbit_get_sup_prot_ops,
689 };
690
691 static struct cxgb4_uld_info cxgbit_uld_info = {
692 .name = DRV_NAME,
693 .nrxq = MAX_ULD_QSETS,
694 .ntxq = MAX_ULD_QSETS,
695 .rxq_size = 1024,
696 .lro = true,
697 .add = cxgbit_uld_add,
698 .state_change = cxgbit_uld_state_change,
699 .lro_rx_handler = cxgbit_uld_lro_rx_handler,
700 .lro_flush = cxgbit_uld_lro_flush,
701 };
702
703 #ifdef CONFIG_CHELSIO_T4_DCB
704 static struct notifier_block cxgbit_dcbevent_nb = {
705 .notifier_call = cxgbit_dcbevent_notify,
706 };
707 #endif
708
cxgbit_init(void)709 static int __init cxgbit_init(void)
710 {
711 cxgb4_register_uld(CXGB4_ULD_ISCSIT, &cxgbit_uld_info);
712 iscsit_register_transport(&cxgbit_transport);
713
714 #ifdef CONFIG_CHELSIO_T4_DCB
715 pr_info("%s dcb enabled.\n", DRV_NAME);
716 register_dcbevent_notifier(&cxgbit_dcbevent_nb);
717 #endif
718 BUILD_BUG_ON(sizeof_field(struct sk_buff, cb) <
719 sizeof(union cxgbit_skb_cb));
720 return 0;
721 }
722
cxgbit_exit(void)723 static void __exit cxgbit_exit(void)
724 {
725 struct cxgbit_device *cdev, *tmp;
726
727 #ifdef CONFIG_CHELSIO_T4_DCB
728 unregister_dcbevent_notifier(&cxgbit_dcbevent_nb);
729 #endif
730 mutex_lock(&cdev_list_lock);
731 list_for_each_entry_safe(cdev, tmp, &cdev_list_head, list) {
732 list_del(&cdev->list);
733 cxgbit_put_cdev(cdev);
734 }
735 mutex_unlock(&cdev_list_lock);
736 iscsit_unregister_transport(&cxgbit_transport);
737 cxgb4_unregister_uld(CXGB4_ULD_ISCSIT);
738 }
739
740 module_init(cxgbit_init);
741 module_exit(cxgbit_exit);
742
743 MODULE_DESCRIPTION("Chelsio iSCSI target offload driver");
744 MODULE_AUTHOR("Chelsio Communications");
745 MODULE_VERSION(DRV_VERSION);
746 MODULE_LICENSE("GPL");
747