1 /*
2 * Copyright (c) 2007, 2008, 2009, 2010 QLogic Corporation. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/spinlock.h>
34 #include <linux/netdevice.h>
35
36 #include "qib.h"
37 #include "qib_common.h"
38
39 /* default pio off, sdma on */
40 static ushort sdma_descq_cnt = 256;
41 module_param_named(sdma_descq_cnt, sdma_descq_cnt, ushort, S_IRUGO);
42 MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries");
43
44 /*
45 * Bits defined in the send DMA descriptor.
46 */
47 #define SDMA_DESC_LAST (1ULL << 11)
48 #define SDMA_DESC_FIRST (1ULL << 12)
49 #define SDMA_DESC_DMA_HEAD (1ULL << 13)
50 #define SDMA_DESC_USE_LARGE_BUF (1ULL << 14)
51 #define SDMA_DESC_INTR (1ULL << 15)
52 #define SDMA_DESC_COUNT_LSB 16
53 #define SDMA_DESC_GEN_LSB 30
54
55 char *qib_sdma_state_names[] = {
56 [qib_sdma_state_s00_hw_down] = "s00_HwDown",
57 [qib_sdma_state_s10_hw_start_up_wait] = "s10_HwStartUpWait",
58 [qib_sdma_state_s20_idle] = "s20_Idle",
59 [qib_sdma_state_s30_sw_clean_up_wait] = "s30_SwCleanUpWait",
60 [qib_sdma_state_s40_hw_clean_up_wait] = "s40_HwCleanUpWait",
61 [qib_sdma_state_s50_hw_halt_wait] = "s50_HwHaltWait",
62 [qib_sdma_state_s99_running] = "s99_Running",
63 };
64
65 char *qib_sdma_event_names[] = {
66 [qib_sdma_event_e00_go_hw_down] = "e00_GoHwDown",
67 [qib_sdma_event_e10_go_hw_start] = "e10_GoHwStart",
68 [qib_sdma_event_e20_hw_started] = "e20_HwStarted",
69 [qib_sdma_event_e30_go_running] = "e30_GoRunning",
70 [qib_sdma_event_e40_sw_cleaned] = "e40_SwCleaned",
71 [qib_sdma_event_e50_hw_cleaned] = "e50_HwCleaned",
72 [qib_sdma_event_e60_hw_halted] = "e60_HwHalted",
73 [qib_sdma_event_e70_go_idle] = "e70_GoIdle",
74 [qib_sdma_event_e7220_err_halted] = "e7220_ErrHalted",
75 [qib_sdma_event_e7322_err_halted] = "e7322_ErrHalted",
76 [qib_sdma_event_e90_timer_tick] = "e90_TimerTick",
77 };
78
79 /* declare all statics here rather than keep sorting */
80 static int alloc_sdma(struct qib_pportdata *);
81 static void sdma_complete(struct kref *);
82 static void sdma_finalput(struct qib_sdma_state *);
83 static void sdma_get(struct qib_sdma_state *);
84 static void sdma_put(struct qib_sdma_state *);
85 static void sdma_set_state(struct qib_pportdata *, enum qib_sdma_states);
86 static void sdma_start_sw_clean_up(struct qib_pportdata *);
87 static void sdma_sw_clean_up_task(unsigned long);
88 static void unmap_desc(struct qib_pportdata *, unsigned);
89
sdma_get(struct qib_sdma_state * ss)90 static void sdma_get(struct qib_sdma_state *ss)
91 {
92 kref_get(&ss->kref);
93 }
94
sdma_complete(struct kref * kref)95 static void sdma_complete(struct kref *kref)
96 {
97 struct qib_sdma_state *ss =
98 container_of(kref, struct qib_sdma_state, kref);
99
100 complete(&ss->comp);
101 }
102
sdma_put(struct qib_sdma_state * ss)103 static void sdma_put(struct qib_sdma_state *ss)
104 {
105 kref_put(&ss->kref, sdma_complete);
106 }
107
sdma_finalput(struct qib_sdma_state * ss)108 static void sdma_finalput(struct qib_sdma_state *ss)
109 {
110 sdma_put(ss);
111 wait_for_completion(&ss->comp);
112 }
113
114 /*
115 * Complete all the sdma requests on the active list, in the correct
116 * order, and with appropriate processing. Called when cleaning up
117 * after sdma shutdown, and when new sdma requests are submitted for
118 * a link that is down. This matches what is done for requests
119 * that complete normally, it's just the full list.
120 *
121 * Must be called with sdma_lock held
122 */
clear_sdma_activelist(struct qib_pportdata * ppd)123 static void clear_sdma_activelist(struct qib_pportdata *ppd)
124 {
125 struct qib_sdma_txreq *txp, *txp_next;
126
127 list_for_each_entry_safe(txp, txp_next, &ppd->sdma_activelist, list) {
128 list_del_init(&txp->list);
129 if (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) {
130 unsigned idx;
131
132 idx = txp->start_idx;
133 while (idx != txp->next_descq_idx) {
134 unmap_desc(ppd, idx);
135 if (++idx == ppd->sdma_descq_cnt)
136 idx = 0;
137 }
138 }
139 if (txp->callback)
140 (*txp->callback)(txp, QIB_SDMA_TXREQ_S_ABORTED);
141 }
142 }
143
sdma_sw_clean_up_task(unsigned long opaque)144 static void sdma_sw_clean_up_task(unsigned long opaque)
145 {
146 struct qib_pportdata *ppd = (struct qib_pportdata *) opaque;
147 unsigned long flags;
148
149 spin_lock_irqsave(&ppd->sdma_lock, flags);
150
151 /*
152 * At this point, the following should always be true:
153 * - We are halted, so no more descriptors are getting retired.
154 * - We are not running, so no one is submitting new work.
155 * - Only we can send the e40_sw_cleaned, so we can't start
156 * running again until we say so. So, the active list and
157 * descq are ours to play with.
158 */
159
160 /* Process all retired requests. */
161 qib_sdma_make_progress(ppd);
162
163 clear_sdma_activelist(ppd);
164
165 /*
166 * Resync count of added and removed. It is VERY important that
167 * sdma_descq_removed NEVER decrement - user_sdma depends on it.
168 */
169 ppd->sdma_descq_removed = ppd->sdma_descq_added;
170
171 /*
172 * Reset our notion of head and tail.
173 * Note that the HW registers will be reset when switching states
174 * due to calling __qib_sdma_process_event() below.
175 */
176 ppd->sdma_descq_tail = 0;
177 ppd->sdma_descq_head = 0;
178 ppd->sdma_head_dma[0] = 0;
179 ppd->sdma_generation = 0;
180
181 __qib_sdma_process_event(ppd, qib_sdma_event_e40_sw_cleaned);
182
183 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
184 }
185
186 /*
187 * This is called when changing to state qib_sdma_state_s10_hw_start_up_wait
188 * as a result of send buffer errors or send DMA descriptor errors.
189 * We want to disarm the buffers in these cases.
190 */
sdma_hw_start_up(struct qib_pportdata * ppd)191 static void sdma_hw_start_up(struct qib_pportdata *ppd)
192 {
193 struct qib_sdma_state *ss = &ppd->sdma_state;
194 unsigned bufno;
195
196 for (bufno = ss->first_sendbuf; bufno < ss->last_sendbuf; ++bufno)
197 ppd->dd->f_sendctrl(ppd, QIB_SENDCTRL_DISARM_BUF(bufno));
198
199 ppd->dd->f_sdma_hw_start_up(ppd);
200 }
201
sdma_sw_tear_down(struct qib_pportdata * ppd)202 static void sdma_sw_tear_down(struct qib_pportdata *ppd)
203 {
204 struct qib_sdma_state *ss = &ppd->sdma_state;
205
206 /* Releasing this reference means the state machine has stopped. */
207 sdma_put(ss);
208 }
209
sdma_start_sw_clean_up(struct qib_pportdata * ppd)210 static void sdma_start_sw_clean_up(struct qib_pportdata *ppd)
211 {
212 tasklet_hi_schedule(&ppd->sdma_sw_clean_up_task);
213 }
214
sdma_set_state(struct qib_pportdata * ppd,enum qib_sdma_states next_state)215 static void sdma_set_state(struct qib_pportdata *ppd,
216 enum qib_sdma_states next_state)
217 {
218 struct qib_sdma_state *ss = &ppd->sdma_state;
219 struct sdma_set_state_action *action = ss->set_state_action;
220 unsigned op = 0;
221
222 /* debugging bookkeeping */
223 ss->previous_state = ss->current_state;
224 ss->previous_op = ss->current_op;
225
226 ss->current_state = next_state;
227
228 if (action[next_state].op_enable)
229 op |= QIB_SDMA_SENDCTRL_OP_ENABLE;
230
231 if (action[next_state].op_intenable)
232 op |= QIB_SDMA_SENDCTRL_OP_INTENABLE;
233
234 if (action[next_state].op_halt)
235 op |= QIB_SDMA_SENDCTRL_OP_HALT;
236
237 if (action[next_state].op_drain)
238 op |= QIB_SDMA_SENDCTRL_OP_DRAIN;
239
240 if (action[next_state].go_s99_running_tofalse)
241 ss->go_s99_running = 0;
242
243 if (action[next_state].go_s99_running_totrue)
244 ss->go_s99_running = 1;
245
246 ss->current_op = op;
247
248 ppd->dd->f_sdma_sendctrl(ppd, ss->current_op);
249 }
250
unmap_desc(struct qib_pportdata * ppd,unsigned head)251 static void unmap_desc(struct qib_pportdata *ppd, unsigned head)
252 {
253 __le64 *descqp = &ppd->sdma_descq[head].qw[0];
254 u64 desc[2];
255 dma_addr_t addr;
256 size_t len;
257
258 desc[0] = le64_to_cpu(descqp[0]);
259 desc[1] = le64_to_cpu(descqp[1]);
260
261 addr = (desc[1] << 32) | (desc[0] >> 32);
262 len = (desc[0] >> 14) & (0x7ffULL << 2);
263 dma_unmap_single(&ppd->dd->pcidev->dev, addr, len, DMA_TO_DEVICE);
264 }
265
alloc_sdma(struct qib_pportdata * ppd)266 static int alloc_sdma(struct qib_pportdata *ppd)
267 {
268 ppd->sdma_descq_cnt = sdma_descq_cnt;
269 if (!ppd->sdma_descq_cnt)
270 ppd->sdma_descq_cnt = 256;
271
272 /* Allocate memory for SendDMA descriptor FIFO */
273 ppd->sdma_descq = dma_alloc_coherent(&ppd->dd->pcidev->dev,
274 ppd->sdma_descq_cnt * sizeof(u64[2]), &ppd->sdma_descq_phys,
275 GFP_KERNEL);
276
277 if (!ppd->sdma_descq) {
278 qib_dev_err(ppd->dd, "failed to allocate SendDMA descriptor "
279 "FIFO memory\n");
280 goto bail;
281 }
282
283 /* Allocate memory for DMA of head register to memory */
284 ppd->sdma_head_dma = dma_alloc_coherent(&ppd->dd->pcidev->dev,
285 PAGE_SIZE, &ppd->sdma_head_phys, GFP_KERNEL);
286 if (!ppd->sdma_head_dma) {
287 qib_dev_err(ppd->dd, "failed to allocate SendDMA "
288 "head memory\n");
289 goto cleanup_descq;
290 }
291 ppd->sdma_head_dma[0] = 0;
292 return 0;
293
294 cleanup_descq:
295 dma_free_coherent(&ppd->dd->pcidev->dev,
296 ppd->sdma_descq_cnt * sizeof(u64[2]), (void *)ppd->sdma_descq,
297 ppd->sdma_descq_phys);
298 ppd->sdma_descq = NULL;
299 ppd->sdma_descq_phys = 0;
300 bail:
301 ppd->sdma_descq_cnt = 0;
302 return -ENOMEM;
303 }
304
free_sdma(struct qib_pportdata * ppd)305 static void free_sdma(struct qib_pportdata *ppd)
306 {
307 struct qib_devdata *dd = ppd->dd;
308
309 if (ppd->sdma_head_dma) {
310 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
311 (void *)ppd->sdma_head_dma,
312 ppd->sdma_head_phys);
313 ppd->sdma_head_dma = NULL;
314 ppd->sdma_head_phys = 0;
315 }
316
317 if (ppd->sdma_descq) {
318 dma_free_coherent(&dd->pcidev->dev,
319 ppd->sdma_descq_cnt * sizeof(u64[2]),
320 ppd->sdma_descq, ppd->sdma_descq_phys);
321 ppd->sdma_descq = NULL;
322 ppd->sdma_descq_phys = 0;
323 }
324 }
325
make_sdma_desc(struct qib_pportdata * ppd,u64 * sdmadesc,u64 addr,u64 dwlen,u64 dwoffset)326 static inline void make_sdma_desc(struct qib_pportdata *ppd,
327 u64 *sdmadesc, u64 addr, u64 dwlen,
328 u64 dwoffset)
329 {
330
331 WARN_ON(addr & 3);
332 /* SDmaPhyAddr[47:32] */
333 sdmadesc[1] = addr >> 32;
334 /* SDmaPhyAddr[31:0] */
335 sdmadesc[0] = (addr & 0xfffffffcULL) << 32;
336 /* SDmaGeneration[1:0] */
337 sdmadesc[0] |= (ppd->sdma_generation & 3ULL) <<
338 SDMA_DESC_GEN_LSB;
339 /* SDmaDwordCount[10:0] */
340 sdmadesc[0] |= (dwlen & 0x7ffULL) << SDMA_DESC_COUNT_LSB;
341 /* SDmaBufOffset[12:2] */
342 sdmadesc[0] |= dwoffset & 0x7ffULL;
343 }
344
345 /* sdma_lock must be held */
qib_sdma_make_progress(struct qib_pportdata * ppd)346 int qib_sdma_make_progress(struct qib_pportdata *ppd)
347 {
348 struct list_head *lp = NULL;
349 struct qib_sdma_txreq *txp = NULL;
350 struct qib_devdata *dd = ppd->dd;
351 int progress = 0;
352 u16 hwhead;
353 u16 idx = 0;
354
355 hwhead = dd->f_sdma_gethead(ppd);
356
357 /* The reason for some of the complexity of this code is that
358 * not all descriptors have corresponding txps. So, we have to
359 * be able to skip over descs until we wander into the range of
360 * the next txp on the list.
361 */
362
363 if (!list_empty(&ppd->sdma_activelist)) {
364 lp = ppd->sdma_activelist.next;
365 txp = list_entry(lp, struct qib_sdma_txreq, list);
366 idx = txp->start_idx;
367 }
368
369 while (ppd->sdma_descq_head != hwhead) {
370 /* if desc is part of this txp, unmap if needed */
371 if (txp && (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) &&
372 (idx == ppd->sdma_descq_head)) {
373 unmap_desc(ppd, ppd->sdma_descq_head);
374 if (++idx == ppd->sdma_descq_cnt)
375 idx = 0;
376 }
377
378 /* increment dequed desc count */
379 ppd->sdma_descq_removed++;
380
381 /* advance head, wrap if needed */
382 if (++ppd->sdma_descq_head == ppd->sdma_descq_cnt)
383 ppd->sdma_descq_head = 0;
384
385 /* if now past this txp's descs, do the callback */
386 if (txp && txp->next_descq_idx == ppd->sdma_descq_head) {
387 /* remove from active list */
388 list_del_init(&txp->list);
389 if (txp->callback)
390 (*txp->callback)(txp, QIB_SDMA_TXREQ_S_OK);
391 /* see if there is another txp */
392 if (list_empty(&ppd->sdma_activelist))
393 txp = NULL;
394 else {
395 lp = ppd->sdma_activelist.next;
396 txp = list_entry(lp, struct qib_sdma_txreq,
397 list);
398 idx = txp->start_idx;
399 }
400 }
401 progress = 1;
402 }
403 if (progress)
404 qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd));
405 return progress;
406 }
407
408 /*
409 * This is called from interrupt context.
410 */
qib_sdma_intr(struct qib_pportdata * ppd)411 void qib_sdma_intr(struct qib_pportdata *ppd)
412 {
413 unsigned long flags;
414
415 spin_lock_irqsave(&ppd->sdma_lock, flags);
416
417 __qib_sdma_intr(ppd);
418
419 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
420 }
421
__qib_sdma_intr(struct qib_pportdata * ppd)422 void __qib_sdma_intr(struct qib_pportdata *ppd)
423 {
424 if (__qib_sdma_running(ppd))
425 qib_sdma_make_progress(ppd);
426 }
427
qib_setup_sdma(struct qib_pportdata * ppd)428 int qib_setup_sdma(struct qib_pportdata *ppd)
429 {
430 struct qib_devdata *dd = ppd->dd;
431 unsigned long flags;
432 int ret = 0;
433
434 ret = alloc_sdma(ppd);
435 if (ret)
436 goto bail;
437
438 /* set consistent sdma state */
439 ppd->dd->f_sdma_init_early(ppd);
440 spin_lock_irqsave(&ppd->sdma_lock, flags);
441 sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
442 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
443
444 /* set up reference counting */
445 kref_init(&ppd->sdma_state.kref);
446 init_completion(&ppd->sdma_state.comp);
447
448 ppd->sdma_generation = 0;
449 ppd->sdma_descq_head = 0;
450 ppd->sdma_descq_removed = 0;
451 ppd->sdma_descq_added = 0;
452
453 INIT_LIST_HEAD(&ppd->sdma_activelist);
454
455 tasklet_init(&ppd->sdma_sw_clean_up_task, sdma_sw_clean_up_task,
456 (unsigned long)ppd);
457
458 ret = dd->f_init_sdma_regs(ppd);
459 if (ret)
460 goto bail_alloc;
461
462 qib_sdma_process_event(ppd, qib_sdma_event_e10_go_hw_start);
463
464 return 0;
465
466 bail_alloc:
467 qib_teardown_sdma(ppd);
468 bail:
469 return ret;
470 }
471
qib_teardown_sdma(struct qib_pportdata * ppd)472 void qib_teardown_sdma(struct qib_pportdata *ppd)
473 {
474 qib_sdma_process_event(ppd, qib_sdma_event_e00_go_hw_down);
475
476 /*
477 * This waits for the state machine to exit so it is not
478 * necessary to kill the sdma_sw_clean_up_task to make sure
479 * it is not running.
480 */
481 sdma_finalput(&ppd->sdma_state);
482
483 free_sdma(ppd);
484 }
485
qib_sdma_running(struct qib_pportdata * ppd)486 int qib_sdma_running(struct qib_pportdata *ppd)
487 {
488 unsigned long flags;
489 int ret;
490
491 spin_lock_irqsave(&ppd->sdma_lock, flags);
492 ret = __qib_sdma_running(ppd);
493 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
494
495 return ret;
496 }
497
498 /*
499 * Complete a request when sdma not running; likely only request
500 * but to simplify the code, always queue it, then process the full
501 * activelist. We process the entire list to ensure that this particular
502 * request does get it's callback, but in the correct order.
503 * Must be called with sdma_lock held
504 */
complete_sdma_err_req(struct qib_pportdata * ppd,struct qib_verbs_txreq * tx)505 static void complete_sdma_err_req(struct qib_pportdata *ppd,
506 struct qib_verbs_txreq *tx)
507 {
508 atomic_inc(&tx->qp->s_dma_busy);
509 /* no sdma descriptors, so no unmap_desc */
510 tx->txreq.start_idx = 0;
511 tx->txreq.next_descq_idx = 0;
512 list_add_tail(&tx->txreq.list, &ppd->sdma_activelist);
513 clear_sdma_activelist(ppd);
514 }
515
516 /*
517 * This function queues one IB packet onto the send DMA queue per call.
518 * The caller is responsible for checking:
519 * 1) The number of send DMA descriptor entries is less than the size of
520 * the descriptor queue.
521 * 2) The IB SGE addresses and lengths are 32-bit aligned
522 * (except possibly the last SGE's length)
523 * 3) The SGE addresses are suitable for passing to dma_map_single().
524 */
qib_sdma_verbs_send(struct qib_pportdata * ppd,struct qib_sge_state * ss,u32 dwords,struct qib_verbs_txreq * tx)525 int qib_sdma_verbs_send(struct qib_pportdata *ppd,
526 struct qib_sge_state *ss, u32 dwords,
527 struct qib_verbs_txreq *tx)
528 {
529 unsigned long flags;
530 struct qib_sge *sge;
531 struct qib_qp *qp;
532 int ret = 0;
533 u16 tail;
534 __le64 *descqp;
535 u64 sdmadesc[2];
536 u32 dwoffset;
537 dma_addr_t addr;
538
539 spin_lock_irqsave(&ppd->sdma_lock, flags);
540
541 retry:
542 if (unlikely(!__qib_sdma_running(ppd))) {
543 complete_sdma_err_req(ppd, tx);
544 goto unlock;
545 }
546
547 if (tx->txreq.sg_count > qib_sdma_descq_freecnt(ppd)) {
548 if (qib_sdma_make_progress(ppd))
549 goto retry;
550 if (ppd->dd->flags & QIB_HAS_SDMA_TIMEOUT)
551 ppd->dd->f_sdma_set_desc_cnt(ppd,
552 ppd->sdma_descq_cnt / 2);
553 goto busy;
554 }
555
556 dwoffset = tx->hdr_dwords;
557 make_sdma_desc(ppd, sdmadesc, (u64) tx->txreq.addr, dwoffset, 0);
558
559 sdmadesc[0] |= SDMA_DESC_FIRST;
560 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF)
561 sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF;
562
563 /* write to the descq */
564 tail = ppd->sdma_descq_tail;
565 descqp = &ppd->sdma_descq[tail].qw[0];
566 *descqp++ = cpu_to_le64(sdmadesc[0]);
567 *descqp++ = cpu_to_le64(sdmadesc[1]);
568
569 /* increment the tail */
570 if (++tail == ppd->sdma_descq_cnt) {
571 tail = 0;
572 descqp = &ppd->sdma_descq[0].qw[0];
573 ++ppd->sdma_generation;
574 }
575
576 tx->txreq.start_idx = tail;
577
578 sge = &ss->sge;
579 while (dwords) {
580 u32 dw;
581 u32 len;
582
583 len = dwords << 2;
584 if (len > sge->length)
585 len = sge->length;
586 if (len > sge->sge_length)
587 len = sge->sge_length;
588 BUG_ON(len == 0);
589 dw = (len + 3) >> 2;
590 addr = dma_map_single(&ppd->dd->pcidev->dev, sge->vaddr,
591 dw << 2, DMA_TO_DEVICE);
592 if (dma_mapping_error(&ppd->dd->pcidev->dev, addr))
593 goto unmap;
594 sdmadesc[0] = 0;
595 make_sdma_desc(ppd, sdmadesc, (u64) addr, dw, dwoffset);
596 /* SDmaUseLargeBuf has to be set in every descriptor */
597 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF)
598 sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF;
599 /* write to the descq */
600 *descqp++ = cpu_to_le64(sdmadesc[0]);
601 *descqp++ = cpu_to_le64(sdmadesc[1]);
602
603 /* increment the tail */
604 if (++tail == ppd->sdma_descq_cnt) {
605 tail = 0;
606 descqp = &ppd->sdma_descq[0].qw[0];
607 ++ppd->sdma_generation;
608 }
609 sge->vaddr += len;
610 sge->length -= len;
611 sge->sge_length -= len;
612 if (sge->sge_length == 0) {
613 if (--ss->num_sge)
614 *sge = *ss->sg_list++;
615 } else if (sge->length == 0 && sge->mr->lkey) {
616 if (++sge->n >= QIB_SEGSZ) {
617 if (++sge->m >= sge->mr->mapsz)
618 break;
619 sge->n = 0;
620 }
621 sge->vaddr =
622 sge->mr->map[sge->m]->segs[sge->n].vaddr;
623 sge->length =
624 sge->mr->map[sge->m]->segs[sge->n].length;
625 }
626
627 dwoffset += dw;
628 dwords -= dw;
629 }
630
631 if (!tail)
632 descqp = &ppd->sdma_descq[ppd->sdma_descq_cnt].qw[0];
633 descqp -= 2;
634 descqp[0] |= cpu_to_le64(SDMA_DESC_LAST);
635 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_HEADTOHOST)
636 descqp[0] |= cpu_to_le64(SDMA_DESC_DMA_HEAD);
637 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_INTREQ)
638 descqp[0] |= cpu_to_le64(SDMA_DESC_INTR);
639
640 atomic_inc(&tx->qp->s_dma_busy);
641 tx->txreq.next_descq_idx = tail;
642 ppd->dd->f_sdma_update_tail(ppd, tail);
643 ppd->sdma_descq_added += tx->txreq.sg_count;
644 list_add_tail(&tx->txreq.list, &ppd->sdma_activelist);
645 goto unlock;
646
647 unmap:
648 for (;;) {
649 if (!tail)
650 tail = ppd->sdma_descq_cnt - 1;
651 else
652 tail--;
653 if (tail == ppd->sdma_descq_tail)
654 break;
655 unmap_desc(ppd, tail);
656 }
657 qp = tx->qp;
658 qib_put_txreq(tx);
659 spin_lock(&qp->r_lock);
660 spin_lock(&qp->s_lock);
661 if (qp->ibqp.qp_type == IB_QPT_RC) {
662 /* XXX what about error sending RDMA read responses? */
663 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)
664 qib_error_qp(qp, IB_WC_GENERAL_ERR);
665 } else if (qp->s_wqe)
666 qib_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR);
667 spin_unlock(&qp->s_lock);
668 spin_unlock(&qp->r_lock);
669 /* return zero to process the next send work request */
670 goto unlock;
671
672 busy:
673 qp = tx->qp;
674 spin_lock(&qp->s_lock);
675 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) {
676 struct qib_ibdev *dev;
677
678 /*
679 * If we couldn't queue the DMA request, save the info
680 * and try again later rather than destroying the
681 * buffer and undoing the side effects of the copy.
682 */
683 tx->ss = ss;
684 tx->dwords = dwords;
685 qp->s_tx = tx;
686 dev = &ppd->dd->verbs_dev;
687 spin_lock(&dev->pending_lock);
688 if (list_empty(&qp->iowait)) {
689 struct qib_ibport *ibp;
690
691 ibp = &ppd->ibport_data;
692 ibp->n_dmawait++;
693 qp->s_flags |= QIB_S_WAIT_DMA_DESC;
694 list_add_tail(&qp->iowait, &dev->dmawait);
695 }
696 spin_unlock(&dev->pending_lock);
697 qp->s_flags &= ~QIB_S_BUSY;
698 spin_unlock(&qp->s_lock);
699 ret = -EBUSY;
700 } else {
701 spin_unlock(&qp->s_lock);
702 qib_put_txreq(tx);
703 }
704 unlock:
705 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
706 return ret;
707 }
708
qib_sdma_process_event(struct qib_pportdata * ppd,enum qib_sdma_events event)709 void qib_sdma_process_event(struct qib_pportdata *ppd,
710 enum qib_sdma_events event)
711 {
712 unsigned long flags;
713
714 spin_lock_irqsave(&ppd->sdma_lock, flags);
715
716 __qib_sdma_process_event(ppd, event);
717
718 if (ppd->sdma_state.current_state == qib_sdma_state_s99_running)
719 qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd));
720
721 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
722 }
723
__qib_sdma_process_event(struct qib_pportdata * ppd,enum qib_sdma_events event)724 void __qib_sdma_process_event(struct qib_pportdata *ppd,
725 enum qib_sdma_events event)
726 {
727 struct qib_sdma_state *ss = &ppd->sdma_state;
728
729 switch (ss->current_state) {
730 case qib_sdma_state_s00_hw_down:
731 switch (event) {
732 case qib_sdma_event_e00_go_hw_down:
733 break;
734 case qib_sdma_event_e30_go_running:
735 /*
736 * If down, but running requested (usually result
737 * of link up, then we need to start up.
738 * This can happen when hw down is requested while
739 * bringing the link up with traffic active on
740 * 7220, e.g. */
741 ss->go_s99_running = 1;
742 /* fall through and start dma engine */
743 case qib_sdma_event_e10_go_hw_start:
744 /* This reference means the state machine is started */
745 sdma_get(&ppd->sdma_state);
746 sdma_set_state(ppd,
747 qib_sdma_state_s10_hw_start_up_wait);
748 break;
749 case qib_sdma_event_e20_hw_started:
750 break;
751 case qib_sdma_event_e40_sw_cleaned:
752 sdma_sw_tear_down(ppd);
753 break;
754 case qib_sdma_event_e50_hw_cleaned:
755 break;
756 case qib_sdma_event_e60_hw_halted:
757 break;
758 case qib_sdma_event_e70_go_idle:
759 break;
760 case qib_sdma_event_e7220_err_halted:
761 break;
762 case qib_sdma_event_e7322_err_halted:
763 break;
764 case qib_sdma_event_e90_timer_tick:
765 break;
766 }
767 break;
768
769 case qib_sdma_state_s10_hw_start_up_wait:
770 switch (event) {
771 case qib_sdma_event_e00_go_hw_down:
772 sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
773 sdma_sw_tear_down(ppd);
774 break;
775 case qib_sdma_event_e10_go_hw_start:
776 break;
777 case qib_sdma_event_e20_hw_started:
778 sdma_set_state(ppd, ss->go_s99_running ?
779 qib_sdma_state_s99_running :
780 qib_sdma_state_s20_idle);
781 break;
782 case qib_sdma_event_e30_go_running:
783 ss->go_s99_running = 1;
784 break;
785 case qib_sdma_event_e40_sw_cleaned:
786 break;
787 case qib_sdma_event_e50_hw_cleaned:
788 break;
789 case qib_sdma_event_e60_hw_halted:
790 break;
791 case qib_sdma_event_e70_go_idle:
792 ss->go_s99_running = 0;
793 break;
794 case qib_sdma_event_e7220_err_halted:
795 break;
796 case qib_sdma_event_e7322_err_halted:
797 break;
798 case qib_sdma_event_e90_timer_tick:
799 break;
800 }
801 break;
802
803 case qib_sdma_state_s20_idle:
804 switch (event) {
805 case qib_sdma_event_e00_go_hw_down:
806 sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
807 sdma_sw_tear_down(ppd);
808 break;
809 case qib_sdma_event_e10_go_hw_start:
810 break;
811 case qib_sdma_event_e20_hw_started:
812 break;
813 case qib_sdma_event_e30_go_running:
814 sdma_set_state(ppd, qib_sdma_state_s99_running);
815 ss->go_s99_running = 1;
816 break;
817 case qib_sdma_event_e40_sw_cleaned:
818 break;
819 case qib_sdma_event_e50_hw_cleaned:
820 break;
821 case qib_sdma_event_e60_hw_halted:
822 break;
823 case qib_sdma_event_e70_go_idle:
824 break;
825 case qib_sdma_event_e7220_err_halted:
826 break;
827 case qib_sdma_event_e7322_err_halted:
828 break;
829 case qib_sdma_event_e90_timer_tick:
830 break;
831 }
832 break;
833
834 case qib_sdma_state_s30_sw_clean_up_wait:
835 switch (event) {
836 case qib_sdma_event_e00_go_hw_down:
837 sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
838 break;
839 case qib_sdma_event_e10_go_hw_start:
840 break;
841 case qib_sdma_event_e20_hw_started:
842 break;
843 case qib_sdma_event_e30_go_running:
844 ss->go_s99_running = 1;
845 break;
846 case qib_sdma_event_e40_sw_cleaned:
847 sdma_set_state(ppd,
848 qib_sdma_state_s10_hw_start_up_wait);
849 sdma_hw_start_up(ppd);
850 break;
851 case qib_sdma_event_e50_hw_cleaned:
852 break;
853 case qib_sdma_event_e60_hw_halted:
854 break;
855 case qib_sdma_event_e70_go_idle:
856 ss->go_s99_running = 0;
857 break;
858 case qib_sdma_event_e7220_err_halted:
859 break;
860 case qib_sdma_event_e7322_err_halted:
861 break;
862 case qib_sdma_event_e90_timer_tick:
863 break;
864 }
865 break;
866
867 case qib_sdma_state_s40_hw_clean_up_wait:
868 switch (event) {
869 case qib_sdma_event_e00_go_hw_down:
870 sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
871 sdma_start_sw_clean_up(ppd);
872 break;
873 case qib_sdma_event_e10_go_hw_start:
874 break;
875 case qib_sdma_event_e20_hw_started:
876 break;
877 case qib_sdma_event_e30_go_running:
878 ss->go_s99_running = 1;
879 break;
880 case qib_sdma_event_e40_sw_cleaned:
881 break;
882 case qib_sdma_event_e50_hw_cleaned:
883 sdma_set_state(ppd,
884 qib_sdma_state_s30_sw_clean_up_wait);
885 sdma_start_sw_clean_up(ppd);
886 break;
887 case qib_sdma_event_e60_hw_halted:
888 break;
889 case qib_sdma_event_e70_go_idle:
890 ss->go_s99_running = 0;
891 break;
892 case qib_sdma_event_e7220_err_halted:
893 break;
894 case qib_sdma_event_e7322_err_halted:
895 break;
896 case qib_sdma_event_e90_timer_tick:
897 break;
898 }
899 break;
900
901 case qib_sdma_state_s50_hw_halt_wait:
902 switch (event) {
903 case qib_sdma_event_e00_go_hw_down:
904 sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
905 sdma_start_sw_clean_up(ppd);
906 break;
907 case qib_sdma_event_e10_go_hw_start:
908 break;
909 case qib_sdma_event_e20_hw_started:
910 break;
911 case qib_sdma_event_e30_go_running:
912 ss->go_s99_running = 1;
913 break;
914 case qib_sdma_event_e40_sw_cleaned:
915 break;
916 case qib_sdma_event_e50_hw_cleaned:
917 break;
918 case qib_sdma_event_e60_hw_halted:
919 sdma_set_state(ppd,
920 qib_sdma_state_s40_hw_clean_up_wait);
921 ppd->dd->f_sdma_hw_clean_up(ppd);
922 break;
923 case qib_sdma_event_e70_go_idle:
924 ss->go_s99_running = 0;
925 break;
926 case qib_sdma_event_e7220_err_halted:
927 break;
928 case qib_sdma_event_e7322_err_halted:
929 break;
930 case qib_sdma_event_e90_timer_tick:
931 break;
932 }
933 break;
934
935 case qib_sdma_state_s99_running:
936 switch (event) {
937 case qib_sdma_event_e00_go_hw_down:
938 sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
939 sdma_start_sw_clean_up(ppd);
940 break;
941 case qib_sdma_event_e10_go_hw_start:
942 break;
943 case qib_sdma_event_e20_hw_started:
944 break;
945 case qib_sdma_event_e30_go_running:
946 break;
947 case qib_sdma_event_e40_sw_cleaned:
948 break;
949 case qib_sdma_event_e50_hw_cleaned:
950 break;
951 case qib_sdma_event_e60_hw_halted:
952 sdma_set_state(ppd,
953 qib_sdma_state_s30_sw_clean_up_wait);
954 sdma_start_sw_clean_up(ppd);
955 break;
956 case qib_sdma_event_e70_go_idle:
957 sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait);
958 ss->go_s99_running = 0;
959 break;
960 case qib_sdma_event_e7220_err_halted:
961 sdma_set_state(ppd,
962 qib_sdma_state_s30_sw_clean_up_wait);
963 sdma_start_sw_clean_up(ppd);
964 break;
965 case qib_sdma_event_e7322_err_halted:
966 sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait);
967 break;
968 case qib_sdma_event_e90_timer_tick:
969 break;
970 }
971 break;
972 }
973
974 ss->last_event = event;
975 }
976