1 /*
2 * Copyright (C) 2005-2006 by Texas Instruments
3 *
4 * This file implements a DMA interface using TI's CPPI DMA.
5 * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB.
6 * The TUSB6020, using VLYNQ, has CPPI that looks much like DaVinci.
7 */
8
9 #include <linux/platform_device.h>
10 #include <linux/slab.h>
11 #include <linux/usb.h>
12
13 #include "musb_core.h"
14 #include "musb_debug.h"
15 #include "cppi_dma.h"
16
17
18 /* CPPI DMA status 7-mar-2006:
19 *
20 * - See musb_{host,gadget}.c for more info
21 *
22 * - Correct RX DMA generally forces the engine into irq-per-packet mode,
23 * which can easily saturate the CPU under non-mass-storage loads.
24 *
25 * NOTES 24-aug-2006 (2.6.18-rc4):
26 *
27 * - peripheral RXDMA wedged in a test with packets of length 512/512/1.
28 * evidently after the 1 byte packet was received and acked, the queue
29 * of BDs got garbaged so it wouldn't empty the fifo. (rxcsr 0x2003,
30 * and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401
31 * 004001ff 00000001 .. 8feff860) Host was just getting NAKed on tx
32 * of its next (512 byte) packet. IRQ issues?
33 *
34 * REVISIT: the "transfer DMA" glue between CPPI and USB fifos will
35 * evidently also directly update the RX and TX CSRs ... so audit all
36 * host and peripheral side DMA code to avoid CSR access after DMA has
37 * been started.
38 */
39
40 /* REVISIT now we can avoid preallocating these descriptors; or
41 * more simply, switch to a global freelist not per-channel ones.
42 * Note: at full speed, 64 descriptors == 4K bulk data.
43 */
44 #define NUM_TXCHAN_BD 64
45 #define NUM_RXCHAN_BD 64
46
cpu_drain_writebuffer(void)47 static inline void cpu_drain_writebuffer(void)
48 {
49 wmb();
50 #ifdef CONFIG_CPU_ARM926T
51 /* REVISIT this "should not be needed",
52 * but lack of it sure seemed to hurt ...
53 */
54 asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n");
55 #endif
56 }
57
cppi_bd_alloc(struct cppi_channel * c)58 static inline struct cppi_descriptor *cppi_bd_alloc(struct cppi_channel *c)
59 {
60 struct cppi_descriptor *bd = c->freelist;
61
62 if (bd)
63 c->freelist = bd->next;
64 return bd;
65 }
66
67 static inline void
cppi_bd_free(struct cppi_channel * c,struct cppi_descriptor * bd)68 cppi_bd_free(struct cppi_channel *c, struct cppi_descriptor *bd)
69 {
70 if (!bd)
71 return;
72 bd->next = c->freelist;
73 c->freelist = bd;
74 }
75
76 /*
77 * Start DMA controller
78 *
79 * Initialize the DMA controller as necessary.
80 */
81
82 /* zero out entire rx state RAM entry for the channel */
cppi_reset_rx(struct cppi_rx_stateram __iomem * rx)83 static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx)
84 {
85 musb_writel(&rx->rx_skipbytes, 0, 0);
86 musb_writel(&rx->rx_head, 0, 0);
87 musb_writel(&rx->rx_sop, 0, 0);
88 musb_writel(&rx->rx_current, 0, 0);
89 musb_writel(&rx->rx_buf_current, 0, 0);
90 musb_writel(&rx->rx_len_len, 0, 0);
91 musb_writel(&rx->rx_cnt_cnt, 0, 0);
92 }
93
94 /* zero out entire tx state RAM entry for the channel */
cppi_reset_tx(struct cppi_tx_stateram __iomem * tx,u32 ptr)95 static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr)
96 {
97 musb_writel(&tx->tx_head, 0, 0);
98 musb_writel(&tx->tx_buf, 0, 0);
99 musb_writel(&tx->tx_current, 0, 0);
100 musb_writel(&tx->tx_buf_current, 0, 0);
101 musb_writel(&tx->tx_info, 0, 0);
102 musb_writel(&tx->tx_rem_len, 0, 0);
103 /* musb_writel(&tx->tx_dummy, 0, 0); */
104 musb_writel(&tx->tx_complete, 0, ptr);
105 }
106
cppi_pool_init(struct cppi * cppi,struct cppi_channel * c)107 static void __init cppi_pool_init(struct cppi *cppi, struct cppi_channel *c)
108 {
109 int j;
110
111 /* initialize channel fields */
112 c->head = NULL;
113 c->tail = NULL;
114 c->last_processed = NULL;
115 c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
116 c->controller = cppi;
117 c->is_rndis = 0;
118 c->freelist = NULL;
119
120 /* build the BD Free list for the channel */
121 for (j = 0; j < NUM_TXCHAN_BD + 1; j++) {
122 struct cppi_descriptor *bd;
123 dma_addr_t dma;
124
125 bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma);
126 bd->dma = dma;
127 cppi_bd_free(c, bd);
128 }
129 }
130
131 static int cppi_channel_abort(struct dma_channel *);
132
cppi_pool_free(struct cppi_channel * c)133 static void cppi_pool_free(struct cppi_channel *c)
134 {
135 struct cppi *cppi = c->controller;
136 struct cppi_descriptor *bd;
137
138 (void) cppi_channel_abort(&c->channel);
139 c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
140 c->controller = NULL;
141
142 /* free all its bds */
143 bd = c->last_processed;
144 do {
145 if (bd)
146 dma_pool_free(cppi->pool, bd, bd->dma);
147 bd = cppi_bd_alloc(c);
148 } while (bd);
149 c->last_processed = NULL;
150 }
151
cppi_controller_start(struct dma_controller * c)152 static int __init cppi_controller_start(struct dma_controller *c)
153 {
154 struct cppi *controller;
155 void __iomem *tibase;
156 int i;
157
158 controller = container_of(c, struct cppi, controller);
159
160 /* do whatever is necessary to start controller */
161 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
162 controller->tx[i].transmit = true;
163 controller->tx[i].index = i;
164 }
165 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
166 controller->rx[i].transmit = false;
167 controller->rx[i].index = i;
168 }
169
170 /* setup BD list on a per channel basis */
171 for (i = 0; i < ARRAY_SIZE(controller->tx); i++)
172 cppi_pool_init(controller, controller->tx + i);
173 for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
174 cppi_pool_init(controller, controller->rx + i);
175
176 tibase = controller->tibase;
177 INIT_LIST_HEAD(&controller->tx_complete);
178
179 /* initialise tx/rx channel head pointers to zero */
180 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
181 struct cppi_channel *tx_ch = controller->tx + i;
182 struct cppi_tx_stateram __iomem *tx;
183
184 INIT_LIST_HEAD(&tx_ch->tx_complete);
185
186 tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i);
187 tx_ch->state_ram = tx;
188 cppi_reset_tx(tx, 0);
189 }
190 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
191 struct cppi_channel *rx_ch = controller->rx + i;
192 struct cppi_rx_stateram __iomem *rx;
193
194 INIT_LIST_HEAD(&rx_ch->tx_complete);
195
196 rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i);
197 rx_ch->state_ram = rx;
198 cppi_reset_rx(rx);
199 }
200
201 /* enable individual cppi channels */
202 musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG,
203 DAVINCI_DMA_ALL_CHANNELS_ENABLE);
204 musb_writel(tibase, DAVINCI_RXCPPI_INTENAB_REG,
205 DAVINCI_DMA_ALL_CHANNELS_ENABLE);
206
207 /* enable tx/rx CPPI control */
208 musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
209 musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
210
211 /* disable RNDIS mode, also host rx RNDIS autorequest */
212 musb_writel(tibase, DAVINCI_RNDIS_REG, 0);
213 musb_writel(tibase, DAVINCI_AUTOREQ_REG, 0);
214
215 return 0;
216 }
217
218 /*
219 * Stop DMA controller
220 *
221 * De-Init the DMA controller as necessary.
222 */
223
cppi_controller_stop(struct dma_controller * c)224 static int cppi_controller_stop(struct dma_controller *c)
225 {
226 struct cppi *controller;
227 void __iomem *tibase;
228 int i;
229 struct musb *musb;
230
231 controller = container_of(c, struct cppi, controller);
232 musb = controller->musb;
233
234 tibase = controller->tibase;
235 /* DISABLE INDIVIDUAL CHANNEL Interrupts */
236 musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG,
237 DAVINCI_DMA_ALL_CHANNELS_ENABLE);
238 musb_writel(tibase, DAVINCI_RXCPPI_INTCLR_REG,
239 DAVINCI_DMA_ALL_CHANNELS_ENABLE);
240
241 dev_dbg(musb->controller, "Tearing down RX and TX Channels\n");
242 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
243 /* FIXME restructure of txdma to use bds like rxdma */
244 controller->tx[i].last_processed = NULL;
245 cppi_pool_free(controller->tx + i);
246 }
247 for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
248 cppi_pool_free(controller->rx + i);
249
250 /* in Tx Case proper teardown is supported. We resort to disabling
251 * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is
252 * complete TX CPPI cannot be disabled.
253 */
254 /*disable tx/rx cppi */
255 musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
256 musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
257
258 return 0;
259 }
260
261 /* While dma channel is allocated, we only want the core irqs active
262 * for fault reports, otherwise we'd get irqs that we don't care about.
263 * Except for TX irqs, where dma done != fifo empty and reusable ...
264 *
265 * NOTE: docs don't say either way, but irq masking **enables** irqs.
266 *
267 * REVISIT same issue applies to pure PIO usage too, and non-cppi dma...
268 */
core_rxirq_disable(void __iomem * tibase,unsigned epnum)269 static inline void core_rxirq_disable(void __iomem *tibase, unsigned epnum)
270 {
271 musb_writel(tibase, DAVINCI_USB_INT_MASK_CLR_REG, 1 << (epnum + 8));
272 }
273
core_rxirq_enable(void __iomem * tibase,unsigned epnum)274 static inline void core_rxirq_enable(void __iomem *tibase, unsigned epnum)
275 {
276 musb_writel(tibase, DAVINCI_USB_INT_MASK_SET_REG, 1 << (epnum + 8));
277 }
278
279
280 /*
281 * Allocate a CPPI Channel for DMA. With CPPI, channels are bound to
282 * each transfer direction of a non-control endpoint, so allocating
283 * (and deallocating) is mostly a way to notice bad housekeeping on
284 * the software side. We assume the irqs are always active.
285 */
286 static struct dma_channel *
cppi_channel_allocate(struct dma_controller * c,struct musb_hw_ep * ep,u8 transmit)287 cppi_channel_allocate(struct dma_controller *c,
288 struct musb_hw_ep *ep, u8 transmit)
289 {
290 struct cppi *controller;
291 u8 index;
292 struct cppi_channel *cppi_ch;
293 void __iomem *tibase;
294 struct musb *musb;
295
296 controller = container_of(c, struct cppi, controller);
297 tibase = controller->tibase;
298 musb = controller->musb;
299
300 /* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */
301 index = ep->epnum - 1;
302
303 /* return the corresponding CPPI Channel Handle, and
304 * probably disable the non-CPPI irq until we need it.
305 */
306 if (transmit) {
307 if (index >= ARRAY_SIZE(controller->tx)) {
308 dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'T', index);
309 return NULL;
310 }
311 cppi_ch = controller->tx + index;
312 } else {
313 if (index >= ARRAY_SIZE(controller->rx)) {
314 dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'R', index);
315 return NULL;
316 }
317 cppi_ch = controller->rx + index;
318 core_rxirq_disable(tibase, ep->epnum);
319 }
320
321 /* REVISIT make this an error later once the same driver code works
322 * with the other DMA engine too
323 */
324 if (cppi_ch->hw_ep)
325 dev_dbg(musb->controller, "re-allocating DMA%d %cX channel %p\n",
326 index, transmit ? 'T' : 'R', cppi_ch);
327 cppi_ch->hw_ep = ep;
328 cppi_ch->channel.status = MUSB_DMA_STATUS_FREE;
329 cppi_ch->channel.max_len = 0x7fffffff;
330
331 dev_dbg(musb->controller, "Allocate CPPI%d %cX\n", index, transmit ? 'T' : 'R');
332 return &cppi_ch->channel;
333 }
334
335 /* Release a CPPI Channel. */
cppi_channel_release(struct dma_channel * channel)336 static void cppi_channel_release(struct dma_channel *channel)
337 {
338 struct cppi_channel *c;
339 void __iomem *tibase;
340
341 /* REVISIT: for paranoia, check state and abort if needed... */
342
343 c = container_of(channel, struct cppi_channel, channel);
344 tibase = c->controller->tibase;
345 if (!c->hw_ep)
346 dev_dbg(c->controller->musb->controller,
347 "releasing idle DMA channel %p\n", c);
348 else if (!c->transmit)
349 core_rxirq_enable(tibase, c->index + 1);
350
351 /* for now, leave its cppi IRQ enabled (we won't trigger it) */
352 c->hw_ep = NULL;
353 channel->status = MUSB_DMA_STATUS_UNKNOWN;
354 }
355
356 /* Context: controller irqlocked */
357 static void
cppi_dump_rx(int level,struct cppi_channel * c,const char * tag)358 cppi_dump_rx(int level, struct cppi_channel *c, const char *tag)
359 {
360 void __iomem *base = c->controller->mregs;
361 struct cppi_rx_stateram __iomem *rx = c->state_ram;
362
363 musb_ep_select(base, c->index + 1);
364
365 dev_dbg(c->controller->musb->controller,
366 "RX DMA%d%s: %d left, csr %04x, "
367 "%08x H%08x S%08x C%08x, "
368 "B%08x L%08x %08x .. %08x"
369 "\n",
370 c->index, tag,
371 musb_readl(c->controller->tibase,
372 DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index),
373 musb_readw(c->hw_ep->regs, MUSB_RXCSR),
374
375 musb_readl(&rx->rx_skipbytes, 0),
376 musb_readl(&rx->rx_head, 0),
377 musb_readl(&rx->rx_sop, 0),
378 musb_readl(&rx->rx_current, 0),
379
380 musb_readl(&rx->rx_buf_current, 0),
381 musb_readl(&rx->rx_len_len, 0),
382 musb_readl(&rx->rx_cnt_cnt, 0),
383 musb_readl(&rx->rx_complete, 0)
384 );
385 }
386
387 /* Context: controller irqlocked */
388 static void
cppi_dump_tx(int level,struct cppi_channel * c,const char * tag)389 cppi_dump_tx(int level, struct cppi_channel *c, const char *tag)
390 {
391 void __iomem *base = c->controller->mregs;
392 struct cppi_tx_stateram __iomem *tx = c->state_ram;
393
394 musb_ep_select(base, c->index + 1);
395
396 dev_dbg(c->controller->musb->controller,
397 "TX DMA%d%s: csr %04x, "
398 "H%08x S%08x C%08x %08x, "
399 "F%08x L%08x .. %08x"
400 "\n",
401 c->index, tag,
402 musb_readw(c->hw_ep->regs, MUSB_TXCSR),
403
404 musb_readl(&tx->tx_head, 0),
405 musb_readl(&tx->tx_buf, 0),
406 musb_readl(&tx->tx_current, 0),
407 musb_readl(&tx->tx_buf_current, 0),
408
409 musb_readl(&tx->tx_info, 0),
410 musb_readl(&tx->tx_rem_len, 0),
411 /* dummy/unused word 6 */
412 musb_readl(&tx->tx_complete, 0)
413 );
414 }
415
416 /* Context: controller irqlocked */
417 static inline void
cppi_rndis_update(struct cppi_channel * c,int is_rx,void __iomem * tibase,int is_rndis)418 cppi_rndis_update(struct cppi_channel *c, int is_rx,
419 void __iomem *tibase, int is_rndis)
420 {
421 /* we may need to change the rndis flag for this cppi channel */
422 if (c->is_rndis != is_rndis) {
423 u32 value = musb_readl(tibase, DAVINCI_RNDIS_REG);
424 u32 temp = 1 << (c->index);
425
426 if (is_rx)
427 temp <<= 16;
428 if (is_rndis)
429 value |= temp;
430 else
431 value &= ~temp;
432 musb_writel(tibase, DAVINCI_RNDIS_REG, value);
433 c->is_rndis = is_rndis;
434 }
435 }
436
437 #ifdef CONFIG_USB_MUSB_DEBUG
cppi_dump_rxbd(const char * tag,struct cppi_descriptor * bd)438 static void cppi_dump_rxbd(const char *tag, struct cppi_descriptor *bd)
439 {
440 pr_debug("RXBD/%s %08x: "
441 "nxt %08x buf %08x off.blen %08x opt.plen %08x\n",
442 tag, bd->dma,
443 bd->hw_next, bd->hw_bufp, bd->hw_off_len,
444 bd->hw_options);
445 }
446 #endif
447
cppi_dump_rxq(int level,const char * tag,struct cppi_channel * rx)448 static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx)
449 {
450 #ifdef CONFIG_USB_MUSB_DEBUG
451 struct cppi_descriptor *bd;
452
453 if (!_dbg_level(level))
454 return;
455 cppi_dump_rx(level, rx, tag);
456 if (rx->last_processed)
457 cppi_dump_rxbd("last", rx->last_processed);
458 for (bd = rx->head; bd; bd = bd->next)
459 cppi_dump_rxbd("active", bd);
460 #endif
461 }
462
463
464 /* NOTE: DaVinci autoreq is ignored except for host side "RNDIS" mode RX;
465 * so we won't ever use it (see "CPPI RX Woes" below).
466 */
cppi_autoreq_update(struct cppi_channel * rx,void __iomem * tibase,int onepacket,unsigned n_bds)467 static inline int cppi_autoreq_update(struct cppi_channel *rx,
468 void __iomem *tibase, int onepacket, unsigned n_bds)
469 {
470 u32 val;
471
472 #ifdef RNDIS_RX_IS_USABLE
473 u32 tmp;
474 /* assert(is_host_active(musb)) */
475
476 /* start from "AutoReq never" */
477 tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
478 val = tmp & ~((0x3) << (rx->index * 2));
479
480 /* HCD arranged reqpkt for packet #1. we arrange int
481 * for all but the last one, maybe in two segments.
482 */
483 if (!onepacket) {
484 #if 0
485 /* use two segments, autoreq "all" then the last "never" */
486 val |= ((0x3) << (rx->index * 2));
487 n_bds--;
488 #else
489 /* one segment, autoreq "all-but-last" */
490 val |= ((0x1) << (rx->index * 2));
491 #endif
492 }
493
494 if (val != tmp) {
495 int n = 100;
496
497 /* make sure that autoreq is updated before continuing */
498 musb_writel(tibase, DAVINCI_AUTOREQ_REG, val);
499 do {
500 tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
501 if (tmp == val)
502 break;
503 cpu_relax();
504 } while (n-- > 0);
505 }
506 #endif
507
508 /* REQPKT is turned off after each segment */
509 if (n_bds && rx->channel.actual_len) {
510 void __iomem *regs = rx->hw_ep->regs;
511
512 val = musb_readw(regs, MUSB_RXCSR);
513 if (!(val & MUSB_RXCSR_H_REQPKT)) {
514 val |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS;
515 musb_writew(regs, MUSB_RXCSR, val);
516 /* flush writebuffer */
517 val = musb_readw(regs, MUSB_RXCSR);
518 }
519 }
520 return n_bds;
521 }
522
523
524 /* Buffer enqueuing Logic:
525 *
526 * - RX builds new queues each time, to help handle routine "early
527 * termination" cases (faults, including errors and short reads)
528 * more correctly.
529 *
530 * - for now, TX reuses the same queue of BDs every time
531 *
532 * REVISIT long term, we want a normal dynamic model.
533 * ... the goal will be to append to the
534 * existing queue, processing completed "dma buffers" (segments) on the fly.
535 *
536 * Otherwise we force an IRQ latency between requests, which slows us a lot
537 * (especially in "transparent" dma). Unfortunately that model seems to be
538 * inherent in the DMA model from the Mentor code, except in the rare case
539 * of transfers big enough (~128+ KB) that we could append "middle" segments
540 * in the TX paths. (RX can't do this, see below.)
541 *
542 * That's true even in the CPPI- friendly iso case, where most urbs have
543 * several small segments provided in a group and where the "packet at a time"
544 * "transparent" DMA model is always correct, even on the RX side.
545 */
546
547 /*
548 * CPPI TX:
549 * ========
550 * TX is a lot more reasonable than RX; it doesn't need to run in
551 * irq-per-packet mode very often. RNDIS mode seems to behave too
552 * (except how it handles the exactly-N-packets case). Building a
553 * txdma queue with multiple requests (urb or usb_request) looks
554 * like it would work ... but fault handling would need much testing.
555 *
556 * The main issue with TX mode RNDIS relates to transfer lengths that
557 * are an exact multiple of the packet length. It appears that there's
558 * a hiccup in that case (maybe the DMA completes before the ZLP gets
559 * written?) boiling down to not being able to rely on CPPI writing any
560 * terminating zero length packet before the next transfer is written.
561 * So that's punted to PIO; better yet, gadget drivers can avoid it.
562 *
563 * Plus, there's allegedly an undocumented constraint that rndis transfer
564 * length be a multiple of 64 bytes ... but the chip doesn't act that
565 * way, and we really don't _want_ that behavior anyway.
566 *
567 * On TX, "transparent" mode works ... although experiments have shown
568 * problems trying to use the SOP/EOP bits in different USB packets.
569 *
570 * REVISIT try to handle terminating zero length packets using CPPI
571 * instead of doing it by PIO after an IRQ. (Meanwhile, make Ethernet
572 * links avoid that issue by forcing them to avoid zlps.)
573 */
574 static void
cppi_next_tx_segment(struct musb * musb,struct cppi_channel * tx)575 cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx)
576 {
577 unsigned maxpacket = tx->maxpacket;
578 dma_addr_t addr = tx->buf_dma + tx->offset;
579 size_t length = tx->buf_len - tx->offset;
580 struct cppi_descriptor *bd;
581 unsigned n_bds;
582 unsigned i;
583 struct cppi_tx_stateram __iomem *tx_ram = tx->state_ram;
584 int rndis;
585
586 /* TX can use the CPPI "rndis" mode, where we can probably fit this
587 * transfer in one BD and one IRQ. The only time we would NOT want
588 * to use it is when hardware constraints prevent it, or if we'd
589 * trigger the "send a ZLP?" confusion.
590 */
591 rndis = (maxpacket & 0x3f) == 0
592 && length > maxpacket
593 && length < 0xffff
594 && (length % maxpacket) != 0;
595
596 if (rndis) {
597 maxpacket = length;
598 n_bds = 1;
599 } else {
600 n_bds = length / maxpacket;
601 if (!length || (length % maxpacket))
602 n_bds++;
603 n_bds = min(n_bds, (unsigned) NUM_TXCHAN_BD);
604 length = min(n_bds * maxpacket, length);
605 }
606
607 dev_dbg(musb->controller, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u\n",
608 tx->index,
609 maxpacket,
610 rndis ? "rndis" : "transparent",
611 n_bds,
612 (unsigned long long)addr, length);
613
614 cppi_rndis_update(tx, 0, musb->ctrl_base, rndis);
615
616 /* assuming here that channel_program is called during
617 * transfer initiation ... current code maintains state
618 * for one outstanding request only (no queues, not even
619 * the implicit ones of an iso urb).
620 */
621
622 bd = tx->freelist;
623 tx->head = bd;
624 tx->last_processed = NULL;
625
626 /* FIXME use BD pool like RX side does, and just queue
627 * the minimum number for this request.
628 */
629
630 /* Prepare queue of BDs first, then hand it to hardware.
631 * All BDs except maybe the last should be of full packet
632 * size; for RNDIS there _is_ only that last packet.
633 */
634 for (i = 0; i < n_bds; ) {
635 if (++i < n_bds && bd->next)
636 bd->hw_next = bd->next->dma;
637 else
638 bd->hw_next = 0;
639
640 bd->hw_bufp = tx->buf_dma + tx->offset;
641
642 /* FIXME set EOP only on the last packet,
643 * SOP only on the first ... avoid IRQs
644 */
645 if ((tx->offset + maxpacket) <= tx->buf_len) {
646 tx->offset += maxpacket;
647 bd->hw_off_len = maxpacket;
648 bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
649 | CPPI_OWN_SET | maxpacket;
650 } else {
651 /* only this one may be a partial USB Packet */
652 u32 partial_len;
653
654 partial_len = tx->buf_len - tx->offset;
655 tx->offset = tx->buf_len;
656 bd->hw_off_len = partial_len;
657
658 bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
659 | CPPI_OWN_SET | partial_len;
660 if (partial_len == 0)
661 bd->hw_options |= CPPI_ZERO_SET;
662 }
663
664 dev_dbg(musb->controller, "TXBD %p: nxt %08x buf %08x len %04x opt %08x\n",
665 bd, bd->hw_next, bd->hw_bufp,
666 bd->hw_off_len, bd->hw_options);
667
668 /* update the last BD enqueued to the list */
669 tx->tail = bd;
670 bd = bd->next;
671 }
672
673 /* BDs live in DMA-coherent memory, but writes might be pending */
674 cpu_drain_writebuffer();
675
676 /* Write to the HeadPtr in state RAM to trigger */
677 musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma);
678
679 cppi_dump_tx(5, tx, "/S");
680 }
681
682 /*
683 * CPPI RX Woes:
684 * =============
685 * Consider a 1KB bulk RX buffer in two scenarios: (a) it's fed two 300 byte
686 * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back.
687 * (Full speed transfers have similar scenarios.)
688 *
689 * The correct behavior for Linux is that (a) fills the buffer with 300 bytes,
690 * and the next packet goes into a buffer that's queued later; while (b) fills
691 * the buffer with 1024 bytes. How to do that with CPPI?
692 *
693 * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but
694 * (b) loses **BADLY** because nothing (!) happens when that second packet
695 * fills the buffer, much less when a third one arrives. (Which makes this
696 * not a "true" RNDIS mode. In the RNDIS protocol short-packet termination
697 * is optional, and it's fine if peripherals -- not hosts! -- pad messages
698 * out to end-of-buffer. Standard PCI host controller DMA descriptors
699 * implement that mode by default ... which is no accident.)
700 *
701 * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have
702 * converse problems: (b) is handled right, but (a) loses badly. CPPI RX
703 * ignores SOP/EOP markings and processes both of those BDs; so both packets
704 * are loaded into the buffer (with a 212 byte gap between them), and the next
705 * buffer queued will NOT get its 300 bytes of data. (It seems like SOP/EOP
706 * are intended as outputs for RX queues, not inputs...)
707 *
708 * - A variant of "transparent" mode -- one BD at a time -- is the only way to
709 * reliably make both cases work, with software handling both cases correctly
710 * and at the significant penalty of needing an IRQ per packet. (The lack of
711 * I/O overlap can be slightly ameliorated by enabling double buffering.)
712 *
713 * So how to get rid of IRQ-per-packet? The transparent multi-BD case could
714 * be used in special cases like mass storage, which sets URB_SHORT_NOT_OK
715 * (or maybe its peripheral side counterpart) to flag (a) scenarios as errors
716 * with guaranteed driver level fault recovery and scrubbing out what's left
717 * of that garbaged datastream.
718 *
719 * But there seems to be no way to identify the cases where CPPI RNDIS mode
720 * is appropriate -- which do NOT include RNDIS host drivers, but do include
721 * the CDC Ethernet driver! -- and the documentation is incomplete/wrong.
722 * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic
723 * that applies best on the peripheral side (and which could fail rudely).
724 *
725 * Leaving only "transparent" mode; we avoid multi-bd modes in almost all
726 * cases other than mass storage class. Otherwise we're correct but slow,
727 * since CPPI penalizes our need for a "true RNDIS" default mode.
728 */
729
730
731 /* Heuristic, intended to kick in for ethernet/rndis peripheral ONLY
732 *
733 * IFF
734 * (a) peripheral mode ... since rndis peripherals could pad their
735 * writes to hosts, causing i/o failure; or we'd have to cope with
736 * a largely unknowable variety of host side protocol variants
737 * (b) and short reads are NOT errors ... since full reads would
738 * cause those same i/o failures
739 * (c) and read length is
740 * - less than 64KB (max per cppi descriptor)
741 * - not a multiple of 4096 (g_zero default, full reads typical)
742 * - N (>1) packets long, ditto (full reads not EXPECTED)
743 * THEN
744 * try rx rndis mode
745 *
746 * Cost of heuristic failing: RXDMA wedges at the end of transfers that
747 * fill out the whole buffer. Buggy host side usb network drivers could
748 * trigger that, but "in the field" such bugs seem to be all but unknown.
749 *
750 * So this module parameter lets the heuristic be disabled. When using
751 * gadgetfs, the heuristic will probably need to be disabled.
752 */
753 static bool cppi_rx_rndis = 1;
754
755 module_param(cppi_rx_rndis, bool, 0);
756 MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic");
757
758
759 /**
760 * cppi_next_rx_segment - dma read for the next chunk of a buffer
761 * @musb: the controller
762 * @rx: dma channel
763 * @onepacket: true unless caller treats short reads as errors, and
764 * performs fault recovery above usbcore.
765 * Context: controller irqlocked
766 *
767 * See above notes about why we can't use multi-BD RX queues except in
768 * rare cases (mass storage class), and can never use the hardware "rndis"
769 * mode (since it's not a "true" RNDIS mode) with complete safety..
770 *
771 * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in
772 * code to recover from corrupted datastreams after each short transfer.
773 */
774 static void
cppi_next_rx_segment(struct musb * musb,struct cppi_channel * rx,int onepacket)775 cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket)
776 {
777 unsigned maxpacket = rx->maxpacket;
778 dma_addr_t addr = rx->buf_dma + rx->offset;
779 size_t length = rx->buf_len - rx->offset;
780 struct cppi_descriptor *bd, *tail;
781 unsigned n_bds;
782 unsigned i;
783 void __iomem *tibase = musb->ctrl_base;
784 int is_rndis = 0;
785 struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram;
786
787 if (onepacket) {
788 /* almost every USB driver, host or peripheral side */
789 n_bds = 1;
790
791 /* maybe apply the heuristic above */
792 if (cppi_rx_rndis
793 && is_peripheral_active(musb)
794 && length > maxpacket
795 && (length & ~0xffff) == 0
796 && (length & 0x0fff) != 0
797 && (length & (maxpacket - 1)) == 0) {
798 maxpacket = length;
799 is_rndis = 1;
800 }
801 } else {
802 /* virtually nothing except mass storage class */
803 if (length > 0xffff) {
804 n_bds = 0xffff / maxpacket;
805 length = n_bds * maxpacket;
806 } else {
807 n_bds = length / maxpacket;
808 if (length % maxpacket)
809 n_bds++;
810 }
811 if (n_bds == 1)
812 onepacket = 1;
813 else
814 n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD);
815 }
816
817 /* In host mode, autorequest logic can generate some IN tokens; it's
818 * tricky since we can't leave REQPKT set in RXCSR after the transfer
819 * finishes. So: multipacket transfers involve two or more segments.
820 * And always at least two IRQs ... RNDIS mode is not an option.
821 */
822 if (is_host_active(musb))
823 n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds);
824
825 cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis);
826
827 length = min(n_bds * maxpacket, length);
828
829 dev_dbg(musb->controller, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) "
830 "dma 0x%llx len %u %u/%u\n",
831 rx->index, maxpacket,
832 onepacket
833 ? (is_rndis ? "rndis" : "onepacket")
834 : "multipacket",
835 n_bds,
836 musb_readl(tibase,
837 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
838 & 0xffff,
839 (unsigned long long)addr, length,
840 rx->channel.actual_len, rx->buf_len);
841
842 /* only queue one segment at a time, since the hardware prevents
843 * correct queue shutdown after unexpected short packets
844 */
845 bd = cppi_bd_alloc(rx);
846 rx->head = bd;
847
848 /* Build BDs for all packets in this segment */
849 for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) {
850 u32 bd_len;
851
852 if (i) {
853 bd = cppi_bd_alloc(rx);
854 if (!bd)
855 break;
856 tail->next = bd;
857 tail->hw_next = bd->dma;
858 }
859 bd->hw_next = 0;
860
861 /* all but the last packet will be maxpacket size */
862 if (maxpacket < length)
863 bd_len = maxpacket;
864 else
865 bd_len = length;
866
867 bd->hw_bufp = addr;
868 addr += bd_len;
869 rx->offset += bd_len;
870
871 bd->hw_off_len = (0 /*offset*/ << 16) + bd_len;
872 bd->buflen = bd_len;
873
874 bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0);
875 length -= bd_len;
876 }
877
878 /* we always expect at least one reusable BD! */
879 if (!tail) {
880 WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds);
881 return;
882 } else if (i < n_bds)
883 WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds);
884
885 tail->next = NULL;
886 tail->hw_next = 0;
887
888 bd = rx->head;
889 rx->tail = tail;
890
891 /* short reads and other faults should terminate this entire
892 * dma segment. we want one "dma packet" per dma segment, not
893 * one per USB packet, terminating the whole queue at once...
894 * NOTE that current hardware seems to ignore SOP and EOP.
895 */
896 bd->hw_options |= CPPI_SOP_SET;
897 tail->hw_options |= CPPI_EOP_SET;
898
899 #ifdef CONFIG_USB_MUSB_DEBUG
900 if (_dbg_level(5)) {
901 struct cppi_descriptor *d;
902
903 for (d = rx->head; d; d = d->next)
904 cppi_dump_rxbd("S", d);
905 }
906 #endif
907
908 /* in case the preceding transfer left some state... */
909 tail = rx->last_processed;
910 if (tail) {
911 tail->next = bd;
912 tail->hw_next = bd->dma;
913 }
914
915 core_rxirq_enable(tibase, rx->index + 1);
916
917 /* BDs live in DMA-coherent memory, but writes might be pending */
918 cpu_drain_writebuffer();
919
920 /* REVISIT specs say to write this AFTER the BUFCNT register
921 * below ... but that loses badly.
922 */
923 musb_writel(&rx_ram->rx_head, 0, bd->dma);
924
925 /* bufferCount must be at least 3, and zeroes on completion
926 * unless it underflows below zero, or stops at two, or keeps
927 * growing ... grr.
928 */
929 i = musb_readl(tibase,
930 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
931 & 0xffff;
932
933 if (!i)
934 musb_writel(tibase,
935 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
936 n_bds + 2);
937 else if (n_bds > (i - 3))
938 musb_writel(tibase,
939 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
940 n_bds - (i - 3));
941
942 i = musb_readl(tibase,
943 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
944 & 0xffff;
945 if (i < (2 + n_bds)) {
946 dev_dbg(musb->controller, "bufcnt%d underrun - %d (for %d)\n",
947 rx->index, i, n_bds);
948 musb_writel(tibase,
949 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
950 n_bds + 2);
951 }
952
953 cppi_dump_rx(4, rx, "/S");
954 }
955
956 /**
957 * cppi_channel_program - program channel for data transfer
958 * @ch: the channel
959 * @maxpacket: max packet size
960 * @mode: For RX, 1 unless the usb protocol driver promised to treat
961 * all short reads as errors and kick in high level fault recovery.
962 * For TX, ignored because of RNDIS mode races/glitches.
963 * @dma_addr: dma address of buffer
964 * @len: length of buffer
965 * Context: controller irqlocked
966 */
cppi_channel_program(struct dma_channel * ch,u16 maxpacket,u8 mode,dma_addr_t dma_addr,u32 len)967 static int cppi_channel_program(struct dma_channel *ch,
968 u16 maxpacket, u8 mode,
969 dma_addr_t dma_addr, u32 len)
970 {
971 struct cppi_channel *cppi_ch;
972 struct cppi *controller;
973 struct musb *musb;
974
975 cppi_ch = container_of(ch, struct cppi_channel, channel);
976 controller = cppi_ch->controller;
977 musb = controller->musb;
978
979 switch (ch->status) {
980 case MUSB_DMA_STATUS_BUS_ABORT:
981 case MUSB_DMA_STATUS_CORE_ABORT:
982 /* fault irq handler should have handled cleanup */
983 WARNING("%cX DMA%d not cleaned up after abort!\n",
984 cppi_ch->transmit ? 'T' : 'R',
985 cppi_ch->index);
986 /* WARN_ON(1); */
987 break;
988 case MUSB_DMA_STATUS_BUSY:
989 WARNING("program active channel? %cX DMA%d\n",
990 cppi_ch->transmit ? 'T' : 'R',
991 cppi_ch->index);
992 /* WARN_ON(1); */
993 break;
994 case MUSB_DMA_STATUS_UNKNOWN:
995 dev_dbg(musb->controller, "%cX DMA%d not allocated!\n",
996 cppi_ch->transmit ? 'T' : 'R',
997 cppi_ch->index);
998 /* FALLTHROUGH */
999 case MUSB_DMA_STATUS_FREE:
1000 break;
1001 }
1002
1003 ch->status = MUSB_DMA_STATUS_BUSY;
1004
1005 /* set transfer parameters, then queue up its first segment */
1006 cppi_ch->buf_dma = dma_addr;
1007 cppi_ch->offset = 0;
1008 cppi_ch->maxpacket = maxpacket;
1009 cppi_ch->buf_len = len;
1010 cppi_ch->channel.actual_len = 0;
1011
1012 /* TX channel? or RX? */
1013 if (cppi_ch->transmit)
1014 cppi_next_tx_segment(musb, cppi_ch);
1015 else
1016 cppi_next_rx_segment(musb, cppi_ch, mode);
1017
1018 return true;
1019 }
1020
cppi_rx_scan(struct cppi * cppi,unsigned ch)1021 static bool cppi_rx_scan(struct cppi *cppi, unsigned ch)
1022 {
1023 struct cppi_channel *rx = &cppi->rx[ch];
1024 struct cppi_rx_stateram __iomem *state = rx->state_ram;
1025 struct cppi_descriptor *bd;
1026 struct cppi_descriptor *last = rx->last_processed;
1027 bool completed = false;
1028 bool acked = false;
1029 int i;
1030 dma_addr_t safe2ack;
1031 void __iomem *regs = rx->hw_ep->regs;
1032 struct musb *musb = cppi->musb;
1033
1034 cppi_dump_rx(6, rx, "/K");
1035
1036 bd = last ? last->next : rx->head;
1037 if (!bd)
1038 return false;
1039
1040 /* run through all completed BDs */
1041 for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0);
1042 (safe2ack || completed) && bd && i < NUM_RXCHAN_BD;
1043 i++, bd = bd->next) {
1044 u16 len;
1045
1046 /* catch latest BD writes from CPPI */
1047 rmb();
1048 if (!completed && (bd->hw_options & CPPI_OWN_SET))
1049 break;
1050
1051 dev_dbg(musb->controller, "C/RXBD %llx: nxt %08x buf %08x "
1052 "off.len %08x opt.len %08x (%d)\n",
1053 (unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp,
1054 bd->hw_off_len, bd->hw_options,
1055 rx->channel.actual_len);
1056
1057 /* actual packet received length */
1058 if ((bd->hw_options & CPPI_SOP_SET) && !completed)
1059 len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK;
1060 else
1061 len = 0;
1062
1063 if (bd->hw_options & CPPI_EOQ_MASK)
1064 completed = true;
1065
1066 if (!completed && len < bd->buflen) {
1067 /* NOTE: when we get a short packet, RXCSR_H_REQPKT
1068 * must have been cleared, and no more DMA packets may
1069 * active be in the queue... TI docs didn't say, but
1070 * CPPI ignores those BDs even though OWN is still set.
1071 */
1072 completed = true;
1073 dev_dbg(musb->controller, "rx short %d/%d (%d)\n",
1074 len, bd->buflen,
1075 rx->channel.actual_len);
1076 }
1077
1078 /* If we got here, we expect to ack at least one BD; meanwhile
1079 * CPPI may completing other BDs while we scan this list...
1080 *
1081 * RACE: we can notice OWN cleared before CPPI raises the
1082 * matching irq by writing that BD as the completion pointer.
1083 * In such cases, stop scanning and wait for the irq, avoiding
1084 * lost acks and states where BD ownership is unclear.
1085 */
1086 if (bd->dma == safe2ack) {
1087 musb_writel(&state->rx_complete, 0, safe2ack);
1088 safe2ack = musb_readl(&state->rx_complete, 0);
1089 acked = true;
1090 if (bd->dma == safe2ack)
1091 safe2ack = 0;
1092 }
1093
1094 rx->channel.actual_len += len;
1095
1096 cppi_bd_free(rx, last);
1097 last = bd;
1098
1099 /* stop scanning on end-of-segment */
1100 if (bd->hw_next == 0)
1101 completed = true;
1102 }
1103 rx->last_processed = last;
1104
1105 /* dma abort, lost ack, or ... */
1106 if (!acked && last) {
1107 int csr;
1108
1109 if (safe2ack == 0 || safe2ack == rx->last_processed->dma)
1110 musb_writel(&state->rx_complete, 0, safe2ack);
1111 if (safe2ack == 0) {
1112 cppi_bd_free(rx, last);
1113 rx->last_processed = NULL;
1114
1115 /* if we land here on the host side, H_REQPKT will
1116 * be clear and we need to restart the queue...
1117 */
1118 WARN_ON(rx->head);
1119 }
1120 musb_ep_select(cppi->mregs, rx->index + 1);
1121 csr = musb_readw(regs, MUSB_RXCSR);
1122 if (csr & MUSB_RXCSR_DMAENAB) {
1123 dev_dbg(musb->controller, "list%d %p/%p, last %llx%s, csr %04x\n",
1124 rx->index,
1125 rx->head, rx->tail,
1126 rx->last_processed
1127 ? (unsigned long long)
1128 rx->last_processed->dma
1129 : 0,
1130 completed ? ", completed" : "",
1131 csr);
1132 cppi_dump_rxq(4, "/what?", rx);
1133 }
1134 }
1135 if (!completed) {
1136 int csr;
1137
1138 rx->head = bd;
1139
1140 /* REVISIT seems like "autoreq all but EOP" doesn't...
1141 * setting it here "should" be racey, but seems to work
1142 */
1143 csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
1144 if (is_host_active(cppi->musb)
1145 && bd
1146 && !(csr & MUSB_RXCSR_H_REQPKT)) {
1147 csr |= MUSB_RXCSR_H_REQPKT;
1148 musb_writew(regs, MUSB_RXCSR,
1149 MUSB_RXCSR_H_WZC_BITS | csr);
1150 csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
1151 }
1152 } else {
1153 rx->head = NULL;
1154 rx->tail = NULL;
1155 }
1156
1157 cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned");
1158 return completed;
1159 }
1160
cppi_interrupt(int irq,void * dev_id)1161 irqreturn_t cppi_interrupt(int irq, void *dev_id)
1162 {
1163 struct musb *musb = dev_id;
1164 struct cppi *cppi;
1165 void __iomem *tibase;
1166 struct musb_hw_ep *hw_ep = NULL;
1167 u32 rx, tx;
1168 int i, index;
1169 unsigned long uninitialized_var(flags);
1170
1171 cppi = container_of(musb->dma_controller, struct cppi, controller);
1172 if (cppi->irq)
1173 spin_lock_irqsave(&musb->lock, flags);
1174
1175 tibase = musb->ctrl_base;
1176
1177 tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG);
1178 rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG);
1179
1180 if (!tx && !rx) {
1181 if (cppi->irq)
1182 spin_unlock_irqrestore(&musb->lock, flags);
1183 return IRQ_NONE;
1184 }
1185
1186 dev_dbg(musb->controller, "CPPI IRQ Tx%x Rx%x\n", tx, rx);
1187
1188 /* process TX channels */
1189 for (index = 0; tx; tx = tx >> 1, index++) {
1190 struct cppi_channel *tx_ch;
1191 struct cppi_tx_stateram __iomem *tx_ram;
1192 bool completed = false;
1193 struct cppi_descriptor *bd;
1194
1195 if (!(tx & 1))
1196 continue;
1197
1198 tx_ch = cppi->tx + index;
1199 tx_ram = tx_ch->state_ram;
1200
1201 /* FIXME need a cppi_tx_scan() routine, which
1202 * can also be called from abort code
1203 */
1204
1205 cppi_dump_tx(5, tx_ch, "/E");
1206
1207 bd = tx_ch->head;
1208
1209 /*
1210 * If Head is null then this could mean that a abort interrupt
1211 * that needs to be acknowledged.
1212 */
1213 if (NULL == bd) {
1214 dev_dbg(musb->controller, "null BD\n");
1215 musb_writel(&tx_ram->tx_complete, 0, 0);
1216 continue;
1217 }
1218
1219 /* run through all completed BDs */
1220 for (i = 0; !completed && bd && i < NUM_TXCHAN_BD;
1221 i++, bd = bd->next) {
1222 u16 len;
1223
1224 /* catch latest BD writes from CPPI */
1225 rmb();
1226 if (bd->hw_options & CPPI_OWN_SET)
1227 break;
1228
1229 dev_dbg(musb->controller, "C/TXBD %p n %x b %x off %x opt %x\n",
1230 bd, bd->hw_next, bd->hw_bufp,
1231 bd->hw_off_len, bd->hw_options);
1232
1233 len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK;
1234 tx_ch->channel.actual_len += len;
1235
1236 tx_ch->last_processed = bd;
1237
1238 /* write completion register to acknowledge
1239 * processing of completed BDs, and possibly
1240 * release the IRQ; EOQ might not be set ...
1241 *
1242 * REVISIT use the same ack strategy as rx
1243 *
1244 * REVISIT have observed bit 18 set; huh??
1245 */
1246 /* if ((bd->hw_options & CPPI_EOQ_MASK)) */
1247 musb_writel(&tx_ram->tx_complete, 0, bd->dma);
1248
1249 /* stop scanning on end-of-segment */
1250 if (bd->hw_next == 0)
1251 completed = true;
1252 }
1253
1254 /* on end of segment, maybe go to next one */
1255 if (completed) {
1256 /* cppi_dump_tx(4, tx_ch, "/complete"); */
1257
1258 /* transfer more, or report completion */
1259 if (tx_ch->offset >= tx_ch->buf_len) {
1260 tx_ch->head = NULL;
1261 tx_ch->tail = NULL;
1262 tx_ch->channel.status = MUSB_DMA_STATUS_FREE;
1263
1264 hw_ep = tx_ch->hw_ep;
1265
1266 musb_dma_completion(musb, index + 1, 1);
1267
1268 } else {
1269 /* Bigger transfer than we could fit in
1270 * that first batch of descriptors...
1271 */
1272 cppi_next_tx_segment(musb, tx_ch);
1273 }
1274 } else
1275 tx_ch->head = bd;
1276 }
1277
1278 /* Start processing the RX block */
1279 for (index = 0; rx; rx = rx >> 1, index++) {
1280
1281 if (rx & 1) {
1282 struct cppi_channel *rx_ch;
1283
1284 rx_ch = cppi->rx + index;
1285
1286 /* let incomplete dma segments finish */
1287 if (!cppi_rx_scan(cppi, index))
1288 continue;
1289
1290 /* start another dma segment if needed */
1291 if (rx_ch->channel.actual_len != rx_ch->buf_len
1292 && rx_ch->channel.actual_len
1293 == rx_ch->offset) {
1294 cppi_next_rx_segment(musb, rx_ch, 1);
1295 continue;
1296 }
1297
1298 /* all segments completed! */
1299 rx_ch->channel.status = MUSB_DMA_STATUS_FREE;
1300
1301 hw_ep = rx_ch->hw_ep;
1302
1303 core_rxirq_disable(tibase, index + 1);
1304 musb_dma_completion(musb, index + 1, 0);
1305 }
1306 }
1307
1308 /* write to CPPI EOI register to re-enable interrupts */
1309 musb_writel(tibase, DAVINCI_CPPI_EOI_REG, 0);
1310
1311 if (cppi->irq)
1312 spin_unlock_irqrestore(&musb->lock, flags);
1313
1314 return IRQ_HANDLED;
1315 }
1316 EXPORT_SYMBOL_GPL(cppi_interrupt);
1317
1318 /* Instantiate a software object representing a DMA controller. */
1319 struct dma_controller *__init
dma_controller_create(struct musb * musb,void __iomem * mregs)1320 dma_controller_create(struct musb *musb, void __iomem *mregs)
1321 {
1322 struct cppi *controller;
1323 struct device *dev = musb->controller;
1324 struct platform_device *pdev = to_platform_device(dev);
1325 int irq = platform_get_irq_byname(pdev, "dma");
1326
1327 controller = kzalloc(sizeof *controller, GFP_KERNEL);
1328 if (!controller)
1329 return NULL;
1330
1331 controller->mregs = mregs;
1332 controller->tibase = mregs - DAVINCI_BASE_OFFSET;
1333
1334 controller->musb = musb;
1335 controller->controller.start = cppi_controller_start;
1336 controller->controller.stop = cppi_controller_stop;
1337 controller->controller.channel_alloc = cppi_channel_allocate;
1338 controller->controller.channel_release = cppi_channel_release;
1339 controller->controller.channel_program = cppi_channel_program;
1340 controller->controller.channel_abort = cppi_channel_abort;
1341
1342 /* NOTE: allocating from on-chip SRAM would give the least
1343 * contention for memory access, if that ever matters here.
1344 */
1345
1346 /* setup BufferPool */
1347 controller->pool = dma_pool_create("cppi",
1348 controller->musb->controller,
1349 sizeof(struct cppi_descriptor),
1350 CPPI_DESCRIPTOR_ALIGN, 0);
1351 if (!controller->pool) {
1352 kfree(controller);
1353 return NULL;
1354 }
1355
1356 if (irq > 0) {
1357 if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) {
1358 dev_err(dev, "request_irq %d failed!\n", irq);
1359 dma_controller_destroy(&controller->controller);
1360 return NULL;
1361 }
1362 controller->irq = irq;
1363 }
1364
1365 return &controller->controller;
1366 }
1367
1368 /*
1369 * Destroy a previously-instantiated DMA controller.
1370 */
dma_controller_destroy(struct dma_controller * c)1371 void dma_controller_destroy(struct dma_controller *c)
1372 {
1373 struct cppi *cppi;
1374
1375 cppi = container_of(c, struct cppi, controller);
1376
1377 if (cppi->irq)
1378 free_irq(cppi->irq, cppi->musb);
1379
1380 /* assert: caller stopped the controller first */
1381 dma_pool_destroy(cppi->pool);
1382
1383 kfree(cppi);
1384 }
1385
1386 /*
1387 * Context: controller irqlocked, endpoint selected
1388 */
cppi_channel_abort(struct dma_channel * channel)1389 static int cppi_channel_abort(struct dma_channel *channel)
1390 {
1391 struct cppi_channel *cppi_ch;
1392 struct cppi *controller;
1393 void __iomem *mbase;
1394 void __iomem *tibase;
1395 void __iomem *regs;
1396 u32 value;
1397 struct cppi_descriptor *queue;
1398
1399 cppi_ch = container_of(channel, struct cppi_channel, channel);
1400
1401 controller = cppi_ch->controller;
1402
1403 switch (channel->status) {
1404 case MUSB_DMA_STATUS_BUS_ABORT:
1405 case MUSB_DMA_STATUS_CORE_ABORT:
1406 /* from RX or TX fault irq handler */
1407 case MUSB_DMA_STATUS_BUSY:
1408 /* the hardware needs shutting down */
1409 regs = cppi_ch->hw_ep->regs;
1410 break;
1411 case MUSB_DMA_STATUS_UNKNOWN:
1412 case MUSB_DMA_STATUS_FREE:
1413 return 0;
1414 default:
1415 return -EINVAL;
1416 }
1417
1418 if (!cppi_ch->transmit && cppi_ch->head)
1419 cppi_dump_rxq(3, "/abort", cppi_ch);
1420
1421 mbase = controller->mregs;
1422 tibase = controller->tibase;
1423
1424 queue = cppi_ch->head;
1425 cppi_ch->head = NULL;
1426 cppi_ch->tail = NULL;
1427
1428 /* REVISIT should rely on caller having done this,
1429 * and caller should rely on us not changing it.
1430 * peripheral code is safe ... check host too.
1431 */
1432 musb_ep_select(mbase, cppi_ch->index + 1);
1433
1434 if (cppi_ch->transmit) {
1435 struct cppi_tx_stateram __iomem *tx_ram;
1436 /* REVISIT put timeouts on these controller handshakes */
1437
1438 cppi_dump_tx(6, cppi_ch, " (teardown)");
1439
1440 /* teardown DMA engine then usb core */
1441 do {
1442 value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG);
1443 } while (!(value & CPPI_TEAR_READY));
1444 musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index);
1445
1446 tx_ram = cppi_ch->state_ram;
1447 do {
1448 value = musb_readl(&tx_ram->tx_complete, 0);
1449 } while (0xFFFFFFFC != value);
1450
1451 /* FIXME clean up the transfer state ... here?
1452 * the completion routine should get called with
1453 * an appropriate status code.
1454 */
1455
1456 value = musb_readw(regs, MUSB_TXCSR);
1457 value &= ~MUSB_TXCSR_DMAENAB;
1458 value |= MUSB_TXCSR_FLUSHFIFO;
1459 musb_writew(regs, MUSB_TXCSR, value);
1460 musb_writew(regs, MUSB_TXCSR, value);
1461
1462 /*
1463 * 1. Write to completion Ptr value 0x1(bit 0 set)
1464 * (write back mode)
1465 * 2. Wait for abort interrupt and then put the channel in
1466 * compare mode by writing 1 to the tx_complete register.
1467 */
1468 cppi_reset_tx(tx_ram, 1);
1469 cppi_ch->head = NULL;
1470 musb_writel(&tx_ram->tx_complete, 0, 1);
1471 cppi_dump_tx(5, cppi_ch, " (done teardown)");
1472
1473 /* REVISIT tx side _should_ clean up the same way
1474 * as the RX side ... this does no cleanup at all!
1475 */
1476
1477 } else /* RX */ {
1478 u16 csr;
1479
1480 /* NOTE: docs don't guarantee any of this works ... we
1481 * expect that if the usb core stops telling the cppi core
1482 * to pull more data from it, then it'll be safe to flush
1483 * current RX DMA state iff any pending fifo transfer is done.
1484 */
1485
1486 core_rxirq_disable(tibase, cppi_ch->index + 1);
1487
1488 /* for host, ensure ReqPkt is never set again */
1489 if (is_host_active(cppi_ch->controller->musb)) {
1490 value = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
1491 value &= ~((0x3) << (cppi_ch->index * 2));
1492 musb_writel(tibase, DAVINCI_AUTOREQ_REG, value);
1493 }
1494
1495 csr = musb_readw(regs, MUSB_RXCSR);
1496
1497 /* for host, clear (just) ReqPkt at end of current packet(s) */
1498 if (is_host_active(cppi_ch->controller->musb)) {
1499 csr |= MUSB_RXCSR_H_WZC_BITS;
1500 csr &= ~MUSB_RXCSR_H_REQPKT;
1501 } else
1502 csr |= MUSB_RXCSR_P_WZC_BITS;
1503
1504 /* clear dma enable */
1505 csr &= ~(MUSB_RXCSR_DMAENAB);
1506 musb_writew(regs, MUSB_RXCSR, csr);
1507 csr = musb_readw(regs, MUSB_RXCSR);
1508
1509 /* Quiesce: wait for current dma to finish (if not cleanup).
1510 * We can't use bit zero of stateram->rx_sop, since that
1511 * refers to an entire "DMA packet" not just emptying the
1512 * current fifo. Most segments need multiple usb packets.
1513 */
1514 if (channel->status == MUSB_DMA_STATUS_BUSY)
1515 udelay(50);
1516
1517 /* scan the current list, reporting any data that was
1518 * transferred and acking any IRQ
1519 */
1520 cppi_rx_scan(controller, cppi_ch->index);
1521
1522 /* clobber the existing state once it's idle
1523 *
1524 * NOTE: arguably, we should also wait for all the other
1525 * RX channels to quiesce (how??) and then temporarily
1526 * disable RXCPPI_CTRL_REG ... but it seems that we can
1527 * rely on the controller restarting from state ram, with
1528 * only RXCPPI_BUFCNT state being bogus. BUFCNT will
1529 * correct itself after the next DMA transfer though.
1530 *
1531 * REVISIT does using rndis mode change that?
1532 */
1533 cppi_reset_rx(cppi_ch->state_ram);
1534
1535 /* next DMA request _should_ load cppi head ptr */
1536
1537 /* ... we don't "free" that list, only mutate it in place. */
1538 cppi_dump_rx(5, cppi_ch, " (done abort)");
1539
1540 /* clean up previously pending bds */
1541 cppi_bd_free(cppi_ch, cppi_ch->last_processed);
1542 cppi_ch->last_processed = NULL;
1543
1544 while (queue) {
1545 struct cppi_descriptor *tmp = queue->next;
1546
1547 cppi_bd_free(cppi_ch, queue);
1548 queue = tmp;
1549 }
1550 }
1551
1552 channel->status = MUSB_DMA_STATUS_FREE;
1553 cppi_ch->buf_dma = 0;
1554 cppi_ch->offset = 0;
1555 cppi_ch->buf_len = 0;
1556 cppi_ch->maxpacket = 0;
1557 return 0;
1558 }
1559
1560 /* TBD Queries:
1561 *
1562 * Power Management ... probably turn off cppi during suspend, restart;
1563 * check state ram? Clocking is presumably shared with usb core.
1564 */
1565