1 /*
2 * Copyright (c) 2010 Broadcom Corporation
3 *
4 * Permission to use, copy, modify, and/or distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
11 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
13 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
14 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
16
17 #include <linux/kernel.h>
18 #include <linux/string.h>
19 #include <linux/netdevice.h>
20 #include <linux/pci.h>
21 #include <bcmdefs.h>
22 #include <bcmdevs.h>
23 #include <hndsoc.h>
24 #include <bcmutils.h>
25 #include <siutils.h>
26
27 #include <sbhnddma.h>
28 #include <hnddma.h>
29
30 #if defined(__mips__)
31 #include <asm/addrspace.h>
32 #endif
33
34 #ifdef BRCM_FULLMAC
35 #error "hnddma.c shouldn't be needed for FULLMAC"
36 #endif
37
38 /* debug/trace */
39 #ifdef BCMDBG
40 #define DMA_ERROR(args) \
41 do { \
42 if (!(*di->msg_level & 1)) \
43 ; \
44 else \
45 printk args; \
46 } while (0)
47 #define DMA_TRACE(args) \
48 do { \
49 if (!(*di->msg_level & 2)) \
50 ; \
51 else \
52 printk args; \
53 } while (0)
54 #else
55 #define DMA_ERROR(args)
56 #define DMA_TRACE(args)
57 #endif /* BCMDBG */
58
59 #define DMA_NONE(args)
60
61 #define d64txregs dregs.d64_u.txregs_64
62 #define d64rxregs dregs.d64_u.rxregs_64
63 #define txd64 dregs.d64_u.txd_64
64 #define rxd64 dregs.d64_u.rxd_64
65
66 /* default dma message level (if input msg_level pointer is null in dma_attach()) */
67 static uint dma_msg_level;
68
69 #define MAXNAMEL 8 /* 8 char names */
70
71 #define DI_INFO(dmah) ((dma_info_t *)dmah)
72
73 #define R_SM(r) (*(r))
74 #define W_SM(r, v) (*(r) = (v))
75
76 /* dma engine software state */
77 typedef struct dma_info {
78 struct hnddma_pub hnddma; /* exported structure */
79 uint *msg_level; /* message level pointer */
80 char name[MAXNAMEL]; /* callers name for diag msgs */
81
82 void *pbus; /* bus handle */
83
84 bool dma64; /* this dma engine is operating in 64-bit mode */
85 bool addrext; /* this dma engine supports DmaExtendedAddrChanges */
86
87 union {
88 struct {
89 dma64regs_t *txregs_64; /* 64-bit dma tx engine registers */
90 dma64regs_t *rxregs_64; /* 64-bit dma rx engine registers */
91 dma64dd_t *txd_64; /* pointer to dma64 tx descriptor ring */
92 dma64dd_t *rxd_64; /* pointer to dma64 rx descriptor ring */
93 } d64_u;
94 } dregs;
95
96 u16 dmadesc_align; /* alignment requirement for dma descriptors */
97
98 u16 ntxd; /* # tx descriptors tunable */
99 u16 txin; /* index of next descriptor to reclaim */
100 u16 txout; /* index of next descriptor to post */
101 void **txp; /* pointer to parallel array of pointers to packets */
102 hnddma_seg_map_t *txp_dmah; /* DMA MAP meta-data handle */
103 dmaaddr_t txdpa; /* Aligned physical address of descriptor ring */
104 dmaaddr_t txdpaorig; /* Original physical address of descriptor ring */
105 u16 txdalign; /* #bytes added to alloc'd mem to align txd */
106 u32 txdalloc; /* #bytes allocated for the ring */
107 u32 xmtptrbase; /* When using unaligned descriptors, the ptr register
108 * is not just an index, it needs all 13 bits to be
109 * an offset from the addr register.
110 */
111
112 u16 nrxd; /* # rx descriptors tunable */
113 u16 rxin; /* index of next descriptor to reclaim */
114 u16 rxout; /* index of next descriptor to post */
115 void **rxp; /* pointer to parallel array of pointers to packets */
116 hnddma_seg_map_t *rxp_dmah; /* DMA MAP meta-data handle */
117 dmaaddr_t rxdpa; /* Aligned physical address of descriptor ring */
118 dmaaddr_t rxdpaorig; /* Original physical address of descriptor ring */
119 u16 rxdalign; /* #bytes added to alloc'd mem to align rxd */
120 u32 rxdalloc; /* #bytes allocated for the ring */
121 u32 rcvptrbase; /* Base for ptr reg when using unaligned descriptors */
122
123 /* tunables */
124 unsigned int rxbufsize; /* rx buffer size in bytes,
125 * not including the extra headroom
126 */
127 uint rxextrahdrroom; /* extra rx headroom, reverseved to assist upper stack
128 * e.g. some rx pkt buffers will be bridged to tx side
129 * without byte copying. The extra headroom needs to be
130 * large enough to fit txheader needs.
131 * Some dongle driver may not need it.
132 */
133 uint nrxpost; /* # rx buffers to keep posted */
134 unsigned int rxoffset; /* rxcontrol offset */
135 uint ddoffsetlow; /* add to get dma address of descriptor ring, low 32 bits */
136 uint ddoffsethigh; /* high 32 bits */
137 uint dataoffsetlow; /* add to get dma address of data buffer, low 32 bits */
138 uint dataoffsethigh; /* high 32 bits */
139 bool aligndesc_4k; /* descriptor base need to be aligned or not */
140 } dma_info_t;
141
142 /* DMA Scatter-gather list is supported. Note this is limited to TX direction only */
143 #ifdef BCMDMASGLISTOSL
144 #define DMASGLIST_ENAB true
145 #else
146 #define DMASGLIST_ENAB false
147 #endif /* BCMDMASGLISTOSL */
148
149 /* descriptor bumping macros */
150 #define XXD(x, n) ((x) & ((n) - 1)) /* faster than %, but n must be power of 2 */
151 #define TXD(x) XXD((x), di->ntxd)
152 #define RXD(x) XXD((x), di->nrxd)
153 #define NEXTTXD(i) TXD((i) + 1)
154 #define PREVTXD(i) TXD((i) - 1)
155 #define NEXTRXD(i) RXD((i) + 1)
156 #define PREVRXD(i) RXD((i) - 1)
157
158 #define NTXDACTIVE(h, t) TXD((t) - (h))
159 #define NRXDACTIVE(h, t) RXD((t) - (h))
160
161 /* macros to convert between byte offsets and indexes */
162 #define B2I(bytes, type) ((bytes) / sizeof(type))
163 #define I2B(index, type) ((index) * sizeof(type))
164
165 #define PCI32ADDR_HIGH 0xc0000000 /* address[31:30] */
166 #define PCI32ADDR_HIGH_SHIFT 30 /* address[31:30] */
167
168 #define PCI64ADDR_HIGH 0x80000000 /* address[63] */
169 #define PCI64ADDR_HIGH_SHIFT 31 /* address[63] */
170
171 /* Common prototypes */
172 static bool _dma_isaddrext(dma_info_t *di);
173 static bool _dma_descriptor_align(dma_info_t *di);
174 static bool _dma_alloc(dma_info_t *di, uint direction);
175 static void _dma_detach(dma_info_t *di);
176 static void _dma_ddtable_init(dma_info_t *di, uint direction, dmaaddr_t pa);
177 static void _dma_rxinit(dma_info_t *di);
178 static void *_dma_rx(dma_info_t *di);
179 static bool _dma_rxfill(dma_info_t *di);
180 static void _dma_rxreclaim(dma_info_t *di);
181 static void _dma_rxenable(dma_info_t *di);
182 static void *_dma_getnextrxp(dma_info_t *di, bool forceall);
183 static void _dma_rx_param_get(dma_info_t *di, u16 *rxoffset,
184 u16 *rxbufsize);
185
186 static void _dma_txblock(dma_info_t *di);
187 static void _dma_txunblock(dma_info_t *di);
188 static uint _dma_txactive(dma_info_t *di);
189 static uint _dma_rxactive(dma_info_t *di);
190 static uint _dma_txpending(dma_info_t *di);
191 static uint _dma_txcommitted(dma_info_t *di);
192
193 static void *_dma_peeknexttxp(dma_info_t *di);
194 static void *_dma_peeknextrxp(dma_info_t *di);
195 static unsigned long _dma_getvar(dma_info_t *di, const char *name);
196 static void _dma_counterreset(dma_info_t *di);
197 static void _dma_fifoloopbackenable(dma_info_t *di);
198 static uint _dma_ctrlflags(dma_info_t *di, uint mask, uint flags);
199 static u8 dma_align_sizetobits(uint size);
200 static void *dma_ringalloc(dma_info_t *di, u32 boundary, uint size,
201 u16 *alignbits, uint *alloced,
202 dmaaddr_t *descpa);
203
204 /* Prototypes for 64-bit routines */
205 static bool dma64_alloc(dma_info_t *di, uint direction);
206 static bool dma64_txreset(dma_info_t *di);
207 static bool dma64_rxreset(dma_info_t *di);
208 static bool dma64_txsuspendedidle(dma_info_t *di);
209 static int dma64_txfast(dma_info_t *di, struct sk_buff *p0, bool commit);
210 static int dma64_txunframed(dma_info_t *di, void *p0, uint len, bool commit);
211 static void *dma64_getpos(dma_info_t *di, bool direction);
212 static void *dma64_getnexttxp(dma_info_t *di, txd_range_t range);
213 static void *dma64_getnextrxp(dma_info_t *di, bool forceall);
214 static void dma64_txrotate(dma_info_t *di);
215
216 static bool dma64_rxidle(dma_info_t *di);
217 static void dma64_txinit(dma_info_t *di);
218 static bool dma64_txenabled(dma_info_t *di);
219 static void dma64_txsuspend(dma_info_t *di);
220 static void dma64_txresume(dma_info_t *di);
221 static bool dma64_txsuspended(dma_info_t *di);
222 static void dma64_txreclaim(dma_info_t *di, txd_range_t range);
223 static bool dma64_txstopped(dma_info_t *di);
224 static bool dma64_rxstopped(dma_info_t *di);
225 static bool dma64_rxenabled(dma_info_t *di);
226 static bool _dma64_addrext(dma64regs_t *dma64regs);
227
228 static inline u32 parity32(u32 data);
229
230 const di_fcn_t dma64proc = {
231 (di_detach_t) _dma_detach,
232 (di_txinit_t) dma64_txinit,
233 (di_txreset_t) dma64_txreset,
234 (di_txenabled_t) dma64_txenabled,
235 (di_txsuspend_t) dma64_txsuspend,
236 (di_txresume_t) dma64_txresume,
237 (di_txsuspended_t) dma64_txsuspended,
238 (di_txsuspendedidle_t) dma64_txsuspendedidle,
239 (di_txfast_t) dma64_txfast,
240 (di_txunframed_t) dma64_txunframed,
241 (di_getpos_t) dma64_getpos,
242 (di_txstopped_t) dma64_txstopped,
243 (di_txreclaim_t) dma64_txreclaim,
244 (di_getnexttxp_t) dma64_getnexttxp,
245 (di_peeknexttxp_t) _dma_peeknexttxp,
246 (di_txblock_t) _dma_txblock,
247 (di_txunblock_t) _dma_txunblock,
248 (di_txactive_t) _dma_txactive,
249 (di_txrotate_t) dma64_txrotate,
250
251 (di_rxinit_t) _dma_rxinit,
252 (di_rxreset_t) dma64_rxreset,
253 (di_rxidle_t) dma64_rxidle,
254 (di_rxstopped_t) dma64_rxstopped,
255 (di_rxenable_t) _dma_rxenable,
256 (di_rxenabled_t) dma64_rxenabled,
257 (di_rx_t) _dma_rx,
258 (di_rxfill_t) _dma_rxfill,
259 (di_rxreclaim_t) _dma_rxreclaim,
260 (di_getnextrxp_t) _dma_getnextrxp,
261 (di_peeknextrxp_t) _dma_peeknextrxp,
262 (di_rxparam_get_t) _dma_rx_param_get,
263
264 (di_fifoloopbackenable_t) _dma_fifoloopbackenable,
265 (di_getvar_t) _dma_getvar,
266 (di_counterreset_t) _dma_counterreset,
267 (di_ctrlflags_t) _dma_ctrlflags,
268 NULL,
269 NULL,
270 NULL,
271 (di_rxactive_t) _dma_rxactive,
272 (di_txpending_t) _dma_txpending,
273 (di_txcommitted_t) _dma_txcommitted,
274 39
275 };
276
dma_attach(char * name,si_t * sih,void * dmaregstx,void * dmaregsrx,uint ntxd,uint nrxd,uint rxbufsize,int rxextheadroom,uint nrxpost,uint rxoffset,uint * msg_level)277 struct hnddma_pub *dma_attach(char *name, si_t *sih,
278 void *dmaregstx, void *dmaregsrx, uint ntxd,
279 uint nrxd, uint rxbufsize, int rxextheadroom,
280 uint nrxpost, uint rxoffset, uint *msg_level)
281 {
282 dma_info_t *di;
283 uint size;
284
285 /* allocate private info structure */
286 di = kzalloc(sizeof(dma_info_t), GFP_ATOMIC);
287 if (di == NULL) {
288 #ifdef BCMDBG
289 printk(KERN_ERR "dma_attach: out of memory\n");
290 #endif
291 return NULL;
292 }
293
294 di->msg_level = msg_level ? msg_level : &dma_msg_level;
295
296 /* old chips w/o sb is no longer supported */
297 ASSERT(sih != NULL);
298
299 di->dma64 = ((si_core_sflags(sih, 0, 0) & SISF_DMA64) == SISF_DMA64);
300
301 /* check arguments */
302 ASSERT(ISPOWEROF2(ntxd));
303 ASSERT(ISPOWEROF2(nrxd));
304
305 if (nrxd == 0)
306 ASSERT(dmaregsrx == NULL);
307 if (ntxd == 0)
308 ASSERT(dmaregstx == NULL);
309
310 /* init dma reg pointer */
311 ASSERT(ntxd <= D64MAXDD);
312 ASSERT(nrxd <= D64MAXDD);
313 di->d64txregs = (dma64regs_t *) dmaregstx;
314 di->d64rxregs = (dma64regs_t *) dmaregsrx;
315 di->hnddma.di_fn = (const di_fcn_t *)&dma64proc;
316
317 /* Default flags (which can be changed by the driver calling dma_ctrlflags
318 * before enable): For backwards compatibility both Rx Overflow Continue
319 * and Parity are DISABLED.
320 * supports it.
321 */
322 di->hnddma.di_fn->ctrlflags(&di->hnddma, DMA_CTRL_ROC | DMA_CTRL_PEN,
323 0);
324
325 DMA_TRACE(("%s: dma_attach: %s flags 0x%x ntxd %d nrxd %d "
326 "rxbufsize %d rxextheadroom %d nrxpost %d rxoffset %d "
327 "dmaregstx %p dmaregsrx %p\n", name, "DMA64",
328 di->hnddma.dmactrlflags, ntxd, nrxd, rxbufsize,
329 rxextheadroom, nrxpost, rxoffset, dmaregstx, dmaregsrx));
330
331 /* make a private copy of our callers name */
332 strncpy(di->name, name, MAXNAMEL);
333 di->name[MAXNAMEL - 1] = '\0';
334
335 di->pbus = ((struct si_info *)sih)->pbus;
336
337 /* save tunables */
338 di->ntxd = (u16) ntxd;
339 di->nrxd = (u16) nrxd;
340
341 /* the actual dma size doesn't include the extra headroom */
342 di->rxextrahdrroom =
343 (rxextheadroom == -1) ? BCMEXTRAHDROOM : rxextheadroom;
344 if (rxbufsize > BCMEXTRAHDROOM)
345 di->rxbufsize = (u16) (rxbufsize - di->rxextrahdrroom);
346 else
347 di->rxbufsize = (u16) rxbufsize;
348
349 di->nrxpost = (u16) nrxpost;
350 di->rxoffset = (u8) rxoffset;
351
352 /*
353 * figure out the DMA physical address offset for dd and data
354 * PCI/PCIE: they map silicon backplace address to zero based memory, need offset
355 * Other bus: use zero
356 * SI_BUS BIGENDIAN kludge: use sdram swapped region for data buffer, not descriptor
357 */
358 di->ddoffsetlow = 0;
359 di->dataoffsetlow = 0;
360 /* for pci bus, add offset */
361 if (sih->bustype == PCI_BUS) {
362 /* pcie with DMA64 */
363 di->ddoffsetlow = 0;
364 di->ddoffsethigh = SI_PCIE_DMA_H32;
365 di->dataoffsetlow = di->ddoffsetlow;
366 di->dataoffsethigh = di->ddoffsethigh;
367 }
368 #if defined(__mips__) && defined(IL_BIGENDIAN)
369 di->dataoffsetlow = di->dataoffsetlow + SI_SDRAM_SWAPPED;
370 #endif /* defined(__mips__) && defined(IL_BIGENDIAN) */
371 /* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */
372 if ((si_coreid(sih) == SDIOD_CORE_ID)
373 && ((si_corerev(sih) > 0) && (si_corerev(sih) <= 2)))
374 di->addrext = 0;
375 else if ((si_coreid(sih) == I2S_CORE_ID) &&
376 ((si_corerev(sih) == 0) || (si_corerev(sih) == 1)))
377 di->addrext = 0;
378 else
379 di->addrext = _dma_isaddrext(di);
380
381 /* does the descriptors need to be aligned and if yes, on 4K/8K or not */
382 di->aligndesc_4k = _dma_descriptor_align(di);
383 if (di->aligndesc_4k) {
384 di->dmadesc_align = D64RINGALIGN_BITS;
385 if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2)) {
386 /* for smaller dd table, HW relax alignment reqmnt */
387 di->dmadesc_align = D64RINGALIGN_BITS - 1;
388 }
389 } else
390 di->dmadesc_align = 4; /* 16 byte alignment */
391
392 DMA_NONE(("DMA descriptor align_needed %d, align %d\n",
393 di->aligndesc_4k, di->dmadesc_align));
394
395 /* allocate tx packet pointer vector */
396 if (ntxd) {
397 size = ntxd * sizeof(void *);
398 di->txp = kzalloc(size, GFP_ATOMIC);
399 if (di->txp == NULL) {
400 DMA_ERROR(("%s: dma_attach: out of tx memory\n", di->name));
401 goto fail;
402 }
403 }
404
405 /* allocate rx packet pointer vector */
406 if (nrxd) {
407 size = nrxd * sizeof(void *);
408 di->rxp = kzalloc(size, GFP_ATOMIC);
409 if (di->rxp == NULL) {
410 DMA_ERROR(("%s: dma_attach: out of rx memory\n", di->name));
411 goto fail;
412 }
413 }
414
415 /* allocate transmit descriptor ring, only need ntxd descriptors but it must be aligned */
416 if (ntxd) {
417 if (!_dma_alloc(di, DMA_TX))
418 goto fail;
419 }
420
421 /* allocate receive descriptor ring, only need nrxd descriptors but it must be aligned */
422 if (nrxd) {
423 if (!_dma_alloc(di, DMA_RX))
424 goto fail;
425 }
426
427 if ((di->ddoffsetlow != 0) && !di->addrext) {
428 if (PHYSADDRLO(di->txdpa) > SI_PCI_DMA_SZ) {
429 DMA_ERROR(("%s: dma_attach: txdpa 0x%x: addrext not supported\n", di->name, (u32) PHYSADDRLO(di->txdpa)));
430 goto fail;
431 }
432 if (PHYSADDRLO(di->rxdpa) > SI_PCI_DMA_SZ) {
433 DMA_ERROR(("%s: dma_attach: rxdpa 0x%x: addrext not supported\n", di->name, (u32) PHYSADDRLO(di->rxdpa)));
434 goto fail;
435 }
436 }
437
438 DMA_TRACE(("ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh " "0x%x addrext %d\n", di->ddoffsetlow, di->ddoffsethigh, di->dataoffsetlow, di->dataoffsethigh, di->addrext));
439
440 /* allocate DMA mapping vectors */
441 if (DMASGLIST_ENAB) {
442 if (ntxd) {
443 size = ntxd * sizeof(hnddma_seg_map_t);
444 di->txp_dmah = kzalloc(size, GFP_ATOMIC);
445 if (di->txp_dmah == NULL)
446 goto fail;
447 }
448
449 if (nrxd) {
450 size = nrxd * sizeof(hnddma_seg_map_t);
451 di->rxp_dmah = kzalloc(size, GFP_ATOMIC);
452 if (di->rxp_dmah == NULL)
453 goto fail;
454 }
455 }
456
457 return (struct hnddma_pub *) di;
458
459 fail:
460 _dma_detach(di);
461 return NULL;
462 }
463
464 /* Check for odd number of 1's */
parity32(u32 data)465 static inline u32 parity32(u32 data)
466 {
467 data ^= data >> 16;
468 data ^= data >> 8;
469 data ^= data >> 4;
470 data ^= data >> 2;
471 data ^= data >> 1;
472
473 return data & 1;
474 }
475
476 #define DMA64_DD_PARITY(dd) parity32((dd)->addrlow ^ (dd)->addrhigh ^ (dd)->ctrl1 ^ (dd)->ctrl2)
477
478 static inline void
dma64_dd_upd(dma_info_t * di,dma64dd_t * ddring,dmaaddr_t pa,uint outidx,u32 * flags,u32 bufcount)479 dma64_dd_upd(dma_info_t *di, dma64dd_t *ddring, dmaaddr_t pa, uint outidx,
480 u32 *flags, u32 bufcount)
481 {
482 u32 ctrl2 = bufcount & D64_CTRL2_BC_MASK;
483
484 /* PCI bus with big(>1G) physical address, use address extension */
485 #if defined(__mips__) && defined(IL_BIGENDIAN)
486 if ((di->dataoffsetlow == SI_SDRAM_SWAPPED)
487 || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) {
488 #else
489 if ((di->dataoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) {
490 #endif /* defined(__mips__) && defined(IL_BIGENDIAN) */
491 ASSERT((PHYSADDRHI(pa) & PCI64ADDR_HIGH) == 0);
492
493 W_SM(&ddring[outidx].addrlow,
494 BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow));
495 W_SM(&ddring[outidx].addrhigh,
496 BUS_SWAP32(PHYSADDRHI(pa) + di->dataoffsethigh));
497 W_SM(&ddring[outidx].ctrl1, BUS_SWAP32(*flags));
498 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2));
499 } else {
500 /* address extension for 32-bit PCI */
501 u32 ae;
502 ASSERT(di->addrext);
503
504 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
505 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH;
506 ASSERT(PHYSADDRHI(pa) == 0);
507
508 ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE;
509 W_SM(&ddring[outidx].addrlow,
510 BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow));
511 W_SM(&ddring[outidx].addrhigh,
512 BUS_SWAP32(0 + di->dataoffsethigh));
513 W_SM(&ddring[outidx].ctrl1, BUS_SWAP32(*flags));
514 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2));
515 }
516 if (di->hnddma.dmactrlflags & DMA_CTRL_PEN) {
517 if (DMA64_DD_PARITY(&ddring[outidx])) {
518 W_SM(&ddring[outidx].ctrl2,
519 BUS_SWAP32(ctrl2 | D64_CTRL2_PARITY));
520 }
521 }
522 }
523
524 static bool _dma_alloc(dma_info_t *di, uint direction)
525 {
526 return dma64_alloc(di, direction);
527 }
528
529 void *dma_alloc_consistent(struct pci_dev *pdev, uint size, u16 align_bits,
530 uint *alloced, unsigned long *pap)
531 {
532 if (align_bits) {
533 u16 align = (1 << align_bits);
534 if (!IS_ALIGNED(PAGE_SIZE, align))
535 size += align;
536 *alloced = size;
537 }
538 return pci_alloc_consistent(pdev, size, (dma_addr_t *) pap);
539 }
540
541 /* !! may be called with core in reset */
542 static void _dma_detach(dma_info_t *di)
543 {
544
545 DMA_TRACE(("%s: dma_detach\n", di->name));
546
547 /* shouldn't be here if descriptors are unreclaimed */
548 ASSERT(di->txin == di->txout);
549 ASSERT(di->rxin == di->rxout);
550
551 /* free dma descriptor rings */
552 if (di->txd64)
553 pci_free_consistent(di->pbus, di->txdalloc,
554 ((s8 *)di->txd64 - di->txdalign),
555 (di->txdpaorig));
556 if (di->rxd64)
557 pci_free_consistent(di->pbus, di->rxdalloc,
558 ((s8 *)di->rxd64 - di->rxdalign),
559 (di->rxdpaorig));
560
561 /* free packet pointer vectors */
562 kfree(di->txp);
563 kfree(di->rxp);
564
565 /* free tx packet DMA handles */
566 kfree(di->txp_dmah);
567
568 /* free rx packet DMA handles */
569 kfree(di->rxp_dmah);
570
571 /* free our private info structure */
572 kfree(di);
573
574 }
575
576 static bool _dma_descriptor_align(dma_info_t *di)
577 {
578 u32 addrl;
579
580 /* Check to see if the descriptors need to be aligned on 4K/8K or not */
581 if (di->d64txregs != NULL) {
582 W_REG(&di->d64txregs->addrlow, 0xff0);
583 addrl = R_REG(&di->d64txregs->addrlow);
584 if (addrl != 0)
585 return false;
586 } else if (di->d64rxregs != NULL) {
587 W_REG(&di->d64rxregs->addrlow, 0xff0);
588 addrl = R_REG(&di->d64rxregs->addrlow);
589 if (addrl != 0)
590 return false;
591 }
592 return true;
593 }
594
595 /* return true if this dma engine supports DmaExtendedAddrChanges, otherwise false */
596 static bool _dma_isaddrext(dma_info_t *di)
597 {
598 /* DMA64 supports full 32- or 64-bit operation. AE is always valid */
599
600 /* not all tx or rx channel are available */
601 if (di->d64txregs != NULL) {
602 if (!_dma64_addrext(di->d64txregs)) {
603 DMA_ERROR(("%s: _dma_isaddrext: DMA64 tx doesn't have "
604 "AE set\n", di->name));
605 ASSERT(0);
606 }
607 return true;
608 } else if (di->d64rxregs != NULL) {
609 if (!_dma64_addrext(di->d64rxregs)) {
610 DMA_ERROR(("%s: _dma_isaddrext: DMA64 rx doesn't have "
611 "AE set\n", di->name));
612 ASSERT(0);
613 }
614 return true;
615 }
616 return false;
617 }
618
619 /* initialize descriptor table base address */
620 static void _dma_ddtable_init(dma_info_t *di, uint direction, dmaaddr_t pa)
621 {
622 if (!di->aligndesc_4k) {
623 if (direction == DMA_TX)
624 di->xmtptrbase = PHYSADDRLO(pa);
625 else
626 di->rcvptrbase = PHYSADDRLO(pa);
627 }
628
629 if ((di->ddoffsetlow == 0)
630 || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) {
631 if (direction == DMA_TX) {
632 W_REG(&di->d64txregs->addrlow,
633 (PHYSADDRLO(pa) + di->ddoffsetlow));
634 W_REG(&di->d64txregs->addrhigh,
635 (PHYSADDRHI(pa) + di->ddoffsethigh));
636 } else {
637 W_REG(&di->d64rxregs->addrlow,
638 (PHYSADDRLO(pa) + di->ddoffsetlow));
639 W_REG(&di->d64rxregs->addrhigh,
640 (PHYSADDRHI(pa) + di->ddoffsethigh));
641 }
642 } else {
643 /* DMA64 32bits address extension */
644 u32 ae;
645 ASSERT(di->addrext);
646 ASSERT(PHYSADDRHI(pa) == 0);
647
648 /* shift the high bit(s) from pa to ae */
649 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >>
650 PCI32ADDR_HIGH_SHIFT;
651 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH;
652
653 if (direction == DMA_TX) {
654 W_REG(&di->d64txregs->addrlow,
655 (PHYSADDRLO(pa) + di->ddoffsetlow));
656 W_REG(&di->d64txregs->addrhigh,
657 di->ddoffsethigh);
658 SET_REG(&di->d64txregs->control,
659 D64_XC_AE, (ae << D64_XC_AE_SHIFT));
660 } else {
661 W_REG(&di->d64rxregs->addrlow,
662 (PHYSADDRLO(pa) + di->ddoffsetlow));
663 W_REG(&di->d64rxregs->addrhigh,
664 di->ddoffsethigh);
665 SET_REG(&di->d64rxregs->control,
666 D64_RC_AE, (ae << D64_RC_AE_SHIFT));
667 }
668 }
669 }
670
671 static void _dma_fifoloopbackenable(dma_info_t *di)
672 {
673 DMA_TRACE(("%s: dma_fifoloopbackenable\n", di->name));
674
675 OR_REG(&di->d64txregs->control, D64_XC_LE);
676 }
677
678 static void _dma_rxinit(dma_info_t *di)
679 {
680 DMA_TRACE(("%s: dma_rxinit\n", di->name));
681
682 if (di->nrxd == 0)
683 return;
684
685 di->rxin = di->rxout = 0;
686
687 /* clear rx descriptor ring */
688 memset((void *)di->rxd64, '\0',
689 (di->nrxd * sizeof(dma64dd_t)));
690
691 /* DMA engine with out alignment requirement requires table to be inited
692 * before enabling the engine
693 */
694 if (!di->aligndesc_4k)
695 _dma_ddtable_init(di, DMA_RX, di->rxdpa);
696
697 _dma_rxenable(di);
698
699 if (di->aligndesc_4k)
700 _dma_ddtable_init(di, DMA_RX, di->rxdpa);
701 }
702
703 static void _dma_rxenable(dma_info_t *di)
704 {
705 uint dmactrlflags = di->hnddma.dmactrlflags;
706 u32 control;
707
708 DMA_TRACE(("%s: dma_rxenable\n", di->name));
709
710 control =
711 (R_REG(&di->d64rxregs->control) & D64_RC_AE) |
712 D64_RC_RE;
713
714 if ((dmactrlflags & DMA_CTRL_PEN) == 0)
715 control |= D64_RC_PD;
716
717 if (dmactrlflags & DMA_CTRL_ROC)
718 control |= D64_RC_OC;
719
720 W_REG(&di->d64rxregs->control,
721 ((di->rxoffset << D64_RC_RO_SHIFT) | control));
722 }
723
724 static void
725 _dma_rx_param_get(dma_info_t *di, u16 *rxoffset, u16 *rxbufsize)
726 {
727 /* the normal values fit into 16 bits */
728 *rxoffset = (u16) di->rxoffset;
729 *rxbufsize = (u16) di->rxbufsize;
730 }
731
732 /* !! rx entry routine
733 * returns a pointer to the next frame received, or NULL if there are no more
734 * if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is supported
735 * with pkts chain
736 * otherwise, it's treated as giant pkt and will be tossed.
737 * The DMA scattering starts with normal DMA header, followed by first buffer data.
738 * After it reaches the max size of buffer, the data continues in next DMA descriptor
739 * buffer WITHOUT DMA header
740 */
741 static void *BCMFASTPATH _dma_rx(dma_info_t *di)
742 {
743 struct sk_buff *p, *head, *tail;
744 uint len;
745 uint pkt_len;
746 int resid = 0;
747
748 next_frame:
749 head = _dma_getnextrxp(di, false);
750 if (head == NULL)
751 return NULL;
752
753 len = le16_to_cpu(*(u16 *) (head->data));
754 DMA_TRACE(("%s: dma_rx len %d\n", di->name, len));
755
756 #if defined(__mips__)
757 #define OSL_UNCACHED(va) ((void *)KSEG1ADDR((va)))
758 if (!len) {
759 while (!(len = *(u16 *) OSL_UNCACHED(head->data)))
760 udelay(1);
761
762 *(u16 *) (head->data) = cpu_to_le16((u16) len);
763 }
764 #endif /* defined(__mips__) */
765
766 /* set actual length */
767 pkt_len = min((di->rxoffset + len), di->rxbufsize);
768 __skb_trim(head, pkt_len);
769 resid = len - (di->rxbufsize - di->rxoffset);
770
771 /* check for single or multi-buffer rx */
772 if (resid > 0) {
773 tail = head;
774 while ((resid > 0) && (p = _dma_getnextrxp(di, false))) {
775 tail->next = p;
776 pkt_len = min(resid, (int)di->rxbufsize);
777 __skb_trim(p, pkt_len);
778
779 tail = p;
780 resid -= di->rxbufsize;
781 }
782
783 #ifdef BCMDBG
784 if (resid > 0) {
785 uint cur;
786 ASSERT(p == NULL);
787 cur =
788 B2I(((R_REG(&di->d64rxregs->status0) &
789 D64_RS0_CD_MASK) -
790 di->rcvptrbase) & D64_RS0_CD_MASK,
791 dma64dd_t);
792 DMA_ERROR(("_dma_rx, rxin %d rxout %d, hw_curr %d\n",
793 di->rxin, di->rxout, cur));
794 }
795 #endif /* BCMDBG */
796
797 if ((di->hnddma.dmactrlflags & DMA_CTRL_RXMULTI) == 0) {
798 DMA_ERROR(("%s: dma_rx: bad frame length (%d)\n",
799 di->name, len));
800 pkt_buf_free_skb(head);
801 di->hnddma.rxgiants++;
802 goto next_frame;
803 }
804 }
805
806 return head;
807 }
808
809 /* post receive buffers
810 * return false is refill failed completely and ring is empty
811 * this will stall the rx dma and user might want to call rxfill again asap
812 * This unlikely happens on memory-rich NIC, but often on memory-constrained dongle
813 */
814 static bool BCMFASTPATH _dma_rxfill(dma_info_t *di)
815 {
816 struct sk_buff *p;
817 u16 rxin, rxout;
818 u32 flags = 0;
819 uint n;
820 uint i;
821 dmaaddr_t pa;
822 uint extra_offset = 0;
823 bool ring_empty;
824
825 ring_empty = false;
826
827 /*
828 * Determine how many receive buffers we're lacking
829 * from the full complement, allocate, initialize,
830 * and post them, then update the chip rx lastdscr.
831 */
832
833 rxin = di->rxin;
834 rxout = di->rxout;
835
836 n = di->nrxpost - NRXDACTIVE(rxin, rxout);
837
838 DMA_TRACE(("%s: dma_rxfill: post %d\n", di->name, n));
839
840 if (di->rxbufsize > BCMEXTRAHDROOM)
841 extra_offset = di->rxextrahdrroom;
842
843 for (i = 0; i < n; i++) {
844 /* the di->rxbufsize doesn't include the extra headroom, we need to add it to the
845 size to be allocated
846 */
847
848 p = pkt_buf_get_skb(di->rxbufsize + extra_offset);
849
850 if (p == NULL) {
851 DMA_ERROR(("%s: dma_rxfill: out of rxbufs\n",
852 di->name));
853 if (i == 0 && dma64_rxidle(di)) {
854 DMA_ERROR(("%s: rxfill64: ring is empty !\n",
855 di->name));
856 ring_empty = true;
857 }
858 di->hnddma.rxnobuf++;
859 break;
860 }
861 /* reserve an extra headroom, if applicable */
862 if (extra_offset)
863 skb_pull(p, extra_offset);
864
865 /* Do a cached write instead of uncached write since DMA_MAP
866 * will flush the cache.
867 */
868 *(u32 *) (p->data) = 0;
869
870 if (DMASGLIST_ENAB)
871 memset(&di->rxp_dmah[rxout], 0,
872 sizeof(hnddma_seg_map_t));
873
874 pa = pci_map_single(di->pbus, p->data,
875 di->rxbufsize, PCI_DMA_FROMDEVICE);
876
877 ASSERT(IS_ALIGNED(PHYSADDRLO(pa), 4));
878
879 /* save the free packet pointer */
880 ASSERT(di->rxp[rxout] == NULL);
881 di->rxp[rxout] = p;
882
883 /* reset flags for each descriptor */
884 flags = 0;
885 if (rxout == (di->nrxd - 1))
886 flags = D64_CTRL1_EOT;
887
888 dma64_dd_upd(di, di->rxd64, pa, rxout, &flags,
889 di->rxbufsize);
890 rxout = NEXTRXD(rxout);
891 }
892
893 di->rxout = rxout;
894
895 /* update the chip lastdscr pointer */
896 W_REG(&di->d64rxregs->ptr,
897 di->rcvptrbase + I2B(rxout, dma64dd_t));
898
899 return ring_empty;
900 }
901
902 /* like getnexttxp but no reclaim */
903 static void *_dma_peeknexttxp(dma_info_t *di)
904 {
905 uint end, i;
906
907 if (di->ntxd == 0)
908 return NULL;
909
910 end =
911 B2I(((R_REG(&di->d64txregs->status0) &
912 D64_XS0_CD_MASK) - di->xmtptrbase) & D64_XS0_CD_MASK,
913 dma64dd_t);
914
915 for (i = di->txin; i != end; i = NEXTTXD(i))
916 if (di->txp[i])
917 return di->txp[i];
918
919 return NULL;
920 }
921
922 /* like getnextrxp but not take off the ring */
923 static void *_dma_peeknextrxp(dma_info_t *di)
924 {
925 uint end, i;
926
927 if (di->nrxd == 0)
928 return NULL;
929
930 end =
931 B2I(((R_REG(&di->d64rxregs->status0) &
932 D64_RS0_CD_MASK) - di->rcvptrbase) & D64_RS0_CD_MASK,
933 dma64dd_t);
934
935 for (i = di->rxin; i != end; i = NEXTRXD(i))
936 if (di->rxp[i])
937 return di->rxp[i];
938
939 return NULL;
940 }
941
942 static void _dma_rxreclaim(dma_info_t *di)
943 {
944 void *p;
945
946 DMA_TRACE(("%s: dma_rxreclaim\n", di->name));
947
948 while ((p = _dma_getnextrxp(di, true)))
949 pkt_buf_free_skb(p);
950 }
951
952 static void *BCMFASTPATH _dma_getnextrxp(dma_info_t *di, bool forceall)
953 {
954 if (di->nrxd == 0)
955 return NULL;
956
957 return dma64_getnextrxp(di, forceall);
958 }
959
960 static void _dma_txblock(dma_info_t *di)
961 {
962 di->hnddma.txavail = 0;
963 }
964
965 static void _dma_txunblock(dma_info_t *di)
966 {
967 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
968 }
969
970 static uint _dma_txactive(dma_info_t *di)
971 {
972 return NTXDACTIVE(di->txin, di->txout);
973 }
974
975 static uint _dma_txpending(dma_info_t *di)
976 {
977 uint curr;
978
979 curr =
980 B2I(((R_REG(&di->d64txregs->status0) &
981 D64_XS0_CD_MASK) - di->xmtptrbase) & D64_XS0_CD_MASK,
982 dma64dd_t);
983
984 return NTXDACTIVE(curr, di->txout);
985 }
986
987 static uint _dma_txcommitted(dma_info_t *di)
988 {
989 uint ptr;
990 uint txin = di->txin;
991
992 if (txin == di->txout)
993 return 0;
994
995 ptr = B2I(R_REG(&di->d64txregs->ptr), dma64dd_t);
996
997 return NTXDACTIVE(di->txin, ptr);
998 }
999
1000 static uint _dma_rxactive(dma_info_t *di)
1001 {
1002 return NRXDACTIVE(di->rxin, di->rxout);
1003 }
1004
1005 static void _dma_counterreset(dma_info_t *di)
1006 {
1007 /* reset all software counter */
1008 di->hnddma.rxgiants = 0;
1009 di->hnddma.rxnobuf = 0;
1010 di->hnddma.txnobuf = 0;
1011 }
1012
1013 static uint _dma_ctrlflags(dma_info_t *di, uint mask, uint flags)
1014 {
1015 uint dmactrlflags = di->hnddma.dmactrlflags;
1016
1017 if (di == NULL) {
1018 DMA_ERROR(("%s: _dma_ctrlflags: NULL dma handle\n", di->name));
1019 return 0;
1020 }
1021
1022 ASSERT((flags & ~mask) == 0);
1023
1024 dmactrlflags &= ~mask;
1025 dmactrlflags |= flags;
1026
1027 /* If trying to enable parity, check if parity is actually supported */
1028 if (dmactrlflags & DMA_CTRL_PEN) {
1029 u32 control;
1030
1031 control = R_REG(&di->d64txregs->control);
1032 W_REG(&di->d64txregs->control,
1033 control | D64_XC_PD);
1034 if (R_REG(&di->d64txregs->control) & D64_XC_PD) {
1035 /* We *can* disable it so it is supported,
1036 * restore control register
1037 */
1038 W_REG(&di->d64txregs->control,
1039 control);
1040 } else {
1041 /* Not supported, don't allow it to be enabled */
1042 dmactrlflags &= ~DMA_CTRL_PEN;
1043 }
1044 }
1045
1046 di->hnddma.dmactrlflags = dmactrlflags;
1047
1048 return dmactrlflags;
1049 }
1050
1051 /* get the address of the var in order to change later */
1052 static unsigned long _dma_getvar(dma_info_t *di, const char *name)
1053 {
1054 if (!strcmp(name, "&txavail"))
1055 return (unsigned long)&(di->hnddma.txavail);
1056 else {
1057 ASSERT(0);
1058 }
1059 return 0;
1060 }
1061
1062 static
1063 u8 dma_align_sizetobits(uint size)
1064 {
1065 u8 bitpos = 0;
1066 ASSERT(size);
1067 ASSERT(!(size & (size - 1)));
1068 while (size >>= 1) {
1069 bitpos++;
1070 }
1071 return bitpos;
1072 }
1073
1074 /* This function ensures that the DMA descriptor ring will not get allocated
1075 * across Page boundary. If the allocation is done across the page boundary
1076 * at the first time, then it is freed and the allocation is done at
1077 * descriptor ring size aligned location. This will ensure that the ring will
1078 * not cross page boundary
1079 */
1080 static void *dma_ringalloc(dma_info_t *di, u32 boundary, uint size,
1081 u16 *alignbits, uint *alloced,
1082 dmaaddr_t *descpa)
1083 {
1084 void *va;
1085 u32 desc_strtaddr;
1086 u32 alignbytes = 1 << *alignbits;
1087
1088 va = dma_alloc_consistent(di->pbus, size, *alignbits, alloced, descpa);
1089
1090 if (NULL == va)
1091 return NULL;
1092
1093 desc_strtaddr = (u32) roundup((unsigned long)va, alignbytes);
1094 if (((desc_strtaddr + size - 1) & boundary) != (desc_strtaddr
1095 & boundary)) {
1096 *alignbits = dma_align_sizetobits(size);
1097 pci_free_consistent(di->pbus, size, va, *descpa);
1098 va = dma_alloc_consistent(di->pbus, size, *alignbits,
1099 alloced, descpa);
1100 }
1101 return va;
1102 }
1103
1104 /* 64-bit DMA functions */
1105
1106 static void dma64_txinit(dma_info_t *di)
1107 {
1108 u32 control = D64_XC_XE;
1109
1110 DMA_TRACE(("%s: dma_txinit\n", di->name));
1111
1112 if (di->ntxd == 0)
1113 return;
1114
1115 di->txin = di->txout = 0;
1116 di->hnddma.txavail = di->ntxd - 1;
1117
1118 /* clear tx descriptor ring */
1119 memset((void *)di->txd64, '\0', (di->ntxd * sizeof(dma64dd_t)));
1120
1121 /* DMA engine with out alignment requirement requires table to be inited
1122 * before enabling the engine
1123 */
1124 if (!di->aligndesc_4k)
1125 _dma_ddtable_init(di, DMA_TX, di->txdpa);
1126
1127 if ((di->hnddma.dmactrlflags & DMA_CTRL_PEN) == 0)
1128 control |= D64_XC_PD;
1129 OR_REG(&di->d64txregs->control, control);
1130
1131 /* DMA engine with alignment requirement requires table to be inited
1132 * before enabling the engine
1133 */
1134 if (di->aligndesc_4k)
1135 _dma_ddtable_init(di, DMA_TX, di->txdpa);
1136 }
1137
1138 static bool dma64_txenabled(dma_info_t *di)
1139 {
1140 u32 xc;
1141
1142 /* If the chip is dead, it is not enabled :-) */
1143 xc = R_REG(&di->d64txregs->control);
1144 return (xc != 0xffffffff) && (xc & D64_XC_XE);
1145 }
1146
1147 static void dma64_txsuspend(dma_info_t *di)
1148 {
1149 DMA_TRACE(("%s: dma_txsuspend\n", di->name));
1150
1151 if (di->ntxd == 0)
1152 return;
1153
1154 OR_REG(&di->d64txregs->control, D64_XC_SE);
1155 }
1156
1157 static void dma64_txresume(dma_info_t *di)
1158 {
1159 DMA_TRACE(("%s: dma_txresume\n", di->name));
1160
1161 if (di->ntxd == 0)
1162 return;
1163
1164 AND_REG(&di->d64txregs->control, ~D64_XC_SE);
1165 }
1166
1167 static bool dma64_txsuspended(dma_info_t *di)
1168 {
1169 return (di->ntxd == 0) ||
1170 ((R_REG(&di->d64txregs->control) & D64_XC_SE) ==
1171 D64_XC_SE);
1172 }
1173
1174 static void BCMFASTPATH dma64_txreclaim(dma_info_t *di, txd_range_t range)
1175 {
1176 void *p;
1177
1178 DMA_TRACE(("%s: dma_txreclaim %s\n", di->name,
1179 (range == HNDDMA_RANGE_ALL) ? "all" :
1180 ((range ==
1181 HNDDMA_RANGE_TRANSMITTED) ? "transmitted" :
1182 "transferred")));
1183
1184 if (di->txin == di->txout)
1185 return;
1186
1187 while ((p = dma64_getnexttxp(di, range))) {
1188 /* For unframed data, we don't have any packets to free */
1189 if (!(di->hnddma.dmactrlflags & DMA_CTRL_UNFRAMED))
1190 pkt_buf_free_skb(p);
1191 }
1192 }
1193
1194 static bool dma64_txstopped(dma_info_t *di)
1195 {
1196 return ((R_REG(&di->d64txregs->status0) & D64_XS0_XS_MASK) ==
1197 D64_XS0_XS_STOPPED);
1198 }
1199
1200 static bool dma64_rxstopped(dma_info_t *di)
1201 {
1202 return ((R_REG(&di->d64rxregs->status0) & D64_RS0_RS_MASK) ==
1203 D64_RS0_RS_STOPPED);
1204 }
1205
1206 static bool dma64_alloc(dma_info_t *di, uint direction)
1207 {
1208 u16 size;
1209 uint ddlen;
1210 void *va;
1211 uint alloced = 0;
1212 u16 align;
1213 u16 align_bits;
1214
1215 ddlen = sizeof(dma64dd_t);
1216
1217 size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
1218 align_bits = di->dmadesc_align;
1219 align = (1 << align_bits);
1220
1221 if (direction == DMA_TX) {
1222 va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
1223 &alloced, &di->txdpaorig);
1224 if (va == NULL) {
1225 DMA_ERROR(("%s: dma64_alloc: DMA_ALLOC_CONSISTENT(ntxd) failed\n", di->name));
1226 return false;
1227 }
1228 align = (1 << align_bits);
1229 di->txd64 = (dma64dd_t *) roundup((unsigned long)va, align);
1230 di->txdalign = (uint) ((s8 *)di->txd64 - (s8 *) va);
1231 PHYSADDRLOSET(di->txdpa,
1232 PHYSADDRLO(di->txdpaorig) + di->txdalign);
1233 /* Make sure that alignment didn't overflow */
1234 ASSERT(PHYSADDRLO(di->txdpa) >= PHYSADDRLO(di->txdpaorig));
1235
1236 PHYSADDRHISET(di->txdpa, PHYSADDRHI(di->txdpaorig));
1237 di->txdalloc = alloced;
1238 ASSERT(IS_ALIGNED((unsigned long)di->txd64, align));
1239 } else {
1240 va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
1241 &alloced, &di->rxdpaorig);
1242 if (va == NULL) {
1243 DMA_ERROR(("%s: dma64_alloc: DMA_ALLOC_CONSISTENT(nrxd) failed\n", di->name));
1244 return false;
1245 }
1246 align = (1 << align_bits);
1247 di->rxd64 = (dma64dd_t *) roundup((unsigned long)va, align);
1248 di->rxdalign = (uint) ((s8 *)di->rxd64 - (s8 *) va);
1249 PHYSADDRLOSET(di->rxdpa,
1250 PHYSADDRLO(di->rxdpaorig) + di->rxdalign);
1251 /* Make sure that alignment didn't overflow */
1252 ASSERT(PHYSADDRLO(di->rxdpa) >= PHYSADDRLO(di->rxdpaorig));
1253
1254 PHYSADDRHISET(di->rxdpa, PHYSADDRHI(di->rxdpaorig));
1255 di->rxdalloc = alloced;
1256 ASSERT(IS_ALIGNED((unsigned long)di->rxd64, align));
1257 }
1258
1259 return true;
1260 }
1261
1262 static bool dma64_txreset(dma_info_t *di)
1263 {
1264 u32 status;
1265
1266 if (di->ntxd == 0)
1267 return true;
1268
1269 /* suspend tx DMA first */
1270 W_REG(&di->d64txregs->control, D64_XC_SE);
1271 SPINWAIT(((status =
1272 (R_REG(&di->d64txregs->status0) & D64_XS0_XS_MASK))
1273 != D64_XS0_XS_DISABLED) && (status != D64_XS0_XS_IDLE)
1274 && (status != D64_XS0_XS_STOPPED), 10000);
1275
1276 W_REG(&di->d64txregs->control, 0);
1277 SPINWAIT(((status =
1278 (R_REG(&di->d64txregs->status0) & D64_XS0_XS_MASK))
1279 != D64_XS0_XS_DISABLED), 10000);
1280
1281 /* wait for the last transaction to complete */
1282 udelay(300);
1283
1284 return status == D64_XS0_XS_DISABLED;
1285 }
1286
1287 static bool dma64_rxidle(dma_info_t *di)
1288 {
1289 DMA_TRACE(("%s: dma_rxidle\n", di->name));
1290
1291 if (di->nrxd == 0)
1292 return true;
1293
1294 return ((R_REG(&di->d64rxregs->status0) & D64_RS0_CD_MASK) ==
1295 (R_REG(&di->d64rxregs->ptr) & D64_RS0_CD_MASK));
1296 }
1297
1298 static bool dma64_rxreset(dma_info_t *di)
1299 {
1300 u32 status;
1301
1302 if (di->nrxd == 0)
1303 return true;
1304
1305 W_REG(&di->d64rxregs->control, 0);
1306 SPINWAIT(((status =
1307 (R_REG(&di->d64rxregs->status0) & D64_RS0_RS_MASK))
1308 != D64_RS0_RS_DISABLED), 10000);
1309
1310 return status == D64_RS0_RS_DISABLED;
1311 }
1312
1313 static bool dma64_rxenabled(dma_info_t *di)
1314 {
1315 u32 rc;
1316
1317 rc = R_REG(&di->d64rxregs->control);
1318 return (rc != 0xffffffff) && (rc & D64_RC_RE);
1319 }
1320
1321 static bool dma64_txsuspendedidle(dma_info_t *di)
1322 {
1323
1324 if (di->ntxd == 0)
1325 return true;
1326
1327 if (!(R_REG(&di->d64txregs->control) & D64_XC_SE))
1328 return 0;
1329
1330 if ((R_REG(&di->d64txregs->status0) & D64_XS0_XS_MASK) ==
1331 D64_XS0_XS_IDLE)
1332 return 1;
1333
1334 return 0;
1335 }
1336
1337 /* Useful when sending unframed data. This allows us to get a progress report from the DMA.
1338 * We return a pointer to the beginning of the DATA buffer of the current descriptor.
1339 * If DMA is idle, we return NULL.
1340 */
1341 static void *dma64_getpos(dma_info_t *di, bool direction)
1342 {
1343 void *va;
1344 bool idle;
1345 u32 cd_offset;
1346
1347 if (direction == DMA_TX) {
1348 cd_offset =
1349 R_REG(&di->d64txregs->status0) & D64_XS0_CD_MASK;
1350 idle = !NTXDACTIVE(di->txin, di->txout);
1351 va = di->txp[B2I(cd_offset, dma64dd_t)];
1352 } else {
1353 cd_offset =
1354 R_REG(&di->d64rxregs->status0) & D64_XS0_CD_MASK;
1355 idle = !NRXDACTIVE(di->rxin, di->rxout);
1356 va = di->rxp[B2I(cd_offset, dma64dd_t)];
1357 }
1358
1359 /* If DMA is IDLE, return NULL */
1360 if (idle) {
1361 DMA_TRACE(("%s: DMA idle, return NULL\n", __func__));
1362 va = NULL;
1363 }
1364
1365 return va;
1366 }
1367
1368 /* TX of unframed data
1369 *
1370 * Adds a DMA ring descriptor for the data pointed to by "buf".
1371 * This is for DMA of a buffer of data and is unlike other hnddma TX functions
1372 * that take a pointer to a "packet"
1373 * Each call to this is results in a single descriptor being added for "len" bytes of
1374 * data starting at "buf", it doesn't handle chained buffers.
1375 */
1376 static int dma64_txunframed(dma_info_t *di, void *buf, uint len, bool commit)
1377 {
1378 u16 txout;
1379 u32 flags = 0;
1380 dmaaddr_t pa; /* phys addr */
1381
1382 txout = di->txout;
1383
1384 /* return nonzero if out of tx descriptors */
1385 if (NEXTTXD(txout) == di->txin)
1386 goto outoftxd;
1387
1388 if (len == 0)
1389 return 0;
1390
1391 pa = pci_map_single(di->pbus, buf, len, PCI_DMA_TODEVICE);
1392
1393 flags = (D64_CTRL1_SOF | D64_CTRL1_IOC | D64_CTRL1_EOF);
1394
1395 if (txout == (di->ntxd - 1))
1396 flags |= D64_CTRL1_EOT;
1397
1398 dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);
1399 ASSERT(di->txp[txout] == NULL);
1400
1401 /* save the buffer pointer - used by dma_getpos */
1402 di->txp[txout] = buf;
1403
1404 txout = NEXTTXD(txout);
1405 /* bump the tx descriptor index */
1406 di->txout = txout;
1407
1408 /* kick the chip */
1409 if (commit) {
1410 W_REG(&di->d64txregs->ptr,
1411 di->xmtptrbase + I2B(txout, dma64dd_t));
1412 }
1413
1414 /* tx flow control */
1415 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1416
1417 return 0;
1418
1419 outoftxd:
1420 DMA_ERROR(("%s: %s: out of txds !!!\n", di->name, __func__));
1421 di->hnddma.txavail = 0;
1422 di->hnddma.txnobuf++;
1423 return -1;
1424 }
1425
1426 /* !! tx entry routine
1427 * WARNING: call must check the return value for error.
1428 * the error(toss frames) could be fatal and cause many subsequent hard to debug problems
1429 */
1430 static int BCMFASTPATH dma64_txfast(dma_info_t *di, struct sk_buff *p0,
1431 bool commit)
1432 {
1433 struct sk_buff *p, *next;
1434 unsigned char *data;
1435 uint len;
1436 u16 txout;
1437 u32 flags = 0;
1438 dmaaddr_t pa;
1439
1440 DMA_TRACE(("%s: dma_txfast\n", di->name));
1441
1442 txout = di->txout;
1443
1444 /*
1445 * Walk the chain of packet buffers
1446 * allocating and initializing transmit descriptor entries.
1447 */
1448 for (p = p0; p; p = next) {
1449 uint nsegs, j;
1450 hnddma_seg_map_t *map;
1451
1452 data = p->data;
1453 len = p->len;
1454 #ifdef BCM_DMAPAD
1455 len += PKTDMAPAD(di->osh, p);
1456 #endif /* BCM_DMAPAD */
1457 next = p->next;
1458
1459 /* return nonzero if out of tx descriptors */
1460 if (NEXTTXD(txout) == di->txin)
1461 goto outoftxd;
1462
1463 if (len == 0)
1464 continue;
1465
1466 /* get physical address of buffer start */
1467 if (DMASGLIST_ENAB)
1468 memset(&di->txp_dmah[txout], 0,
1469 sizeof(hnddma_seg_map_t));
1470
1471 pa = pci_map_single(di->pbus, data, len, PCI_DMA_TODEVICE);
1472
1473 if (DMASGLIST_ENAB) {
1474 map = &di->txp_dmah[txout];
1475
1476 /* See if all the segments can be accounted for */
1477 if (map->nsegs >
1478 (uint) (di->ntxd - NTXDACTIVE(di->txin, di->txout) -
1479 1))
1480 goto outoftxd;
1481
1482 nsegs = map->nsegs;
1483 } else
1484 nsegs = 1;
1485
1486 for (j = 1; j <= nsegs; j++) {
1487 flags = 0;
1488 if (p == p0 && j == 1)
1489 flags |= D64_CTRL1_SOF;
1490
1491 /* With a DMA segment list, Descriptor table is filled
1492 * using the segment list instead of looping over
1493 * buffers in multi-chain DMA. Therefore, EOF for SGLIST is when
1494 * end of segment list is reached.
1495 */
1496 if ((!DMASGLIST_ENAB && next == NULL) ||
1497 (DMASGLIST_ENAB && j == nsegs))
1498 flags |= (D64_CTRL1_IOC | D64_CTRL1_EOF);
1499 if (txout == (di->ntxd - 1))
1500 flags |= D64_CTRL1_EOT;
1501
1502 if (DMASGLIST_ENAB) {
1503 len = map->segs[j - 1].length;
1504 pa = map->segs[j - 1].addr;
1505 }
1506 dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);
1507 ASSERT(di->txp[txout] == NULL);
1508
1509 txout = NEXTTXD(txout);
1510 }
1511
1512 /* See above. No need to loop over individual buffers */
1513 if (DMASGLIST_ENAB)
1514 break;
1515 }
1516
1517 /* if last txd eof not set, fix it */
1518 if (!(flags & D64_CTRL1_EOF))
1519 W_SM(&di->txd64[PREVTXD(txout)].ctrl1,
1520 BUS_SWAP32(flags | D64_CTRL1_IOC | D64_CTRL1_EOF));
1521
1522 /* save the packet */
1523 di->txp[PREVTXD(txout)] = p0;
1524
1525 /* bump the tx descriptor index */
1526 di->txout = txout;
1527
1528 /* kick the chip */
1529 if (commit)
1530 W_REG(&di->d64txregs->ptr,
1531 di->xmtptrbase + I2B(txout, dma64dd_t));
1532
1533 /* tx flow control */
1534 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1535
1536 return 0;
1537
1538 outoftxd:
1539 DMA_ERROR(("%s: dma_txfast: out of txds !!!\n", di->name));
1540 pkt_buf_free_skb(p0);
1541 di->hnddma.txavail = 0;
1542 di->hnddma.txnobuf++;
1543 return -1;
1544 }
1545
1546 /*
1547 * Reclaim next completed txd (txds if using chained buffers) in the range
1548 * specified and return associated packet.
1549 * If range is HNDDMA_RANGE_TRANSMITTED, reclaim descriptors that have be
1550 * transmitted as noted by the hardware "CurrDescr" pointer.
1551 * If range is HNDDMA_RANGE_TRANSFERED, reclaim descriptors that have be
1552 * transferred by the DMA as noted by the hardware "ActiveDescr" pointer.
1553 * If range is HNDDMA_RANGE_ALL, reclaim all txd(s) posted to the ring and
1554 * return associated packet regardless of the value of hardware pointers.
1555 */
1556 static void *BCMFASTPATH dma64_getnexttxp(dma_info_t *di, txd_range_t range)
1557 {
1558 u16 start, end, i;
1559 u16 active_desc;
1560 void *txp;
1561
1562 DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name,
1563 (range == HNDDMA_RANGE_ALL) ? "all" :
1564 ((range ==
1565 HNDDMA_RANGE_TRANSMITTED) ? "transmitted" :
1566 "transferred")));
1567
1568 if (di->ntxd == 0)
1569 return NULL;
1570
1571 txp = NULL;
1572
1573 start = di->txin;
1574 if (range == HNDDMA_RANGE_ALL)
1575 end = di->txout;
1576 else {
1577 dma64regs_t *dregs = di->d64txregs;
1578
1579 end =
1580 (u16) (B2I
1581 (((R_REG(&dregs->status0) &
1582 D64_XS0_CD_MASK) -
1583 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t));
1584
1585 if (range == HNDDMA_RANGE_TRANSFERED) {
1586 active_desc =
1587 (u16) (R_REG(&dregs->status1) &
1588 D64_XS1_AD_MASK);
1589 active_desc =
1590 (active_desc - di->xmtptrbase) & D64_XS0_CD_MASK;
1591 active_desc = B2I(active_desc, dma64dd_t);
1592 if (end != active_desc)
1593 end = PREVTXD(active_desc);
1594 }
1595 }
1596
1597 if ((start == 0) && (end > di->txout))
1598 goto bogus;
1599
1600 for (i = start; i != end && !txp; i = NEXTTXD(i)) {
1601 dmaaddr_t pa;
1602 hnddma_seg_map_t *map = NULL;
1603 uint size, j, nsegs;
1604
1605 PHYSADDRLOSET(pa,
1606 (BUS_SWAP32(R_SM(&di->txd64[i].addrlow)) -
1607 di->dataoffsetlow));
1608 PHYSADDRHISET(pa,
1609 (BUS_SWAP32(R_SM(&di->txd64[i].addrhigh)) -
1610 di->dataoffsethigh));
1611
1612 if (DMASGLIST_ENAB) {
1613 map = &di->txp_dmah[i];
1614 size = map->origsize;
1615 nsegs = map->nsegs;
1616 } else {
1617 size =
1618 (BUS_SWAP32(R_SM(&di->txd64[i].ctrl2)) &
1619 D64_CTRL2_BC_MASK);
1620 nsegs = 1;
1621 }
1622
1623 for (j = nsegs; j > 0; j--) {
1624 W_SM(&di->txd64[i].addrlow, 0xdeadbeef);
1625 W_SM(&di->txd64[i].addrhigh, 0xdeadbeef);
1626
1627 txp = di->txp[i];
1628 di->txp[i] = NULL;
1629 if (j > 1)
1630 i = NEXTTXD(i);
1631 }
1632
1633 pci_unmap_single(di->pbus, pa, size, PCI_DMA_TODEVICE);
1634 }
1635
1636 di->txin = i;
1637
1638 /* tx flow control */
1639 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1640
1641 return txp;
1642
1643 bogus:
1644 DMA_NONE(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n", start, end, di->txout, forceall));
1645 return NULL;
1646 }
1647
1648 static void *BCMFASTPATH dma64_getnextrxp(dma_info_t *di, bool forceall)
1649 {
1650 uint i, curr;
1651 void *rxp;
1652 dmaaddr_t pa;
1653
1654 /* if forcing, dma engine must be disabled */
1655 ASSERT(!forceall || !dma64_rxenabled(di));
1656
1657 i = di->rxin;
1658
1659 /* return if no packets posted */
1660 if (i == di->rxout)
1661 return NULL;
1662
1663 curr =
1664 B2I(((R_REG(&di->d64rxregs->status0) & D64_RS0_CD_MASK) -
1665 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t);
1666
1667 /* ignore curr if forceall */
1668 if (!forceall && (i == curr))
1669 return NULL;
1670
1671 /* get the packet pointer that corresponds to the rx descriptor */
1672 rxp = di->rxp[i];
1673 ASSERT(rxp);
1674 di->rxp[i] = NULL;
1675
1676 PHYSADDRLOSET(pa,
1677 (BUS_SWAP32(R_SM(&di->rxd64[i].addrlow)) -
1678 di->dataoffsetlow));
1679 PHYSADDRHISET(pa,
1680 (BUS_SWAP32(R_SM(&di->rxd64[i].addrhigh)) -
1681 di->dataoffsethigh));
1682
1683 /* clear this packet from the descriptor ring */
1684 pci_unmap_single(di->pbus, pa, di->rxbufsize, PCI_DMA_FROMDEVICE);
1685
1686 W_SM(&di->rxd64[i].addrlow, 0xdeadbeef);
1687 W_SM(&di->rxd64[i].addrhigh, 0xdeadbeef);
1688
1689 di->rxin = NEXTRXD(i);
1690
1691 return rxp;
1692 }
1693
1694 static bool _dma64_addrext(dma64regs_t *dma64regs)
1695 {
1696 u32 w;
1697 OR_REG(&dma64regs->control, D64_XC_AE);
1698 w = R_REG(&dma64regs->control);
1699 AND_REG(&dma64regs->control, ~D64_XC_AE);
1700 return (w & D64_XC_AE) == D64_XC_AE;
1701 }
1702
1703 /*
1704 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin).
1705 */
1706 static void dma64_txrotate(dma_info_t *di)
1707 {
1708 u16 ad;
1709 uint nactive;
1710 uint rot;
1711 u16 old, new;
1712 u32 w;
1713 u16 first, last;
1714
1715 ASSERT(dma64_txsuspendedidle(di));
1716
1717 nactive = _dma_txactive(di);
1718 ad = (u16) (B2I
1719 ((((R_REG(&di->d64txregs->status1) &
1720 D64_XS1_AD_MASK)
1721 - di->xmtptrbase) & D64_XS1_AD_MASK), dma64dd_t));
1722 rot = TXD(ad - di->txin);
1723
1724 ASSERT(rot < di->ntxd);
1725
1726 /* full-ring case is a lot harder - don't worry about this */
1727 if (rot >= (di->ntxd - nactive)) {
1728 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name));
1729 return;
1730 }
1731
1732 first = di->txin;
1733 last = PREVTXD(di->txout);
1734
1735 /* move entries starting at last and moving backwards to first */
1736 for (old = last; old != PREVTXD(first); old = PREVTXD(old)) {
1737 new = TXD(old + rot);
1738
1739 /*
1740 * Move the tx dma descriptor.
1741 * EOT is set only in the last entry in the ring.
1742 */
1743 w = BUS_SWAP32(R_SM(&di->txd64[old].ctrl1)) & ~D64_CTRL1_EOT;
1744 if (new == (di->ntxd - 1))
1745 w |= D64_CTRL1_EOT;
1746 W_SM(&di->txd64[new].ctrl1, BUS_SWAP32(w));
1747
1748 w = BUS_SWAP32(R_SM(&di->txd64[old].ctrl2));
1749 W_SM(&di->txd64[new].ctrl2, BUS_SWAP32(w));
1750
1751 W_SM(&di->txd64[new].addrlow, R_SM(&di->txd64[old].addrlow));
1752 W_SM(&di->txd64[new].addrhigh, R_SM(&di->txd64[old].addrhigh));
1753
1754 /* zap the old tx dma descriptor address field */
1755 W_SM(&di->txd64[old].addrlow, BUS_SWAP32(0xdeadbeef));
1756 W_SM(&di->txd64[old].addrhigh, BUS_SWAP32(0xdeadbeef));
1757
1758 /* move the corresponding txp[] entry */
1759 ASSERT(di->txp[new] == NULL);
1760 di->txp[new] = di->txp[old];
1761
1762 /* Move the map */
1763 if (DMASGLIST_ENAB) {
1764 memcpy(&di->txp_dmah[new], &di->txp_dmah[old],
1765 sizeof(hnddma_seg_map_t));
1766 memset(&di->txp_dmah[old], 0, sizeof(hnddma_seg_map_t));
1767 }
1768
1769 di->txp[old] = NULL;
1770 }
1771
1772 /* update txin and txout */
1773 di->txin = ad;
1774 di->txout = TXD(di->txout + rot);
1775 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1776
1777 /* kick the chip */
1778 W_REG(&di->d64txregs->ptr,
1779 di->xmtptrbase + I2B(di->txout, dma64dd_t));
1780 }
1781
1782 uint dma_addrwidth(si_t *sih, void *dmaregs)
1783 {
1784 /* Perform 64-bit checks only if we want to advertise 64-bit (> 32bit) capability) */
1785 /* DMA engine is 64-bit capable */
1786 if ((si_core_sflags(sih, 0, 0) & SISF_DMA64) == SISF_DMA64) {
1787 /* backplane are 64-bit capable */
1788 if (si_backplane64(sih))
1789 /* If bus is System Backplane or PCIE then we can access 64-bits */
1790 if ((sih->bustype == SI_BUS) ||
1791 ((sih->bustype == PCI_BUS) &&
1792 (sih->buscoretype == PCIE_CORE_ID)))
1793 return DMADDRWIDTH_64;
1794 }
1795 ASSERT(0); /* DMA hardware not supported by this driver*/
1796 return DMADDRWIDTH_64;
1797 }
1798
1799 /*
1800 * Mac80211 initiated actions sometimes require packets in the DMA queue to be
1801 * modified. The modified portion of the packet is not under control of the DMA
1802 * engine. This function calls a caller-supplied function for each packet in
1803 * the caller specified dma chain.
1804 */
1805 void dma_walk_packets(struct hnddma_pub *dmah, void (*callback_fnc)
1806 (void *pkt, void *arg_a), void *arg_a)
1807 {
1808 dma_info_t *di = (dma_info_t *) dmah;
1809 uint i = di->txin;
1810 uint end = di->txout;
1811 struct sk_buff *skb;
1812 struct ieee80211_tx_info *tx_info;
1813
1814 while (i != end) {
1815 skb = (struct sk_buff *)di->txp[i];
1816 if (skb != NULL) {
1817 tx_info = (struct ieee80211_tx_info *)skb->cb;
1818 (callback_fnc)(tx_info, arg_a);
1819 }
1820 i = NEXTTXD(i);
1821 }
1822 }
1823