1 /*
2  * Copyright (c) 2010 Broadcom Corporation
3  *
4  * Permission to use, copy, modify, and/or distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
11  * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
13  * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
14  * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18 
19 #include <linux/slab.h>
20 #include <linux/delay.h>
21 #include <linux/pci.h>
22 
23 #include <brcmu_utils.h>
24 #include <aiutils.h>
25 #include "types.h"
26 #include "dma.h"
27 #include "soc.h"
28 
29 /*
30  * dma register field offset calculation
31  */
32 #define DMA64REGOFFS(field)		offsetof(struct dma64regs, field)
33 #define DMA64TXREGOFFS(di, field)	(di->d64txregbase + DMA64REGOFFS(field))
34 #define DMA64RXREGOFFS(di, field)	(di->d64rxregbase + DMA64REGOFFS(field))
35 
36 /*
37  * DMA hardware requires each descriptor ring to be 8kB aligned, and fit within
38  * a contiguous 8kB physical address.
39  */
40 #define D64RINGALIGN_BITS	13
41 #define	D64MAXRINGSZ		(1 << D64RINGALIGN_BITS)
42 #define	D64RINGALIGN		(1 << D64RINGALIGN_BITS)
43 
44 #define	D64MAXDD	(D64MAXRINGSZ / sizeof(struct dma64desc))
45 
46 /* transmit channel control */
47 #define	D64_XC_XE		0x00000001	/* transmit enable */
48 #define	D64_XC_SE		0x00000002	/* transmit suspend request */
49 #define	D64_XC_LE		0x00000004	/* loopback enable */
50 #define	D64_XC_FL		0x00000010	/* flush request */
51 #define	D64_XC_PD		0x00000800	/* parity check disable */
52 #define	D64_XC_AE		0x00030000	/* address extension bits */
53 #define	D64_XC_AE_SHIFT		16
54 
55 /* transmit descriptor table pointer */
56 #define	D64_XP_LD_MASK		0x00000fff	/* last valid descriptor */
57 
58 /* transmit channel status */
59 #define	D64_XS0_CD_MASK		0x00001fff	/* current descriptor pointer */
60 #define	D64_XS0_XS_MASK		0xf0000000	/* transmit state */
61 #define	D64_XS0_XS_SHIFT		28
62 #define	D64_XS0_XS_DISABLED	0x00000000	/* disabled */
63 #define	D64_XS0_XS_ACTIVE	0x10000000	/* active */
64 #define	D64_XS0_XS_IDLE		0x20000000	/* idle wait */
65 #define	D64_XS0_XS_STOPPED	0x30000000	/* stopped */
66 #define	D64_XS0_XS_SUSP		0x40000000	/* suspend pending */
67 
68 #define	D64_XS1_AD_MASK		0x00001fff	/* active descriptor */
69 #define	D64_XS1_XE_MASK		0xf0000000	/* transmit errors */
70 #define	D64_XS1_XE_SHIFT		28
71 #define	D64_XS1_XE_NOERR	0x00000000	/* no error */
72 #define	D64_XS1_XE_DPE		0x10000000	/* descriptor protocol error */
73 #define	D64_XS1_XE_DFU		0x20000000	/* data fifo underrun */
74 #define	D64_XS1_XE_DTE		0x30000000	/* data transfer error */
75 #define	D64_XS1_XE_DESRE	0x40000000	/* descriptor read error */
76 #define	D64_XS1_XE_COREE	0x50000000	/* core error */
77 
78 /* receive channel control */
79 /* receive enable */
80 #define	D64_RC_RE		0x00000001
81 /* receive frame offset */
82 #define	D64_RC_RO_MASK		0x000000fe
83 #define	D64_RC_RO_SHIFT		1
84 /* direct fifo receive (pio) mode */
85 #define	D64_RC_FM		0x00000100
86 /* separate rx header descriptor enable */
87 #define	D64_RC_SH		0x00000200
88 /* overflow continue */
89 #define	D64_RC_OC		0x00000400
90 /* parity check disable */
91 #define	D64_RC_PD		0x00000800
92 /* address extension bits */
93 #define	D64_RC_AE		0x00030000
94 #define	D64_RC_AE_SHIFT		16
95 
96 /* flags for dma controller */
97 /* partity enable */
98 #define DMA_CTRL_PEN		(1 << 0)
99 /* rx overflow continue */
100 #define DMA_CTRL_ROC		(1 << 1)
101 /* allow rx scatter to multiple descriptors */
102 #define DMA_CTRL_RXMULTI	(1 << 2)
103 /* Unframed Rx/Tx data */
104 #define DMA_CTRL_UNFRAMED	(1 << 3)
105 
106 /* receive descriptor table pointer */
107 #define	D64_RP_LD_MASK		0x00000fff	/* last valid descriptor */
108 
109 /* receive channel status */
110 #define	D64_RS0_CD_MASK		0x00001fff	/* current descriptor pointer */
111 #define	D64_RS0_RS_MASK		0xf0000000	/* receive state */
112 #define	D64_RS0_RS_SHIFT		28
113 #define	D64_RS0_RS_DISABLED	0x00000000	/* disabled */
114 #define	D64_RS0_RS_ACTIVE	0x10000000	/* active */
115 #define	D64_RS0_RS_IDLE		0x20000000	/* idle wait */
116 #define	D64_RS0_RS_STOPPED	0x30000000	/* stopped */
117 #define	D64_RS0_RS_SUSP		0x40000000	/* suspend pending */
118 
119 #define	D64_RS1_AD_MASK		0x0001ffff	/* active descriptor */
120 #define	D64_RS1_RE_MASK		0xf0000000	/* receive errors */
121 #define	D64_RS1_RE_SHIFT		28
122 #define	D64_RS1_RE_NOERR	0x00000000	/* no error */
123 #define	D64_RS1_RE_DPO		0x10000000	/* descriptor protocol error */
124 #define	D64_RS1_RE_DFU		0x20000000	/* data fifo overflow */
125 #define	D64_RS1_RE_DTE		0x30000000	/* data transfer error */
126 #define	D64_RS1_RE_DESRE	0x40000000	/* descriptor read error */
127 #define	D64_RS1_RE_COREE	0x50000000	/* core error */
128 
129 /* fifoaddr */
130 #define	D64_FA_OFF_MASK		0xffff	/* offset */
131 #define	D64_FA_SEL_MASK		0xf0000	/* select */
132 #define	D64_FA_SEL_SHIFT	16
133 #define	D64_FA_SEL_XDD		0x00000	/* transmit dma data */
134 #define	D64_FA_SEL_XDP		0x10000	/* transmit dma pointers */
135 #define	D64_FA_SEL_RDD		0x40000	/* receive dma data */
136 #define	D64_FA_SEL_RDP		0x50000	/* receive dma pointers */
137 #define	D64_FA_SEL_XFD		0x80000	/* transmit fifo data */
138 #define	D64_FA_SEL_XFP		0x90000	/* transmit fifo pointers */
139 #define	D64_FA_SEL_RFD		0xc0000	/* receive fifo data */
140 #define	D64_FA_SEL_RFP		0xd0000	/* receive fifo pointers */
141 #define	D64_FA_SEL_RSD		0xe0000	/* receive frame status data */
142 #define	D64_FA_SEL_RSP		0xf0000	/* receive frame status pointers */
143 
144 /* descriptor control flags 1 */
145 #define D64_CTRL_COREFLAGS	0x0ff00000	/* core specific flags */
146 #define	D64_CTRL1_EOT		((u32)1 << 28)	/* end of descriptor table */
147 #define	D64_CTRL1_IOC		((u32)1 << 29)	/* interrupt on completion */
148 #define	D64_CTRL1_EOF		((u32)1 << 30)	/* end of frame */
149 #define	D64_CTRL1_SOF		((u32)1 << 31)	/* start of frame */
150 
151 /* descriptor control flags 2 */
152 /* buffer byte count. real data len must <= 16KB */
153 #define	D64_CTRL2_BC_MASK	0x00007fff
154 /* address extension bits */
155 #define	D64_CTRL2_AE		0x00030000
156 #define	D64_CTRL2_AE_SHIFT	16
157 /* parity bit */
158 #define D64_CTRL2_PARITY	0x00040000
159 
160 /* control flags in the range [27:20] are core-specific and not defined here */
161 #define	D64_CTRL_CORE_MASK	0x0ff00000
162 
163 #define D64_RX_FRM_STS_LEN	0x0000ffff	/* frame length mask */
164 #define D64_RX_FRM_STS_OVFL	0x00800000	/* RxOverFlow */
165 #define D64_RX_FRM_STS_DSCRCNT	0x0f000000  /* no. of descriptors used - 1 */
166 #define D64_RX_FRM_STS_DATATYPE	0xf0000000	/* core-dependent data type */
167 
168 /*
169  * packet headroom necessary to accommodate the largest header
170  * in the system, (i.e TXOFF). By doing, we avoid the need to
171  * allocate an extra buffer for the header when bridging to WL.
172  * There is a compile time check in wlc.c which ensure that this
173  * value is at least as big as TXOFF. This value is used in
174  * dma_rxfill().
175  */
176 
177 #define BCMEXTRAHDROOM 172
178 
179 /* debug/trace */
180 #ifdef DEBUG
181 #define	DMA_ERROR(fmt, ...)					\
182 do {								\
183 	if (*di->msg_level & 1)					\
184 		pr_debug("%s: " fmt, __func__, ##__VA_ARGS__);	\
185 } while (0)
186 #define	DMA_TRACE(fmt, ...)					\
187 do {								\
188 	if (*di->msg_level & 2)					\
189 		pr_debug("%s: " fmt, __func__, ##__VA_ARGS__);	\
190 } while (0)
191 #else
192 #define	DMA_ERROR(fmt, ...)			\
193 	no_printk(fmt, ##__VA_ARGS__)
194 #define	DMA_TRACE(fmt, ...)			\
195 	no_printk(fmt, ##__VA_ARGS__)
196 #endif				/* DEBUG */
197 
198 #define	DMA_NONE(fmt, ...)			\
199 	no_printk(fmt, ##__VA_ARGS__)
200 
201 #define	MAXNAMEL	8	/* 8 char names */
202 
203 /* macros to convert between byte offsets and indexes */
204 #define	B2I(bytes, type)	((bytes) / sizeof(type))
205 #define	I2B(index, type)	((index) * sizeof(type))
206 
207 #define	PCI32ADDR_HIGH		0xc0000000	/* address[31:30] */
208 #define	PCI32ADDR_HIGH_SHIFT	30	/* address[31:30] */
209 
210 #define	PCI64ADDR_HIGH		0x80000000	/* address[63] */
211 #define	PCI64ADDR_HIGH_SHIFT	31	/* address[63] */
212 
213 /*
214  * DMA Descriptor
215  * Descriptors are only read by the hardware, never written back.
216  */
217 struct dma64desc {
218 	__le32 ctrl1;	/* misc control bits & bufcount */
219 	__le32 ctrl2;	/* buffer count and address extension */
220 	__le32 addrlow;	/* memory address of the date buffer, bits 31:0 */
221 	__le32 addrhigh; /* memory address of the date buffer, bits 63:32 */
222 };
223 
224 /* dma engine software state */
225 struct dma_info {
226 	struct dma_pub dma; /* exported structure */
227 	uint *msg_level;	/* message level pointer */
228 	char name[MAXNAMEL];	/* callers name for diag msgs */
229 
230 	struct bcma_device *core;
231 	struct device *dmadev;
232 
233 	bool dma64;	/* this dma engine is operating in 64-bit mode */
234 	bool addrext;	/* this dma engine supports DmaExtendedAddrChanges */
235 
236 	/* 64-bit dma tx engine registers */
237 	uint d64txregbase;
238 	/* 64-bit dma rx engine registers */
239 	uint d64rxregbase;
240 	/* pointer to dma64 tx descriptor ring */
241 	struct dma64desc *txd64;
242 	/* pointer to dma64 rx descriptor ring */
243 	struct dma64desc *rxd64;
244 
245 	u16 dmadesc_align;	/* alignment requirement for dma descriptors */
246 
247 	u16 ntxd;		/* # tx descriptors tunable */
248 	u16 txin;		/* index of next descriptor to reclaim */
249 	u16 txout;		/* index of next descriptor to post */
250 	/* pointer to parallel array of pointers to packets */
251 	struct sk_buff **txp;
252 	/* Aligned physical address of descriptor ring */
253 	dma_addr_t txdpa;
254 	/* Original physical address of descriptor ring */
255 	dma_addr_t txdpaorig;
256 	u16 txdalign;	/* #bytes added to alloc'd mem to align txd */
257 	u32 txdalloc;	/* #bytes allocated for the ring */
258 	u32 xmtptrbase;	/* When using unaligned descriptors, the ptr register
259 			 * is not just an index, it needs all 13 bits to be
260 			 * an offset from the addr register.
261 			 */
262 
263 	u16 nrxd;	/* # rx descriptors tunable */
264 	u16 rxin;	/* index of next descriptor to reclaim */
265 	u16 rxout;	/* index of next descriptor to post */
266 	/* pointer to parallel array of pointers to packets */
267 	struct sk_buff **rxp;
268 	/* Aligned physical address of descriptor ring */
269 	dma_addr_t rxdpa;
270 	/* Original physical address of descriptor ring */
271 	dma_addr_t rxdpaorig;
272 	u16 rxdalign;	/* #bytes added to alloc'd mem to align rxd */
273 	u32 rxdalloc;	/* #bytes allocated for the ring */
274 	u32 rcvptrbase;	/* Base for ptr reg when using unaligned descriptors */
275 
276 	/* tunables */
277 	unsigned int rxbufsize;	/* rx buffer size in bytes, not including
278 				 * the extra headroom
279 				 */
280 	uint rxextrahdrroom;	/* extra rx headroom, reverseved to assist upper
281 				 * stack, e.g. some rx pkt buffers will be
282 				 * bridged to tx side without byte copying.
283 				 * The extra headroom needs to be large enough
284 				 * to fit txheader needs. Some dongle driver may
285 				 * not need it.
286 				 */
287 	uint nrxpost;		/* # rx buffers to keep posted */
288 	unsigned int rxoffset;	/* rxcontrol offset */
289 	/* add to get dma address of descriptor ring, low 32 bits */
290 	uint ddoffsetlow;
291 	/*   high 32 bits */
292 	uint ddoffsethigh;
293 	/* add to get dma address of data buffer, low 32 bits */
294 	uint dataoffsetlow;
295 	/*   high 32 bits */
296 	uint dataoffsethigh;
297 	/* descriptor base need to be aligned or not */
298 	bool aligndesc_4k;
299 };
300 
301 /*
302  * default dma message level (if input msg_level
303  * pointer is null in dma_attach())
304  */
305 static uint dma_msg_level;
306 
307 /* Check for odd number of 1's */
parity32(__le32 data)308 static u32 parity32(__le32 data)
309 {
310 	/* no swap needed for counting 1's */
311 	u32 par_data = *(u32 *)&data;
312 
313 	par_data ^= par_data >> 16;
314 	par_data ^= par_data >> 8;
315 	par_data ^= par_data >> 4;
316 	par_data ^= par_data >> 2;
317 	par_data ^= par_data >> 1;
318 
319 	return par_data & 1;
320 }
321 
dma64_dd_parity(struct dma64desc * dd)322 static bool dma64_dd_parity(struct dma64desc *dd)
323 {
324 	return parity32(dd->addrlow ^ dd->addrhigh ^ dd->ctrl1 ^ dd->ctrl2);
325 }
326 
327 /* descriptor bumping functions */
328 
xxd(uint x,uint n)329 static uint xxd(uint x, uint n)
330 {
331 	return x & (n - 1); /* faster than %, but n must be power of 2 */
332 }
333 
txd(struct dma_info * di,uint x)334 static uint txd(struct dma_info *di, uint x)
335 {
336 	return xxd(x, di->ntxd);
337 }
338 
rxd(struct dma_info * di,uint x)339 static uint rxd(struct dma_info *di, uint x)
340 {
341 	return xxd(x, di->nrxd);
342 }
343 
nexttxd(struct dma_info * di,uint i)344 static uint nexttxd(struct dma_info *di, uint i)
345 {
346 	return txd(di, i + 1);
347 }
348 
prevtxd(struct dma_info * di,uint i)349 static uint prevtxd(struct dma_info *di, uint i)
350 {
351 	return txd(di, i - 1);
352 }
353 
nextrxd(struct dma_info * di,uint i)354 static uint nextrxd(struct dma_info *di, uint i)
355 {
356 	return txd(di, i + 1);
357 }
358 
ntxdactive(struct dma_info * di,uint h,uint t)359 static uint ntxdactive(struct dma_info *di, uint h, uint t)
360 {
361 	return txd(di, t-h);
362 }
363 
nrxdactive(struct dma_info * di,uint h,uint t)364 static uint nrxdactive(struct dma_info *di, uint h, uint t)
365 {
366 	return rxd(di, t-h);
367 }
368 
_dma_ctrlflags(struct dma_info * di,uint mask,uint flags)369 static uint _dma_ctrlflags(struct dma_info *di, uint mask, uint flags)
370 {
371 	uint dmactrlflags;
372 
373 	if (di == NULL) {
374 		DMA_ERROR("NULL dma handle\n");
375 		return 0;
376 	}
377 
378 	dmactrlflags = di->dma.dmactrlflags;
379 	dmactrlflags &= ~mask;
380 	dmactrlflags |= flags;
381 
382 	/* If trying to enable parity, check if parity is actually supported */
383 	if (dmactrlflags & DMA_CTRL_PEN) {
384 		u32 control;
385 
386 		control = bcma_read32(di->core, DMA64TXREGOFFS(di, control));
387 		bcma_write32(di->core, DMA64TXREGOFFS(di, control),
388 		      control | D64_XC_PD);
389 		if (bcma_read32(di->core, DMA64TXREGOFFS(di, control)) &
390 		    D64_XC_PD)
391 			/* We *can* disable it so it is supported,
392 			 * restore control register
393 			 */
394 			bcma_write32(di->core, DMA64TXREGOFFS(di, control),
395 				     control);
396 		else
397 			/* Not supported, don't allow it to be enabled */
398 			dmactrlflags &= ~DMA_CTRL_PEN;
399 	}
400 
401 	di->dma.dmactrlflags = dmactrlflags;
402 
403 	return dmactrlflags;
404 }
405 
_dma64_addrext(struct dma_info * di,uint ctrl_offset)406 static bool _dma64_addrext(struct dma_info *di, uint ctrl_offset)
407 {
408 	u32 w;
409 	bcma_set32(di->core, ctrl_offset, D64_XC_AE);
410 	w = bcma_read32(di->core, ctrl_offset);
411 	bcma_mask32(di->core, ctrl_offset, ~D64_XC_AE);
412 	return (w & D64_XC_AE) == D64_XC_AE;
413 }
414 
415 /*
416  * return true if this dma engine supports DmaExtendedAddrChanges,
417  * otherwise false
418  */
_dma_isaddrext(struct dma_info * di)419 static bool _dma_isaddrext(struct dma_info *di)
420 {
421 	/* DMA64 supports full 32- or 64-bit operation. AE is always valid */
422 
423 	/* not all tx or rx channel are available */
424 	if (di->d64txregbase != 0) {
425 		if (!_dma64_addrext(di, DMA64TXREGOFFS(di, control)))
426 			DMA_ERROR("%s: DMA64 tx doesn't have AE set\n",
427 				  di->name);
428 		return true;
429 	} else if (di->d64rxregbase != 0) {
430 		if (!_dma64_addrext(di, DMA64RXREGOFFS(di, control)))
431 			DMA_ERROR("%s: DMA64 rx doesn't have AE set\n",
432 				  di->name);
433 		return true;
434 	}
435 
436 	return false;
437 }
438 
_dma_descriptor_align(struct dma_info * di)439 static bool _dma_descriptor_align(struct dma_info *di)
440 {
441 	u32 addrl;
442 
443 	/* Check to see if the descriptors need to be aligned on 4K/8K or not */
444 	if (di->d64txregbase != 0) {
445 		bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow), 0xff0);
446 		addrl = bcma_read32(di->core, DMA64TXREGOFFS(di, addrlow));
447 		if (addrl != 0)
448 			return false;
449 	} else if (di->d64rxregbase != 0) {
450 		bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow), 0xff0);
451 		addrl = bcma_read32(di->core, DMA64RXREGOFFS(di, addrlow));
452 		if (addrl != 0)
453 			return false;
454 	}
455 	return true;
456 }
457 
458 /*
459  * Descriptor table must start at the DMA hardware dictated alignment, so
460  * allocated memory must be large enough to support this requirement.
461  */
dma_alloc_consistent(struct dma_info * di,uint size,u16 align_bits,uint * alloced,dma_addr_t * pap)462 static void *dma_alloc_consistent(struct dma_info *di, uint size,
463 				  u16 align_bits, uint *alloced,
464 				  dma_addr_t *pap)
465 {
466 	if (align_bits) {
467 		u16 align = (1 << align_bits);
468 		if (!IS_ALIGNED(PAGE_SIZE, align))
469 			size += align;
470 		*alloced = size;
471 	}
472 	return dma_alloc_coherent(di->dmadev, size, pap, GFP_ATOMIC);
473 }
474 
475 static
dma_align_sizetobits(uint size)476 u8 dma_align_sizetobits(uint size)
477 {
478 	u8 bitpos = 0;
479 	while (size >>= 1)
480 		bitpos++;
481 	return bitpos;
482 }
483 
484 /* This function ensures that the DMA descriptor ring will not get allocated
485  * across Page boundary. If the allocation is done across the page boundary
486  * at the first time, then it is freed and the allocation is done at
487  * descriptor ring size aligned location. This will ensure that the ring will
488  * not cross page boundary
489  */
dma_ringalloc(struct dma_info * di,u32 boundary,uint size,u16 * alignbits,uint * alloced,dma_addr_t * descpa)490 static void *dma_ringalloc(struct dma_info *di, u32 boundary, uint size,
491 			   u16 *alignbits, uint *alloced,
492 			   dma_addr_t *descpa)
493 {
494 	void *va;
495 	u32 desc_strtaddr;
496 	u32 alignbytes = 1 << *alignbits;
497 
498 	va = dma_alloc_consistent(di, size, *alignbits, alloced, descpa);
499 
500 	if (NULL == va)
501 		return NULL;
502 
503 	desc_strtaddr = (u32) roundup((unsigned long)va, alignbytes);
504 	if (((desc_strtaddr + size - 1) & boundary) != (desc_strtaddr
505 							& boundary)) {
506 		*alignbits = dma_align_sizetobits(size);
507 		dma_free_coherent(di->dmadev, size, va, *descpa);
508 		va = dma_alloc_consistent(di, size, *alignbits,
509 			alloced, descpa);
510 	}
511 	return va;
512 }
513 
dma64_alloc(struct dma_info * di,uint direction)514 static bool dma64_alloc(struct dma_info *di, uint direction)
515 {
516 	u16 size;
517 	uint ddlen;
518 	void *va;
519 	uint alloced = 0;
520 	u16 align;
521 	u16 align_bits;
522 
523 	ddlen = sizeof(struct dma64desc);
524 
525 	size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
526 	align_bits = di->dmadesc_align;
527 	align = (1 << align_bits);
528 
529 	if (direction == DMA_TX) {
530 		va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
531 			&alloced, &di->txdpaorig);
532 		if (va == NULL) {
533 			DMA_ERROR("%s: DMA_ALLOC_CONSISTENT(ntxd) failed\n",
534 				  di->name);
535 			return false;
536 		}
537 		align = (1 << align_bits);
538 		di->txd64 = (struct dma64desc *)
539 					roundup((unsigned long)va, align);
540 		di->txdalign = (uint) ((s8 *)di->txd64 - (s8 *) va);
541 		di->txdpa = di->txdpaorig + di->txdalign;
542 		di->txdalloc = alloced;
543 	} else {
544 		va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
545 			&alloced, &di->rxdpaorig);
546 		if (va == NULL) {
547 			DMA_ERROR("%s: DMA_ALLOC_CONSISTENT(nrxd) failed\n",
548 				  di->name);
549 			return false;
550 		}
551 		align = (1 << align_bits);
552 		di->rxd64 = (struct dma64desc *)
553 					roundup((unsigned long)va, align);
554 		di->rxdalign = (uint) ((s8 *)di->rxd64 - (s8 *) va);
555 		di->rxdpa = di->rxdpaorig + di->rxdalign;
556 		di->rxdalloc = alloced;
557 	}
558 
559 	return true;
560 }
561 
_dma_alloc(struct dma_info * di,uint direction)562 static bool _dma_alloc(struct dma_info *di, uint direction)
563 {
564 	return dma64_alloc(di, direction);
565 }
566 
dma_attach(char * name,struct si_pub * sih,struct bcma_device * core,uint txregbase,uint rxregbase,uint ntxd,uint nrxd,uint rxbufsize,int rxextheadroom,uint nrxpost,uint rxoffset,uint * msg_level)567 struct dma_pub *dma_attach(char *name, struct si_pub *sih,
568 			   struct bcma_device *core,
569 			   uint txregbase, uint rxregbase, uint ntxd, uint nrxd,
570 			   uint rxbufsize, int rxextheadroom,
571 			   uint nrxpost, uint rxoffset, uint *msg_level)
572 {
573 	struct dma_info *di;
574 	u8 rev = core->id.rev;
575 	uint size;
576 
577 	/* allocate private info structure */
578 	di = kzalloc(sizeof(struct dma_info), GFP_ATOMIC);
579 	if (di == NULL)
580 		return NULL;
581 
582 	di->msg_level = msg_level ? msg_level : &dma_msg_level;
583 
584 
585 	di->dma64 =
586 		((bcma_aread32(core, BCMA_IOST) & SISF_DMA64) == SISF_DMA64);
587 
588 	/* init dma reg info */
589 	di->core = core;
590 	di->d64txregbase = txregbase;
591 	di->d64rxregbase = rxregbase;
592 
593 	/*
594 	 * Default flags (which can be changed by the driver calling
595 	 * dma_ctrlflags before enable): For backwards compatibility
596 	 * both Rx Overflow Continue and Parity are DISABLED.
597 	 */
598 	_dma_ctrlflags(di, DMA_CTRL_ROC | DMA_CTRL_PEN, 0);
599 
600 	DMA_TRACE("%s: %s flags 0x%x ntxd %d nrxd %d "
601 		  "rxbufsize %d rxextheadroom %d nrxpost %d rxoffset %d "
602 		  "txregbase %u rxregbase %u\n", name, "DMA64",
603 		  di->dma.dmactrlflags, ntxd, nrxd, rxbufsize,
604 		  rxextheadroom, nrxpost, rxoffset, txregbase, rxregbase);
605 
606 	/* make a private copy of our callers name */
607 	strncpy(di->name, name, MAXNAMEL);
608 	di->name[MAXNAMEL - 1] = '\0';
609 
610 	di->dmadev = core->dma_dev;
611 
612 	/* save tunables */
613 	di->ntxd = (u16) ntxd;
614 	di->nrxd = (u16) nrxd;
615 
616 	/* the actual dma size doesn't include the extra headroom */
617 	di->rxextrahdrroom =
618 	    (rxextheadroom == -1) ? BCMEXTRAHDROOM : rxextheadroom;
619 	if (rxbufsize > BCMEXTRAHDROOM)
620 		di->rxbufsize = (u16) (rxbufsize - di->rxextrahdrroom);
621 	else
622 		di->rxbufsize = (u16) rxbufsize;
623 
624 	di->nrxpost = (u16) nrxpost;
625 	di->rxoffset = (u8) rxoffset;
626 
627 	/*
628 	 * figure out the DMA physical address offset for dd and data
629 	 *     PCI/PCIE: they map silicon backplace address to zero
630 	 *     based memory, need offset
631 	 *     Other bus: use zero SI_BUS BIGENDIAN kludge: use sdram
632 	 *     swapped region for data buffer, not descriptor
633 	 */
634 	di->ddoffsetlow = 0;
635 	di->dataoffsetlow = 0;
636 	/* add offset for pcie with DMA64 bus */
637 	di->ddoffsetlow = 0;
638 	di->ddoffsethigh = SI_PCIE_DMA_H32;
639 	di->dataoffsetlow = di->ddoffsetlow;
640 	di->dataoffsethigh = di->ddoffsethigh;
641 	/* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */
642 	if ((core->id.id == SDIOD_CORE_ID)
643 	    && ((rev > 0) && (rev <= 2)))
644 		di->addrext = false;
645 	else if ((core->id.id == I2S_CORE_ID) &&
646 		 ((rev == 0) || (rev == 1)))
647 		di->addrext = false;
648 	else
649 		di->addrext = _dma_isaddrext(di);
650 
651 	/* does the descriptor need to be aligned and if yes, on 4K/8K or not */
652 	di->aligndesc_4k = _dma_descriptor_align(di);
653 	if (di->aligndesc_4k) {
654 		di->dmadesc_align = D64RINGALIGN_BITS;
655 		if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2))
656 			/* for smaller dd table, HW relax alignment reqmnt */
657 			di->dmadesc_align = D64RINGALIGN_BITS - 1;
658 	} else {
659 		di->dmadesc_align = 4;	/* 16 byte alignment */
660 	}
661 
662 	DMA_NONE("DMA descriptor align_needed %d, align %d\n",
663 		 di->aligndesc_4k, di->dmadesc_align);
664 
665 	/* allocate tx packet pointer vector */
666 	if (ntxd) {
667 		size = ntxd * sizeof(void *);
668 		di->txp = kzalloc(size, GFP_ATOMIC);
669 		if (di->txp == NULL)
670 			goto fail;
671 	}
672 
673 	/* allocate rx packet pointer vector */
674 	if (nrxd) {
675 		size = nrxd * sizeof(void *);
676 		di->rxp = kzalloc(size, GFP_ATOMIC);
677 		if (di->rxp == NULL)
678 			goto fail;
679 	}
680 
681 	/*
682 	 * allocate transmit descriptor ring, only need ntxd descriptors
683 	 * but it must be aligned
684 	 */
685 	if (ntxd) {
686 		if (!_dma_alloc(di, DMA_TX))
687 			goto fail;
688 	}
689 
690 	/*
691 	 * allocate receive descriptor ring, only need nrxd descriptors
692 	 * but it must be aligned
693 	 */
694 	if (nrxd) {
695 		if (!_dma_alloc(di, DMA_RX))
696 			goto fail;
697 	}
698 
699 	if ((di->ddoffsetlow != 0) && !di->addrext) {
700 		if (di->txdpa > SI_PCI_DMA_SZ) {
701 			DMA_ERROR("%s: txdpa 0x%x: addrext not supported\n",
702 				  di->name, (u32)di->txdpa);
703 			goto fail;
704 		}
705 		if (di->rxdpa > SI_PCI_DMA_SZ) {
706 			DMA_ERROR("%s: rxdpa 0x%x: addrext not supported\n",
707 				  di->name, (u32)di->rxdpa);
708 			goto fail;
709 		}
710 	}
711 
712 	DMA_TRACE("ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh 0x%x addrext %d\n",
713 		  di->ddoffsetlow, di->ddoffsethigh,
714 		  di->dataoffsetlow, di->dataoffsethigh,
715 		  di->addrext);
716 
717 	return (struct dma_pub *) di;
718 
719  fail:
720 	dma_detach((struct dma_pub *)di);
721 	return NULL;
722 }
723 
724 static inline void
dma64_dd_upd(struct dma_info * di,struct dma64desc * ddring,dma_addr_t pa,uint outidx,u32 * flags,u32 bufcount)725 dma64_dd_upd(struct dma_info *di, struct dma64desc *ddring,
726 	     dma_addr_t pa, uint outidx, u32 *flags, u32 bufcount)
727 {
728 	u32 ctrl2 = bufcount & D64_CTRL2_BC_MASK;
729 
730 	/* PCI bus with big(>1G) physical address, use address extension */
731 	if ((di->dataoffsetlow == 0) || !(pa & PCI32ADDR_HIGH)) {
732 		ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);
733 		ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);
734 		ddring[outidx].ctrl1 = cpu_to_le32(*flags);
735 		ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);
736 	} else {
737 		/* address extension for 32-bit PCI */
738 		u32 ae;
739 
740 		ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
741 		pa &= ~PCI32ADDR_HIGH;
742 
743 		ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE;
744 		ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);
745 		ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);
746 		ddring[outidx].ctrl1 = cpu_to_le32(*flags);
747 		ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);
748 	}
749 	if (di->dma.dmactrlflags & DMA_CTRL_PEN) {
750 		if (dma64_dd_parity(&ddring[outidx]))
751 			ddring[outidx].ctrl2 =
752 			     cpu_to_le32(ctrl2 | D64_CTRL2_PARITY);
753 	}
754 }
755 
756 /* !! may be called with core in reset */
dma_detach(struct dma_pub * pub)757 void dma_detach(struct dma_pub *pub)
758 {
759 	struct dma_info *di = (struct dma_info *)pub;
760 
761 	DMA_TRACE("%s:\n", di->name);
762 
763 	/* free dma descriptor rings */
764 	if (di->txd64)
765 		dma_free_coherent(di->dmadev, di->txdalloc,
766 				  ((s8 *)di->txd64 - di->txdalign),
767 				  (di->txdpaorig));
768 	if (di->rxd64)
769 		dma_free_coherent(di->dmadev, di->rxdalloc,
770 				  ((s8 *)di->rxd64 - di->rxdalign),
771 				  (di->rxdpaorig));
772 
773 	/* free packet pointer vectors */
774 	kfree(di->txp);
775 	kfree(di->rxp);
776 
777 	/* free our private info structure */
778 	kfree(di);
779 
780 }
781 
782 /* initialize descriptor table base address */
783 static void
_dma_ddtable_init(struct dma_info * di,uint direction,dma_addr_t pa)784 _dma_ddtable_init(struct dma_info *di, uint direction, dma_addr_t pa)
785 {
786 	if (!di->aligndesc_4k) {
787 		if (direction == DMA_TX)
788 			di->xmtptrbase = pa;
789 		else
790 			di->rcvptrbase = pa;
791 	}
792 
793 	if ((di->ddoffsetlow == 0)
794 	    || !(pa & PCI32ADDR_HIGH)) {
795 		if (direction == DMA_TX) {
796 			bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),
797 				     pa + di->ddoffsetlow);
798 			bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),
799 				     di->ddoffsethigh);
800 		} else {
801 			bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),
802 				     pa + di->ddoffsetlow);
803 			bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),
804 				     di->ddoffsethigh);
805 		}
806 	} else {
807 		/* DMA64 32bits address extension */
808 		u32 ae;
809 
810 		/* shift the high bit(s) from pa to ae */
811 		ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
812 		pa &= ~PCI32ADDR_HIGH;
813 
814 		if (direction == DMA_TX) {
815 			bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),
816 				     pa + di->ddoffsetlow);
817 			bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),
818 				     di->ddoffsethigh);
819 			bcma_maskset32(di->core, DMA64TXREGOFFS(di, control),
820 				       D64_XC_AE, (ae << D64_XC_AE_SHIFT));
821 		} else {
822 			bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),
823 				     pa + di->ddoffsetlow);
824 			bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),
825 				     di->ddoffsethigh);
826 			bcma_maskset32(di->core, DMA64RXREGOFFS(di, control),
827 				       D64_RC_AE, (ae << D64_RC_AE_SHIFT));
828 		}
829 	}
830 }
831 
_dma_rxenable(struct dma_info * di)832 static void _dma_rxenable(struct dma_info *di)
833 {
834 	uint dmactrlflags = di->dma.dmactrlflags;
835 	u32 control;
836 
837 	DMA_TRACE("%s:\n", di->name);
838 
839 	control = D64_RC_RE | (bcma_read32(di->core,
840 					   DMA64RXREGOFFS(di, control)) &
841 			       D64_RC_AE);
842 
843 	if ((dmactrlflags & DMA_CTRL_PEN) == 0)
844 		control |= D64_RC_PD;
845 
846 	if (dmactrlflags & DMA_CTRL_ROC)
847 		control |= D64_RC_OC;
848 
849 	bcma_write32(di->core, DMA64RXREGOFFS(di, control),
850 		((di->rxoffset << D64_RC_RO_SHIFT) | control));
851 }
852 
dma_rxinit(struct dma_pub * pub)853 void dma_rxinit(struct dma_pub *pub)
854 {
855 	struct dma_info *di = (struct dma_info *)pub;
856 
857 	DMA_TRACE("%s:\n", di->name);
858 
859 	if (di->nrxd == 0)
860 		return;
861 
862 	di->rxin = di->rxout = 0;
863 
864 	/* clear rx descriptor ring */
865 	memset(di->rxd64, '\0', di->nrxd * sizeof(struct dma64desc));
866 
867 	/* DMA engine with out alignment requirement requires table to be inited
868 	 * before enabling the engine
869 	 */
870 	if (!di->aligndesc_4k)
871 		_dma_ddtable_init(di, DMA_RX, di->rxdpa);
872 
873 	_dma_rxenable(di);
874 
875 	if (di->aligndesc_4k)
876 		_dma_ddtable_init(di, DMA_RX, di->rxdpa);
877 }
878 
dma64_getnextrxp(struct dma_info * di,bool forceall)879 static struct sk_buff *dma64_getnextrxp(struct dma_info *di, bool forceall)
880 {
881 	uint i, curr;
882 	struct sk_buff *rxp;
883 	dma_addr_t pa;
884 
885 	i = di->rxin;
886 
887 	/* return if no packets posted */
888 	if (i == di->rxout)
889 		return NULL;
890 
891 	curr =
892 	    B2I(((bcma_read32(di->core,
893 			      DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) -
894 		 di->rcvptrbase) & D64_RS0_CD_MASK, struct dma64desc);
895 
896 	/* ignore curr if forceall */
897 	if (!forceall && (i == curr))
898 		return NULL;
899 
900 	/* get the packet pointer that corresponds to the rx descriptor */
901 	rxp = di->rxp[i];
902 	di->rxp[i] = NULL;
903 
904 	pa = le32_to_cpu(di->rxd64[i].addrlow) - di->dataoffsetlow;
905 
906 	/* clear this packet from the descriptor ring */
907 	dma_unmap_single(di->dmadev, pa, di->rxbufsize, DMA_FROM_DEVICE);
908 
909 	di->rxd64[i].addrlow = cpu_to_le32(0xdeadbeef);
910 	di->rxd64[i].addrhigh = cpu_to_le32(0xdeadbeef);
911 
912 	di->rxin = nextrxd(di, i);
913 
914 	return rxp;
915 }
916 
_dma_getnextrxp(struct dma_info * di,bool forceall)917 static struct sk_buff *_dma_getnextrxp(struct dma_info *di, bool forceall)
918 {
919 	if (di->nrxd == 0)
920 		return NULL;
921 
922 	return dma64_getnextrxp(di, forceall);
923 }
924 
925 /*
926  * !! rx entry routine
927  * returns the number packages in the next frame, or 0 if there are no more
928  *   if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is
929  *   supported with pkts chain
930  *   otherwise, it's treated as giant pkt and will be tossed.
931  *   The DMA scattering starts with normal DMA header, followed by first
932  *   buffer data. After it reaches the max size of buffer, the data continues
933  *   in next DMA descriptor buffer WITHOUT DMA header
934  */
dma_rx(struct dma_pub * pub,struct sk_buff_head * skb_list)935 int dma_rx(struct dma_pub *pub, struct sk_buff_head *skb_list)
936 {
937 	struct dma_info *di = (struct dma_info *)pub;
938 	struct sk_buff_head dma_frames;
939 	struct sk_buff *p, *next;
940 	uint len;
941 	uint pkt_len;
942 	int resid = 0;
943 	int pktcnt = 1;
944 
945 	skb_queue_head_init(&dma_frames);
946  next_frame:
947 	p = _dma_getnextrxp(di, false);
948 	if (p == NULL)
949 		return 0;
950 
951 	len = le16_to_cpu(*(__le16 *) (p->data));
952 	DMA_TRACE("%s: dma_rx len %d\n", di->name, len);
953 	dma_spin_for_len(len, p);
954 
955 	/* set actual length */
956 	pkt_len = min((di->rxoffset + len), di->rxbufsize);
957 	__skb_trim(p, pkt_len);
958 	skb_queue_tail(&dma_frames, p);
959 	resid = len - (di->rxbufsize - di->rxoffset);
960 
961 	/* check for single or multi-buffer rx */
962 	if (resid > 0) {
963 		while ((resid > 0) && (p = _dma_getnextrxp(di, false))) {
964 			pkt_len = min_t(uint, resid, di->rxbufsize);
965 			__skb_trim(p, pkt_len);
966 			skb_queue_tail(&dma_frames, p);
967 			resid -= di->rxbufsize;
968 			pktcnt++;
969 		}
970 
971 #ifdef DEBUG
972 		if (resid > 0) {
973 			uint cur;
974 			cur =
975 			    B2I(((bcma_read32(di->core,
976 					      DMA64RXREGOFFS(di, status0)) &
977 				  D64_RS0_CD_MASK) - di->rcvptrbase) &
978 				D64_RS0_CD_MASK, struct dma64desc);
979 			DMA_ERROR("rxin %d rxout %d, hw_curr %d\n",
980 				   di->rxin, di->rxout, cur);
981 		}
982 #endif				/* DEBUG */
983 
984 		if ((di->dma.dmactrlflags & DMA_CTRL_RXMULTI) == 0) {
985 			DMA_ERROR("%s: bad frame length (%d)\n",
986 				  di->name, len);
987 			skb_queue_walk_safe(&dma_frames, p, next) {
988 				skb_unlink(p, &dma_frames);
989 				brcmu_pkt_buf_free_skb(p);
990 			}
991 			di->dma.rxgiants++;
992 			pktcnt = 1;
993 			goto next_frame;
994 		}
995 	}
996 
997 	skb_queue_splice_tail(&dma_frames, skb_list);
998 	return pktcnt;
999 }
1000 
dma64_rxidle(struct dma_info * di)1001 static bool dma64_rxidle(struct dma_info *di)
1002 {
1003 	DMA_TRACE("%s:\n", di->name);
1004 
1005 	if (di->nrxd == 0)
1006 		return true;
1007 
1008 	return ((bcma_read32(di->core,
1009 			     DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) ==
1010 		(bcma_read32(di->core, DMA64RXREGOFFS(di, ptr)) &
1011 		 D64_RS0_CD_MASK));
1012 }
1013 
1014 /*
1015  * post receive buffers
1016  *  Return false if refill failed completely or dma mapping failed. The ring
1017  *  is empty, which will stall the rx dma and user might want to call rxfill
1018  *  again asap. This is unlikely to happen on a memory-rich NIC, but often on
1019  *  memory-constrained dongle.
1020  */
dma_rxfill(struct dma_pub * pub)1021 bool dma_rxfill(struct dma_pub *pub)
1022 {
1023 	struct dma_info *di = (struct dma_info *)pub;
1024 	struct sk_buff *p;
1025 	u16 rxin, rxout;
1026 	u32 flags = 0;
1027 	uint n;
1028 	uint i;
1029 	dma_addr_t pa;
1030 	uint extra_offset = 0;
1031 	bool ring_empty;
1032 
1033 	ring_empty = false;
1034 
1035 	/*
1036 	 * Determine how many receive buffers we're lacking
1037 	 * from the full complement, allocate, initialize,
1038 	 * and post them, then update the chip rx lastdscr.
1039 	 */
1040 
1041 	rxin = di->rxin;
1042 	rxout = di->rxout;
1043 
1044 	n = di->nrxpost - nrxdactive(di, rxin, rxout);
1045 
1046 	DMA_TRACE("%s: post %d\n", di->name, n);
1047 
1048 	if (di->rxbufsize > BCMEXTRAHDROOM)
1049 		extra_offset = di->rxextrahdrroom;
1050 
1051 	for (i = 0; i < n; i++) {
1052 		/*
1053 		 * the di->rxbufsize doesn't include the extra headroom,
1054 		 * we need to add it to the size to be allocated
1055 		 */
1056 		p = brcmu_pkt_buf_get_skb(di->rxbufsize + extra_offset);
1057 
1058 		if (p == NULL) {
1059 			DMA_ERROR("%s: out of rxbufs\n", di->name);
1060 			if (i == 0 && dma64_rxidle(di)) {
1061 				DMA_ERROR("%s: ring is empty !\n", di->name);
1062 				ring_empty = true;
1063 			}
1064 			di->dma.rxnobuf++;
1065 			break;
1066 		}
1067 		/* reserve an extra headroom, if applicable */
1068 		if (extra_offset)
1069 			skb_pull(p, extra_offset);
1070 
1071 		/* Do a cached write instead of uncached write since DMA_MAP
1072 		 * will flush the cache.
1073 		 */
1074 		*(u32 *) (p->data) = 0;
1075 
1076 		pa = dma_map_single(di->dmadev, p->data, di->rxbufsize,
1077 				    DMA_FROM_DEVICE);
1078 		if (dma_mapping_error(di->dmadev, pa))
1079 			return false;
1080 
1081 		/* save the free packet pointer */
1082 		di->rxp[rxout] = p;
1083 
1084 		/* reset flags for each descriptor */
1085 		flags = 0;
1086 		if (rxout == (di->nrxd - 1))
1087 			flags = D64_CTRL1_EOT;
1088 
1089 		dma64_dd_upd(di, di->rxd64, pa, rxout, &flags,
1090 			     di->rxbufsize);
1091 		rxout = nextrxd(di, rxout);
1092 	}
1093 
1094 	di->rxout = rxout;
1095 
1096 	/* update the chip lastdscr pointer */
1097 	bcma_write32(di->core, DMA64RXREGOFFS(di, ptr),
1098 	      di->rcvptrbase + I2B(rxout, struct dma64desc));
1099 
1100 	return ring_empty;
1101 }
1102 
dma_rxreclaim(struct dma_pub * pub)1103 void dma_rxreclaim(struct dma_pub *pub)
1104 {
1105 	struct dma_info *di = (struct dma_info *)pub;
1106 	struct sk_buff *p;
1107 
1108 	DMA_TRACE("%s:\n", di->name);
1109 
1110 	while ((p = _dma_getnextrxp(di, true)))
1111 		brcmu_pkt_buf_free_skb(p);
1112 }
1113 
dma_counterreset(struct dma_pub * pub)1114 void dma_counterreset(struct dma_pub *pub)
1115 {
1116 	/* reset all software counters */
1117 	pub->rxgiants = 0;
1118 	pub->rxnobuf = 0;
1119 	pub->txnobuf = 0;
1120 }
1121 
1122 /* get the address of the var in order to change later */
dma_getvar(struct dma_pub * pub,const char * name)1123 unsigned long dma_getvar(struct dma_pub *pub, const char *name)
1124 {
1125 	struct dma_info *di = (struct dma_info *)pub;
1126 
1127 	if (!strcmp(name, "&txavail"))
1128 		return (unsigned long)&(di->dma.txavail);
1129 	return 0;
1130 }
1131 
1132 /* 64-bit DMA functions */
1133 
dma_txinit(struct dma_pub * pub)1134 void dma_txinit(struct dma_pub *pub)
1135 {
1136 	struct dma_info *di = (struct dma_info *)pub;
1137 	u32 control = D64_XC_XE;
1138 
1139 	DMA_TRACE("%s:\n", di->name);
1140 
1141 	if (di->ntxd == 0)
1142 		return;
1143 
1144 	di->txin = di->txout = 0;
1145 	di->dma.txavail = di->ntxd - 1;
1146 
1147 	/* clear tx descriptor ring */
1148 	memset(di->txd64, '\0', (di->ntxd * sizeof(struct dma64desc)));
1149 
1150 	/* DMA engine with out alignment requirement requires table to be inited
1151 	 * before enabling the engine
1152 	 */
1153 	if (!di->aligndesc_4k)
1154 		_dma_ddtable_init(di, DMA_TX, di->txdpa);
1155 
1156 	if ((di->dma.dmactrlflags & DMA_CTRL_PEN) == 0)
1157 		control |= D64_XC_PD;
1158 	bcma_set32(di->core, DMA64TXREGOFFS(di, control), control);
1159 
1160 	/* DMA engine with alignment requirement requires table to be inited
1161 	 * before enabling the engine
1162 	 */
1163 	if (di->aligndesc_4k)
1164 		_dma_ddtable_init(di, DMA_TX, di->txdpa);
1165 }
1166 
dma_txsuspend(struct dma_pub * pub)1167 void dma_txsuspend(struct dma_pub *pub)
1168 {
1169 	struct dma_info *di = (struct dma_info *)pub;
1170 
1171 	DMA_TRACE("%s:\n", di->name);
1172 
1173 	if (di->ntxd == 0)
1174 		return;
1175 
1176 	bcma_set32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);
1177 }
1178 
dma_txresume(struct dma_pub * pub)1179 void dma_txresume(struct dma_pub *pub)
1180 {
1181 	struct dma_info *di = (struct dma_info *)pub;
1182 
1183 	DMA_TRACE("%s:\n", di->name);
1184 
1185 	if (di->ntxd == 0)
1186 		return;
1187 
1188 	bcma_mask32(di->core, DMA64TXREGOFFS(di, control), ~D64_XC_SE);
1189 }
1190 
dma_txsuspended(struct dma_pub * pub)1191 bool dma_txsuspended(struct dma_pub *pub)
1192 {
1193 	struct dma_info *di = (struct dma_info *)pub;
1194 
1195 	return (di->ntxd == 0) ||
1196 	       ((bcma_read32(di->core,
1197 			     DMA64TXREGOFFS(di, control)) & D64_XC_SE) ==
1198 		D64_XC_SE);
1199 }
1200 
dma_txreclaim(struct dma_pub * pub,enum txd_range range)1201 void dma_txreclaim(struct dma_pub *pub, enum txd_range range)
1202 {
1203 	struct dma_info *di = (struct dma_info *)pub;
1204 	struct sk_buff *p;
1205 
1206 	DMA_TRACE("%s: %s\n",
1207 		  di->name,
1208 		  range == DMA_RANGE_ALL ? "all" :
1209 		  range == DMA_RANGE_TRANSMITTED ? "transmitted" :
1210 		  "transferred");
1211 
1212 	if (di->txin == di->txout)
1213 		return;
1214 
1215 	while ((p = dma_getnexttxp(pub, range))) {
1216 		/* For unframed data, we don't have any packets to free */
1217 		if (!(di->dma.dmactrlflags & DMA_CTRL_UNFRAMED))
1218 			brcmu_pkt_buf_free_skb(p);
1219 	}
1220 }
1221 
dma_txreset(struct dma_pub * pub)1222 bool dma_txreset(struct dma_pub *pub)
1223 {
1224 	struct dma_info *di = (struct dma_info *)pub;
1225 	u32 status;
1226 
1227 	if (di->ntxd == 0)
1228 		return true;
1229 
1230 	/* suspend tx DMA first */
1231 	bcma_write32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);
1232 	SPINWAIT(((status =
1233 		   (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &
1234 		    D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED) &&
1235 		  (status != D64_XS0_XS_IDLE) && (status != D64_XS0_XS_STOPPED),
1236 		 10000);
1237 
1238 	bcma_write32(di->core, DMA64TXREGOFFS(di, control), 0);
1239 	SPINWAIT(((status =
1240 		   (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &
1241 		    D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED), 10000);
1242 
1243 	/* wait for the last transaction to complete */
1244 	udelay(300);
1245 
1246 	return status == D64_XS0_XS_DISABLED;
1247 }
1248 
dma_rxreset(struct dma_pub * pub)1249 bool dma_rxreset(struct dma_pub *pub)
1250 {
1251 	struct dma_info *di = (struct dma_info *)pub;
1252 	u32 status;
1253 
1254 	if (di->nrxd == 0)
1255 		return true;
1256 
1257 	bcma_write32(di->core, DMA64RXREGOFFS(di, control), 0);
1258 	SPINWAIT(((status =
1259 		   (bcma_read32(di->core, DMA64RXREGOFFS(di, status0)) &
1260 		    D64_RS0_RS_MASK)) != D64_RS0_RS_DISABLED), 10000);
1261 
1262 	return status == D64_RS0_RS_DISABLED;
1263 }
1264 
1265 /*
1266  * !! tx entry routine
1267  * WARNING: call must check the return value for error.
1268  *   the error(toss frames) could be fatal and cause many subsequent hard
1269  *   to debug problems
1270  */
dma_txfast(struct dma_pub * pub,struct sk_buff * p,bool commit)1271 int dma_txfast(struct dma_pub *pub, struct sk_buff *p, bool commit)
1272 {
1273 	struct dma_info *di = (struct dma_info *)pub;
1274 	unsigned char *data;
1275 	uint len;
1276 	u16 txout;
1277 	u32 flags = 0;
1278 	dma_addr_t pa;
1279 
1280 	DMA_TRACE("%s:\n", di->name);
1281 
1282 	txout = di->txout;
1283 
1284 	/*
1285 	 * obtain and initialize transmit descriptor entry.
1286 	 */
1287 	data = p->data;
1288 	len = p->len;
1289 
1290 	/* no use to transmit a zero length packet */
1291 	if (len == 0)
1292 		return 0;
1293 
1294 	/* return nonzero if out of tx descriptors */
1295 	if (nexttxd(di, txout) == di->txin)
1296 		goto outoftxd;
1297 
1298 	/* get physical address of buffer start */
1299 	pa = dma_map_single(di->dmadev, data, len, DMA_TO_DEVICE);
1300 	/* if mapping failed, free skb */
1301 	if (dma_mapping_error(di->dmadev, pa)) {
1302 		brcmu_pkt_buf_free_skb(p);
1303 		return;
1304 	}
1305 	/* With a DMA segment list, Descriptor table is filled
1306 	 * using the segment list instead of looping over
1307 	 * buffers in multi-chain DMA. Therefore, EOF for SGLIST
1308 	 * is when end of segment list is reached.
1309 	 */
1310 	flags = D64_CTRL1_SOF | D64_CTRL1_IOC | D64_CTRL1_EOF;
1311 	if (txout == (di->ntxd - 1))
1312 		flags |= D64_CTRL1_EOT;
1313 
1314 	dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);
1315 
1316 	txout = nexttxd(di, txout);
1317 
1318 	/* save the packet */
1319 	di->txp[prevtxd(di, txout)] = p;
1320 
1321 	/* bump the tx descriptor index */
1322 	di->txout = txout;
1323 
1324 	/* kick the chip */
1325 	if (commit)
1326 		bcma_write32(di->core, DMA64TXREGOFFS(di, ptr),
1327 		      di->xmtptrbase + I2B(txout, struct dma64desc));
1328 
1329 	/* tx flow control */
1330 	di->dma.txavail = di->ntxd - ntxdactive(di, di->txin, di->txout) - 1;
1331 
1332 	return 0;
1333 
1334  outoftxd:
1335 	DMA_ERROR("%s: out of txds !!!\n", di->name);
1336 	brcmu_pkt_buf_free_skb(p);
1337 	di->dma.txavail = 0;
1338 	di->dma.txnobuf++;
1339 	return -1;
1340 }
1341 
1342 /*
1343  * Reclaim next completed txd (txds if using chained buffers) in the range
1344  * specified and return associated packet.
1345  * If range is DMA_RANGE_TRANSMITTED, reclaim descriptors that have be
1346  * transmitted as noted by the hardware "CurrDescr" pointer.
1347  * If range is DMA_RANGE_TRANSFERED, reclaim descriptors that have be
1348  * transferred by the DMA as noted by the hardware "ActiveDescr" pointer.
1349  * If range is DMA_RANGE_ALL, reclaim all txd(s) posted to the ring and
1350  * return associated packet regardless of the value of hardware pointers.
1351  */
dma_getnexttxp(struct dma_pub * pub,enum txd_range range)1352 struct sk_buff *dma_getnexttxp(struct dma_pub *pub, enum txd_range range)
1353 {
1354 	struct dma_info *di = (struct dma_info *)pub;
1355 	u16 start, end, i;
1356 	u16 active_desc;
1357 	struct sk_buff *txp;
1358 
1359 	DMA_TRACE("%s: %s\n",
1360 		  di->name,
1361 		  range == DMA_RANGE_ALL ? "all" :
1362 		  range == DMA_RANGE_TRANSMITTED ? "transmitted" :
1363 		  "transferred");
1364 
1365 	if (di->ntxd == 0)
1366 		return NULL;
1367 
1368 	txp = NULL;
1369 
1370 	start = di->txin;
1371 	if (range == DMA_RANGE_ALL)
1372 		end = di->txout;
1373 	else {
1374 		end = (u16) (B2I(((bcma_read32(di->core,
1375 					       DMA64TXREGOFFS(di, status0)) &
1376 				   D64_XS0_CD_MASK) - di->xmtptrbase) &
1377 				 D64_XS0_CD_MASK, struct dma64desc));
1378 
1379 		if (range == DMA_RANGE_TRANSFERED) {
1380 			active_desc =
1381 				(u16)(bcma_read32(di->core,
1382 						  DMA64TXREGOFFS(di, status1)) &
1383 				      D64_XS1_AD_MASK);
1384 			active_desc =
1385 			    (active_desc - di->xmtptrbase) & D64_XS0_CD_MASK;
1386 			active_desc = B2I(active_desc, struct dma64desc);
1387 			if (end != active_desc)
1388 				end = prevtxd(di, active_desc);
1389 		}
1390 	}
1391 
1392 	if ((start == 0) && (end > di->txout))
1393 		goto bogus;
1394 
1395 	for (i = start; i != end && !txp; i = nexttxd(di, i)) {
1396 		dma_addr_t pa;
1397 		uint size;
1398 
1399 		pa = le32_to_cpu(di->txd64[i].addrlow) - di->dataoffsetlow;
1400 
1401 		size =
1402 		    (le32_to_cpu(di->txd64[i].ctrl2) &
1403 		     D64_CTRL2_BC_MASK);
1404 
1405 		di->txd64[i].addrlow = cpu_to_le32(0xdeadbeef);
1406 		di->txd64[i].addrhigh = cpu_to_le32(0xdeadbeef);
1407 
1408 		txp = di->txp[i];
1409 		di->txp[i] = NULL;
1410 
1411 		dma_unmap_single(di->dmadev, pa, size, DMA_TO_DEVICE);
1412 	}
1413 
1414 	di->txin = i;
1415 
1416 	/* tx flow control */
1417 	di->dma.txavail = di->ntxd - ntxdactive(di, di->txin, di->txout) - 1;
1418 
1419 	return txp;
1420 
1421  bogus:
1422 	DMA_NONE("bogus curr: start %d end %d txout %d\n",
1423 		 start, end, di->txout);
1424 	return NULL;
1425 }
1426 
1427 /*
1428  * Mac80211 initiated actions sometimes require packets in the DMA queue to be
1429  * modified. The modified portion of the packet is not under control of the DMA
1430  * engine. This function calls a caller-supplied function for each packet in
1431  * the caller specified dma chain.
1432  */
dma_walk_packets(struct dma_pub * dmah,void (* callback_fnc)(void * pkt,void * arg_a),void * arg_a)1433 void dma_walk_packets(struct dma_pub *dmah, void (*callback_fnc)
1434 		      (void *pkt, void *arg_a), void *arg_a)
1435 {
1436 	struct dma_info *di = (struct dma_info *) dmah;
1437 	uint i =   di->txin;
1438 	uint end = di->txout;
1439 	struct sk_buff *skb;
1440 	struct ieee80211_tx_info *tx_info;
1441 
1442 	while (i != end) {
1443 		skb = (struct sk_buff *)di->txp[i];
1444 		if (skb != NULL) {
1445 			tx_info = (struct ieee80211_tx_info *)skb->cb;
1446 			(callback_fnc)(tx_info, arg_a);
1447 		}
1448 		i = nexttxd(di, i);
1449 	}
1450 }
1451