1 /*
2 * bfin_dma_5xx.c - Blackfin DMA implementation
3 *
4 * Copyright 2004-2008 Analog Devices Inc.
5 *
6 * Licensed under the GPL-2 or later.
7 */
8
9 #include <linux/errno.h>
10 #include <linux/interrupt.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/param.h>
14 #include <linux/proc_fs.h>
15 #include <linux/sched.h>
16 #include <linux/seq_file.h>
17 #include <linux/spinlock.h>
18
19 #include <asm/blackfin.h>
20 #include <asm/cacheflush.h>
21 #include <asm/dma.h>
22 #include <asm/uaccess.h>
23 #include <asm/early_printk.h>
24
25 /*
26 * To make sure we work around 05000119 - we always check DMA_DONE bit,
27 * never the DMA_RUN bit
28 */
29
30 struct dma_channel dma_ch[MAX_DMA_CHANNELS];
31 EXPORT_SYMBOL(dma_ch);
32
blackfin_dma_init(void)33 static int __init blackfin_dma_init(void)
34 {
35 int i;
36
37 printk(KERN_INFO "Blackfin DMA Controller\n");
38
39 for (i = 0; i < MAX_DMA_CHANNELS; i++) {
40 atomic_set(&dma_ch[i].chan_status, 0);
41 dma_ch[i].regs = dma_io_base_addr[i];
42 }
43 /* Mark MEMDMA Channel 0 as requested since we're using it internally */
44 request_dma(CH_MEM_STREAM0_DEST, "Blackfin dma_memcpy");
45 request_dma(CH_MEM_STREAM0_SRC, "Blackfin dma_memcpy");
46
47 #if defined(CONFIG_DEB_DMA_URGENT)
48 bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
49 | DEB1_URGENT | DEB2_URGENT | DEB3_URGENT);
50 #endif
51
52 return 0;
53 }
54 arch_initcall(blackfin_dma_init);
55
56 #ifdef CONFIG_PROC_FS
proc_dma_show(struct seq_file * m,void * v)57 static int proc_dma_show(struct seq_file *m, void *v)
58 {
59 int i;
60
61 for (i = 0; i < MAX_DMA_CHANNELS; ++i)
62 if (dma_channel_active(i))
63 seq_printf(m, "%2d: %s\n", i, dma_ch[i].device_id);
64
65 return 0;
66 }
67
proc_dma_open(struct inode * inode,struct file * file)68 static int proc_dma_open(struct inode *inode, struct file *file)
69 {
70 return single_open(file, proc_dma_show, NULL);
71 }
72
73 static const struct file_operations proc_dma_operations = {
74 .open = proc_dma_open,
75 .read = seq_read,
76 .llseek = seq_lseek,
77 .release = single_release,
78 };
79
proc_dma_init(void)80 static int __init proc_dma_init(void)
81 {
82 return proc_create("dma", 0, NULL, &proc_dma_operations) != NULL;
83 }
84 late_initcall(proc_dma_init);
85 #endif
86
set_dma_peripheral_map(unsigned int channel,const char * device_id)87 static void set_dma_peripheral_map(unsigned int channel, const char *device_id)
88 {
89 #ifdef CONFIG_BF54x
90 unsigned int per_map;
91
92 switch (channel) {
93 case CH_UART2_RX: per_map = 0xC << 12; break;
94 case CH_UART2_TX: per_map = 0xD << 12; break;
95 case CH_UART3_RX: per_map = 0xE << 12; break;
96 case CH_UART3_TX: per_map = 0xF << 12; break;
97 default: return;
98 }
99
100 if (strncmp(device_id, "BFIN_UART", 9) == 0)
101 dma_ch[channel].regs->peripheral_map = per_map;
102 #endif
103 }
104
105 /**
106 * request_dma - request a DMA channel
107 *
108 * Request the specific DMA channel from the system if it's available.
109 */
request_dma(unsigned int channel,const char * device_id)110 int request_dma(unsigned int channel, const char *device_id)
111 {
112 pr_debug("request_dma() : BEGIN\n");
113
114 if (device_id == NULL)
115 printk(KERN_WARNING "request_dma(%u): no device_id given\n", channel);
116
117 #if defined(CONFIG_BF561) && ANOMALY_05000182
118 if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) {
119 if (get_cclk() > 500000000) {
120 printk(KERN_WARNING
121 "Request IMDMA failed due to ANOMALY 05000182\n");
122 return -EFAULT;
123 }
124 }
125 #endif
126
127 if (atomic_cmpxchg(&dma_ch[channel].chan_status, 0, 1)) {
128 pr_debug("DMA CHANNEL IN USE\n");
129 return -EBUSY;
130 }
131
132 set_dma_peripheral_map(channel, device_id);
133 dma_ch[channel].device_id = device_id;
134 dma_ch[channel].irq = 0;
135
136 /* This is to be enabled by putting a restriction -
137 * you have to request DMA, before doing any operations on
138 * descriptor/channel
139 */
140 pr_debug("request_dma() : END\n");
141 return 0;
142 }
143 EXPORT_SYMBOL(request_dma);
144
set_dma_callback(unsigned int channel,irq_handler_t callback,void * data)145 int set_dma_callback(unsigned int channel, irq_handler_t callback, void *data)
146 {
147 int ret;
148 unsigned int irq;
149
150 BUG_ON(channel >= MAX_DMA_CHANNELS || !callback ||
151 !atomic_read(&dma_ch[channel].chan_status));
152
153 irq = channel2irq(channel);
154 ret = request_irq(irq, callback, 0, dma_ch[channel].device_id, data);
155 if (ret)
156 return ret;
157
158 dma_ch[channel].irq = irq;
159 dma_ch[channel].data = data;
160
161 return 0;
162 }
163 EXPORT_SYMBOL(set_dma_callback);
164
165 /**
166 * clear_dma_buffer - clear DMA fifos for specified channel
167 *
168 * Set the Buffer Clear bit in the Configuration register of specific DMA
169 * channel. This will stop the descriptor based DMA operation.
170 */
clear_dma_buffer(unsigned int channel)171 static void clear_dma_buffer(unsigned int channel)
172 {
173 dma_ch[channel].regs->cfg |= RESTART;
174 SSYNC();
175 dma_ch[channel].regs->cfg &= ~RESTART;
176 }
177
free_dma(unsigned int channel)178 void free_dma(unsigned int channel)
179 {
180 pr_debug("freedma() : BEGIN\n");
181 BUG_ON(channel >= MAX_DMA_CHANNELS ||
182 !atomic_read(&dma_ch[channel].chan_status));
183
184 /* Halt the DMA */
185 disable_dma(channel);
186 clear_dma_buffer(channel);
187
188 if (dma_ch[channel].irq)
189 free_irq(dma_ch[channel].irq, dma_ch[channel].data);
190
191 /* Clear the DMA Variable in the Channel */
192 atomic_set(&dma_ch[channel].chan_status, 0);
193
194 pr_debug("freedma() : END\n");
195 }
196 EXPORT_SYMBOL(free_dma);
197
198 #ifdef CONFIG_PM
199 # ifndef MAX_DMA_SUSPEND_CHANNELS
200 # define MAX_DMA_SUSPEND_CHANNELS MAX_DMA_CHANNELS
201 # endif
blackfin_dma_suspend(void)202 int blackfin_dma_suspend(void)
203 {
204 int i;
205
206 for (i = 0; i < MAX_DMA_CHANNELS; ++i) {
207 if (dma_ch[i].regs->cfg & DMAEN) {
208 printk(KERN_ERR "DMA Channel %d failed to suspend\n", i);
209 return -EBUSY;
210 }
211
212 if (i < MAX_DMA_SUSPEND_CHANNELS)
213 dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map;
214 }
215
216 return 0;
217 }
218
blackfin_dma_resume(void)219 void blackfin_dma_resume(void)
220 {
221 int i;
222
223 for (i = 0; i < MAX_DMA_CHANNELS; ++i) {
224 dma_ch[i].regs->cfg = 0;
225
226 if (i < MAX_DMA_SUSPEND_CHANNELS)
227 dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map;
228 }
229 }
230 #endif
231
232 /**
233 * blackfin_dma_early_init - minimal DMA init
234 *
235 * Setup a few DMA registers so we can safely do DMA transfers early on in
236 * the kernel booting process. Really this just means using dma_memcpy().
237 */
blackfin_dma_early_init(void)238 void __init blackfin_dma_early_init(void)
239 {
240 early_shadow_stamp();
241 bfin_write_MDMA_S0_CONFIG(0);
242 bfin_write_MDMA_S1_CONFIG(0);
243 }
244
early_dma_memcpy(void * pdst,const void * psrc,size_t size)245 void __init early_dma_memcpy(void *pdst, const void *psrc, size_t size)
246 {
247 unsigned long dst = (unsigned long)pdst;
248 unsigned long src = (unsigned long)psrc;
249 struct dma_register *dst_ch, *src_ch;
250
251 early_shadow_stamp();
252
253 /* We assume that everything is 4 byte aligned, so include
254 * a basic sanity check
255 */
256 BUG_ON(dst % 4);
257 BUG_ON(src % 4);
258 BUG_ON(size % 4);
259
260 src_ch = 0;
261 /* Find an avalible memDMA channel */
262 while (1) {
263 if (src_ch == (struct dma_register *)MDMA_S0_NEXT_DESC_PTR) {
264 dst_ch = (struct dma_register *)MDMA_D1_NEXT_DESC_PTR;
265 src_ch = (struct dma_register *)MDMA_S1_NEXT_DESC_PTR;
266 } else {
267 dst_ch = (struct dma_register *)MDMA_D0_NEXT_DESC_PTR;
268 src_ch = (struct dma_register *)MDMA_S0_NEXT_DESC_PTR;
269 }
270
271 if (!bfin_read16(&src_ch->cfg))
272 break;
273 else if (bfin_read16(&dst_ch->irq_status) & DMA_DONE) {
274 bfin_write16(&src_ch->cfg, 0);
275 break;
276 }
277 }
278
279 /* Force a sync in case a previous config reset on this channel
280 * occurred. This is needed so subsequent writes to DMA registers
281 * are not spuriously lost/corrupted.
282 */
283 __builtin_bfin_ssync();
284
285 /* Destination */
286 bfin_write32(&dst_ch->start_addr, dst);
287 bfin_write16(&dst_ch->x_count, size >> 2);
288 bfin_write16(&dst_ch->x_modify, 1 << 2);
289 bfin_write16(&dst_ch->irq_status, DMA_DONE | DMA_ERR);
290
291 /* Source */
292 bfin_write32(&src_ch->start_addr, src);
293 bfin_write16(&src_ch->x_count, size >> 2);
294 bfin_write16(&src_ch->x_modify, 1 << 2);
295 bfin_write16(&src_ch->irq_status, DMA_DONE | DMA_ERR);
296
297 /* Enable */
298 bfin_write16(&src_ch->cfg, DMAEN | WDSIZE_32);
299 bfin_write16(&dst_ch->cfg, WNR | DI_EN | DMAEN | WDSIZE_32);
300
301 /* Since we are atomic now, don't use the workaround ssync */
302 __builtin_bfin_ssync();
303 }
304
early_dma_memcpy_done(void)305 void __init early_dma_memcpy_done(void)
306 {
307 early_shadow_stamp();
308
309 while ((bfin_read_MDMA_S0_CONFIG() && !(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE)) ||
310 (bfin_read_MDMA_S1_CONFIG() && !(bfin_read_MDMA_D1_IRQ_STATUS() & DMA_DONE)))
311 continue;
312
313 bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
314 bfin_write_MDMA_D1_IRQ_STATUS(DMA_DONE | DMA_ERR);
315 /*
316 * Now that DMA is done, we would normally flush cache, but
317 * i/d cache isn't running this early, so we don't bother,
318 * and just clear out the DMA channel for next time
319 */
320 bfin_write_MDMA_S0_CONFIG(0);
321 bfin_write_MDMA_S1_CONFIG(0);
322 bfin_write_MDMA_D0_CONFIG(0);
323 bfin_write_MDMA_D1_CONFIG(0);
324
325 __builtin_bfin_ssync();
326 }
327
328 /**
329 * __dma_memcpy - program the MDMA registers
330 *
331 * Actually program MDMA0 and wait for the transfer to finish. Disable IRQs
332 * while programming registers so that everything is fully configured. Wait
333 * for DMA to finish with IRQs enabled. If interrupted, the initial DMA_DONE
334 * check will make sure we don't clobber any existing transfer.
335 */
__dma_memcpy(u32 daddr,s16 dmod,u32 saddr,s16 smod,size_t cnt,u32 conf)336 static void __dma_memcpy(u32 daddr, s16 dmod, u32 saddr, s16 smod, size_t cnt, u32 conf)
337 {
338 static DEFINE_SPINLOCK(mdma_lock);
339 unsigned long flags;
340
341 spin_lock_irqsave(&mdma_lock, flags);
342
343 /* Force a sync in case a previous config reset on this channel
344 * occurred. This is needed so subsequent writes to DMA registers
345 * are not spuriously lost/corrupted. Do it under irq lock and
346 * without the anomaly version (because we are atomic already).
347 */
348 __builtin_bfin_ssync();
349
350 if (bfin_read_MDMA_S0_CONFIG())
351 while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
352 continue;
353
354 if (conf & DMA2D) {
355 /* For larger bit sizes, we've already divided down cnt so it
356 * is no longer a multiple of 64k. So we have to break down
357 * the limit here so it is a multiple of the incoming size.
358 * There is no limitation here in terms of total size other
359 * than the hardware though as the bits lost in the shift are
360 * made up by MODIFY (== we can hit the whole address space).
361 * X: (2^(16 - 0)) * 1 == (2^(16 - 1)) * 2 == (2^(16 - 2)) * 4
362 */
363 u32 shift = abs(dmod) >> 1;
364 size_t ycnt = cnt >> (16 - shift);
365 cnt = 1 << (16 - shift);
366 bfin_write_MDMA_D0_Y_COUNT(ycnt);
367 bfin_write_MDMA_S0_Y_COUNT(ycnt);
368 bfin_write_MDMA_D0_Y_MODIFY(dmod);
369 bfin_write_MDMA_S0_Y_MODIFY(smod);
370 }
371
372 bfin_write_MDMA_D0_START_ADDR(daddr);
373 bfin_write_MDMA_D0_X_COUNT(cnt);
374 bfin_write_MDMA_D0_X_MODIFY(dmod);
375 bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
376
377 bfin_write_MDMA_S0_START_ADDR(saddr);
378 bfin_write_MDMA_S0_X_COUNT(cnt);
379 bfin_write_MDMA_S0_X_MODIFY(smod);
380 bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
381
382 bfin_write_MDMA_S0_CONFIG(DMAEN | conf);
383 bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | conf);
384
385 spin_unlock_irqrestore(&mdma_lock, flags);
386
387 SSYNC();
388
389 while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
390 if (bfin_read_MDMA_S0_CONFIG())
391 continue;
392 else
393 return;
394
395 bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
396
397 bfin_write_MDMA_S0_CONFIG(0);
398 bfin_write_MDMA_D0_CONFIG(0);
399 }
400
401 /**
402 * _dma_memcpy - translate C memcpy settings into MDMA settings
403 *
404 * Handle all the high level steps before we touch the MDMA registers. So
405 * handle direction, tweaking of sizes, and formatting of addresses.
406 */
_dma_memcpy(void * pdst,const void * psrc,size_t size)407 static void *_dma_memcpy(void *pdst, const void *psrc, size_t size)
408 {
409 u32 conf, shift;
410 s16 mod;
411 unsigned long dst = (unsigned long)pdst;
412 unsigned long src = (unsigned long)psrc;
413
414 if (size == 0)
415 return NULL;
416
417 if (dst % 4 == 0 && src % 4 == 0 && size % 4 == 0) {
418 conf = WDSIZE_32;
419 shift = 2;
420 } else if (dst % 2 == 0 && src % 2 == 0 && size % 2 == 0) {
421 conf = WDSIZE_16;
422 shift = 1;
423 } else {
424 conf = WDSIZE_8;
425 shift = 0;
426 }
427
428 /* If the two memory regions have a chance of overlapping, make
429 * sure the memcpy still works as expected. Do this by having the
430 * copy run backwards instead.
431 */
432 mod = 1 << shift;
433 if (src < dst) {
434 mod *= -1;
435 dst += size + mod;
436 src += size + mod;
437 }
438 size >>= shift;
439
440 if (size > 0x10000)
441 conf |= DMA2D;
442
443 __dma_memcpy(dst, mod, src, mod, size, conf);
444
445 return pdst;
446 }
447
448 /**
449 * dma_memcpy - DMA memcpy under mutex lock
450 *
451 * Do not check arguments before starting the DMA memcpy. Break the transfer
452 * up into two pieces. The first transfer is in multiples of 64k and the
453 * second transfer is the piece smaller than 64k.
454 */
dma_memcpy(void * pdst,const void * psrc,size_t size)455 void *dma_memcpy(void *pdst, const void *psrc, size_t size)
456 {
457 unsigned long dst = (unsigned long)pdst;
458 unsigned long src = (unsigned long)psrc;
459
460 if (bfin_addr_dcacheable(src))
461 blackfin_dcache_flush_range(src, src + size);
462
463 if (bfin_addr_dcacheable(dst))
464 blackfin_dcache_invalidate_range(dst, dst + size);
465
466 return dma_memcpy_nocache(pdst, psrc, size);
467 }
468 EXPORT_SYMBOL(dma_memcpy);
469
470 /**
471 * dma_memcpy_nocache - DMA memcpy under mutex lock
472 * - No cache flush/invalidate
473 *
474 * Do not check arguments before starting the DMA memcpy. Break the transfer
475 * up into two pieces. The first transfer is in multiples of 64k and the
476 * second transfer is the piece smaller than 64k.
477 */
dma_memcpy_nocache(void * pdst,const void * psrc,size_t size)478 void *dma_memcpy_nocache(void *pdst, const void *psrc, size_t size)
479 {
480 size_t bulk, rest;
481
482 bulk = size & ~0xffff;
483 rest = size - bulk;
484 if (bulk)
485 _dma_memcpy(pdst, psrc, bulk);
486 _dma_memcpy(pdst + bulk, psrc + bulk, rest);
487 return pdst;
488 }
489 EXPORT_SYMBOL(dma_memcpy_nocache);
490
491 /**
492 * safe_dma_memcpy - DMA memcpy w/argument checking
493 *
494 * Verify arguments are safe before heading to dma_memcpy().
495 */
safe_dma_memcpy(void * dst,const void * src,size_t size)496 void *safe_dma_memcpy(void *dst, const void *src, size_t size)
497 {
498 if (!access_ok(VERIFY_WRITE, dst, size))
499 return NULL;
500 if (!access_ok(VERIFY_READ, src, size))
501 return NULL;
502 return dma_memcpy(dst, src, size);
503 }
504 EXPORT_SYMBOL(safe_dma_memcpy);
505
_dma_out(unsigned long addr,unsigned long buf,unsigned short len,u16 size,u16 dma_size)506 static void _dma_out(unsigned long addr, unsigned long buf, unsigned short len,
507 u16 size, u16 dma_size)
508 {
509 blackfin_dcache_flush_range(buf, buf + len * size);
510 __dma_memcpy(addr, 0, buf, size, len, dma_size);
511 }
512
_dma_in(unsigned long addr,unsigned long buf,unsigned short len,u16 size,u16 dma_size)513 static void _dma_in(unsigned long addr, unsigned long buf, unsigned short len,
514 u16 size, u16 dma_size)
515 {
516 blackfin_dcache_invalidate_range(buf, buf + len * size);
517 __dma_memcpy(buf, size, addr, 0, len, dma_size);
518 }
519
520 #define MAKE_DMA_IO(io, bwl, isize, dmasize, cnst) \
521 void dma_##io##s##bwl(unsigned long addr, cnst void *buf, unsigned short len) \
522 { \
523 _dma_##io(addr, (unsigned long)buf, len, isize, WDSIZE_##dmasize); \
524 } \
525 EXPORT_SYMBOL(dma_##io##s##bwl)
526 MAKE_DMA_IO(out, b, 1, 8, const);
527 MAKE_DMA_IO(in, b, 1, 8, );
528 MAKE_DMA_IO(out, w, 2, 16, const);
529 MAKE_DMA_IO(in, w, 2, 16, );
530 MAKE_DMA_IO(out, l, 4, 32, const);
531 MAKE_DMA_IO(in, l, 4, 32, );
532