1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2018-2019 MediaTek Inc.
3
4 /*
5 * Driver for MediaTek Command-Queue DMA Controller
6 *
7 * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com>
8 *
9 */
10
11 #include <linux/bitops.h>
12 #include <linux/clk.h>
13 #include <linux/dmaengine.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/err.h>
16 #include <linux/iopoll.h>
17 #include <linux/interrupt.h>
18 #include <linux/list.h>
19 #include <linux/module.h>
20 #include <linux/of.h>
21 #include <linux/of_device.h>
22 #include <linux/of_dma.h>
23 #include <linux/platform_device.h>
24 #include <linux/pm_runtime.h>
25 #include <linux/refcount.h>
26 #include <linux/slab.h>
27
28 #include "../virt-dma.h"
29
30 #define MTK_CQDMA_USEC_POLL 10
31 #define MTK_CQDMA_TIMEOUT_POLL 1000
32 #define MTK_CQDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
33 #define MTK_CQDMA_ALIGN_SIZE 1
34
35 /* The default number of virtual channel */
36 #define MTK_CQDMA_NR_VCHANS 32
37
38 /* The default number of physical channel */
39 #define MTK_CQDMA_NR_PCHANS 3
40
41 /* Registers for underlying dma manipulation */
42 #define MTK_CQDMA_INT_FLAG 0x0
43 #define MTK_CQDMA_INT_EN 0x4
44 #define MTK_CQDMA_EN 0x8
45 #define MTK_CQDMA_RESET 0xc
46 #define MTK_CQDMA_FLUSH 0x14
47 #define MTK_CQDMA_SRC 0x1c
48 #define MTK_CQDMA_DST 0x20
49 #define MTK_CQDMA_LEN1 0x24
50 #define MTK_CQDMA_LEN2 0x28
51 #define MTK_CQDMA_SRC2 0x60
52 #define MTK_CQDMA_DST2 0x64
53
54 /* Registers setting */
55 #define MTK_CQDMA_EN_BIT BIT(0)
56 #define MTK_CQDMA_INT_FLAG_BIT BIT(0)
57 #define MTK_CQDMA_INT_EN_BIT BIT(0)
58 #define MTK_CQDMA_FLUSH_BIT BIT(0)
59
60 #define MTK_CQDMA_WARM_RST_BIT BIT(0)
61 #define MTK_CQDMA_HARD_RST_BIT BIT(1)
62
63 #define MTK_CQDMA_MAX_LEN GENMASK(27, 0)
64 #define MTK_CQDMA_ADDR_LIMIT GENMASK(31, 0)
65 #define MTK_CQDMA_ADDR2_SHFIT (32)
66
67 /**
68 * struct mtk_cqdma_vdesc - The struct holding info describing virtual
69 * descriptor (CVD)
70 * @vd: An instance for struct virt_dma_desc
71 * @len: The total data size device wants to move
72 * @residue: The remaining data size device will move
73 * @dest: The destination address device wants to move to
74 * @src: The source address device wants to move from
75 * @ch: The pointer to the corresponding dma channel
76 * @node: The lise_head struct to build link-list for VDs
77 * @parent: The pointer to the parent CVD
78 */
79 struct mtk_cqdma_vdesc {
80 struct virt_dma_desc vd;
81 size_t len;
82 size_t residue;
83 dma_addr_t dest;
84 dma_addr_t src;
85 struct dma_chan *ch;
86
87 struct list_head node;
88 struct mtk_cqdma_vdesc *parent;
89 };
90
91 /**
92 * struct mtk_cqdma_pchan - The struct holding info describing physical
93 * channel (PC)
94 * @queue: Queue for the PDs issued to this PC
95 * @base: The mapped register I/O base of this PC
96 * @irq: The IRQ that this PC are using
97 * @refcnt: Track how many VCs are using this PC
98 * @tasklet: Tasklet for this PC
99 * @lock: Lock protect agaisting multiple VCs access PC
100 */
101 struct mtk_cqdma_pchan {
102 struct list_head queue;
103 void __iomem *base;
104 u32 irq;
105
106 refcount_t refcnt;
107
108 struct tasklet_struct tasklet;
109
110 /* lock to protect PC */
111 spinlock_t lock;
112 };
113
114 /**
115 * struct mtk_cqdma_vchan - The struct holding info describing virtual
116 * channel (VC)
117 * @vc: An instance for struct virt_dma_chan
118 * @pc: The pointer to the underlying PC
119 * @issue_completion: The wait for all issued descriptors completited
120 * @issue_synchronize: Bool indicating channel synchronization starts
121 */
122 struct mtk_cqdma_vchan {
123 struct virt_dma_chan vc;
124 struct mtk_cqdma_pchan *pc;
125 struct completion issue_completion;
126 bool issue_synchronize;
127 };
128
129 /**
130 * struct mtk_cqdma_device - The struct holding info describing CQDMA
131 * device
132 * @ddev: An instance for struct dma_device
133 * @clk: The clock that device internal is using
134 * @dma_requests: The number of VCs the device supports to
135 * @dma_channels: The number of PCs the device supports to
136 * @vc: The pointer to all available VCs
137 * @pc: The pointer to all the underlying PCs
138 */
139 struct mtk_cqdma_device {
140 struct dma_device ddev;
141 struct clk *clk;
142
143 u32 dma_requests;
144 u32 dma_channels;
145 struct mtk_cqdma_vchan *vc;
146 struct mtk_cqdma_pchan **pc;
147 };
148
to_cqdma_dev(struct dma_chan * chan)149 static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan)
150 {
151 return container_of(chan->device, struct mtk_cqdma_device, ddev);
152 }
153
to_cqdma_vchan(struct dma_chan * chan)154 static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan)
155 {
156 return container_of(chan, struct mtk_cqdma_vchan, vc.chan);
157 }
158
to_cqdma_vdesc(struct virt_dma_desc * vd)159 static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd)
160 {
161 return container_of(vd, struct mtk_cqdma_vdesc, vd);
162 }
163
cqdma2dev(struct mtk_cqdma_device * cqdma)164 static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma)
165 {
166 return cqdma->ddev.dev;
167 }
168
mtk_dma_read(struct mtk_cqdma_pchan * pc,u32 reg)169 static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg)
170 {
171 return readl(pc->base + reg);
172 }
173
mtk_dma_write(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)174 static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
175 {
176 writel_relaxed(val, pc->base + reg);
177 }
178
mtk_dma_rmw(struct mtk_cqdma_pchan * pc,u32 reg,u32 mask,u32 set)179 static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg,
180 u32 mask, u32 set)
181 {
182 u32 val;
183
184 val = mtk_dma_read(pc, reg);
185 val &= ~mask;
186 val |= set;
187 mtk_dma_write(pc, reg, val);
188 }
189
mtk_dma_set(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)190 static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
191 {
192 mtk_dma_rmw(pc, reg, 0, val);
193 }
194
mtk_dma_clr(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)195 static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
196 {
197 mtk_dma_rmw(pc, reg, val, 0);
198 }
199
mtk_cqdma_vdesc_free(struct virt_dma_desc * vd)200 static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd)
201 {
202 kfree(to_cqdma_vdesc(vd));
203 }
204
mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan * pc,bool atomic)205 static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic)
206 {
207 u32 status = 0;
208
209 if (!atomic)
210 return readl_poll_timeout(pc->base + MTK_CQDMA_EN,
211 status,
212 !(status & MTK_CQDMA_EN_BIT),
213 MTK_CQDMA_USEC_POLL,
214 MTK_CQDMA_TIMEOUT_POLL);
215
216 return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN,
217 status,
218 !(status & MTK_CQDMA_EN_BIT),
219 MTK_CQDMA_USEC_POLL,
220 MTK_CQDMA_TIMEOUT_POLL);
221 }
222
mtk_cqdma_hard_reset(struct mtk_cqdma_pchan * pc)223 static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc)
224 {
225 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
226 mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
227
228 return mtk_cqdma_poll_engine_done(pc, true);
229 }
230
mtk_cqdma_start(struct mtk_cqdma_pchan * pc,struct mtk_cqdma_vdesc * cvd)231 static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc,
232 struct mtk_cqdma_vdesc *cvd)
233 {
234 /* wait for the previous transaction done */
235 if (mtk_cqdma_poll_engine_done(pc, true) < 0)
236 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n");
237
238 /* warm reset the dma engine for the new transaction */
239 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT);
240 if (mtk_cqdma_poll_engine_done(pc, true) < 0)
241 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n");
242
243 /* setup the source */
244 mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT);
245 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
246 mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT);
247 #else
248 mtk_dma_set(pc, MTK_CQDMA_SRC2, 0);
249 #endif
250
251 /* setup the destination */
252 mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT);
253 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
254 mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT);
255 #else
256 mtk_dma_set(pc, MTK_CQDMA_DST2, 0);
257 #endif
258
259 /* setup the length */
260 mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len);
261
262 /* start dma engine */
263 mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT);
264 }
265
mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan * cvc)266 static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc)
267 {
268 struct virt_dma_desc *vd, *vd2;
269 struct mtk_cqdma_pchan *pc = cvc->pc;
270 struct mtk_cqdma_vdesc *cvd;
271 bool trigger_engine = false;
272
273 lockdep_assert_held(&cvc->vc.lock);
274 lockdep_assert_held(&pc->lock);
275
276 list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) {
277 /* need to trigger dma engine if PC's queue is empty */
278 if (list_empty(&pc->queue))
279 trigger_engine = true;
280
281 cvd = to_cqdma_vdesc(vd);
282
283 /* add VD into PC's queue */
284 list_add_tail(&cvd->node, &pc->queue);
285
286 /* start the dma engine */
287 if (trigger_engine)
288 mtk_cqdma_start(pc, cvd);
289
290 /* remove VD from list desc_issued */
291 list_del(&vd->node);
292 }
293 }
294
295 /*
296 * return true if this VC is active,
297 * meaning that there are VDs under processing by the PC
298 */
mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan * cvc)299 static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc)
300 {
301 struct mtk_cqdma_vdesc *cvd;
302
303 list_for_each_entry(cvd, &cvc->pc->queue, node)
304 if (cvc == to_cqdma_vchan(cvd->ch))
305 return true;
306
307 return false;
308 }
309
310 /*
311 * return the pointer of the CVD that is just consumed by the PC
312 */
313 static struct mtk_cqdma_vdesc
mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan * pc)314 *mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc)
315 {
316 struct mtk_cqdma_vchan *cvc;
317 struct mtk_cqdma_vdesc *cvd, *ret = NULL;
318
319 /* consume a CVD from PC's queue */
320 cvd = list_first_entry_or_null(&pc->queue,
321 struct mtk_cqdma_vdesc, node);
322 if (unlikely(!cvd || !cvd->parent))
323 return NULL;
324
325 cvc = to_cqdma_vchan(cvd->ch);
326 ret = cvd;
327
328 /* update residue of the parent CVD */
329 cvd->parent->residue -= cvd->len;
330
331 /* delete CVD from PC's queue */
332 list_del(&cvd->node);
333
334 spin_lock(&cvc->vc.lock);
335
336 /* check whether all the child CVDs completed */
337 if (!cvd->parent->residue) {
338 /* add the parent VD into list desc_completed */
339 vchan_cookie_complete(&cvd->parent->vd);
340
341 /* setup completion if this VC is under synchronization */
342 if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) {
343 complete(&cvc->issue_completion);
344 cvc->issue_synchronize = false;
345 }
346 }
347
348 spin_unlock(&cvc->vc.lock);
349
350 /* start transaction for next CVD in the queue */
351 cvd = list_first_entry_or_null(&pc->queue,
352 struct mtk_cqdma_vdesc, node);
353 if (cvd)
354 mtk_cqdma_start(pc, cvd);
355
356 return ret;
357 }
358
mtk_cqdma_tasklet_cb(struct tasklet_struct * t)359 static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t)
360 {
361 struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet);
362 struct mtk_cqdma_vdesc *cvd = NULL;
363 unsigned long flags;
364
365 spin_lock_irqsave(&pc->lock, flags);
366 /* consume the queue */
367 cvd = mtk_cqdma_consume_work_queue(pc);
368 spin_unlock_irqrestore(&pc->lock, flags);
369
370 /* submit the next CVD */
371 if (cvd) {
372 dma_run_dependencies(&cvd->vd.tx);
373
374 /*
375 * free child CVD after completion.
376 * the parent CVD would be freed with desc_free by user.
377 */
378 if (cvd->parent != cvd)
379 kfree(cvd);
380 }
381
382 /* re-enable interrupt before leaving tasklet */
383 enable_irq(pc->irq);
384 }
385
mtk_cqdma_irq(int irq,void * devid)386 static irqreturn_t mtk_cqdma_irq(int irq, void *devid)
387 {
388 struct mtk_cqdma_device *cqdma = devid;
389 irqreturn_t ret = IRQ_NONE;
390 bool schedule_tasklet = false;
391 u32 i;
392
393 /* clear interrupt flags for each PC */
394 for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) {
395 spin_lock(&cqdma->pc[i]->lock);
396 if (mtk_dma_read(cqdma->pc[i],
397 MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) {
398 /* clear interrupt */
399 mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG,
400 MTK_CQDMA_INT_FLAG_BIT);
401
402 schedule_tasklet = true;
403 ret = IRQ_HANDLED;
404 }
405 spin_unlock(&cqdma->pc[i]->lock);
406
407 if (schedule_tasklet) {
408 /* disable interrupt */
409 disable_irq_nosync(cqdma->pc[i]->irq);
410
411 /* schedule the tasklet to handle the transactions */
412 tasklet_schedule(&cqdma->pc[i]->tasklet);
413 }
414 }
415
416 return ret;
417 }
418
mtk_cqdma_find_active_desc(struct dma_chan * c,dma_cookie_t cookie)419 static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c,
420 dma_cookie_t cookie)
421 {
422 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
423 struct virt_dma_desc *vd;
424 unsigned long flags;
425
426 spin_lock_irqsave(&cvc->pc->lock, flags);
427 list_for_each_entry(vd, &cvc->pc->queue, node)
428 if (vd->tx.cookie == cookie) {
429 spin_unlock_irqrestore(&cvc->pc->lock, flags);
430 return vd;
431 }
432 spin_unlock_irqrestore(&cvc->pc->lock, flags);
433
434 list_for_each_entry(vd, &cvc->vc.desc_issued, node)
435 if (vd->tx.cookie == cookie)
436 return vd;
437
438 return NULL;
439 }
440
mtk_cqdma_tx_status(struct dma_chan * c,dma_cookie_t cookie,struct dma_tx_state * txstate)441 static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c,
442 dma_cookie_t cookie,
443 struct dma_tx_state *txstate)
444 {
445 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
446 struct mtk_cqdma_vdesc *cvd;
447 struct virt_dma_desc *vd;
448 enum dma_status ret;
449 unsigned long flags;
450 size_t bytes = 0;
451
452 ret = dma_cookie_status(c, cookie, txstate);
453 if (ret == DMA_COMPLETE || !txstate)
454 return ret;
455
456 spin_lock_irqsave(&cvc->vc.lock, flags);
457 vd = mtk_cqdma_find_active_desc(c, cookie);
458 spin_unlock_irqrestore(&cvc->vc.lock, flags);
459
460 if (vd) {
461 cvd = to_cqdma_vdesc(vd);
462 bytes = cvd->residue;
463 }
464
465 dma_set_residue(txstate, bytes);
466
467 return ret;
468 }
469
mtk_cqdma_issue_pending(struct dma_chan * c)470 static void mtk_cqdma_issue_pending(struct dma_chan *c)
471 {
472 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
473 unsigned long pc_flags;
474 unsigned long vc_flags;
475
476 /* acquire PC's lock before VS's lock for lock dependency in tasklet */
477 spin_lock_irqsave(&cvc->pc->lock, pc_flags);
478 spin_lock_irqsave(&cvc->vc.lock, vc_flags);
479
480 if (vchan_issue_pending(&cvc->vc))
481 mtk_cqdma_issue_vchan_pending(cvc);
482
483 spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
484 spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
485 }
486
487 static struct dma_async_tx_descriptor *
mtk_cqdma_prep_dma_memcpy(struct dma_chan * c,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)488 mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
489 dma_addr_t src, size_t len, unsigned long flags)
490 {
491 struct mtk_cqdma_vdesc **cvd;
492 struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL;
493 size_t i, tlen, nr_vd;
494
495 /*
496 * In the case that trsanction length is larger than the
497 * DMA engine supports, a single memcpy transaction needs
498 * to be separated into several DMA transactions.
499 * Each DMA transaction would be described by a CVD,
500 * and the first one is referred as the parent CVD,
501 * while the others are child CVDs.
502 * The parent CVD's tx descriptor is the only tx descriptor
503 * returned to the DMA user, and it should not be completed
504 * until all the child CVDs completed.
505 */
506 nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN);
507 cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT);
508 if (!cvd)
509 return NULL;
510
511 for (i = 0; i < nr_vd; ++i) {
512 cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT);
513 if (!cvd[i]) {
514 for (; i > 0; --i)
515 kfree(cvd[i - 1]);
516 return NULL;
517 }
518
519 /* setup dma channel */
520 cvd[i]->ch = c;
521
522 /* setup sourece, destination, and length */
523 tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len;
524 cvd[i]->len = tlen;
525 cvd[i]->src = src;
526 cvd[i]->dest = dest;
527
528 /* setup tx descriptor */
529 tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags);
530 tx->next = NULL;
531
532 if (!i) {
533 cvd[0]->residue = len;
534 } else {
535 prev_tx->next = tx;
536 cvd[i]->residue = tlen;
537 }
538
539 cvd[i]->parent = cvd[0];
540
541 /* update the src, dest, len, prev_tx for the next CVD */
542 src += tlen;
543 dest += tlen;
544 len -= tlen;
545 prev_tx = tx;
546 }
547
548 return &cvd[0]->vd.tx;
549 }
550
mtk_cqdma_free_inactive_desc(struct dma_chan * c)551 static void mtk_cqdma_free_inactive_desc(struct dma_chan *c)
552 {
553 struct virt_dma_chan *vc = to_virt_chan(c);
554 unsigned long flags;
555 LIST_HEAD(head);
556
557 /*
558 * set desc_allocated, desc_submitted,
559 * and desc_issued as the candicates to be freed
560 */
561 spin_lock_irqsave(&vc->lock, flags);
562 list_splice_tail_init(&vc->desc_allocated, &head);
563 list_splice_tail_init(&vc->desc_submitted, &head);
564 list_splice_tail_init(&vc->desc_issued, &head);
565 spin_unlock_irqrestore(&vc->lock, flags);
566
567 /* free descriptor lists */
568 vchan_dma_desc_free_list(vc, &head);
569 }
570
mtk_cqdma_free_active_desc(struct dma_chan * c)571 static void mtk_cqdma_free_active_desc(struct dma_chan *c)
572 {
573 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
574 bool sync_needed = false;
575 unsigned long pc_flags;
576 unsigned long vc_flags;
577
578 /* acquire PC's lock first due to lock dependency in dma ISR */
579 spin_lock_irqsave(&cvc->pc->lock, pc_flags);
580 spin_lock_irqsave(&cvc->vc.lock, vc_flags);
581
582 /* synchronization is required if this VC is active */
583 if (mtk_cqdma_is_vchan_active(cvc)) {
584 cvc->issue_synchronize = true;
585 sync_needed = true;
586 }
587
588 spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
589 spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
590
591 /* waiting for the completion of this VC */
592 if (sync_needed)
593 wait_for_completion(&cvc->issue_completion);
594
595 /* free all descriptors in list desc_completed */
596 vchan_synchronize(&cvc->vc);
597
598 WARN_ONCE(!list_empty(&cvc->vc.desc_completed),
599 "Desc pending still in list desc_completed\n");
600 }
601
mtk_cqdma_terminate_all(struct dma_chan * c)602 static int mtk_cqdma_terminate_all(struct dma_chan *c)
603 {
604 /* free descriptors not processed yet by hardware */
605 mtk_cqdma_free_inactive_desc(c);
606
607 /* free descriptors being processed by hardware */
608 mtk_cqdma_free_active_desc(c);
609
610 return 0;
611 }
612
mtk_cqdma_alloc_chan_resources(struct dma_chan * c)613 static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c)
614 {
615 struct mtk_cqdma_device *cqdma = to_cqdma_dev(c);
616 struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c);
617 struct mtk_cqdma_pchan *pc = NULL;
618 u32 i, min_refcnt = U32_MAX, refcnt;
619 unsigned long flags;
620
621 /* allocate PC with the minimun refcount */
622 for (i = 0; i < cqdma->dma_channels; ++i) {
623 refcnt = refcount_read(&cqdma->pc[i]->refcnt);
624 if (refcnt < min_refcnt) {
625 pc = cqdma->pc[i];
626 min_refcnt = refcnt;
627 }
628 }
629
630 if (!pc)
631 return -ENOSPC;
632
633 spin_lock_irqsave(&pc->lock, flags);
634
635 if (!refcount_read(&pc->refcnt)) {
636 /* allocate PC when the refcount is zero */
637 mtk_cqdma_hard_reset(pc);
638
639 /* enable interrupt for this PC */
640 mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
641
642 /*
643 * refcount_inc would complain increment on 0; use-after-free.
644 * Thus, we need to explicitly set it as 1 initially.
645 */
646 refcount_set(&pc->refcnt, 1);
647 } else {
648 refcount_inc(&pc->refcnt);
649 }
650
651 spin_unlock_irqrestore(&pc->lock, flags);
652
653 vc->pc = pc;
654
655 return 0;
656 }
657
mtk_cqdma_free_chan_resources(struct dma_chan * c)658 static void mtk_cqdma_free_chan_resources(struct dma_chan *c)
659 {
660 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
661 unsigned long flags;
662
663 /* free all descriptors in all lists on the VC */
664 mtk_cqdma_terminate_all(c);
665
666 spin_lock_irqsave(&cvc->pc->lock, flags);
667
668 /* PC is not freed until there is no VC mapped to it */
669 if (refcount_dec_and_test(&cvc->pc->refcnt)) {
670 /* start the flush operation and stop the engine */
671 mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
672
673 /* wait for the completion of flush operation */
674 if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0)
675 dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n");
676
677 /* clear the flush bit and interrupt flag */
678 mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
679 mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG,
680 MTK_CQDMA_INT_FLAG_BIT);
681
682 /* disable interrupt for this PC */
683 mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
684 }
685
686 spin_unlock_irqrestore(&cvc->pc->lock, flags);
687 }
688
mtk_cqdma_hw_init(struct mtk_cqdma_device * cqdma)689 static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma)
690 {
691 unsigned long flags;
692 int err;
693 u32 i;
694
695 pm_runtime_enable(cqdma2dev(cqdma));
696 pm_runtime_get_sync(cqdma2dev(cqdma));
697
698 err = clk_prepare_enable(cqdma->clk);
699
700 if (err) {
701 pm_runtime_put_sync(cqdma2dev(cqdma));
702 pm_runtime_disable(cqdma2dev(cqdma));
703 return err;
704 }
705
706 /* reset all PCs */
707 for (i = 0; i < cqdma->dma_channels; ++i) {
708 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
709 if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) {
710 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
711 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
712
713 clk_disable_unprepare(cqdma->clk);
714 pm_runtime_put_sync(cqdma2dev(cqdma));
715 pm_runtime_disable(cqdma2dev(cqdma));
716 return -EINVAL;
717 }
718 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
719 }
720
721 return 0;
722 }
723
mtk_cqdma_hw_deinit(struct mtk_cqdma_device * cqdma)724 static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma)
725 {
726 unsigned long flags;
727 u32 i;
728
729 /* reset all PCs */
730 for (i = 0; i < cqdma->dma_channels; ++i) {
731 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
732 if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0)
733 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
734 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
735 }
736
737 clk_disable_unprepare(cqdma->clk);
738
739 pm_runtime_put_sync(cqdma2dev(cqdma));
740 pm_runtime_disable(cqdma2dev(cqdma));
741 }
742
743 static const struct of_device_id mtk_cqdma_match[] = {
744 { .compatible = "mediatek,mt6765-cqdma" },
745 { /* sentinel */ }
746 };
747 MODULE_DEVICE_TABLE(of, mtk_cqdma_match);
748
mtk_cqdma_probe(struct platform_device * pdev)749 static int mtk_cqdma_probe(struct platform_device *pdev)
750 {
751 struct mtk_cqdma_device *cqdma;
752 struct mtk_cqdma_vchan *vc;
753 struct dma_device *dd;
754 int err;
755 u32 i;
756
757 cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL);
758 if (!cqdma)
759 return -ENOMEM;
760
761 dd = &cqdma->ddev;
762
763 cqdma->clk = devm_clk_get(&pdev->dev, "cqdma");
764 if (IS_ERR(cqdma->clk)) {
765 dev_err(&pdev->dev, "No clock for %s\n",
766 dev_name(&pdev->dev));
767 return PTR_ERR(cqdma->clk);
768 }
769
770 dma_cap_set(DMA_MEMCPY, dd->cap_mask);
771
772 dd->copy_align = MTK_CQDMA_ALIGN_SIZE;
773 dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources;
774 dd->device_free_chan_resources = mtk_cqdma_free_chan_resources;
775 dd->device_tx_status = mtk_cqdma_tx_status;
776 dd->device_issue_pending = mtk_cqdma_issue_pending;
777 dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy;
778 dd->device_terminate_all = mtk_cqdma_terminate_all;
779 dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
780 dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
781 dd->directions = BIT(DMA_MEM_TO_MEM);
782 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
783 dd->dev = &pdev->dev;
784 INIT_LIST_HEAD(&dd->channels);
785
786 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
787 "dma-requests",
788 &cqdma->dma_requests)) {
789 dev_info(&pdev->dev,
790 "Using %u as missing dma-requests property\n",
791 MTK_CQDMA_NR_VCHANS);
792
793 cqdma->dma_requests = MTK_CQDMA_NR_VCHANS;
794 }
795
796 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
797 "dma-channels",
798 &cqdma->dma_channels)) {
799 dev_info(&pdev->dev,
800 "Using %u as missing dma-channels property\n",
801 MTK_CQDMA_NR_PCHANS);
802
803 cqdma->dma_channels = MTK_CQDMA_NR_PCHANS;
804 }
805
806 cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels,
807 sizeof(*cqdma->pc), GFP_KERNEL);
808 if (!cqdma->pc)
809 return -ENOMEM;
810
811 /* initialization for PCs */
812 for (i = 0; i < cqdma->dma_channels; ++i) {
813 cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1,
814 sizeof(**cqdma->pc), GFP_KERNEL);
815 if (!cqdma->pc[i])
816 return -ENOMEM;
817
818 INIT_LIST_HEAD(&cqdma->pc[i]->queue);
819 spin_lock_init(&cqdma->pc[i]->lock);
820 refcount_set(&cqdma->pc[i]->refcnt, 0);
821 cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i);
822 if (IS_ERR(cqdma->pc[i]->base))
823 return PTR_ERR(cqdma->pc[i]->base);
824
825 /* allocate IRQ resource */
826 err = platform_get_irq(pdev, i);
827 if (err < 0)
828 return err;
829 cqdma->pc[i]->irq = err;
830
831 err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq,
832 mtk_cqdma_irq, 0, dev_name(&pdev->dev),
833 cqdma);
834 if (err) {
835 dev_err(&pdev->dev,
836 "request_irq failed with err %d\n", err);
837 return -EINVAL;
838 }
839 }
840
841 /* allocate resource for VCs */
842 cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests,
843 sizeof(*cqdma->vc), GFP_KERNEL);
844 if (!cqdma->vc)
845 return -ENOMEM;
846
847 for (i = 0; i < cqdma->dma_requests; i++) {
848 vc = &cqdma->vc[i];
849 vc->vc.desc_free = mtk_cqdma_vdesc_free;
850 vchan_init(&vc->vc, dd);
851 init_completion(&vc->issue_completion);
852 }
853
854 err = dma_async_device_register(dd);
855 if (err)
856 return err;
857
858 err = of_dma_controller_register(pdev->dev.of_node,
859 of_dma_xlate_by_chan_id, cqdma);
860 if (err) {
861 dev_err(&pdev->dev,
862 "MediaTek CQDMA OF registration failed %d\n", err);
863 goto err_unregister;
864 }
865
866 err = mtk_cqdma_hw_init(cqdma);
867 if (err) {
868 dev_err(&pdev->dev,
869 "MediaTek CQDMA HW initialization failed %d\n", err);
870 goto err_unregister;
871 }
872
873 platform_set_drvdata(pdev, cqdma);
874
875 /* initialize tasklet for each PC */
876 for (i = 0; i < cqdma->dma_channels; ++i)
877 tasklet_setup(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb);
878
879 dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n");
880
881 return 0;
882
883 err_unregister:
884 dma_async_device_unregister(dd);
885
886 return err;
887 }
888
mtk_cqdma_remove(struct platform_device * pdev)889 static int mtk_cqdma_remove(struct platform_device *pdev)
890 {
891 struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev);
892 struct mtk_cqdma_vchan *vc;
893 unsigned long flags;
894 int i;
895
896 /* kill VC task */
897 for (i = 0; i < cqdma->dma_requests; i++) {
898 vc = &cqdma->vc[i];
899
900 list_del(&vc->vc.chan.device_node);
901 tasklet_kill(&vc->vc.task);
902 }
903
904 /* disable interrupt */
905 for (i = 0; i < cqdma->dma_channels; i++) {
906 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
907 mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN,
908 MTK_CQDMA_INT_EN_BIT);
909 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
910
911 /* Waits for any pending IRQ handlers to complete */
912 synchronize_irq(cqdma->pc[i]->irq);
913
914 tasklet_kill(&cqdma->pc[i]->tasklet);
915 }
916
917 /* disable hardware */
918 mtk_cqdma_hw_deinit(cqdma);
919
920 dma_async_device_unregister(&cqdma->ddev);
921 of_dma_controller_free(pdev->dev.of_node);
922
923 return 0;
924 }
925
926 static struct platform_driver mtk_cqdma_driver = {
927 .probe = mtk_cqdma_probe,
928 .remove = mtk_cqdma_remove,
929 .driver = {
930 .name = KBUILD_MODNAME,
931 .of_match_table = mtk_cqdma_match,
932 },
933 };
934 module_platform_driver(mtk_cqdma_driver);
935
936 MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver");
937 MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>");
938 MODULE_LICENSE("GPL v2");
939