1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2018-2019 MediaTek Inc.
3 
4 /*
5  * Driver for MediaTek Command-Queue DMA Controller
6  *
7  * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com>
8  *
9  */
10 
11 #include <linux/bitops.h>
12 #include <linux/clk.h>
13 #include <linux/dmaengine.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/err.h>
16 #include <linux/iopoll.h>
17 #include <linux/interrupt.h>
18 #include <linux/list.h>
19 #include <linux/module.h>
20 #include <linux/of.h>
21 #include <linux/of_device.h>
22 #include <linux/of_dma.h>
23 #include <linux/platform_device.h>
24 #include <linux/pm_runtime.h>
25 #include <linux/refcount.h>
26 #include <linux/slab.h>
27 
28 #include "../virt-dma.h"
29 
30 #define MTK_CQDMA_USEC_POLL		10
31 #define MTK_CQDMA_TIMEOUT_POLL		1000
32 #define MTK_CQDMA_DMA_BUSWIDTHS		BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
33 #define MTK_CQDMA_ALIGN_SIZE		1
34 
35 /* The default number of virtual channel */
36 #define MTK_CQDMA_NR_VCHANS		32
37 
38 /* The default number of physical channel */
39 #define MTK_CQDMA_NR_PCHANS		3
40 
41 /* Registers for underlying dma manipulation */
42 #define MTK_CQDMA_INT_FLAG		0x0
43 #define MTK_CQDMA_INT_EN		0x4
44 #define MTK_CQDMA_EN			0x8
45 #define MTK_CQDMA_RESET			0xc
46 #define MTK_CQDMA_FLUSH			0x14
47 #define MTK_CQDMA_SRC			0x1c
48 #define MTK_CQDMA_DST			0x20
49 #define MTK_CQDMA_LEN1			0x24
50 #define MTK_CQDMA_LEN2			0x28
51 #define MTK_CQDMA_SRC2			0x60
52 #define MTK_CQDMA_DST2			0x64
53 
54 /* Registers setting */
55 #define MTK_CQDMA_EN_BIT		BIT(0)
56 #define MTK_CQDMA_INT_FLAG_BIT		BIT(0)
57 #define MTK_CQDMA_INT_EN_BIT		BIT(0)
58 #define MTK_CQDMA_FLUSH_BIT		BIT(0)
59 
60 #define MTK_CQDMA_WARM_RST_BIT		BIT(0)
61 #define MTK_CQDMA_HARD_RST_BIT		BIT(1)
62 
63 #define MTK_CQDMA_MAX_LEN		GENMASK(27, 0)
64 #define MTK_CQDMA_ADDR_LIMIT		GENMASK(31, 0)
65 #define MTK_CQDMA_ADDR2_SHFIT		(32)
66 
67 /**
68  * struct mtk_cqdma_vdesc - The struct holding info describing virtual
69  *                         descriptor (CVD)
70  * @vd:                    An instance for struct virt_dma_desc
71  * @len:                   The total data size device wants to move
72  * @residue:               The remaining data size device will move
73  * @dest:                  The destination address device wants to move to
74  * @src:                   The source address device wants to move from
75  * @ch:                    The pointer to the corresponding dma channel
76  * @node:                  The lise_head struct to build link-list for VDs
77  * @parent:                The pointer to the parent CVD
78  */
79 struct mtk_cqdma_vdesc {
80 	struct virt_dma_desc vd;
81 	size_t len;
82 	size_t residue;
83 	dma_addr_t dest;
84 	dma_addr_t src;
85 	struct dma_chan *ch;
86 
87 	struct list_head node;
88 	struct mtk_cqdma_vdesc *parent;
89 };
90 
91 /**
92  * struct mtk_cqdma_pchan - The struct holding info describing physical
93  *                         channel (PC)
94  * @queue:                 Queue for the PDs issued to this PC
95  * @base:                  The mapped register I/O base of this PC
96  * @irq:                   The IRQ that this PC are using
97  * @refcnt:                Track how many VCs are using this PC
98  * @tasklet:               Tasklet for this PC
99  * @lock:                  Lock protect agaisting multiple VCs access PC
100  */
101 struct mtk_cqdma_pchan {
102 	struct list_head queue;
103 	void __iomem *base;
104 	u32 irq;
105 
106 	refcount_t refcnt;
107 
108 	struct tasklet_struct tasklet;
109 
110 	/* lock to protect PC */
111 	spinlock_t lock;
112 };
113 
114 /**
115  * struct mtk_cqdma_vchan - The struct holding info describing virtual
116  *                         channel (VC)
117  * @vc:                    An instance for struct virt_dma_chan
118  * @pc:                    The pointer to the underlying PC
119  * @issue_completion:	   The wait for all issued descriptors completited
120  * @issue_synchronize:	   Bool indicating channel synchronization starts
121  */
122 struct mtk_cqdma_vchan {
123 	struct virt_dma_chan vc;
124 	struct mtk_cqdma_pchan *pc;
125 	struct completion issue_completion;
126 	bool issue_synchronize;
127 };
128 
129 /**
130  * struct mtk_cqdma_device - The struct holding info describing CQDMA
131  *                          device
132  * @ddev:                   An instance for struct dma_device
133  * @clk:                    The clock that device internal is using
134  * @dma_requests:           The number of VCs the device supports to
135  * @dma_channels:           The number of PCs the device supports to
136  * @vc:                     The pointer to all available VCs
137  * @pc:                     The pointer to all the underlying PCs
138  */
139 struct mtk_cqdma_device {
140 	struct dma_device ddev;
141 	struct clk *clk;
142 
143 	u32 dma_requests;
144 	u32 dma_channels;
145 	struct mtk_cqdma_vchan *vc;
146 	struct mtk_cqdma_pchan **pc;
147 };
148 
to_cqdma_dev(struct dma_chan * chan)149 static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan)
150 {
151 	return container_of(chan->device, struct mtk_cqdma_device, ddev);
152 }
153 
to_cqdma_vchan(struct dma_chan * chan)154 static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan)
155 {
156 	return container_of(chan, struct mtk_cqdma_vchan, vc.chan);
157 }
158 
to_cqdma_vdesc(struct virt_dma_desc * vd)159 static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd)
160 {
161 	return container_of(vd, struct mtk_cqdma_vdesc, vd);
162 }
163 
cqdma2dev(struct mtk_cqdma_device * cqdma)164 static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma)
165 {
166 	return cqdma->ddev.dev;
167 }
168 
mtk_dma_read(struct mtk_cqdma_pchan * pc,u32 reg)169 static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg)
170 {
171 	return readl(pc->base + reg);
172 }
173 
mtk_dma_write(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)174 static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
175 {
176 	writel_relaxed(val, pc->base + reg);
177 }
178 
mtk_dma_rmw(struct mtk_cqdma_pchan * pc,u32 reg,u32 mask,u32 set)179 static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg,
180 			u32 mask, u32 set)
181 {
182 	u32 val;
183 
184 	val = mtk_dma_read(pc, reg);
185 	val &= ~mask;
186 	val |= set;
187 	mtk_dma_write(pc, reg, val);
188 }
189 
mtk_dma_set(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)190 static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
191 {
192 	mtk_dma_rmw(pc, reg, 0, val);
193 }
194 
mtk_dma_clr(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)195 static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
196 {
197 	mtk_dma_rmw(pc, reg, val, 0);
198 }
199 
mtk_cqdma_vdesc_free(struct virt_dma_desc * vd)200 static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd)
201 {
202 	kfree(to_cqdma_vdesc(vd));
203 }
204 
mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan * pc,bool atomic)205 static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic)
206 {
207 	u32 status = 0;
208 
209 	if (!atomic)
210 		return readl_poll_timeout(pc->base + MTK_CQDMA_EN,
211 					  status,
212 					  !(status & MTK_CQDMA_EN_BIT),
213 					  MTK_CQDMA_USEC_POLL,
214 					  MTK_CQDMA_TIMEOUT_POLL);
215 
216 	return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN,
217 					 status,
218 					 !(status & MTK_CQDMA_EN_BIT),
219 					 MTK_CQDMA_USEC_POLL,
220 					 MTK_CQDMA_TIMEOUT_POLL);
221 }
222 
mtk_cqdma_hard_reset(struct mtk_cqdma_pchan * pc)223 static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc)
224 {
225 	mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
226 	mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
227 
228 	return mtk_cqdma_poll_engine_done(pc, true);
229 }
230 
mtk_cqdma_start(struct mtk_cqdma_pchan * pc,struct mtk_cqdma_vdesc * cvd)231 static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc,
232 			    struct mtk_cqdma_vdesc *cvd)
233 {
234 	/* wait for the previous transaction done */
235 	if (mtk_cqdma_poll_engine_done(pc, true) < 0)
236 		dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n");
237 
238 	/* warm reset the dma engine for the new transaction */
239 	mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT);
240 	if (mtk_cqdma_poll_engine_done(pc, true) < 0)
241 		dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n");
242 
243 	/* setup the source */
244 	mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT);
245 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
246 	mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT);
247 #else
248 	mtk_dma_set(pc, MTK_CQDMA_SRC2, 0);
249 #endif
250 
251 	/* setup the destination */
252 	mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT);
253 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
254 	mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT);
255 #else
256 	mtk_dma_set(pc, MTK_CQDMA_DST2, 0);
257 #endif
258 
259 	/* setup the length */
260 	mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len);
261 
262 	/* start dma engine */
263 	mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT);
264 }
265 
mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan * cvc)266 static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc)
267 {
268 	struct virt_dma_desc *vd, *vd2;
269 	struct mtk_cqdma_pchan *pc = cvc->pc;
270 	struct mtk_cqdma_vdesc *cvd;
271 	bool trigger_engine = false;
272 
273 	lockdep_assert_held(&cvc->vc.lock);
274 	lockdep_assert_held(&pc->lock);
275 
276 	list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) {
277 		/* need to trigger dma engine if PC's queue is empty */
278 		if (list_empty(&pc->queue))
279 			trigger_engine = true;
280 
281 		cvd = to_cqdma_vdesc(vd);
282 
283 		/* add VD into PC's queue */
284 		list_add_tail(&cvd->node, &pc->queue);
285 
286 		/* start the dma engine */
287 		if (trigger_engine)
288 			mtk_cqdma_start(pc, cvd);
289 
290 		/* remove VD from list desc_issued */
291 		list_del(&vd->node);
292 	}
293 }
294 
295 /*
296  * return true if this VC is active,
297  * meaning that there are VDs under processing by the PC
298  */
mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan * cvc)299 static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc)
300 {
301 	struct mtk_cqdma_vdesc *cvd;
302 
303 	list_for_each_entry(cvd, &cvc->pc->queue, node)
304 		if (cvc == to_cqdma_vchan(cvd->ch))
305 			return true;
306 
307 	return false;
308 }
309 
310 /*
311  * return the pointer of the CVD that is just consumed by the PC
312  */
313 static struct mtk_cqdma_vdesc
mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan * pc)314 *mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc)
315 {
316 	struct mtk_cqdma_vchan *cvc;
317 	struct mtk_cqdma_vdesc *cvd, *ret = NULL;
318 
319 	/* consume a CVD from PC's queue */
320 	cvd = list_first_entry_or_null(&pc->queue,
321 				       struct mtk_cqdma_vdesc, node);
322 	if (unlikely(!cvd || !cvd->parent))
323 		return NULL;
324 
325 	cvc = to_cqdma_vchan(cvd->ch);
326 	ret = cvd;
327 
328 	/* update residue of the parent CVD */
329 	cvd->parent->residue -= cvd->len;
330 
331 	/* delete CVD from PC's queue */
332 	list_del(&cvd->node);
333 
334 	spin_lock(&cvc->vc.lock);
335 
336 	/* check whether all the child CVDs completed */
337 	if (!cvd->parent->residue) {
338 		/* add the parent VD into list desc_completed */
339 		vchan_cookie_complete(&cvd->parent->vd);
340 
341 		/* setup completion if this VC is under synchronization */
342 		if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) {
343 			complete(&cvc->issue_completion);
344 			cvc->issue_synchronize = false;
345 		}
346 	}
347 
348 	spin_unlock(&cvc->vc.lock);
349 
350 	/* start transaction for next CVD in the queue */
351 	cvd = list_first_entry_or_null(&pc->queue,
352 				       struct mtk_cqdma_vdesc, node);
353 	if (cvd)
354 		mtk_cqdma_start(pc, cvd);
355 
356 	return ret;
357 }
358 
mtk_cqdma_tasklet_cb(struct tasklet_struct * t)359 static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t)
360 {
361 	struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet);
362 	struct mtk_cqdma_vdesc *cvd = NULL;
363 	unsigned long flags;
364 
365 	spin_lock_irqsave(&pc->lock, flags);
366 	/* consume the queue */
367 	cvd = mtk_cqdma_consume_work_queue(pc);
368 	spin_unlock_irqrestore(&pc->lock, flags);
369 
370 	/* submit the next CVD */
371 	if (cvd) {
372 		dma_run_dependencies(&cvd->vd.tx);
373 
374 		/*
375 		 * free child CVD after completion.
376 		 * the parent CVD would be freed with desc_free by user.
377 		 */
378 		if (cvd->parent != cvd)
379 			kfree(cvd);
380 	}
381 
382 	/* re-enable interrupt before leaving tasklet */
383 	enable_irq(pc->irq);
384 }
385 
mtk_cqdma_irq(int irq,void * devid)386 static irqreturn_t mtk_cqdma_irq(int irq, void *devid)
387 {
388 	struct mtk_cqdma_device *cqdma = devid;
389 	irqreturn_t ret = IRQ_NONE;
390 	bool schedule_tasklet = false;
391 	u32 i;
392 
393 	/* clear interrupt flags for each PC */
394 	for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) {
395 		spin_lock(&cqdma->pc[i]->lock);
396 		if (mtk_dma_read(cqdma->pc[i],
397 				 MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) {
398 			/* clear interrupt */
399 			mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG,
400 				    MTK_CQDMA_INT_FLAG_BIT);
401 
402 			schedule_tasklet = true;
403 			ret = IRQ_HANDLED;
404 		}
405 		spin_unlock(&cqdma->pc[i]->lock);
406 
407 		if (schedule_tasklet) {
408 			/* disable interrupt */
409 			disable_irq_nosync(cqdma->pc[i]->irq);
410 
411 			/* schedule the tasklet to handle the transactions */
412 			tasklet_schedule(&cqdma->pc[i]->tasklet);
413 		}
414 	}
415 
416 	return ret;
417 }
418 
mtk_cqdma_find_active_desc(struct dma_chan * c,dma_cookie_t cookie)419 static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c,
420 							dma_cookie_t cookie)
421 {
422 	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
423 	struct virt_dma_desc *vd;
424 	unsigned long flags;
425 
426 	spin_lock_irqsave(&cvc->pc->lock, flags);
427 	list_for_each_entry(vd, &cvc->pc->queue, node)
428 		if (vd->tx.cookie == cookie) {
429 			spin_unlock_irqrestore(&cvc->pc->lock, flags);
430 			return vd;
431 		}
432 	spin_unlock_irqrestore(&cvc->pc->lock, flags);
433 
434 	list_for_each_entry(vd, &cvc->vc.desc_issued, node)
435 		if (vd->tx.cookie == cookie)
436 			return vd;
437 
438 	return NULL;
439 }
440 
mtk_cqdma_tx_status(struct dma_chan * c,dma_cookie_t cookie,struct dma_tx_state * txstate)441 static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c,
442 					   dma_cookie_t cookie,
443 					   struct dma_tx_state *txstate)
444 {
445 	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
446 	struct mtk_cqdma_vdesc *cvd;
447 	struct virt_dma_desc *vd;
448 	enum dma_status ret;
449 	unsigned long flags;
450 	size_t bytes = 0;
451 
452 	ret = dma_cookie_status(c, cookie, txstate);
453 	if (ret == DMA_COMPLETE || !txstate)
454 		return ret;
455 
456 	spin_lock_irqsave(&cvc->vc.lock, flags);
457 	vd = mtk_cqdma_find_active_desc(c, cookie);
458 	spin_unlock_irqrestore(&cvc->vc.lock, flags);
459 
460 	if (vd) {
461 		cvd = to_cqdma_vdesc(vd);
462 		bytes = cvd->residue;
463 	}
464 
465 	dma_set_residue(txstate, bytes);
466 
467 	return ret;
468 }
469 
mtk_cqdma_issue_pending(struct dma_chan * c)470 static void mtk_cqdma_issue_pending(struct dma_chan *c)
471 {
472 	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
473 	unsigned long pc_flags;
474 	unsigned long vc_flags;
475 
476 	/* acquire PC's lock before VS's lock for lock dependency in tasklet */
477 	spin_lock_irqsave(&cvc->pc->lock, pc_flags);
478 	spin_lock_irqsave(&cvc->vc.lock, vc_flags);
479 
480 	if (vchan_issue_pending(&cvc->vc))
481 		mtk_cqdma_issue_vchan_pending(cvc);
482 
483 	spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
484 	spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
485 }
486 
487 static struct dma_async_tx_descriptor *
mtk_cqdma_prep_dma_memcpy(struct dma_chan * c,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)488 mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
489 			  dma_addr_t src, size_t len, unsigned long flags)
490 {
491 	struct mtk_cqdma_vdesc **cvd;
492 	struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL;
493 	size_t i, tlen, nr_vd;
494 
495 	/*
496 	 * In the case that trsanction length is larger than the
497 	 * DMA engine supports, a single memcpy transaction needs
498 	 * to be separated into several DMA transactions.
499 	 * Each DMA transaction would be described by a CVD,
500 	 * and the first one is referred as the parent CVD,
501 	 * while the others are child CVDs.
502 	 * The parent CVD's tx descriptor is the only tx descriptor
503 	 * returned to the DMA user, and it should not be completed
504 	 * until all the child CVDs completed.
505 	 */
506 	nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN);
507 	cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT);
508 	if (!cvd)
509 		return NULL;
510 
511 	for (i = 0; i < nr_vd; ++i) {
512 		cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT);
513 		if (!cvd[i]) {
514 			for (; i > 0; --i)
515 				kfree(cvd[i - 1]);
516 			return NULL;
517 		}
518 
519 		/* setup dma channel */
520 		cvd[i]->ch = c;
521 
522 		/* setup sourece, destination, and length */
523 		tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len;
524 		cvd[i]->len = tlen;
525 		cvd[i]->src = src;
526 		cvd[i]->dest = dest;
527 
528 		/* setup tx descriptor */
529 		tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags);
530 		tx->next = NULL;
531 
532 		if (!i) {
533 			cvd[0]->residue = len;
534 		} else {
535 			prev_tx->next = tx;
536 			cvd[i]->residue = tlen;
537 		}
538 
539 		cvd[i]->parent = cvd[0];
540 
541 		/* update the src, dest, len, prev_tx for the next CVD */
542 		src += tlen;
543 		dest += tlen;
544 		len -= tlen;
545 		prev_tx = tx;
546 	}
547 
548 	return &cvd[0]->vd.tx;
549 }
550 
mtk_cqdma_free_inactive_desc(struct dma_chan * c)551 static void mtk_cqdma_free_inactive_desc(struct dma_chan *c)
552 {
553 	struct virt_dma_chan *vc = to_virt_chan(c);
554 	unsigned long flags;
555 	LIST_HEAD(head);
556 
557 	/*
558 	 * set desc_allocated, desc_submitted,
559 	 * and desc_issued as the candicates to be freed
560 	 */
561 	spin_lock_irqsave(&vc->lock, flags);
562 	list_splice_tail_init(&vc->desc_allocated, &head);
563 	list_splice_tail_init(&vc->desc_submitted, &head);
564 	list_splice_tail_init(&vc->desc_issued, &head);
565 	spin_unlock_irqrestore(&vc->lock, flags);
566 
567 	/* free descriptor lists */
568 	vchan_dma_desc_free_list(vc, &head);
569 }
570 
mtk_cqdma_free_active_desc(struct dma_chan * c)571 static void mtk_cqdma_free_active_desc(struct dma_chan *c)
572 {
573 	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
574 	bool sync_needed = false;
575 	unsigned long pc_flags;
576 	unsigned long vc_flags;
577 
578 	/* acquire PC's lock first due to lock dependency in dma ISR */
579 	spin_lock_irqsave(&cvc->pc->lock, pc_flags);
580 	spin_lock_irqsave(&cvc->vc.lock, vc_flags);
581 
582 	/* synchronization is required if this VC is active */
583 	if (mtk_cqdma_is_vchan_active(cvc)) {
584 		cvc->issue_synchronize = true;
585 		sync_needed = true;
586 	}
587 
588 	spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
589 	spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
590 
591 	/* waiting for the completion of this VC */
592 	if (sync_needed)
593 		wait_for_completion(&cvc->issue_completion);
594 
595 	/* free all descriptors in list desc_completed */
596 	vchan_synchronize(&cvc->vc);
597 
598 	WARN_ONCE(!list_empty(&cvc->vc.desc_completed),
599 		  "Desc pending still in list desc_completed\n");
600 }
601 
mtk_cqdma_terminate_all(struct dma_chan * c)602 static int mtk_cqdma_terminate_all(struct dma_chan *c)
603 {
604 	/* free descriptors not processed yet by hardware */
605 	mtk_cqdma_free_inactive_desc(c);
606 
607 	/* free descriptors being processed by hardware */
608 	mtk_cqdma_free_active_desc(c);
609 
610 	return 0;
611 }
612 
mtk_cqdma_alloc_chan_resources(struct dma_chan * c)613 static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c)
614 {
615 	struct mtk_cqdma_device *cqdma = to_cqdma_dev(c);
616 	struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c);
617 	struct mtk_cqdma_pchan *pc = NULL;
618 	u32 i, min_refcnt = U32_MAX, refcnt;
619 	unsigned long flags;
620 
621 	/* allocate PC with the minimun refcount */
622 	for (i = 0; i < cqdma->dma_channels; ++i) {
623 		refcnt = refcount_read(&cqdma->pc[i]->refcnt);
624 		if (refcnt < min_refcnt) {
625 			pc = cqdma->pc[i];
626 			min_refcnt = refcnt;
627 		}
628 	}
629 
630 	if (!pc)
631 		return -ENOSPC;
632 
633 	spin_lock_irqsave(&pc->lock, flags);
634 
635 	if (!refcount_read(&pc->refcnt)) {
636 		/* allocate PC when the refcount is zero */
637 		mtk_cqdma_hard_reset(pc);
638 
639 		/* enable interrupt for this PC */
640 		mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
641 
642 		/*
643 		 * refcount_inc would complain increment on 0; use-after-free.
644 		 * Thus, we need to explicitly set it as 1 initially.
645 		 */
646 		refcount_set(&pc->refcnt, 1);
647 	} else {
648 		refcount_inc(&pc->refcnt);
649 	}
650 
651 	spin_unlock_irqrestore(&pc->lock, flags);
652 
653 	vc->pc = pc;
654 
655 	return 0;
656 }
657 
mtk_cqdma_free_chan_resources(struct dma_chan * c)658 static void mtk_cqdma_free_chan_resources(struct dma_chan *c)
659 {
660 	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
661 	unsigned long flags;
662 
663 	/* free all descriptors in all lists on the VC */
664 	mtk_cqdma_terminate_all(c);
665 
666 	spin_lock_irqsave(&cvc->pc->lock, flags);
667 
668 	/* PC is not freed until there is no VC mapped to it */
669 	if (refcount_dec_and_test(&cvc->pc->refcnt)) {
670 		/* start the flush operation and stop the engine */
671 		mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
672 
673 		/* wait for the completion of flush operation */
674 		if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0)
675 			dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n");
676 
677 		/* clear the flush bit and interrupt flag */
678 		mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
679 		mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG,
680 			    MTK_CQDMA_INT_FLAG_BIT);
681 
682 		/* disable interrupt for this PC */
683 		mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
684 	}
685 
686 	spin_unlock_irqrestore(&cvc->pc->lock, flags);
687 }
688 
mtk_cqdma_hw_init(struct mtk_cqdma_device * cqdma)689 static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma)
690 {
691 	unsigned long flags;
692 	int err;
693 	u32 i;
694 
695 	pm_runtime_enable(cqdma2dev(cqdma));
696 	pm_runtime_get_sync(cqdma2dev(cqdma));
697 
698 	err = clk_prepare_enable(cqdma->clk);
699 
700 	if (err) {
701 		pm_runtime_put_sync(cqdma2dev(cqdma));
702 		pm_runtime_disable(cqdma2dev(cqdma));
703 		return err;
704 	}
705 
706 	/* reset all PCs */
707 	for (i = 0; i < cqdma->dma_channels; ++i) {
708 		spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
709 		if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) {
710 			dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
711 			spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
712 
713 			clk_disable_unprepare(cqdma->clk);
714 			pm_runtime_put_sync(cqdma2dev(cqdma));
715 			pm_runtime_disable(cqdma2dev(cqdma));
716 			return -EINVAL;
717 		}
718 		spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
719 	}
720 
721 	return 0;
722 }
723 
mtk_cqdma_hw_deinit(struct mtk_cqdma_device * cqdma)724 static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma)
725 {
726 	unsigned long flags;
727 	u32 i;
728 
729 	/* reset all PCs */
730 	for (i = 0; i < cqdma->dma_channels; ++i) {
731 		spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
732 		if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0)
733 			dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
734 		spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
735 	}
736 
737 	clk_disable_unprepare(cqdma->clk);
738 
739 	pm_runtime_put_sync(cqdma2dev(cqdma));
740 	pm_runtime_disable(cqdma2dev(cqdma));
741 }
742 
743 static const struct of_device_id mtk_cqdma_match[] = {
744 	{ .compatible = "mediatek,mt6765-cqdma" },
745 	{ /* sentinel */ }
746 };
747 MODULE_DEVICE_TABLE(of, mtk_cqdma_match);
748 
mtk_cqdma_probe(struct platform_device * pdev)749 static int mtk_cqdma_probe(struct platform_device *pdev)
750 {
751 	struct mtk_cqdma_device *cqdma;
752 	struct mtk_cqdma_vchan *vc;
753 	struct dma_device *dd;
754 	int err;
755 	u32 i;
756 
757 	cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL);
758 	if (!cqdma)
759 		return -ENOMEM;
760 
761 	dd = &cqdma->ddev;
762 
763 	cqdma->clk = devm_clk_get(&pdev->dev, "cqdma");
764 	if (IS_ERR(cqdma->clk)) {
765 		dev_err(&pdev->dev, "No clock for %s\n",
766 			dev_name(&pdev->dev));
767 		return PTR_ERR(cqdma->clk);
768 	}
769 
770 	dma_cap_set(DMA_MEMCPY, dd->cap_mask);
771 
772 	dd->copy_align = MTK_CQDMA_ALIGN_SIZE;
773 	dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources;
774 	dd->device_free_chan_resources = mtk_cqdma_free_chan_resources;
775 	dd->device_tx_status = mtk_cqdma_tx_status;
776 	dd->device_issue_pending = mtk_cqdma_issue_pending;
777 	dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy;
778 	dd->device_terminate_all = mtk_cqdma_terminate_all;
779 	dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
780 	dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
781 	dd->directions = BIT(DMA_MEM_TO_MEM);
782 	dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
783 	dd->dev = &pdev->dev;
784 	INIT_LIST_HEAD(&dd->channels);
785 
786 	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
787 						      "dma-requests",
788 						      &cqdma->dma_requests)) {
789 		dev_info(&pdev->dev,
790 			 "Using %u as missing dma-requests property\n",
791 			 MTK_CQDMA_NR_VCHANS);
792 
793 		cqdma->dma_requests = MTK_CQDMA_NR_VCHANS;
794 	}
795 
796 	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
797 						      "dma-channels",
798 						      &cqdma->dma_channels)) {
799 		dev_info(&pdev->dev,
800 			 "Using %u as missing dma-channels property\n",
801 			 MTK_CQDMA_NR_PCHANS);
802 
803 		cqdma->dma_channels = MTK_CQDMA_NR_PCHANS;
804 	}
805 
806 	cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels,
807 				 sizeof(*cqdma->pc), GFP_KERNEL);
808 	if (!cqdma->pc)
809 		return -ENOMEM;
810 
811 	/* initialization for PCs */
812 	for (i = 0; i < cqdma->dma_channels; ++i) {
813 		cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1,
814 					    sizeof(**cqdma->pc), GFP_KERNEL);
815 		if (!cqdma->pc[i])
816 			return -ENOMEM;
817 
818 		INIT_LIST_HEAD(&cqdma->pc[i]->queue);
819 		spin_lock_init(&cqdma->pc[i]->lock);
820 		refcount_set(&cqdma->pc[i]->refcnt, 0);
821 		cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i);
822 		if (IS_ERR(cqdma->pc[i]->base))
823 			return PTR_ERR(cqdma->pc[i]->base);
824 
825 		/* allocate IRQ resource */
826 		err = platform_get_irq(pdev, i);
827 		if (err < 0)
828 			return err;
829 		cqdma->pc[i]->irq = err;
830 
831 		err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq,
832 				       mtk_cqdma_irq, 0, dev_name(&pdev->dev),
833 				       cqdma);
834 		if (err) {
835 			dev_err(&pdev->dev,
836 				"request_irq failed with err %d\n", err);
837 			return -EINVAL;
838 		}
839 	}
840 
841 	/* allocate resource for VCs */
842 	cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests,
843 				 sizeof(*cqdma->vc), GFP_KERNEL);
844 	if (!cqdma->vc)
845 		return -ENOMEM;
846 
847 	for (i = 0; i < cqdma->dma_requests; i++) {
848 		vc = &cqdma->vc[i];
849 		vc->vc.desc_free = mtk_cqdma_vdesc_free;
850 		vchan_init(&vc->vc, dd);
851 		init_completion(&vc->issue_completion);
852 	}
853 
854 	err = dma_async_device_register(dd);
855 	if (err)
856 		return err;
857 
858 	err = of_dma_controller_register(pdev->dev.of_node,
859 					 of_dma_xlate_by_chan_id, cqdma);
860 	if (err) {
861 		dev_err(&pdev->dev,
862 			"MediaTek CQDMA OF registration failed %d\n", err);
863 		goto err_unregister;
864 	}
865 
866 	err = mtk_cqdma_hw_init(cqdma);
867 	if (err) {
868 		dev_err(&pdev->dev,
869 			"MediaTek CQDMA HW initialization failed %d\n", err);
870 		goto err_unregister;
871 	}
872 
873 	platform_set_drvdata(pdev, cqdma);
874 
875 	/* initialize tasklet for each PC */
876 	for (i = 0; i < cqdma->dma_channels; ++i)
877 		tasklet_setup(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb);
878 
879 	dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n");
880 
881 	return 0;
882 
883 err_unregister:
884 	dma_async_device_unregister(dd);
885 
886 	return err;
887 }
888 
mtk_cqdma_remove(struct platform_device * pdev)889 static int mtk_cqdma_remove(struct platform_device *pdev)
890 {
891 	struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev);
892 	struct mtk_cqdma_vchan *vc;
893 	unsigned long flags;
894 	int i;
895 
896 	/* kill VC task */
897 	for (i = 0; i < cqdma->dma_requests; i++) {
898 		vc = &cqdma->vc[i];
899 
900 		list_del(&vc->vc.chan.device_node);
901 		tasklet_kill(&vc->vc.task);
902 	}
903 
904 	/* disable interrupt */
905 	for (i = 0; i < cqdma->dma_channels; i++) {
906 		spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
907 		mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN,
908 			    MTK_CQDMA_INT_EN_BIT);
909 		spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
910 
911 		/* Waits for any pending IRQ handlers to complete */
912 		synchronize_irq(cqdma->pc[i]->irq);
913 
914 		tasklet_kill(&cqdma->pc[i]->tasklet);
915 	}
916 
917 	/* disable hardware */
918 	mtk_cqdma_hw_deinit(cqdma);
919 
920 	dma_async_device_unregister(&cqdma->ddev);
921 	of_dma_controller_free(pdev->dev.of_node);
922 
923 	return 0;
924 }
925 
926 static struct platform_driver mtk_cqdma_driver = {
927 	.probe = mtk_cqdma_probe,
928 	.remove = mtk_cqdma_remove,
929 	.driver = {
930 		.name           = KBUILD_MODNAME,
931 		.of_match_table = mtk_cqdma_match,
932 	},
933 };
934 module_platform_driver(mtk_cqdma_driver);
935 
936 MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver");
937 MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>");
938 MODULE_LICENSE("GPL v2");
939