1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * K3 NAVSS DMA glue interface
4 *
5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
6 *
7 */
8
9 #include <linux/module.h>
10 #include <linux/atomic.h>
11 #include <linux/delay.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/io.h>
14 #include <linux/init.h>
15 #include <linux/of.h>
16 #include <linux/platform_device.h>
17 #include <linux/soc/ti/k3-ringacc.h>
18 #include <linux/dma/ti-cppi5.h>
19 #include <linux/dma/k3-udma-glue.h>
20
21 #include "k3-udma.h"
22 #include "k3-psil-priv.h"
23
24 struct k3_udma_glue_common {
25 struct device *dev;
26 struct device chan_dev;
27 struct udma_dev *udmax;
28 const struct udma_tisci_rm *tisci_rm;
29 struct k3_ringacc *ringacc;
30 u32 src_thread;
31 u32 dst_thread;
32
33 u32 hdesc_size;
34 bool epib;
35 u32 psdata_size;
36 u32 swdata_size;
37 u32 atype_asel;
38 struct psil_endpoint_config *ep_config;
39 };
40
41 struct k3_udma_glue_tx_channel {
42 struct k3_udma_glue_common common;
43
44 struct udma_tchan *udma_tchanx;
45 int udma_tchan_id;
46
47 struct k3_ring *ringtx;
48 struct k3_ring *ringtxcq;
49
50 bool psil_paired;
51
52 int virq;
53
54 atomic_t free_pkts;
55 bool tx_pause_on_err;
56 bool tx_filt_einfo;
57 bool tx_filt_pswords;
58 bool tx_supr_tdpkt;
59
60 int udma_tflow_id;
61 };
62
63 struct k3_udma_glue_rx_flow {
64 struct udma_rflow *udma_rflow;
65 int udma_rflow_id;
66 struct k3_ring *ringrx;
67 struct k3_ring *ringrxfdq;
68
69 int virq;
70 };
71
72 struct k3_udma_glue_rx_channel {
73 struct k3_udma_glue_common common;
74
75 struct udma_rchan *udma_rchanx;
76 int udma_rchan_id;
77 bool remote;
78
79 bool psil_paired;
80
81 u32 swdata_size;
82 int flow_id_base;
83
84 struct k3_udma_glue_rx_flow *flows;
85 u32 flow_num;
86 u32 flows_ready;
87 };
88
k3_udma_chan_dev_release(struct device * dev)89 static void k3_udma_chan_dev_release(struct device *dev)
90 {
91 /* The struct containing the device is devm managed */
92 }
93
94 static struct class k3_udma_glue_devclass = {
95 .name = "k3_udma_glue_chan",
96 .dev_release = k3_udma_chan_dev_release,
97 };
98
99 #define K3_UDMAX_TDOWN_TIMEOUT_US 1000
100
of_k3_udma_glue_parse(struct device_node * udmax_np,struct k3_udma_glue_common * common)101 static int of_k3_udma_glue_parse(struct device_node *udmax_np,
102 struct k3_udma_glue_common *common)
103 {
104 common->udmax = of_xudma_dev_get(udmax_np, NULL);
105 if (IS_ERR(common->udmax))
106 return PTR_ERR(common->udmax);
107
108 common->ringacc = xudma_get_ringacc(common->udmax);
109 common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax);
110
111 return 0;
112 }
113
of_k3_udma_glue_parse_chn(struct device_node * chn_np,const char * name,struct k3_udma_glue_common * common,bool tx_chn)114 static int of_k3_udma_glue_parse_chn(struct device_node *chn_np,
115 const char *name, struct k3_udma_glue_common *common,
116 bool tx_chn)
117 {
118 struct of_phandle_args dma_spec;
119 u32 thread_id;
120 int ret = 0;
121 int index;
122
123 if (unlikely(!name))
124 return -EINVAL;
125
126 index = of_property_match_string(chn_np, "dma-names", name);
127 if (index < 0)
128 return index;
129
130 if (of_parse_phandle_with_args(chn_np, "dmas", "#dma-cells", index,
131 &dma_spec))
132 return -ENOENT;
133
134 ret = of_k3_udma_glue_parse(dma_spec.np, common);
135 if (ret)
136 goto out_put_spec;
137
138 thread_id = dma_spec.args[0];
139 if (dma_spec.args_count == 2) {
140 if (dma_spec.args[1] > 2 && !xudma_is_pktdma(common->udmax)) {
141 dev_err(common->dev, "Invalid channel atype: %u\n",
142 dma_spec.args[1]);
143 ret = -EINVAL;
144 goto out_put_spec;
145 }
146 if (dma_spec.args[1] > 15 && xudma_is_pktdma(common->udmax)) {
147 dev_err(common->dev, "Invalid channel asel: %u\n",
148 dma_spec.args[1]);
149 ret = -EINVAL;
150 goto out_put_spec;
151 }
152
153 common->atype_asel = dma_spec.args[1];
154 }
155
156 if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
157 ret = -EINVAL;
158 goto out_put_spec;
159 }
160
161 if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
162 ret = -EINVAL;
163 goto out_put_spec;
164 }
165
166 /* get psil endpoint config */
167 common->ep_config = psil_get_ep_config(thread_id);
168 if (IS_ERR(common->ep_config)) {
169 dev_err(common->dev,
170 "No configuration for psi-l thread 0x%04x\n",
171 thread_id);
172 ret = PTR_ERR(common->ep_config);
173 goto out_put_spec;
174 }
175
176 common->epib = common->ep_config->needs_epib;
177 common->psdata_size = common->ep_config->psd_size;
178
179 if (tx_chn)
180 common->dst_thread = thread_id;
181 else
182 common->src_thread = thread_id;
183
184 out_put_spec:
185 of_node_put(dma_spec.np);
186 return ret;
187 };
188
k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel * tx_chn)189 static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
190 {
191 struct device *dev = tx_chn->common.dev;
192
193 dev_dbg(dev, "dump_tx_chn:\n"
194 "udma_tchan_id: %d\n"
195 "src_thread: %08x\n"
196 "dst_thread: %08x\n",
197 tx_chn->udma_tchan_id,
198 tx_chn->common.src_thread,
199 tx_chn->common.dst_thread);
200 }
201
k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel * chn,char * mark)202 static void k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel *chn,
203 char *mark)
204 {
205 struct device *dev = chn->common.dev;
206
207 dev_dbg(dev, "=== dump ===> %s\n", mark);
208 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG,
209 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG));
210 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG,
211 xudma_tchanrt_read(chn->udma_tchanx,
212 UDMA_CHAN_RT_PEER_RT_EN_REG));
213 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG,
214 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_PCNT_REG));
215 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG,
216 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_BCNT_REG));
217 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG,
218 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_SBCNT_REG));
219 }
220
k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel * tx_chn)221 static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
222 {
223 const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm;
224 struct ti_sci_msg_rm_udmap_tx_ch_cfg req;
225
226 memset(&req, 0, sizeof(req));
227
228 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |
229 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |
230 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |
231 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
232 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |
233 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
234 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
235 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
236 req.nav_id = tisci_rm->tisci_dev_id;
237 req.index = tx_chn->udma_tchan_id;
238 if (tx_chn->tx_pause_on_err)
239 req.tx_pause_on_err = 1;
240 if (tx_chn->tx_filt_einfo)
241 req.tx_filt_einfo = 1;
242 if (tx_chn->tx_filt_pswords)
243 req.tx_filt_pswords = 1;
244 req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
245 if (tx_chn->tx_supr_tdpkt)
246 req.tx_supr_tdpkt = 1;
247 req.tx_fetch_size = tx_chn->common.hdesc_size >> 2;
248 req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
249 req.tx_atype = tx_chn->common.atype_asel;
250
251 return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req);
252 }
253
k3_udma_glue_request_tx_chn(struct device * dev,const char * name,struct k3_udma_glue_tx_channel_cfg * cfg)254 struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
255 const char *name, struct k3_udma_glue_tx_channel_cfg *cfg)
256 {
257 struct k3_udma_glue_tx_channel *tx_chn;
258 int ret;
259
260 tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL);
261 if (!tx_chn)
262 return ERR_PTR(-ENOMEM);
263
264 tx_chn->common.dev = dev;
265 tx_chn->common.swdata_size = cfg->swdata_size;
266 tx_chn->tx_pause_on_err = cfg->tx_pause_on_err;
267 tx_chn->tx_filt_einfo = cfg->tx_filt_einfo;
268 tx_chn->tx_filt_pswords = cfg->tx_filt_pswords;
269 tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt;
270
271 /* parse of udmap channel */
272 ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
273 &tx_chn->common, true);
274 if (ret)
275 goto err;
276
277 tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib,
278 tx_chn->common.psdata_size,
279 tx_chn->common.swdata_size);
280
281 if (xudma_is_pktdma(tx_chn->common.udmax))
282 tx_chn->udma_tchan_id = tx_chn->common.ep_config->mapped_channel_id;
283 else
284 tx_chn->udma_tchan_id = -1;
285
286 /* request and cfg UDMAP TX channel */
287 tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax,
288 tx_chn->udma_tchan_id);
289 if (IS_ERR(tx_chn->udma_tchanx)) {
290 ret = PTR_ERR(tx_chn->udma_tchanx);
291 dev_err(dev, "UDMAX tchanx get err %d\n", ret);
292 goto err;
293 }
294 tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx);
295
296 tx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
297 tx_chn->common.chan_dev.parent = xudma_get_device(tx_chn->common.udmax);
298 dev_set_name(&tx_chn->common.chan_dev, "tchan%d-0x%04x",
299 tx_chn->udma_tchan_id, tx_chn->common.dst_thread);
300 ret = device_register(&tx_chn->common.chan_dev);
301 if (ret) {
302 dev_err(dev, "Channel Device registration failed %d\n", ret);
303 put_device(&tx_chn->common.chan_dev);
304 tx_chn->common.chan_dev.parent = NULL;
305 goto err;
306 }
307
308 if (xudma_is_pktdma(tx_chn->common.udmax)) {
309 /* prepare the channel device as coherent */
310 tx_chn->common.chan_dev.dma_coherent = true;
311 dma_coerce_mask_and_coherent(&tx_chn->common.chan_dev,
312 DMA_BIT_MASK(48));
313 }
314
315 atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size);
316
317 if (xudma_is_pktdma(tx_chn->common.udmax))
318 tx_chn->udma_tflow_id = tx_chn->common.ep_config->default_flow_id;
319 else
320 tx_chn->udma_tflow_id = tx_chn->udma_tchan_id;
321
322 /* request and cfg rings */
323 ret = k3_ringacc_request_rings_pair(tx_chn->common.ringacc,
324 tx_chn->udma_tflow_id, -1,
325 &tx_chn->ringtx,
326 &tx_chn->ringtxcq);
327 if (ret) {
328 dev_err(dev, "Failed to get TX/TXCQ rings %d\n", ret);
329 goto err;
330 }
331
332 /* Set the dma_dev for the rings to be configured */
333 cfg->tx_cfg.dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn);
334 cfg->txcq_cfg.dma_dev = cfg->tx_cfg.dma_dev;
335
336 /* Set the ASEL value for DMA rings of PKTDMA */
337 if (xudma_is_pktdma(tx_chn->common.udmax)) {
338 cfg->tx_cfg.asel = tx_chn->common.atype_asel;
339 cfg->txcq_cfg.asel = tx_chn->common.atype_asel;
340 }
341
342 ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg);
343 if (ret) {
344 dev_err(dev, "Failed to cfg ringtx %d\n", ret);
345 goto err;
346 }
347
348 ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg);
349 if (ret) {
350 dev_err(dev, "Failed to cfg ringtx %d\n", ret);
351 goto err;
352 }
353
354 /* request and cfg psi-l */
355 tx_chn->common.src_thread =
356 xudma_dev_get_psil_base(tx_chn->common.udmax) +
357 tx_chn->udma_tchan_id;
358
359 ret = k3_udma_glue_cfg_tx_chn(tx_chn);
360 if (ret) {
361 dev_err(dev, "Failed to cfg tchan %d\n", ret);
362 goto err;
363 }
364
365 k3_udma_glue_dump_tx_chn(tx_chn);
366
367 return tx_chn;
368
369 err:
370 k3_udma_glue_release_tx_chn(tx_chn);
371 return ERR_PTR(ret);
372 }
373 EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn);
374
k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel * tx_chn)375 void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
376 {
377 if (tx_chn->psil_paired) {
378 xudma_navss_psil_unpair(tx_chn->common.udmax,
379 tx_chn->common.src_thread,
380 tx_chn->common.dst_thread);
381 tx_chn->psil_paired = false;
382 }
383
384 if (!IS_ERR_OR_NULL(tx_chn->udma_tchanx))
385 xudma_tchan_put(tx_chn->common.udmax,
386 tx_chn->udma_tchanx);
387
388 if (tx_chn->ringtxcq)
389 k3_ringacc_ring_free(tx_chn->ringtxcq);
390
391 if (tx_chn->ringtx)
392 k3_ringacc_ring_free(tx_chn->ringtx);
393
394 if (tx_chn->common.chan_dev.parent) {
395 device_unregister(&tx_chn->common.chan_dev);
396 tx_chn->common.chan_dev.parent = NULL;
397 }
398 }
399 EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn);
400
k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel * tx_chn,struct cppi5_host_desc_t * desc_tx,dma_addr_t desc_dma)401 int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
402 struct cppi5_host_desc_t *desc_tx,
403 dma_addr_t desc_dma)
404 {
405 u32 ringtxcq_id;
406
407 if (!atomic_add_unless(&tx_chn->free_pkts, -1, 0))
408 return -ENOMEM;
409
410 ringtxcq_id = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
411 cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, ringtxcq_id);
412
413 return k3_ringacc_ring_push(tx_chn->ringtx, &desc_dma);
414 }
415 EXPORT_SYMBOL_GPL(k3_udma_glue_push_tx_chn);
416
k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel * tx_chn,dma_addr_t * desc_dma)417 int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
418 dma_addr_t *desc_dma)
419 {
420 int ret;
421
422 ret = k3_ringacc_ring_pop(tx_chn->ringtxcq, desc_dma);
423 if (!ret)
424 atomic_inc(&tx_chn->free_pkts);
425
426 return ret;
427 }
428 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn);
429
k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel * tx_chn)430 int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
431 {
432 int ret;
433
434 ret = xudma_navss_psil_pair(tx_chn->common.udmax,
435 tx_chn->common.src_thread,
436 tx_chn->common.dst_thread);
437 if (ret) {
438 dev_err(tx_chn->common.dev, "PSI-L request err %d\n", ret);
439 return ret;
440 }
441
442 tx_chn->psil_paired = true;
443
444 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
445 UDMA_PEER_RT_EN_ENABLE);
446
447 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG,
448 UDMA_CHAN_RT_CTL_EN);
449
450 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn en");
451 return 0;
452 }
453 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_tx_chn);
454
k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel * tx_chn)455 void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
456 {
457 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis1");
458
459 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, 0);
460
461 xudma_tchanrt_write(tx_chn->udma_tchanx,
462 UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
463 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2");
464
465 if (tx_chn->psil_paired) {
466 xudma_navss_psil_unpair(tx_chn->common.udmax,
467 tx_chn->common.src_thread,
468 tx_chn->common.dst_thread);
469 tx_chn->psil_paired = false;
470 }
471 }
472 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn);
473
k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel * tx_chn,bool sync)474 void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
475 bool sync)
476 {
477 int i = 0;
478 u32 val;
479
480 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown1");
481
482 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG,
483 UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN);
484
485 val = xudma_tchanrt_read(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG);
486
487 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
488 val = xudma_tchanrt_read(tx_chn->udma_tchanx,
489 UDMA_CHAN_RT_CTL_REG);
490 udelay(1);
491 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
492 dev_err(tx_chn->common.dev, "TX tdown timeout\n");
493 break;
494 }
495 i++;
496 }
497
498 val = xudma_tchanrt_read(tx_chn->udma_tchanx,
499 UDMA_CHAN_RT_PEER_RT_EN_REG);
500 if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
501 dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n");
502 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown2");
503 }
504 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_tx_chn);
505
k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel * tx_chn,void * data,void (* cleanup)(void * data,dma_addr_t desc_dma))506 void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
507 void *data,
508 void (*cleanup)(void *data, dma_addr_t desc_dma))
509 {
510 struct device *dev = tx_chn->common.dev;
511 dma_addr_t desc_dma;
512 int occ_tx, i, ret;
513
514 /*
515 * TXQ reset need to be special way as it is input for udma and its
516 * state cached by udma, so:
517 * 1) save TXQ occ
518 * 2) clean up TXQ and call callback .cleanup() for each desc
519 * 3) reset TXQ in a special way
520 */
521 occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx);
522 dev_dbg(dev, "TX reset occ_tx %u\n", occ_tx);
523
524 for (i = 0; i < occ_tx; i++) {
525 ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma);
526 if (ret) {
527 if (ret != -ENODATA)
528 dev_err(dev, "TX reset pop %d\n", ret);
529 break;
530 }
531 cleanup(data, desc_dma);
532 }
533
534 /* reset TXCQ as it is not input for udma - expected to be empty */
535 k3_ringacc_ring_reset(tx_chn->ringtxcq);
536 k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx);
537 }
538 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn);
539
k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel * tx_chn)540 u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn)
541 {
542 return tx_chn->common.hdesc_size;
543 }
544 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_hdesc_size);
545
k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel * tx_chn)546 u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn)
547 {
548 return k3_ringacc_get_ring_id(tx_chn->ringtxcq);
549 }
550 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id);
551
k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel * tx_chn)552 int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn)
553 {
554 if (xudma_is_pktdma(tx_chn->common.udmax)) {
555 tx_chn->virq = xudma_pktdma_tflow_get_irq(tx_chn->common.udmax,
556 tx_chn->udma_tflow_id);
557 } else {
558 tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq);
559 }
560
561 if (!tx_chn->virq)
562 return -ENXIO;
563
564 return tx_chn->virq;
565 }
566 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq);
567
568 struct device *
k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel * tx_chn)569 k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn)
570 {
571 if (xudma_is_pktdma(tx_chn->common.udmax) &&
572 (tx_chn->common.atype_asel == 14 || tx_chn->common.atype_asel == 15))
573 return &tx_chn->common.chan_dev;
574
575 return xudma_get_device(tx_chn->common.udmax);
576 }
577 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_dma_device);
578
k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel * tx_chn,dma_addr_t * addr)579 void k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel *tx_chn,
580 dma_addr_t *addr)
581 {
582 if (!xudma_is_pktdma(tx_chn->common.udmax) ||
583 !tx_chn->common.atype_asel)
584 return;
585
586 *addr |= (u64)tx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
587 }
588 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_dma_to_cppi5_addr);
589
k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel * tx_chn,dma_addr_t * addr)590 void k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel *tx_chn,
591 dma_addr_t *addr)
592 {
593 if (!xudma_is_pktdma(tx_chn->common.udmax) ||
594 !tx_chn->common.atype_asel)
595 return;
596
597 *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
598 }
599 EXPORT_SYMBOL_GPL(k3_udma_glue_tx_cppi5_to_dma_addr);
600
k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel * rx_chn)601 static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
602 {
603 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
604 struct ti_sci_msg_rm_udmap_rx_ch_cfg req;
605 int ret;
606
607 memset(&req, 0, sizeof(req));
608
609 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
610 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
611 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
612 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
613
614 req.nav_id = tisci_rm->tisci_dev_id;
615 req.index = rx_chn->udma_rchan_id;
616 req.rx_fetch_size = rx_chn->common.hdesc_size >> 2;
617 /*
618 * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw
619 * and udmax impl, so just configure it to invalid value.
620 * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx);
621 */
622 req.rxcq_qnum = 0xFFFF;
623 if (!xudma_is_pktdma(rx_chn->common.udmax) && rx_chn->flow_num &&
624 rx_chn->flow_id_base != rx_chn->udma_rchan_id) {
625 /* Default flow + extra ones */
626 req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
627 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
628 req.flowid_start = rx_chn->flow_id_base;
629 req.flowid_cnt = rx_chn->flow_num;
630 }
631 req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
632 req.rx_atype = rx_chn->common.atype_asel;
633
634 ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req);
635 if (ret)
636 dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n",
637 rx_chn->udma_rchan_id, ret);
638
639 return ret;
640 }
641
k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel * rx_chn,u32 flow_num)642 static void k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
643 u32 flow_num)
644 {
645 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
646
647 if (IS_ERR_OR_NULL(flow->udma_rflow))
648 return;
649
650 if (flow->ringrxfdq)
651 k3_ringacc_ring_free(flow->ringrxfdq);
652
653 if (flow->ringrx)
654 k3_ringacc_ring_free(flow->ringrx);
655
656 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
657 flow->udma_rflow = NULL;
658 rx_chn->flows_ready--;
659 }
660
k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel * rx_chn,u32 flow_idx,struct k3_udma_glue_rx_flow_cfg * flow_cfg)661 static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
662 u32 flow_idx,
663 struct k3_udma_glue_rx_flow_cfg *flow_cfg)
664 {
665 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
666 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
667 struct device *dev = rx_chn->common.dev;
668 struct ti_sci_msg_rm_udmap_flow_cfg req;
669 int rx_ring_id;
670 int rx_ringfdq_id;
671 int ret = 0;
672
673 flow->udma_rflow = xudma_rflow_get(rx_chn->common.udmax,
674 flow->udma_rflow_id);
675 if (IS_ERR(flow->udma_rflow)) {
676 ret = PTR_ERR(flow->udma_rflow);
677 dev_err(dev, "UDMAX rflow get err %d\n", ret);
678 return ret;
679 }
680
681 if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) {
682 ret = -ENODEV;
683 goto err_rflow_put;
684 }
685
686 if (xudma_is_pktdma(rx_chn->common.udmax)) {
687 rx_ringfdq_id = flow->udma_rflow_id +
688 xudma_get_rflow_ring_offset(rx_chn->common.udmax);
689 rx_ring_id = 0;
690 } else {
691 rx_ring_id = flow_cfg->ring_rxq_id;
692 rx_ringfdq_id = flow_cfg->ring_rxfdq0_id;
693 }
694
695 /* request and cfg rings */
696 ret = k3_ringacc_request_rings_pair(rx_chn->common.ringacc,
697 rx_ringfdq_id, rx_ring_id,
698 &flow->ringrxfdq,
699 &flow->ringrx);
700 if (ret) {
701 dev_err(dev, "Failed to get RX/RXFDQ rings %d\n", ret);
702 goto err_rflow_put;
703 }
704
705 /* Set the dma_dev for the rings to be configured */
706 flow_cfg->rx_cfg.dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn);
707 flow_cfg->rxfdq_cfg.dma_dev = flow_cfg->rx_cfg.dma_dev;
708
709 /* Set the ASEL value for DMA rings of PKTDMA */
710 if (xudma_is_pktdma(rx_chn->common.udmax)) {
711 flow_cfg->rx_cfg.asel = rx_chn->common.atype_asel;
712 flow_cfg->rxfdq_cfg.asel = rx_chn->common.atype_asel;
713 }
714
715 ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg);
716 if (ret) {
717 dev_err(dev, "Failed to cfg ringrx %d\n", ret);
718 goto err_ringrxfdq_free;
719 }
720
721 ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg);
722 if (ret) {
723 dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret);
724 goto err_ringrxfdq_free;
725 }
726
727 if (rx_chn->remote) {
728 rx_ring_id = TI_SCI_RESOURCE_NULL;
729 rx_ringfdq_id = TI_SCI_RESOURCE_NULL;
730 } else {
731 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
732 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
733 }
734
735 memset(&req, 0, sizeof(req));
736
737 req.valid_params =
738 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
739 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
740 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
741 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
742 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
743 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
744 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
745 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
746 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
747 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
748 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
749 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
750 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
751 req.nav_id = tisci_rm->tisci_dev_id;
752 req.flow_index = flow->udma_rflow_id;
753 if (rx_chn->common.epib)
754 req.rx_einfo_present = 1;
755 if (rx_chn->common.psdata_size)
756 req.rx_psinfo_present = 1;
757 if (flow_cfg->rx_error_handling)
758 req.rx_error_handling = 1;
759 req.rx_desc_type = 0;
760 req.rx_dest_qnum = rx_ring_id;
761 req.rx_src_tag_hi_sel = 0;
762 req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel;
763 req.rx_dest_tag_hi_sel = 0;
764 req.rx_dest_tag_lo_sel = 0;
765 req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
766 req.rx_fdq1_qnum = rx_ringfdq_id;
767 req.rx_fdq2_qnum = rx_ringfdq_id;
768 req.rx_fdq3_qnum = rx_ringfdq_id;
769
770 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
771 if (ret) {
772 dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id,
773 ret);
774 goto err_ringrxfdq_free;
775 }
776
777 rx_chn->flows_ready++;
778 dev_dbg(dev, "flow%d config done. ready:%d\n",
779 flow->udma_rflow_id, rx_chn->flows_ready);
780
781 return 0;
782
783 err_ringrxfdq_free:
784 k3_ringacc_ring_free(flow->ringrxfdq);
785 k3_ringacc_ring_free(flow->ringrx);
786
787 err_rflow_put:
788 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
789 flow->udma_rflow = NULL;
790
791 return ret;
792 }
793
k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel * chn)794 static void k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel *chn)
795 {
796 struct device *dev = chn->common.dev;
797
798 dev_dbg(dev, "dump_rx_chn:\n"
799 "udma_rchan_id: %d\n"
800 "src_thread: %08x\n"
801 "dst_thread: %08x\n"
802 "epib: %d\n"
803 "hdesc_size: %u\n"
804 "psdata_size: %u\n"
805 "swdata_size: %u\n"
806 "flow_id_base: %d\n"
807 "flow_num: %d\n",
808 chn->udma_rchan_id,
809 chn->common.src_thread,
810 chn->common.dst_thread,
811 chn->common.epib,
812 chn->common.hdesc_size,
813 chn->common.psdata_size,
814 chn->common.swdata_size,
815 chn->flow_id_base,
816 chn->flow_num);
817 }
818
k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel * chn,char * mark)819 static void k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel *chn,
820 char *mark)
821 {
822 struct device *dev = chn->common.dev;
823
824 dev_dbg(dev, "=== dump ===> %s\n", mark);
825
826 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG,
827 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG));
828 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG,
829 xudma_rchanrt_read(chn->udma_rchanx,
830 UDMA_CHAN_RT_PEER_RT_EN_REG));
831 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG,
832 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_PCNT_REG));
833 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG,
834 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_BCNT_REG));
835 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG,
836 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_SBCNT_REG));
837 }
838
839 static int
k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel * rx_chn,struct k3_udma_glue_rx_channel_cfg * cfg)840 k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel *rx_chn,
841 struct k3_udma_glue_rx_channel_cfg *cfg)
842 {
843 int ret;
844
845 /* default rflow */
846 if (cfg->flow_id_use_rxchan_id)
847 return 0;
848
849 /* not a GP rflows */
850 if (rx_chn->flow_id_base != -1 &&
851 !xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
852 return 0;
853
854 /* Allocate range of GP rflows */
855 ret = xudma_alloc_gp_rflow_range(rx_chn->common.udmax,
856 rx_chn->flow_id_base,
857 rx_chn->flow_num);
858 if (ret < 0) {
859 dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n",
860 rx_chn->flow_id_base, rx_chn->flow_num, ret);
861 return ret;
862 }
863 rx_chn->flow_id_base = ret;
864
865 return 0;
866 }
867
868 static struct k3_udma_glue_rx_channel *
k3_udma_glue_request_rx_chn_priv(struct device * dev,const char * name,struct k3_udma_glue_rx_channel_cfg * cfg)869 k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
870 struct k3_udma_glue_rx_channel_cfg *cfg)
871 {
872 struct k3_udma_glue_rx_channel *rx_chn;
873 struct psil_endpoint_config *ep_cfg;
874 int ret, i;
875
876 if (cfg->flow_id_num <= 0)
877 return ERR_PTR(-EINVAL);
878
879 if (cfg->flow_id_num != 1 &&
880 (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id))
881 return ERR_PTR(-EINVAL);
882
883 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
884 if (!rx_chn)
885 return ERR_PTR(-ENOMEM);
886
887 rx_chn->common.dev = dev;
888 rx_chn->common.swdata_size = cfg->swdata_size;
889 rx_chn->remote = false;
890
891 /* parse of udmap channel */
892 ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
893 &rx_chn->common, false);
894 if (ret)
895 goto err;
896
897 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
898 rx_chn->common.psdata_size,
899 rx_chn->common.swdata_size);
900
901 ep_cfg = rx_chn->common.ep_config;
902
903 if (xudma_is_pktdma(rx_chn->common.udmax))
904 rx_chn->udma_rchan_id = ep_cfg->mapped_channel_id;
905 else
906 rx_chn->udma_rchan_id = -1;
907
908 /* request and cfg UDMAP RX channel */
909 rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax,
910 rx_chn->udma_rchan_id);
911 if (IS_ERR(rx_chn->udma_rchanx)) {
912 ret = PTR_ERR(rx_chn->udma_rchanx);
913 dev_err(dev, "UDMAX rchanx get err %d\n", ret);
914 goto err;
915 }
916 rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx);
917
918 rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
919 rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
920 dev_set_name(&rx_chn->common.chan_dev, "rchan%d-0x%04x",
921 rx_chn->udma_rchan_id, rx_chn->common.src_thread);
922 ret = device_register(&rx_chn->common.chan_dev);
923 if (ret) {
924 dev_err(dev, "Channel Device registration failed %d\n", ret);
925 put_device(&rx_chn->common.chan_dev);
926 rx_chn->common.chan_dev.parent = NULL;
927 goto err;
928 }
929
930 if (xudma_is_pktdma(rx_chn->common.udmax)) {
931 /* prepare the channel device as coherent */
932 rx_chn->common.chan_dev.dma_coherent = true;
933 dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
934 DMA_BIT_MASK(48));
935 }
936
937 if (xudma_is_pktdma(rx_chn->common.udmax)) {
938 int flow_start = cfg->flow_id_base;
939 int flow_end;
940
941 if (flow_start == -1)
942 flow_start = ep_cfg->flow_start;
943
944 flow_end = flow_start + cfg->flow_id_num - 1;
945 if (flow_start < ep_cfg->flow_start ||
946 flow_end > (ep_cfg->flow_start + ep_cfg->flow_num - 1)) {
947 dev_err(dev, "Invalid flow range requested\n");
948 ret = -EINVAL;
949 goto err;
950 }
951 rx_chn->flow_id_base = flow_start;
952 } else {
953 rx_chn->flow_id_base = cfg->flow_id_base;
954
955 /* Use RX channel id as flow id: target dev can't generate flow_id */
956 if (cfg->flow_id_use_rxchan_id)
957 rx_chn->flow_id_base = rx_chn->udma_rchan_id;
958 }
959
960 rx_chn->flow_num = cfg->flow_id_num;
961
962 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
963 sizeof(*rx_chn->flows), GFP_KERNEL);
964 if (!rx_chn->flows) {
965 ret = -ENOMEM;
966 goto err;
967 }
968
969 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
970 if (ret)
971 goto err;
972
973 for (i = 0; i < rx_chn->flow_num; i++)
974 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
975
976 /* request and cfg psi-l */
977 rx_chn->common.dst_thread =
978 xudma_dev_get_psil_base(rx_chn->common.udmax) +
979 rx_chn->udma_rchan_id;
980
981 ret = k3_udma_glue_cfg_rx_chn(rx_chn);
982 if (ret) {
983 dev_err(dev, "Failed to cfg rchan %d\n", ret);
984 goto err;
985 }
986
987 /* init default RX flow only if flow_num = 1 */
988 if (cfg->def_flow_cfg) {
989 ret = k3_udma_glue_cfg_rx_flow(rx_chn, 0, cfg->def_flow_cfg);
990 if (ret)
991 goto err;
992 }
993
994 k3_udma_glue_dump_rx_chn(rx_chn);
995
996 return rx_chn;
997
998 err:
999 k3_udma_glue_release_rx_chn(rx_chn);
1000 return ERR_PTR(ret);
1001 }
1002
1003 static struct k3_udma_glue_rx_channel *
k3_udma_glue_request_remote_rx_chn(struct device * dev,const char * name,struct k3_udma_glue_rx_channel_cfg * cfg)1004 k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name,
1005 struct k3_udma_glue_rx_channel_cfg *cfg)
1006 {
1007 struct k3_udma_glue_rx_channel *rx_chn;
1008 int ret, i;
1009
1010 if (cfg->flow_id_num <= 0 ||
1011 cfg->flow_id_use_rxchan_id ||
1012 cfg->def_flow_cfg ||
1013 cfg->flow_id_base < 0)
1014 return ERR_PTR(-EINVAL);
1015
1016 /*
1017 * Remote RX channel is under control of Remote CPU core, so
1018 * Linux can only request and manipulate by dedicated RX flows
1019 */
1020
1021 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
1022 if (!rx_chn)
1023 return ERR_PTR(-ENOMEM);
1024
1025 rx_chn->common.dev = dev;
1026 rx_chn->common.swdata_size = cfg->swdata_size;
1027 rx_chn->remote = true;
1028 rx_chn->udma_rchan_id = -1;
1029 rx_chn->flow_num = cfg->flow_id_num;
1030 rx_chn->flow_id_base = cfg->flow_id_base;
1031 rx_chn->psil_paired = false;
1032
1033 /* parse of udmap channel */
1034 ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
1035 &rx_chn->common, false);
1036 if (ret)
1037 goto err;
1038
1039 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
1040 rx_chn->common.psdata_size,
1041 rx_chn->common.swdata_size);
1042
1043 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
1044 sizeof(*rx_chn->flows), GFP_KERNEL);
1045 if (!rx_chn->flows) {
1046 ret = -ENOMEM;
1047 goto err;
1048 }
1049
1050 rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
1051 rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
1052 dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x",
1053 rx_chn->common.src_thread);
1054 ret = device_register(&rx_chn->common.chan_dev);
1055 if (ret) {
1056 dev_err(dev, "Channel Device registration failed %d\n", ret);
1057 put_device(&rx_chn->common.chan_dev);
1058 rx_chn->common.chan_dev.parent = NULL;
1059 goto err;
1060 }
1061
1062 if (xudma_is_pktdma(rx_chn->common.udmax)) {
1063 /* prepare the channel device as coherent */
1064 rx_chn->common.chan_dev.dma_coherent = true;
1065 dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
1066 DMA_BIT_MASK(48));
1067 }
1068
1069 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
1070 if (ret)
1071 goto err;
1072
1073 for (i = 0; i < rx_chn->flow_num; i++)
1074 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
1075
1076 k3_udma_glue_dump_rx_chn(rx_chn);
1077
1078 return rx_chn;
1079
1080 err:
1081 k3_udma_glue_release_rx_chn(rx_chn);
1082 return ERR_PTR(ret);
1083 }
1084
1085 struct k3_udma_glue_rx_channel *
k3_udma_glue_request_rx_chn(struct device * dev,const char * name,struct k3_udma_glue_rx_channel_cfg * cfg)1086 k3_udma_glue_request_rx_chn(struct device *dev, const char *name,
1087 struct k3_udma_glue_rx_channel_cfg *cfg)
1088 {
1089 if (cfg->remote)
1090 return k3_udma_glue_request_remote_rx_chn(dev, name, cfg);
1091 else
1092 return k3_udma_glue_request_rx_chn_priv(dev, name, cfg);
1093 }
1094 EXPORT_SYMBOL_GPL(k3_udma_glue_request_rx_chn);
1095
k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel * rx_chn)1096 void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1097 {
1098 int i;
1099
1100 if (IS_ERR_OR_NULL(rx_chn->common.udmax))
1101 return;
1102
1103 if (rx_chn->psil_paired) {
1104 xudma_navss_psil_unpair(rx_chn->common.udmax,
1105 rx_chn->common.src_thread,
1106 rx_chn->common.dst_thread);
1107 rx_chn->psil_paired = false;
1108 }
1109
1110 for (i = 0; i < rx_chn->flow_num; i++)
1111 k3_udma_glue_release_rx_flow(rx_chn, i);
1112
1113 if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
1114 xudma_free_gp_rflow_range(rx_chn->common.udmax,
1115 rx_chn->flow_id_base,
1116 rx_chn->flow_num);
1117
1118 if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx))
1119 xudma_rchan_put(rx_chn->common.udmax,
1120 rx_chn->udma_rchanx);
1121
1122 if (rx_chn->common.chan_dev.parent) {
1123 device_unregister(&rx_chn->common.chan_dev);
1124 rx_chn->common.chan_dev.parent = NULL;
1125 }
1126 }
1127 EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn);
1128
k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel * rx_chn,u32 flow_idx,struct k3_udma_glue_rx_flow_cfg * flow_cfg)1129 int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn,
1130 u32 flow_idx,
1131 struct k3_udma_glue_rx_flow_cfg *flow_cfg)
1132 {
1133 if (flow_idx >= rx_chn->flow_num)
1134 return -EINVAL;
1135
1136 return k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx, flow_cfg);
1137 }
1138 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_init);
1139
k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel * rx_chn,u32 flow_idx)1140 u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn,
1141 u32 flow_idx)
1142 {
1143 struct k3_udma_glue_rx_flow *flow;
1144
1145 if (flow_idx >= rx_chn->flow_num)
1146 return -EINVAL;
1147
1148 flow = &rx_chn->flows[flow_idx];
1149
1150 return k3_ringacc_get_ring_id(flow->ringrxfdq);
1151 }
1152 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_get_fdq_id);
1153
k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel * rx_chn)1154 u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn)
1155 {
1156 return rx_chn->flow_id_base;
1157 }
1158 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_flow_id_base);
1159
k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel * rx_chn,u32 flow_idx)1160 int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn,
1161 u32 flow_idx)
1162 {
1163 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1164 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1165 struct device *dev = rx_chn->common.dev;
1166 struct ti_sci_msg_rm_udmap_flow_cfg req;
1167 int rx_ring_id;
1168 int rx_ringfdq_id;
1169 int ret = 0;
1170
1171 if (!rx_chn->remote)
1172 return -EINVAL;
1173
1174 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
1175 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
1176
1177 memset(&req, 0, sizeof(req));
1178
1179 req.valid_params =
1180 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1181 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1182 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1183 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1184 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1185 req.nav_id = tisci_rm->tisci_dev_id;
1186 req.flow_index = flow->udma_rflow_id;
1187 req.rx_dest_qnum = rx_ring_id;
1188 req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
1189 req.rx_fdq1_qnum = rx_ringfdq_id;
1190 req.rx_fdq2_qnum = rx_ringfdq_id;
1191 req.rx_fdq3_qnum = rx_ringfdq_id;
1192
1193 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1194 if (ret) {
1195 dev_err(dev, "flow%d enable failed: %d\n", flow->udma_rflow_id,
1196 ret);
1197 }
1198
1199 return ret;
1200 }
1201 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_enable);
1202
k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel * rx_chn,u32 flow_idx)1203 int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn,
1204 u32 flow_idx)
1205 {
1206 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1207 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1208 struct device *dev = rx_chn->common.dev;
1209 struct ti_sci_msg_rm_udmap_flow_cfg req;
1210 int ret = 0;
1211
1212 if (!rx_chn->remote)
1213 return -EINVAL;
1214
1215 memset(&req, 0, sizeof(req));
1216 req.valid_params =
1217 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1218 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1219 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1220 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1221 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1222 req.nav_id = tisci_rm->tisci_dev_id;
1223 req.flow_index = flow->udma_rflow_id;
1224 req.rx_dest_qnum = TI_SCI_RESOURCE_NULL;
1225 req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL;
1226 req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL;
1227 req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL;
1228 req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL;
1229
1230 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1231 if (ret) {
1232 dev_err(dev, "flow%d disable failed: %d\n", flow->udma_rflow_id,
1233 ret);
1234 }
1235
1236 return ret;
1237 }
1238 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable);
1239
k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel * rx_chn)1240 int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1241 {
1242 int ret;
1243
1244 if (rx_chn->remote)
1245 return -EINVAL;
1246
1247 if (rx_chn->flows_ready < rx_chn->flow_num)
1248 return -EINVAL;
1249
1250 ret = xudma_navss_psil_pair(rx_chn->common.udmax,
1251 rx_chn->common.src_thread,
1252 rx_chn->common.dst_thread);
1253 if (ret) {
1254 dev_err(rx_chn->common.dev, "PSI-L request err %d\n", ret);
1255 return ret;
1256 }
1257
1258 rx_chn->psil_paired = true;
1259
1260 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG,
1261 UDMA_CHAN_RT_CTL_EN);
1262
1263 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
1264 UDMA_PEER_RT_EN_ENABLE);
1265
1266 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt en");
1267 return 0;
1268 }
1269 EXPORT_SYMBOL_GPL(k3_udma_glue_enable_rx_chn);
1270
k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel * rx_chn)1271 void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1272 {
1273 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis1");
1274
1275 xudma_rchanrt_write(rx_chn->udma_rchanx,
1276 UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
1277 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, 0);
1278
1279 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2");
1280
1281 if (rx_chn->psil_paired) {
1282 xudma_navss_psil_unpair(rx_chn->common.udmax,
1283 rx_chn->common.src_thread,
1284 rx_chn->common.dst_thread);
1285 rx_chn->psil_paired = false;
1286 }
1287 }
1288 EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn);
1289
k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel * rx_chn,bool sync)1290 void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1291 bool sync)
1292 {
1293 int i = 0;
1294 u32 val;
1295
1296 if (rx_chn->remote)
1297 return;
1298
1299 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown1");
1300
1301 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
1302 UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN);
1303
1304 val = xudma_rchanrt_read(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG);
1305
1306 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
1307 val = xudma_rchanrt_read(rx_chn->udma_rchanx,
1308 UDMA_CHAN_RT_CTL_REG);
1309 udelay(1);
1310 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
1311 dev_err(rx_chn->common.dev, "RX tdown timeout\n");
1312 break;
1313 }
1314 i++;
1315 }
1316
1317 val = xudma_rchanrt_read(rx_chn->udma_rchanx,
1318 UDMA_CHAN_RT_PEER_RT_EN_REG);
1319 if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
1320 dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n");
1321 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown2");
1322 }
1323 EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn);
1324
k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel * rx_chn,u32 flow_num,void * data,void (* cleanup)(void * data,dma_addr_t desc_dma),bool skip_fdq)1325 void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1326 u32 flow_num, void *data,
1327 void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq)
1328 {
1329 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1330 struct device *dev = rx_chn->common.dev;
1331 dma_addr_t desc_dma;
1332 int occ_rx, i, ret;
1333
1334 /* reset RXCQ as it is not input for udma - expected to be empty */
1335 occ_rx = k3_ringacc_ring_get_occ(flow->ringrx);
1336 dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx);
1337
1338 /* Skip RX FDQ in case one FDQ is used for the set of flows */
1339 if (skip_fdq)
1340 goto do_reset;
1341
1342 /*
1343 * RX FDQ reset need to be special way as it is input for udma and its
1344 * state cached by udma, so:
1345 * 1) save RX FDQ occ
1346 * 2) clean up RX FDQ and call callback .cleanup() for each desc
1347 * 3) reset RX FDQ in a special way
1348 */
1349 occ_rx = k3_ringacc_ring_get_occ(flow->ringrxfdq);
1350 dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n", flow_num, occ_rx);
1351
1352 for (i = 0; i < occ_rx; i++) {
1353 ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma);
1354 if (ret) {
1355 if (ret != -ENODATA)
1356 dev_err(dev, "RX reset pop %d\n", ret);
1357 break;
1358 }
1359 cleanup(data, desc_dma);
1360 }
1361
1362 k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx);
1363
1364 do_reset:
1365 k3_ringacc_ring_reset(flow->ringrx);
1366 }
1367 EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn);
1368
k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel * rx_chn,u32 flow_num,struct cppi5_host_desc_t * desc_rx,dma_addr_t desc_dma)1369 int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1370 u32 flow_num, struct cppi5_host_desc_t *desc_rx,
1371 dma_addr_t desc_dma)
1372 {
1373 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1374
1375 return k3_ringacc_ring_push(flow->ringrxfdq, &desc_dma);
1376 }
1377 EXPORT_SYMBOL_GPL(k3_udma_glue_push_rx_chn);
1378
k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel * rx_chn,u32 flow_num,dma_addr_t * desc_dma)1379 int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1380 u32 flow_num, dma_addr_t *desc_dma)
1381 {
1382 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1383
1384 return k3_ringacc_ring_pop(flow->ringrx, desc_dma);
1385 }
1386 EXPORT_SYMBOL_GPL(k3_udma_glue_pop_rx_chn);
1387
k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel * rx_chn,u32 flow_num)1388 int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn,
1389 u32 flow_num)
1390 {
1391 struct k3_udma_glue_rx_flow *flow;
1392
1393 flow = &rx_chn->flows[flow_num];
1394
1395 if (xudma_is_pktdma(rx_chn->common.udmax)) {
1396 flow->virq = xudma_pktdma_rflow_get_irq(rx_chn->common.udmax,
1397 flow->udma_rflow_id);
1398 } else {
1399 flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx);
1400 }
1401
1402 return flow->virq;
1403 }
1404 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq);
1405
1406 struct device *
k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel * rx_chn)1407 k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn)
1408 {
1409 if (xudma_is_pktdma(rx_chn->common.udmax) &&
1410 (rx_chn->common.atype_asel == 14 || rx_chn->common.atype_asel == 15))
1411 return &rx_chn->common.chan_dev;
1412
1413 return xudma_get_device(rx_chn->common.udmax);
1414 }
1415 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_dma_device);
1416
k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel * rx_chn,dma_addr_t * addr)1417 void k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel *rx_chn,
1418 dma_addr_t *addr)
1419 {
1420 if (!xudma_is_pktdma(rx_chn->common.udmax) ||
1421 !rx_chn->common.atype_asel)
1422 return;
1423
1424 *addr |= (u64)rx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
1425 }
1426 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_dma_to_cppi5_addr);
1427
k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel * rx_chn,dma_addr_t * addr)1428 void k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel *rx_chn,
1429 dma_addr_t *addr)
1430 {
1431 if (!xudma_is_pktdma(rx_chn->common.udmax) ||
1432 !rx_chn->common.atype_asel)
1433 return;
1434
1435 *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
1436 }
1437 EXPORT_SYMBOL_GPL(k3_udma_glue_rx_cppi5_to_dma_addr);
1438
k3_udma_glue_class_init(void)1439 static int __init k3_udma_glue_class_init(void)
1440 {
1441 return class_register(&k3_udma_glue_devclass);
1442 }
1443
1444 module_init(k3_udma_glue_class_init);
1445 MODULE_LICENSE("GPL v2");
1446