23 static void ocelot_fdma_writel(struct ocelot *ocelot, u32 reg, u32 data)  in ocelot_fdma_writel()  argument
25 	regmap_write(ocelot->targets[FDMA], reg, data);  in ocelot_fdma_writel()
28 static u32 ocelot_fdma_readl(struct ocelot *ocelot, u32 reg)  in ocelot_fdma_readl()  argument
32 	regmap_read(ocelot->targets[FDMA], reg, &retval);  in ocelot_fdma_readl()
86 static void ocelot_fdma_activate_chan(struct ocelot *ocelot, dma_addr_t dma,  in ocelot_fdma_activate_chan()  argument
89 	ocelot_fdma_writel(ocelot, MSCC_FDMA_DCB_LLP(chan), dma);  in ocelot_fdma_activate_chan()
94 	ocelot_fdma_writel(ocelot, MSCC_FDMA_CH_ACTIVATE, BIT(chan));  in ocelot_fdma_activate_chan()
97 static u32 ocelot_fdma_read_ch_safe(struct ocelot *ocelot)  in ocelot_fdma_read_ch_safe()  argument
99 	return ocelot_fdma_readl(ocelot, MSCC_FDMA_CH_SAFE);  in ocelot_fdma_read_ch_safe()
102 static int ocelot_fdma_wait_chan_safe(struct ocelot *ocelot, int chan)  in ocelot_fdma_wait_chan_safe()  argument
106 	return readx_poll_timeout_atomic(ocelot_fdma_read_ch_safe, ocelot, safe,  in ocelot_fdma_wait_chan_safe()
123 static bool ocelot_fdma_rx_alloc_page(struct ocelot *ocelot,  in ocelot_fdma_rx_alloc_page()  argument
133 	mapping = dma_map_page(ocelot->dev, page, 0, PAGE_SIZE,  in ocelot_fdma_rx_alloc_page()
135 	if (unlikely(dma_mapping_error(ocelot->dev, mapping))) {  in ocelot_fdma_rx_alloc_page()
147 static int ocelot_fdma_alloc_rx_buffs(struct ocelot *ocelot, u16 alloc_cnt)  in ocelot_fdma_alloc_rx_buffs()  argument
149 	struct ocelot_fdma *fdma = ocelot->fdma;  in ocelot_fdma_alloc_rx_buffs()
164 			if (unlikely(!ocelot_fdma_rx_alloc_page(ocelot, rxb))) {  in ocelot_fdma_alloc_rx_buffs()
165 				dev_err_ratelimited(ocelot->dev,  in ocelot_fdma_alloc_rx_buffs()
187 static bool ocelot_fdma_tx_dcb_set_skb(struct ocelot *ocelot,  in ocelot_fdma_tx_dcb_set_skb()  argument
194 	mapping = dma_map_single(ocelot->dev, skb->data, skb->len,  in ocelot_fdma_tx_dcb_set_skb()
196 	if (unlikely(dma_mapping_error(ocelot->dev, mapping)))  in ocelot_fdma_tx_dcb_set_skb()
209 static bool ocelot_fdma_check_stop_rx(struct ocelot *ocelot)  in ocelot_fdma_check_stop_rx()  argument
214 	llp = ocelot_fdma_readl(ocelot, MSCC_FDMA_DCB_LLP(MSCC_FDMA_XTR_CHAN));  in ocelot_fdma_check_stop_rx()
218 	ocelot_fdma_writel(ocelot, MSCC_FDMA_CH_DISABLE,  in ocelot_fdma_check_stop_rx()
235 static void ocelot_fdma_rx_restart(struct ocelot *ocelot)  in ocelot_fdma_rx_restart()  argument
237 	struct ocelot_fdma *fdma = ocelot->fdma;  in ocelot_fdma_rx_restart()
246 	ret = ocelot_fdma_wait_chan_safe(ocelot, chan);  in ocelot_fdma_rx_restart()
248 		dev_err_ratelimited(ocelot->dev,  in ocelot_fdma_rx_restart()
259 	llp_prev = ocelot_fdma_readl(ocelot, MSCC_FDMA_DCB_LLP_PREV(chan));  in ocelot_fdma_rx_restart()
268 	ocelot_fdma_activate_chan(ocelot, new_llp, chan);  in ocelot_fdma_rx_restart()
296 static void ocelot_fdma_reuse_rx_page(struct ocelot *ocelot,  in ocelot_fdma_reuse_rx_page()  argument
299 	struct ocelot_fdma_rx_ring *rx_ring = &ocelot->fdma->rx_ring;  in ocelot_fdma_reuse_rx_page()
310 	dma_sync_single_range_for_device(ocelot->dev, old_rxb->dma_addr,  in ocelot_fdma_reuse_rx_page()
315 static struct sk_buff *ocelot_fdma_get_skb(struct ocelot *ocelot, u32 stat,  in ocelot_fdma_get_skb()  argument
328 			dev_err_ratelimited(ocelot->dev,  in ocelot_fdma_get_skb()
335 	dma_sync_single_range_for_cpu(ocelot->dev, rxb->dma_addr,  in ocelot_fdma_get_skb()
341 		ocelot_fdma_reuse_rx_page(ocelot, rxb);  in ocelot_fdma_get_skb()
344 		dma_unmap_page(ocelot->dev, rxb->dma_addr, PAGE_SIZE,  in ocelot_fdma_get_skb()
354 static bool ocelot_fdma_receive_skb(struct ocelot *ocelot, struct sk_buff *skb)  in ocelot_fdma_receive_skb()  argument
364 	if (unlikely(src_port >= ocelot->num_phys_ports))  in ocelot_fdma_receive_skb()
367 	ndev = ocelot_port_to_netdev(ocelot, src_port);  in ocelot_fdma_receive_skb()
378 	if (ocelot->ptp) {  in ocelot_fdma_receive_skb()
380 		ocelot_ptp_rx_timestamp(ocelot, skb, timestamp);  in ocelot_fdma_receive_skb()
389 static int ocelot_fdma_rx_get(struct ocelot *ocelot, int budget)  in ocelot_fdma_rx_get()  argument
391 	struct ocelot_fdma *fdma = ocelot->fdma;  in ocelot_fdma_rx_get()
422 		skb = ocelot_fdma_get_skb(ocelot, stat, rxb, skb);  in ocelot_fdma_rx_get()
434 			dev_err_ratelimited(ocelot->dev,  in ocelot_fdma_rx_get()
447 		if (unlikely(!ocelot_fdma_receive_skb(ocelot, skb)))  in ocelot_fdma_rx_get()
456 		ocelot_fdma_alloc_rx_buffs(ocelot, cleaned_cnt);  in ocelot_fdma_rx_get()
461 static void ocelot_fdma_wakeup_netdev(struct ocelot *ocelot)  in ocelot_fdma_wakeup_netdev()  argument
468 	for (port = 0; port < ocelot->num_phys_ports; port++) {  in ocelot_fdma_wakeup_netdev()
469 		ocelot_port = ocelot->ports[port];  in ocelot_fdma_wakeup_netdev()
481 static void ocelot_fdma_tx_cleanup(struct ocelot *ocelot, int budget)  in ocelot_fdma_tx_cleanup()  argument
483 	struct ocelot_fdma *fdma = ocelot->fdma;  in ocelot_fdma_tx_cleanup()
508 		dma_unmap_single(ocelot->dev, dma_unmap_addr(buf, dma_addr),  in ocelot_fdma_tx_cleanup()
526 		ocelot_fdma_wakeup_netdev(ocelot);  in ocelot_fdma_tx_cleanup()
534 	ret = ocelot_fdma_wait_chan_safe(ocelot, MSCC_FDMA_INJ_CHAN);  in ocelot_fdma_tx_cleanup()
536 		dev_warn(ocelot->dev,  in ocelot_fdma_tx_cleanup()
548 	ocelot_fdma_activate_chan(ocelot, dma, MSCC_FDMA_INJ_CHAN);  in ocelot_fdma_tx_cleanup()
554 	struct ocelot *ocelot = fdma->ocelot;  in ocelot_fdma_napi_poll()  local
558 	ocelot_fdma_tx_cleanup(ocelot, budget);  in ocelot_fdma_napi_poll()
560 	rx_stopped = ocelot_fdma_check_stop_rx(ocelot);  in ocelot_fdma_napi_poll()
562 	work_done = ocelot_fdma_rx_get(ocelot, budget);  in ocelot_fdma_napi_poll()
565 		ocelot_fdma_rx_restart(ocelot);  in ocelot_fdma_napi_poll()
569 		ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_ENA,  in ocelot_fdma_napi_poll()
580 	struct ocelot *ocelot = dev_id;  in ocelot_fdma_interrupt()  local
582 	ident = ocelot_fdma_readl(ocelot, MSCC_FDMA_INTR_IDENT);  in ocelot_fdma_interrupt()
583 	frm = ocelot_fdma_readl(ocelot, MSCC_FDMA_INTR_FRM);  in ocelot_fdma_interrupt()
584 	llp = ocelot_fdma_readl(ocelot, MSCC_FDMA_INTR_LLP);  in ocelot_fdma_interrupt()
586 	ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_LLP, llp & ident);  in ocelot_fdma_interrupt()
587 	ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_FRM, frm & ident);  in ocelot_fdma_interrupt()
589 		ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_ENA, 0);  in ocelot_fdma_interrupt()
590 		napi_schedule(&ocelot->fdma->napi);  in ocelot_fdma_interrupt()
593 	err = ocelot_fdma_readl(ocelot, MSCC_FDMA_EVT_ERR);  in ocelot_fdma_interrupt()
595 		err_code = ocelot_fdma_readl(ocelot, MSCC_FDMA_EVT_ERR_CODE);  in ocelot_fdma_interrupt()
596 		dev_err_ratelimited(ocelot->dev,  in ocelot_fdma_interrupt()
600 		ocelot_fdma_writel(ocelot, MSCC_FDMA_EVT_ERR, err);  in ocelot_fdma_interrupt()
601 		ocelot_fdma_writel(ocelot, MSCC_FDMA_EVT_ERR_CODE, err_code);  in ocelot_fdma_interrupt()
607 static void ocelot_fdma_send_skb(struct ocelot *ocelot,  in ocelot_fdma_send_skb()  argument
618 	if (!ocelot_fdma_tx_dcb_set_skb(ocelot, tx_buf, dcb, skb)) {  in ocelot_fdma_send_skb()
631 		ocelot_fdma_activate_chan(ocelot, dma, MSCC_FDMA_INJ_CHAN);  in ocelot_fdma_send_skb()
640 static int ocelot_fdma_prepare_skb(struct ocelot *ocelot, int port, u32 rew_op,  in ocelot_fdma_prepare_skb()  argument
674 int ocelot_fdma_inject_frame(struct ocelot *ocelot, int port, u32 rew_op,  in ocelot_fdma_inject_frame()  argument
677 	struct ocelot_fdma *fdma = ocelot->fdma;  in ocelot_fdma_inject_frame()
688 	if (ocelot_fdma_prepare_skb(ocelot, port, rew_op, skb, dev))  in ocelot_fdma_inject_frame()
691 	ocelot_fdma_send_skb(ocelot, fdma, skb);  in ocelot_fdma_inject_frame()
699 static void ocelot_fdma_free_rx_ring(struct ocelot *ocelot)  in ocelot_fdma_free_rx_ring()  argument
701 	struct ocelot_fdma *fdma = ocelot->fdma;  in ocelot_fdma_free_rx_ring()
712 		dma_unmap_page(ocelot->dev, rxb->dma_addr, PAGE_SIZE,  in ocelot_fdma_free_rx_ring()
722 static void ocelot_fdma_free_tx_ring(struct ocelot *ocelot)  in ocelot_fdma_free_tx_ring()  argument
724 	struct ocelot_fdma *fdma = ocelot->fdma;  in ocelot_fdma_free_tx_ring()
736 		dma_unmap_single(ocelot->dev, dma_unmap_addr(txb, dma_addr),  in ocelot_fdma_free_tx_ring()
743 static int ocelot_fdma_rings_alloc(struct ocelot *ocelot)  in ocelot_fdma_rings_alloc()  argument
745 	struct ocelot_fdma *fdma = ocelot->fdma;  in ocelot_fdma_rings_alloc()
752 	fdma->dcbs_base = dmam_alloc_coherent(ocelot->dev,  in ocelot_fdma_rings_alloc()
775 	ret = ocelot_fdma_alloc_rx_buffs(ocelot,  in ocelot_fdma_rings_alloc()
778 		ocelot_fdma_free_rx_ring(ocelot);  in ocelot_fdma_rings_alloc()
790 void ocelot_fdma_netdev_init(struct ocelot *ocelot, struct net_device *dev)  in ocelot_fdma_netdev_init()  argument
792 	struct ocelot_fdma *fdma = ocelot->fdma;  in ocelot_fdma_netdev_init()
805 void ocelot_fdma_netdev_deinit(struct ocelot *ocelot, struct net_device *dev)  in ocelot_fdma_netdev_deinit()  argument
807 	struct ocelot_fdma *fdma = ocelot->fdma;  in ocelot_fdma_netdev_deinit()
815 void ocelot_fdma_init(struct platform_device *pdev, struct ocelot *ocelot)  in ocelot_fdma_init()  argument
817 	struct device *dev = ocelot->dev;  in ocelot_fdma_init()
825 	ocelot->fdma = fdma;  in ocelot_fdma_init()
826 	ocelot->dev->coherent_dma_mask = DMA_BIT_MASK(32);  in ocelot_fdma_init()
828 	ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_ENA, 0);  in ocelot_fdma_init()
830 	fdma->ocelot = ocelot;  in ocelot_fdma_init()
833 			       dev_name(dev), ocelot);  in ocelot_fdma_init()
837 	ret = ocelot_fdma_rings_alloc(ocelot);  in ocelot_fdma_init()
850 	ocelot->fdma = NULL;  in ocelot_fdma_init()
853 void ocelot_fdma_start(struct ocelot *ocelot)  in ocelot_fdma_start()  argument
855 	struct ocelot_fdma *fdma = ocelot->fdma;  in ocelot_fdma_start()
858 	ocelot_write_rix(ocelot, QS_INJ_GRP_CFG_MODE(2), QS_INJ_GRP_CFG, 0);  in ocelot_fdma_start()
859 	ocelot_write_rix(ocelot, QS_INJ_CTRL_GAP_SIZE(0), QS_INJ_CTRL, 0);  in ocelot_fdma_start()
861 	ocelot_write_rix(ocelot, QS_XTR_GRP_CFG_MODE(2), QS_XTR_GRP_CFG, 0);  in ocelot_fdma_start()
863 	ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_LLP, 0xffffffff);  in ocelot_fdma_start()
864 	ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_FRM, 0xffffffff);  in ocelot_fdma_start()
866 	ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_LLP_ENA,  in ocelot_fdma_start()
868 	ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_FRM_ENA,  in ocelot_fdma_start()
870 	ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_ENA,  in ocelot_fdma_start()
875 	ocelot_fdma_activate_chan(ocelot, ocelot->fdma->rx_ring.dcbs_dma,  in ocelot_fdma_start()
879 void ocelot_fdma_deinit(struct ocelot *ocelot)  in ocelot_fdma_deinit()  argument
881 	struct ocelot_fdma *fdma = ocelot->fdma;  in ocelot_fdma_deinit()
883 	ocelot_fdma_writel(ocelot, MSCC_FDMA_INTR_ENA, 0);  in ocelot_fdma_deinit()
884 	ocelot_fdma_writel(ocelot, MSCC_FDMA_CH_FORCEDIS,  in ocelot_fdma_deinit()
886 	ocelot_fdma_writel(ocelot, MSCC_FDMA_CH_FORCEDIS,  in ocelot_fdma_deinit()
891 	ocelot_fdma_free_rx_ring(ocelot);  in ocelot_fdma_deinit()
892 	ocelot_fdma_free_tx_ring(ocelot);  in ocelot_fdma_deinit()