100 static inline void qtnf_init_hdp_irqs(struct qtnf_pcie_pearl_state *ps)  in qtnf_init_hdp_irqs()  argument
104 	spin_lock_irqsave(&ps->irq_lock, flags);  in qtnf_init_hdp_irqs()
105 	ps->pcie_irq_mask = (PCIE_HDP_INT_RX_BITS | PCIE_HDP_INT_TX_BITS);  in qtnf_init_hdp_irqs()
106 	spin_unlock_irqrestore(&ps->irq_lock, flags);  in qtnf_init_hdp_irqs()
109 static inline void qtnf_enable_hdp_irqs(struct qtnf_pcie_pearl_state *ps)  in qtnf_enable_hdp_irqs()  argument
113 	spin_lock_irqsave(&ps->irq_lock, flags);  in qtnf_enable_hdp_irqs()
114 	writel(ps->pcie_irq_mask, PCIE_HDP_INT_EN(ps->pcie_reg_base));  in qtnf_enable_hdp_irqs()
115 	spin_unlock_irqrestore(&ps->irq_lock, flags);  in qtnf_enable_hdp_irqs()
118 static inline void qtnf_disable_hdp_irqs(struct qtnf_pcie_pearl_state *ps)  in qtnf_disable_hdp_irqs()  argument
122 	spin_lock_irqsave(&ps->irq_lock, flags);  in qtnf_disable_hdp_irqs()
123 	writel(0x0, PCIE_HDP_INT_EN(ps->pcie_reg_base));  in qtnf_disable_hdp_irqs()
124 	spin_unlock_irqrestore(&ps->irq_lock, flags);  in qtnf_disable_hdp_irqs()
127 static inline void qtnf_en_rxdone_irq(struct qtnf_pcie_pearl_state *ps)  in qtnf_en_rxdone_irq()  argument
131 	spin_lock_irqsave(&ps->irq_lock, flags);  in qtnf_en_rxdone_irq()
132 	ps->pcie_irq_mask |= PCIE_HDP_INT_RX_BITS;  in qtnf_en_rxdone_irq()
133 	writel(ps->pcie_irq_mask, PCIE_HDP_INT_EN(ps->pcie_reg_base));  in qtnf_en_rxdone_irq()
134 	spin_unlock_irqrestore(&ps->irq_lock, flags);  in qtnf_en_rxdone_irq()
137 static inline void qtnf_dis_rxdone_irq(struct qtnf_pcie_pearl_state *ps)  in qtnf_dis_rxdone_irq()  argument
141 	spin_lock_irqsave(&ps->irq_lock, flags);  in qtnf_dis_rxdone_irq()
142 	ps->pcie_irq_mask &= ~PCIE_HDP_INT_RX_BITS;  in qtnf_dis_rxdone_irq()
143 	writel(ps->pcie_irq_mask, PCIE_HDP_INT_EN(ps->pcie_reg_base));  in qtnf_dis_rxdone_irq()
144 	spin_unlock_irqrestore(&ps->irq_lock, flags);  in qtnf_dis_rxdone_irq()
147 static inline void qtnf_en_txdone_irq(struct qtnf_pcie_pearl_state *ps)  in qtnf_en_txdone_irq()  argument
151 	spin_lock_irqsave(&ps->irq_lock, flags);  in qtnf_en_txdone_irq()
152 	ps->pcie_irq_mask |= PCIE_HDP_INT_TX_BITS;  in qtnf_en_txdone_irq()
153 	writel(ps->pcie_irq_mask, PCIE_HDP_INT_EN(ps->pcie_reg_base));  in qtnf_en_txdone_irq()
154 	spin_unlock_irqrestore(&ps->irq_lock, flags);  in qtnf_en_txdone_irq()
157 static inline void qtnf_dis_txdone_irq(struct qtnf_pcie_pearl_state *ps)  in qtnf_dis_txdone_irq()  argument
161 	spin_lock_irqsave(&ps->irq_lock, flags);  in qtnf_dis_txdone_irq()
162 	ps->pcie_irq_mask &= ~PCIE_HDP_INT_TX_BITS;  in qtnf_dis_txdone_irq()
163 	writel(ps->pcie_irq_mask, PCIE_HDP_INT_EN(ps->pcie_reg_base));  in qtnf_dis_txdone_irq()
164 	spin_unlock_irqrestore(&ps->irq_lock, flags);  in qtnf_dis_txdone_irq()
167 static void qtnf_deassert_intx(struct qtnf_pcie_pearl_state *ps)  in qtnf_deassert_intx()  argument
169 	void __iomem *reg = ps->base.sysctl_bar + PEARL_PCIE_CFG0_OFFSET;  in qtnf_deassert_intx()
177 static void qtnf_pearl_reset_ep(struct qtnf_pcie_pearl_state *ps)  in qtnf_pearl_reset_ep()  argument
180 	void __iomem *reg = ps->base.sysctl_bar +  in qtnf_pearl_reset_ep()
185 	pci_restore_state(ps->base.pdev);  in qtnf_pearl_reset_ep()
190 	const struct qtnf_pcie_pearl_state *ps = arg;  in qtnf_pcie_pearl_ipc_gen_ep_int()  local
192 	void __iomem *reg = ps->base.sysctl_bar +  in qtnf_pcie_pearl_ipc_gen_ep_int()
232 static int pearl_alloc_bd_table(struct qtnf_pcie_pearl_state *ps)  in pearl_alloc_bd_table()  argument
234 	struct qtnf_pcie_bus_priv *priv = &ps->base;  in pearl_alloc_bd_table()
248 	ps->bd_table_vaddr = vaddr;  in pearl_alloc_bd_table()
249 	ps->bd_table_paddr = paddr;  in pearl_alloc_bd_table()
250 	ps->bd_table_len = len;  in pearl_alloc_bd_table()
252 	ps->tx_bd_vbase = vaddr;  in pearl_alloc_bd_table()
253 	ps->tx_bd_pbase = paddr;  in pearl_alloc_bd_table()
265 	ps->rx_bd_vbase = vaddr;  in pearl_alloc_bd_table()
266 	ps->rx_bd_pbase = paddr;  in pearl_alloc_bd_table()
270 	       PCIE_HDP_TX_HOST_Q_BASE_H(ps->pcie_reg_base));  in pearl_alloc_bd_table()
273 	       PCIE_HDP_TX_HOST_Q_BASE_L(ps->pcie_reg_base));  in pearl_alloc_bd_table()
275 	       PCIE_HDP_TX_HOST_Q_SZ_CTRL(ps->pcie_reg_base));  in pearl_alloc_bd_table()
282 static int pearl_skb2rbd_attach(struct qtnf_pcie_pearl_state *ps, u16 index)  in pearl_skb2rbd_attach()  argument
284 	struct qtnf_pcie_bus_priv *priv = &ps->base;  in pearl_skb2rbd_attach()
296 	rxbd = &ps->rx_bd_vbase[index];  in pearl_skb2rbd_attach()
317 	       PCIE_HDP_HHBM_BUF_PTR_H(ps->pcie_reg_base));  in pearl_skb2rbd_attach()
320 	       PCIE_HDP_HHBM_BUF_PTR(ps->pcie_reg_base));  in pearl_skb2rbd_attach()
322 	writel(index, PCIE_HDP_TX_HOST_Q_WR_PTR(ps->pcie_reg_base));  in pearl_skb2rbd_attach()
326 static int pearl_alloc_rx_buffers(struct qtnf_pcie_pearl_state *ps)  in pearl_alloc_rx_buffers()  argument
331 	memset(ps->rx_bd_vbase, 0x0,  in pearl_alloc_rx_buffers()
332 	       ps->base.rx_bd_num * sizeof(struct qtnf_pearl_rx_bd));  in pearl_alloc_rx_buffers()
334 	for (i = 0; i < ps->base.rx_bd_num; i++) {  in pearl_alloc_rx_buffers()
335 		ret = pearl_skb2rbd_attach(ps, i);  in pearl_alloc_rx_buffers()
344 static void qtnf_pearl_free_xfer_buffers(struct qtnf_pcie_pearl_state *ps)  in qtnf_pearl_free_xfer_buffers()  argument
346 	struct qtnf_pcie_bus_priv *priv = &ps->base;  in qtnf_pearl_free_xfer_buffers()
356 			rxbd = &ps->rx_bd_vbase[i];  in qtnf_pearl_free_xfer_buffers()
370 			txbd = &ps->tx_bd_vbase[i];  in qtnf_pearl_free_xfer_buffers()
382 static int pearl_hhbm_init(struct qtnf_pcie_pearl_state *ps)  in pearl_hhbm_init()  argument
386 	val = readl(PCIE_HHBM_CONFIG(ps->pcie_reg_base));  in pearl_hhbm_init()
388 	writel(val, PCIE_HHBM_CONFIG(ps->pcie_reg_base));  in pearl_hhbm_init()
394 	writel(val, PCIE_HHBM_CONFIG(ps->pcie_reg_base));  in pearl_hhbm_init()
395 	writel(ps->base.rx_bd_num, PCIE_HHBM_Q_LIMIT_REG(ps->pcie_reg_base));  in pearl_hhbm_init()
400 static int qtnf_pcie_pearl_init_xfer(struct qtnf_pcie_pearl_state *ps,  in qtnf_pcie_pearl_init_xfer()  argument
404 	struct qtnf_pcie_bus_priv *priv = &ps->base;  in qtnf_pcie_pearl_init_xfer()
437 	ret = pearl_hhbm_init(ps);  in qtnf_pcie_pearl_init_xfer()
449 	ret = pearl_alloc_bd_table(ps);  in qtnf_pcie_pearl_init_xfer()
455 	ret = pearl_alloc_rx_buffers(ps);  in qtnf_pcie_pearl_init_xfer()
464 static void qtnf_pearl_data_tx_reclaim(struct qtnf_pcie_pearl_state *ps)  in qtnf_pearl_data_tx_reclaim()  argument
466 	struct qtnf_pcie_bus_priv *priv = &ps->base;  in qtnf_pearl_data_tx_reclaim()
477 	tx_done_index = readl(PCIE_HDP_RX0DMA_CNT(ps->pcie_reg_base))  in qtnf_pearl_data_tx_reclaim()
485 			txbd = &ps->tx_bd_vbase[i];  in qtnf_pearl_data_tx_reclaim()
516 static int qtnf_tx_queue_ready(struct qtnf_pcie_pearl_state *ps)  in qtnf_tx_queue_ready()  argument
518 	struct qtnf_pcie_bus_priv *priv = &ps->base;  in qtnf_tx_queue_ready()
522 		qtnf_pearl_data_tx_reclaim(ps);  in qtnf_tx_queue_ready()
537 	struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);  in qtnf_pcie_skb_send()  local
538 	struct qtnf_pcie_bus_priv *priv = &ps->base;  in qtnf_pcie_skb_send()
548 	if (!qtnf_tx_queue_ready(ps)) {  in qtnf_pcie_skb_send()
570 	txbd = &ps->tx_bd_vbase[i];  in qtnf_pcie_skb_send()
581 	txbd_paddr = ps->tx_bd_pbase + i * sizeof(struct qtnf_pearl_tx_bd);  in qtnf_pcie_skb_send()
585 	       PCIE_HDP_HOST_WR_DESC0_H(ps->pcie_reg_base));  in qtnf_pcie_skb_send()
588 	       PCIE_HDP_HOST_WR_DESC0(ps->pcie_reg_base));  in qtnf_pcie_skb_send()
606 	qtnf_pearl_data_tx_reclaim(ps);  in qtnf_pcie_skb_send()
646 	struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);  in qtnf_pcie_pearl_interrupt()  local
647 	struct qtnf_pcie_bus_priv *priv = &ps->base;  in qtnf_pcie_pearl_interrupt()
651 	status = readl(PCIE_HDP_INT_STATUS(ps->pcie_reg_base));  in qtnf_pcie_pearl_interrupt()
656 	if (!(status & ps->pcie_irq_mask))  in qtnf_pcie_pearl_interrupt()
660 		ps->pcie_irq_rx_count++;  in qtnf_pcie_pearl_interrupt()
663 		ps->pcie_irq_tx_count++;  in qtnf_pcie_pearl_interrupt()
666 		ps->pcie_irq_uf_count++;  in qtnf_pcie_pearl_interrupt()
669 		qtnf_dis_rxdone_irq(ps);  in qtnf_pcie_pearl_interrupt()
674 		qtnf_dis_txdone_irq(ps);  in qtnf_pcie_pearl_interrupt()
680 	qtnf_non_posted_write(~0U, PCIE_HDP_INT_STATUS(ps->pcie_reg_base));  in qtnf_pcie_pearl_interrupt()
683 		qtnf_deassert_intx(ps);  in qtnf_pcie_pearl_interrupt()
688 static int qtnf_rx_data_ready(struct qtnf_pcie_pearl_state *ps)  in qtnf_rx_data_ready()  argument
690 	u16 index = ps->base.rx_bd_r_index;  in qtnf_rx_data_ready()
694 	rxbd = &ps->rx_bd_vbase[index];  in qtnf_rx_data_ready()
706 	struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);  in qtnf_pcie_pearl_rx_poll()  local
707 	struct qtnf_pcie_bus_priv *priv = &ps->base;  in qtnf_pcie_pearl_rx_poll()
721 		if (!qtnf_rx_data_ready(ps))  in qtnf_pcie_pearl_rx_poll()
725 		rxbd = &ps->rx_bd_vbase[r_idx];  in qtnf_pcie_pearl_rx_poll()
787 			ret = pearl_skb2rbd_attach(ps, w_idx);  in qtnf_pcie_pearl_rx_poll()
801 		qtnf_en_rxdone_irq(ps);  in qtnf_pcie_pearl_rx_poll()
810 	struct qtnf_pcie_pearl_state *ps = (void *)get_bus_priv(bus);  in qtnf_pcie_data_tx_timeout()  local
812 	tasklet_hi_schedule(&ps->base.reclaim_tq);  in qtnf_pcie_data_tx_timeout()
817 	struct qtnf_pcie_pearl_state *ps = (void *)get_bus_priv(bus);  in qtnf_pcie_data_rx_start()  local
819 	qtnf_enable_hdp_irqs(ps);  in qtnf_pcie_data_rx_start()
825 	struct qtnf_pcie_pearl_state *ps = (void *)get_bus_priv(bus);  in qtnf_pcie_data_rx_stop()  local
828 	qtnf_disable_hdp_irqs(ps);  in qtnf_pcie_data_rx_stop()
854 	struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);  in qtnf_dbg_irq_stats()  local
855 	u32 reg = readl(PCIE_HDP_INT_EN(ps->pcie_reg_base));  in qtnf_dbg_irq_stats()
858 	seq_printf(s, "pcie_irq_count(%u)\n", ps->base.pcie_irq_count);  in qtnf_dbg_irq_stats()
859 	seq_printf(s, "pcie_irq_tx_count(%u)\n", ps->pcie_irq_tx_count);  in qtnf_dbg_irq_stats()
863 	seq_printf(s, "pcie_irq_rx_count(%u)\n", ps->pcie_irq_rx_count);  in qtnf_dbg_irq_stats()
867 	seq_printf(s, "pcie_irq_uf_count(%u)\n", ps->pcie_irq_uf_count);  in qtnf_dbg_irq_stats()
878 	struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);  in qtnf_dbg_hdp_stats()  local
879 	struct qtnf_pcie_bus_priv *priv = &ps->base;  in qtnf_dbg_hdp_stats()
888 		   readl(PCIE_HDP_RX0DMA_CNT(ps->pcie_reg_base))  in qtnf_dbg_hdp_stats()
897 		   readl(PCIE_HDP_TX0DMA_CNT(ps->pcie_reg_base))  in qtnf_dbg_hdp_stats()
955 qtnf_ep_fw_load(struct qtnf_pcie_pearl_state *ps, const u8 *fw, u32 fw_size)  in qtnf_ep_fw_load()  argument
972 		len = qtnf_ep_fw_send(ps->base.pdev, fw_size, blk, pblk, fw);  in qtnf_ep_fw_load()
978 			qtnf_set_state(&ps->bda->bda_rc_state,  in qtnf_ep_fw_load()
980 			if (qtnf_poll_state(&ps->bda->bda_ep_state,  in qtnf_ep_fw_load()
987 			qtnf_clear_state(&ps->bda->bda_ep_state,  in qtnf_ep_fw_load()
990 			if (qtnf_is_state(&ps->bda->bda_ep_state,  in qtnf_ep_fw_load()
1003 				qtnf_clear_state(&ps->bda->bda_ep_state,  in qtnf_ep_fw_load()
1010 			qtnf_pearl_data_tx_reclaim(ps);  in qtnf_ep_fw_load()
1024 	struct qtnf_pcie_pearl_state *ps = (void *)get_bus_priv(bus);  in qtnf_pearl_fw_work_handler()  local
1027 	struct pci_dev *pdev = ps->base.pdev;  in qtnf_pearl_fw_work_handler()
1031 	if (ps->base.flashboot) {  in qtnf_pearl_fw_work_handler()
1041 	qtnf_set_state(&ps->bda->bda_rc_state, state);  in qtnf_pearl_fw_work_handler()
1043 	if (qtnf_poll_state(&ps->bda->bda_ep_state, QTN_EP_FW_LOADRDY,  in qtnf_pearl_fw_work_handler()
1047 		if (!ps->base.flashboot)  in qtnf_pearl_fw_work_handler()
1053 	qtnf_clear_state(&ps->bda->bda_ep_state, QTN_EP_FW_LOADRDY);  in qtnf_pearl_fw_work_handler()
1055 	if (ps->base.flashboot) {  in qtnf_pearl_fw_work_handler()
1061 		ret = qtnf_ep_fw_load(ps, fw->data, fw->size);  in qtnf_pearl_fw_work_handler()
1069 	if (qtnf_poll_state(&ps->bda->bda_ep_state, QTN_EP_FW_DONE,  in qtnf_pearl_fw_work_handler()
1075 	if (qtnf_poll_state(&ps->bda->bda_ep_state,  in qtnf_pearl_fw_work_handler()
1096 	struct qtnf_pcie_pearl_state *ps = from_tasklet(ps, t, base.reclaim_tq);  in qtnf_pearl_reclaim_tasklet_fn()  local
1098 	qtnf_pearl_data_tx_reclaim(ps);  in qtnf_pearl_reclaim_tasklet_fn()
1099 	qtnf_en_txdone_irq(ps);  in qtnf_pearl_reclaim_tasklet_fn()
1115 	struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);  in qtnf_pcie_pearl_probe()  local
1116 	struct pci_dev *pdev = ps->base.pdev;  in qtnf_pcie_pearl_probe()
1120 	spin_lock_init(&ps->irq_lock);  in qtnf_pcie_pearl_probe()
1123 	ps->pcie_reg_base = ps->base.dmareg_bar;  in qtnf_pcie_pearl_probe()
1124 	ps->bda = ps->base.epmem_bar;  in qtnf_pcie_pearl_probe()
1125 	writel(ps->base.msi_enabled, &ps->bda->bda_rc_msi_enabled);  in qtnf_pcie_pearl_probe()
1127 	ret = qtnf_pcie_pearl_init_xfer(ps, tx_bd_size, rx_bd_size);  in qtnf_pcie_pearl_probe()
1134 	qtnf_init_hdp_irqs(ps);  in qtnf_pcie_pearl_probe()
1137 	qtnf_disable_hdp_irqs(ps);  in qtnf_pcie_pearl_probe()
1144 		qtnf_pearl_free_xfer_buffers(ps);  in qtnf_pcie_pearl_probe()
1148 	tasklet_setup(&ps->base.reclaim_tq, qtnf_pearl_reclaim_tasklet_fn);  in qtnf_pcie_pearl_probe()
1153 	ipc_int.arg = ps;  in qtnf_pcie_pearl_probe()
1154 	qtnf_pcie_init_shm_ipc(&ps->base, &ps->bda->bda_shm_reg1,  in qtnf_pcie_pearl_probe()
1155 			       &ps->bda->bda_shm_reg2, &ipc_int);  in qtnf_pcie_pearl_probe()
1162 	struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);  in qtnf_pcie_pearl_remove()  local
1164 	qtnf_pearl_reset_ep(ps);  in qtnf_pcie_pearl_remove()
1165 	qtnf_pearl_free_xfer_buffers(ps);  in qtnf_pcie_pearl_remove()
1183 	struct qtnf_pcie_pearl_state *ps;  in qtnf_pcie_pearl_alloc()  local
1185 	bus = devm_kzalloc(&pdev->dev, sizeof(*bus) + sizeof(*ps), GFP_KERNEL);  in qtnf_pcie_pearl_alloc()
1189 	ps = get_bus_priv(bus);  in qtnf_pcie_pearl_alloc()
1190 	ps->base.probe_cb = qtnf_pcie_pearl_probe;  in qtnf_pcie_pearl_alloc()
1191 	ps->base.remove_cb = qtnf_pcie_pearl_remove;  in qtnf_pcie_pearl_alloc()
1192 	ps->base.dma_mask_get_cb = qtnf_pearl_dma_mask_get;  in qtnf_pcie_pearl_alloc()
1194 	ps->base.resume_cb = qtnf_pcie_pearl_resume;  in qtnf_pcie_pearl_alloc()
1195 	ps->base.suspend_cb = qtnf_pcie_pearl_suspend;  in qtnf_pcie_pearl_alloc()