11 static void nfp_nfd3_xsk_tx_bufs_free(struct nfp_net_tx_ring *tx_ring)  in nfp_nfd3_xsk_tx_bufs_free()  argument
16 	while (tx_ring->rd_p != tx_ring->wr_p) {  in nfp_nfd3_xsk_tx_bufs_free()
17 		idx = D_IDX(tx_ring, tx_ring->rd_p);  in nfp_nfd3_xsk_tx_bufs_free()
18 		txbuf = &tx_ring->txbufs[idx];  in nfp_nfd3_xsk_tx_bufs_free()
22 		tx_ring->qcp_rd_p++;  in nfp_nfd3_xsk_tx_bufs_free()
23 		tx_ring->rd_p++;  in nfp_nfd3_xsk_tx_bufs_free()
25 		if (tx_ring->r_vec->xsk_pool) {  in nfp_nfd3_xsk_tx_bufs_free()
29 			xsk_tx_completed(tx_ring->r_vec->xsk_pool, 1);  in nfp_nfd3_xsk_tx_bufs_free()
42 nfp_nfd3_tx_ring_reset(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)  in nfp_nfd3_tx_ring_reset()  argument
47 	while (!tx_ring->is_xdp && tx_ring->rd_p != tx_ring->wr_p) {  in nfp_nfd3_tx_ring_reset()
52 		idx = D_IDX(tx_ring, tx_ring->rd_p);  in nfp_nfd3_tx_ring_reset()
53 		tx_buf = &tx_ring->txbufs[idx];  in nfp_nfd3_tx_ring_reset()
55 		skb = tx_ring->txbufs[idx].skb;  in nfp_nfd3_tx_ring_reset()
77 		tx_ring->qcp_rd_p++;  in nfp_nfd3_tx_ring_reset()
78 		tx_ring->rd_p++;  in nfp_nfd3_tx_ring_reset()
81 	if (tx_ring->is_xdp)  in nfp_nfd3_tx_ring_reset()
82 		nfp_nfd3_xsk_tx_bufs_free(tx_ring);  in nfp_nfd3_tx_ring_reset()
84 	memset(tx_ring->txds, 0, tx_ring->size);  in nfp_nfd3_tx_ring_reset()
85 	tx_ring->wr_p = 0;  in nfp_nfd3_tx_ring_reset()
86 	tx_ring->rd_p = 0;  in nfp_nfd3_tx_ring_reset()
87 	tx_ring->qcp_rd_p = 0;  in nfp_nfd3_tx_ring_reset()
88 	tx_ring->wr_ptr_add = 0;  in nfp_nfd3_tx_ring_reset()
90 	if (tx_ring->is_xdp || !dp->netdev)  in nfp_nfd3_tx_ring_reset()
93 	nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx);  in nfp_nfd3_tx_ring_reset()
101 static void nfp_nfd3_tx_ring_free(struct nfp_net_tx_ring *tx_ring)  in nfp_nfd3_tx_ring_free()  argument
103 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;  in nfp_nfd3_tx_ring_free()
106 	kvfree(tx_ring->txbufs);  in nfp_nfd3_tx_ring_free()
108 	if (tx_ring->txds)  in nfp_nfd3_tx_ring_free()
109 		dma_free_coherent(dp->dev, tx_ring->size,  in nfp_nfd3_tx_ring_free()
110 				  tx_ring->txds, tx_ring->dma);  in nfp_nfd3_tx_ring_free()
112 	tx_ring->cnt = 0;  in nfp_nfd3_tx_ring_free()
113 	tx_ring->txbufs = NULL;  in nfp_nfd3_tx_ring_free()
114 	tx_ring->txds = NULL;  in nfp_nfd3_tx_ring_free()
115 	tx_ring->dma = 0;  in nfp_nfd3_tx_ring_free()
116 	tx_ring->size = 0;  in nfp_nfd3_tx_ring_free()
127 nfp_nfd3_tx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)  in nfp_nfd3_tx_ring_alloc()  argument
129 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;  in nfp_nfd3_tx_ring_alloc()
131 	tx_ring->cnt = dp->txd_cnt;  in nfp_nfd3_tx_ring_alloc()
133 	tx_ring->size = array_size(tx_ring->cnt, sizeof(*tx_ring->txds));  in nfp_nfd3_tx_ring_alloc()
134 	tx_ring->txds = dma_alloc_coherent(dp->dev, tx_ring->size,  in nfp_nfd3_tx_ring_alloc()
135 					   &tx_ring->dma,  in nfp_nfd3_tx_ring_alloc()
137 	if (!tx_ring->txds) {  in nfp_nfd3_tx_ring_alloc()
139 			    tx_ring->cnt);  in nfp_nfd3_tx_ring_alloc()
143 	tx_ring->txbufs = kvcalloc(tx_ring->cnt, sizeof(*tx_ring->txbufs),  in nfp_nfd3_tx_ring_alloc()
145 	if (!tx_ring->txbufs)  in nfp_nfd3_tx_ring_alloc()
148 	if (!tx_ring->is_xdp && dp->netdev)  in nfp_nfd3_tx_ring_alloc()
150 				    tx_ring->idx);  in nfp_nfd3_tx_ring_alloc()
155 	nfp_nfd3_tx_ring_free(tx_ring);  in nfp_nfd3_tx_ring_alloc()
161 			   struct nfp_net_tx_ring *tx_ring)  in nfp_nfd3_tx_ring_bufs_free()  argument
165 	if (!tx_ring->is_xdp)  in nfp_nfd3_tx_ring_bufs_free()
168 	for (i = 0; i < tx_ring->cnt; i++) {  in nfp_nfd3_tx_ring_bufs_free()
169 		if (!tx_ring->txbufs[i].frag)  in nfp_nfd3_tx_ring_bufs_free()
172 		nfp_net_dma_unmap_rx(dp, tx_ring->txbufs[i].dma_addr);  in nfp_nfd3_tx_ring_bufs_free()
173 		__free_page(virt_to_page(tx_ring->txbufs[i].frag));  in nfp_nfd3_tx_ring_bufs_free()
179 			    struct nfp_net_tx_ring *tx_ring)  in nfp_nfd3_tx_ring_bufs_alloc()  argument
181 	struct nfp_nfd3_tx_buf *txbufs = tx_ring->txbufs;  in nfp_nfd3_tx_ring_bufs_alloc()
184 	if (!tx_ring->is_xdp)  in nfp_nfd3_tx_ring_bufs_alloc()
187 	for (i = 0; i < tx_ring->cnt; i++) {  in nfp_nfd3_tx_ring_bufs_alloc()
190 			nfp_nfd3_tx_ring_bufs_free(dp, tx_ring);  in nfp_nfd3_tx_ring_bufs_alloc()
201 			struct nfp_net_tx_ring *tx_ring,  in nfp_nfd3_print_tx_descs()  argument
205 	u32 txd_cnt = tx_ring->cnt;  in nfp_nfd3_print_tx_descs()
212 		txd = &tx_ring->txds[i];  in nfp_nfd3_print_tx_descs()
217 		if (!tx_ring->is_xdp) {  in nfp_nfd3_print_tx_descs()
218 			skb = READ_ONCE(tx_ring->txbufs[i].skb);  in nfp_nfd3_print_tx_descs()
223 			xdp = READ_ONCE(tx_ring->txbufs[i].xdp);  in nfp_nfd3_print_tx_descs()
228 		if (tx_ring->txbufs[i].dma_addr)  in nfp_nfd3_print_tx_descs()
230 				   &tx_ring->txbufs[i].dma_addr);  in nfp_nfd3_print_tx_descs()
232 		if (i == tx_ring->rd_p % txd_cnt)  in nfp_nfd3_print_tx_descs()
234 		if (i == tx_ring->wr_p % txd_cnt)  in nfp_nfd3_print_tx_descs()