17 static int nfp_nfdk_tx_ring_should_wake(struct nfp_net_tx_ring *tx_ring)  in nfp_nfdk_tx_ring_should_wake()  argument
19 	return !nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT * 2);  in nfp_nfdk_tx_ring_should_wake()
22 static int nfp_nfdk_tx_ring_should_stop(struct nfp_net_tx_ring *tx_ring)  in nfp_nfdk_tx_ring_should_stop()  argument
24 	return nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT);  in nfp_nfdk_tx_ring_should_stop()
28 				  struct nfp_net_tx_ring *tx_ring)  in nfp_nfdk_tx_ring_stop()  argument
34 	if (unlikely(nfp_nfdk_tx_ring_should_wake(tx_ring)))  in nfp_nfdk_tx_ring_stop()
127 nfp_nfdk_tx_maybe_close_block(struct nfp_net_tx_ring *tx_ring,  in nfp_nfdk_tx_maybe_close_block()  argument
159 	if (round_down(tx_ring->wr_p, NFDK_TX_DESC_BLOCK_CNT) !=  in nfp_nfdk_tx_maybe_close_block()
160 	    round_down(tx_ring->wr_p + n_descs, NFDK_TX_DESC_BLOCK_CNT))  in nfp_nfdk_tx_maybe_close_block()
163 	if ((u32)tx_ring->data_pending + skb->len > NFDK_TX_MAX_DATA_PER_BLOCK)  in nfp_nfdk_tx_maybe_close_block()
169 	wr_p = tx_ring->wr_p;  in nfp_nfdk_tx_maybe_close_block()
172 	wr_idx = D_IDX(tx_ring, wr_p);  in nfp_nfdk_tx_maybe_close_block()
173 	tx_ring->ktxbufs[wr_idx].skb = NULL;  in nfp_nfdk_tx_maybe_close_block()
174 	txd = &tx_ring->ktxds[wr_idx];  in nfp_nfdk_tx_maybe_close_block()
178 	tx_ring->data_pending = 0;  in nfp_nfdk_tx_maybe_close_block()
179 	tx_ring->wr_p += nop_slots;  in nfp_nfdk_tx_maybe_close_block()
180 	tx_ring->wr_ptr_add += nop_slots;  in nfp_nfdk_tx_maybe_close_block()
251 	struct nfp_net_tx_ring *tx_ring;  in nfp_nfdk_tx()  local
265 	tx_ring = &dp->tx_rings[qidx];  in nfp_nfdk_tx()
266 	r_vec = tx_ring->r_vec;  in nfp_nfdk_tx()
270 	if (unlikely(nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT))) {  in nfp_nfdk_tx()
272 			   qidx, tx_ring->wr_p, tx_ring->rd_p);  in nfp_nfdk_tx()
274 		nfp_net_tx_xmit_more_flush(tx_ring);  in nfp_nfdk_tx()
285 	if (nfp_nfdk_tx_maybe_close_block(tx_ring, skb))  in nfp_nfdk_tx()
293 	wr_idx = D_IDX(tx_ring, tx_ring->wr_p);  in nfp_nfdk_tx()
294 	txd = &tx_ring->ktxds[wr_idx];  in nfp_nfdk_tx()
295 	txbuf = &tx_ring->ktxbufs[wr_idx];  in nfp_nfdk_tx()
396 	cnt = txd - tx_ring->ktxds - wr_idx;  in nfp_nfdk_tx()
403 	tx_ring->wr_p += cnt;  in nfp_nfdk_tx()
404 	if (tx_ring->wr_p % NFDK_TX_DESC_BLOCK_CNT)  in nfp_nfdk_tx()
405 		tx_ring->data_pending += skb->len;  in nfp_nfdk_tx()
407 		tx_ring->data_pending = 0;  in nfp_nfdk_tx()
409 	if (nfp_nfdk_tx_ring_should_stop(tx_ring))  in nfp_nfdk_tx()
410 		nfp_nfdk_tx_ring_stop(nd_q, tx_ring);  in nfp_nfdk_tx()
412 	tx_ring->wr_ptr_add += cnt;  in nfp_nfdk_tx()
414 		nfp_net_tx_xmit_more_flush(tx_ring);  in nfp_nfdk_tx()
427 	txbuf = &tx_ring->ktxbufs[wr_idx + 1];  in nfp_nfdk_tx()
445 	nfp_net_tx_xmit_more_flush(tx_ring);  in nfp_nfdk_tx()
458 static void nfp_nfdk_tx_complete(struct nfp_net_tx_ring *tx_ring, int budget)  in nfp_nfdk_tx_complete()  argument
460 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;  in nfp_nfdk_tx_complete()
469 	rd_p = tx_ring->rd_p;  in nfp_nfdk_tx_complete()
470 	if (tx_ring->wr_p == rd_p)  in nfp_nfdk_tx_complete()
474 	qcp_rd_p = nfp_net_read_tx_cmpl(tx_ring, dp);  in nfp_nfdk_tx_complete()
476 	if (qcp_rd_p == tx_ring->qcp_rd_p)  in nfp_nfdk_tx_complete()
479 	todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p);  in nfp_nfdk_tx_complete()
480 	ktxbufs = tx_ring->ktxbufs;  in nfp_nfdk_tx_complete()
488 		txbuf = &ktxbufs[D_IDX(tx_ring, rd_p)];  in nfp_nfdk_tx_complete()
531 	tx_ring->rd_p = rd_p;  in nfp_nfdk_tx_complete()
532 	tx_ring->qcp_rd_p = qcp_rd_p;  in nfp_nfdk_tx_complete()
542 	nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx);  in nfp_nfdk_tx_complete()
544 	if (nfp_nfdk_tx_ring_should_wake(tx_ring)) {  in nfp_nfdk_tx_complete()
552 	WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt,  in nfp_nfdk_tx_complete()
554 		  tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt);  in nfp_nfdk_tx_complete()
802 static bool nfp_nfdk_xdp_complete(struct nfp_net_tx_ring *tx_ring)  in nfp_nfdk_xdp_complete()  argument
804 	struct nfp_net_r_vector *r_vec = tx_ring->r_vec;  in nfp_nfdk_xdp_complete()
812 	qcp_rd_p = nfp_net_read_tx_cmpl(tx_ring, dp);  in nfp_nfdk_xdp_complete()
813 	if (qcp_rd_p == tx_ring->qcp_rd_p)  in nfp_nfdk_xdp_complete()
816 	todo = D_IDX(tx_ring, qcp_rd_p - tx_ring->qcp_rd_p);  in nfp_nfdk_xdp_complete()
823 		int idx = D_IDX(tx_ring, tx_ring->rd_p + done);  in nfp_nfdk_xdp_complete()
827 		txbuf = &tx_ring->ktxbufs[idx];  in nfp_nfdk_xdp_complete()
855 	tx_ring->qcp_rd_p = D_IDX(tx_ring, tx_ring->qcp_rd_p + done);  in nfp_nfdk_xdp_complete()
856 	tx_ring->rd_p += done;  in nfp_nfdk_xdp_complete()
858 	WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt,  in nfp_nfdk_xdp_complete()
860 		  tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt);  in nfp_nfdk_xdp_complete()
867 		    struct nfp_net_tx_ring *tx_ring,  in nfp_nfdk_tx_xdp_buf()  argument
887 	if (unlikely(nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT))) {  in nfp_nfdk_tx_xdp_buf()
889 			nfp_nfdk_xdp_complete(tx_ring);  in nfp_nfdk_tx_xdp_buf()
893 		if (unlikely(nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT))) {  in nfp_nfdk_tx_xdp_buf()
902 	if ((round_down(tx_ring->wr_p, NFDK_TX_DESC_BLOCK_CNT) !=  in nfp_nfdk_tx_xdp_buf()
903 	     round_down(tx_ring->wr_p + n_descs, NFDK_TX_DESC_BLOCK_CNT)) ||  in nfp_nfdk_tx_xdp_buf()
904 	    ((u32)tx_ring->data_pending + pkt_len >  in nfp_nfdk_tx_xdp_buf()
906 		unsigned int nop_slots = D_BLOCK_CPL(tx_ring->wr_p);  in nfp_nfdk_tx_xdp_buf()
908 		wr_idx = D_IDX(tx_ring, tx_ring->wr_p);  in nfp_nfdk_tx_xdp_buf()
909 		txd = &tx_ring->ktxds[wr_idx];  in nfp_nfdk_tx_xdp_buf()
913 		tx_ring->data_pending = 0;  in nfp_nfdk_tx_xdp_buf()
914 		tx_ring->wr_p += nop_slots;  in nfp_nfdk_tx_xdp_buf()
915 		tx_ring->wr_ptr_add += nop_slots;  in nfp_nfdk_tx_xdp_buf()
918 	wr_idx = D_IDX(tx_ring, tx_ring->wr_p);  in nfp_nfdk_tx_xdp_buf()
920 	txbuf = &tx_ring->ktxbufs[wr_idx];  in nfp_nfdk_tx_xdp_buf()
930 	txd = &tx_ring->ktxds[wr_idx];  in nfp_nfdk_tx_xdp_buf()
972 	cnt = txd - tx_ring->ktxds - wr_idx;  in nfp_nfdk_tx_xdp_buf()
973 	tx_ring->wr_p += cnt;  in nfp_nfdk_tx_xdp_buf()
974 	if (tx_ring->wr_p % NFDK_TX_DESC_BLOCK_CNT)  in nfp_nfdk_tx_xdp_buf()
975 		tx_ring->data_pending += pkt_len;  in nfp_nfdk_tx_xdp_buf()
977 		tx_ring->data_pending = 0;  in nfp_nfdk_tx_xdp_buf()
979 	tx_ring->wr_ptr_add += cnt;  in nfp_nfdk_tx_xdp_buf()
998 	struct nfp_net_tx_ring *tx_ring;  in nfp_nfdk_rx()  local
1011 	tx_ring = r_vec->xdp_ring;  in nfp_nfdk_rx()
1115 								  tx_ring,  in nfp_nfdk_rx()
1205 		if (tx_ring->wr_ptr_add)  in nfp_nfdk_rx()
1206 			nfp_net_tx_xmit_more_flush(tx_ring);  in nfp_nfdk_rx()
1207 		else if (unlikely(tx_ring->wr_p != tx_ring->rd_p) &&  in nfp_nfdk_rx()
1209 			if (!nfp_nfdk_xdp_complete(tx_ring))  in nfp_nfdk_rx()
1229 	if (r_vec->tx_ring)  in nfp_nfdk_poll()
1230 		nfp_nfdk_tx_complete(r_vec->tx_ring, budget);  in nfp_nfdk_poll()
1253 	if (r_vec->nfp_net->tx_coalesce_adapt_on && r_vec->tx_ring) {  in nfp_nfdk_poll()
1279 	struct nfp_net_tx_ring *tx_ring;  in nfp_nfdk_ctrl_tx_one()  local
1289 	tx_ring = r_vec->tx_ring;  in nfp_nfdk_ctrl_tx_one()
1297 	if (unlikely(nfp_net_tx_full(tx_ring, NFDK_TX_DESC_STOP_CNT))) {  in nfp_nfdk_ctrl_tx_one()
1321 	if (nfp_nfdk_tx_maybe_close_block(tx_ring, skb))  in nfp_nfdk_ctrl_tx_one()
1325 	wr_idx = D_IDX(tx_ring, tx_ring->wr_p);  in nfp_nfdk_ctrl_tx_one()
1326 	txd = &tx_ring->ktxds[wr_idx];  in nfp_nfdk_ctrl_tx_one()
1327 	txbuf = &tx_ring->ktxbufs[wr_idx];  in nfp_nfdk_ctrl_tx_one()
1377 	cnt = txd - tx_ring->ktxds - wr_idx;  in nfp_nfdk_ctrl_tx_one()
1382 	tx_ring->wr_p += cnt;  in nfp_nfdk_ctrl_tx_one()
1383 	if (tx_ring->wr_p % NFDK_TX_DESC_BLOCK_CNT)  in nfp_nfdk_ctrl_tx_one()
1384 		tx_ring->data_pending += skb->len;  in nfp_nfdk_ctrl_tx_one()
1386 		tx_ring->data_pending = 0;  in nfp_nfdk_ctrl_tx_one()
1388 	tx_ring->wr_ptr_add += cnt;  in nfp_nfdk_ctrl_tx_one()
1389 	nfp_net_tx_xmit_more_flush(tx_ring);  in nfp_nfdk_ctrl_tx_one()
1530 	nfp_nfdk_tx_complete(r_vec->tx_ring, 0);  in nfp_nfdk_ctrl_poll()