49 	return hfi1_ipoib_txreqs(txq->tx_ring.sent_txreqs,  in hfi1_ipoib_used()
50 				 txq->tx_ring.complete_txreqs);  in hfi1_ipoib_used()
56 	if (atomic_inc_return(&txq->tx_ring.stops) == 1)  in hfi1_ipoib_stop_txq()
63 	if (atomic_dec_and_test(&txq->tx_ring.stops))  in hfi1_ipoib_wake_txq()
70 		     txq->tx_ring.max_items - 1);  in hfi1_ipoib_ring_hwat()
76 		     txq->tx_ring.max_items) >> 1;  in hfi1_ipoib_ring_lwat()
81 	++txq->tx_ring.sent_txreqs;  in hfi1_ipoib_check_queue_depth()
83 	    !atomic_xchg(&txq->tx_ring.ring_full, 1)) {  in hfi1_ipoib_check_queue_depth()
108 	    atomic_xchg(&txq->tx_ring.ring_full, 0)) {  in hfi1_ipoib_check_queue_stopped()
136 	struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;  in hfi1_ipoib_drain_tx_ring()  local
140 	for (i = 0; i < tx_ring->max_items; i++) {  in hfi1_ipoib_drain_tx_ring()
141 		tx = hfi1_txreq_from_idx(tx_ring, i);  in hfi1_ipoib_drain_tx_ring()
147 	tx_ring->head = 0;  in hfi1_ipoib_drain_tx_ring()
148 	tx_ring->tail = 0;  in hfi1_ipoib_drain_tx_ring()
149 	tx_ring->complete_txreqs = 0;  in hfi1_ipoib_drain_tx_ring()
150 	tx_ring->sent_txreqs = 0;  in hfi1_ipoib_drain_tx_ring()
151 	tx_ring->avail = hfi1_ipoib_ring_hwat(txq);  in hfi1_ipoib_drain_tx_ring()
158 	struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;  in hfi1_ipoib_poll_tx_ring()  local
159 	u32 head = tx_ring->head;  in hfi1_ipoib_poll_tx_ring()
160 	u32 max_tx = tx_ring->max_items;  in hfi1_ipoib_poll_tx_ring()
162 	struct ipoib_txreq *tx =  hfi1_txreq_from_idx(tx_ring, head);  in hfi1_ipoib_poll_tx_ring()
173 		tx =  hfi1_txreq_from_idx(tx_ring, head);  in hfi1_ipoib_poll_tx_ring()
175 	tx_ring->complete_txreqs += work_done;  in hfi1_ipoib_poll_tx_ring()
178 	smp_store_release(&tx_ring->head, head);  in hfi1_ipoib_poll_tx_ring()
329 	ohdr->bth[2] = cpu_to_be32(mask_psn((u32)txp->txq->tx_ring.sent_txreqs));  in hfi1_ipoib_build_ib_tx_headers()
354 	struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;  in hfi1_ipoib_send_dma_common()  local
355 	u32 tail = tx_ring->tail;  in hfi1_ipoib_send_dma_common()
358 	if (unlikely(!tx_ring->avail)) {  in hfi1_ipoib_send_dma_common()
365 		head = smp_load_acquire(&tx_ring->head);  in hfi1_ipoib_send_dma_common()
366 		tx_ring->avail =  in hfi1_ipoib_send_dma_common()
368 			      CIRC_CNT(head, tail, tx_ring->max_items));  in hfi1_ipoib_send_dma_common()
370 		tx_ring->avail--;  in hfi1_ipoib_send_dma_common()
372 	tx = hfi1_txreq_from_idx(tx_ring, tail);  in hfi1_ipoib_send_dma_common()
460 	struct hfi1_ipoib_circ_buf *tx_ring;  in hfi1_ipoib_send_dma_single()  local
478 	tx_ring = &txq->tx_ring;  in hfi1_ipoib_send_dma_single()
479 	trace_hfi1_tx_consume(tx, tx_ring->tail);  in hfi1_ipoib_send_dma_single()
481 	smp_store_release(&tx_ring->tail, CIRC_NEXT(tx_ring->tail, tx_ring->max_items));  in hfi1_ipoib_send_dma_single()
511 	struct hfi1_ipoib_circ_buf *tx_ring;  in hfi1_ipoib_send_dma_list()  local
541 	tx_ring = &txq->tx_ring;  in hfi1_ipoib_send_dma_list()
542 	trace_hfi1_tx_consume(tx, tx_ring->tail);  in hfi1_ipoib_send_dma_list()
544 	smp_store_release(&tx_ring->tail, CIRC_NEXT(tx_ring->tail, tx_ring->max_items));  in hfi1_ipoib_send_dma_list()
635 			if (!atomic_xchg(&txq->tx_ring.no_desc, 1)) {  in hfi1_ipoib_sdma_sleep()
678 		if (atomic_xchg(&txq->tx_ring.no_desc, 0))  in hfi1_ipoib_flush_txq()
686 	struct hfi1_ipoib_circ_buf *tx_ring;  in hfi1_ipoib_txreq_init()  local
707 		tx_ring = &txq->tx_ring;  in hfi1_ipoib_txreq_init()
719 		atomic_set(&txq->tx_ring.stops, 0);  in hfi1_ipoib_txreq_init()
720 		atomic_set(&txq->tx_ring.ring_full, 0);  in hfi1_ipoib_txreq_init()
721 		atomic_set(&txq->tx_ring.no_desc, 0);  in hfi1_ipoib_txreq_init()
730 		txq->tx_ring.items =  in hfi1_ipoib_txreq_init()
733 		if (!txq->tx_ring.items)  in hfi1_ipoib_txreq_init()
736 		txq->tx_ring.max_items = tx_ring_size;  in hfi1_ipoib_txreq_init()
737 		txq->tx_ring.shift = ilog2(tx_item_size);  in hfi1_ipoib_txreq_init()
738 		txq->tx_ring.avail = hfi1_ipoib_ring_hwat(txq);  in hfi1_ipoib_txreq_init()
739 		tx_ring = &txq->tx_ring;  in hfi1_ipoib_txreq_init()
741 			hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr =  in hfi1_ipoib_txreq_init()
757 		tx_ring = &txq->tx_ring;  in hfi1_ipoib_txreq_init()
759 			kfree(hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr);  in hfi1_ipoib_txreq_init()
760 		kvfree(tx_ring->items);  in hfi1_ipoib_txreq_init()
781 		txq->tx_ring.complete_txreqs++;  in hfi1_ipoib_drain_tx_list()
788 			    hfi1_ipoib_txreqs(txq->tx_ring.sent_txreqs,  in hfi1_ipoib_drain_tx_list()
789 					      txq->tx_ring.complete_txreqs));  in hfi1_ipoib_drain_tx_list()
798 		struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;  in hfi1_ipoib_txreq_deinit()  local
805 		for (j = 0; j < tx_ring->max_items; j++)  in hfi1_ipoib_txreq_deinit()
806 			kfree(hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr);  in hfi1_ipoib_txreq_deinit()
807 		kvfree(tx_ring->items);  in hfi1_ipoib_txreq_deinit()
847 		    atomic_read(&txq->tx_ring.stops),  in hfi1_ipoib_tx_timeout()
848 		    atomic_read(&txq->tx_ring.no_desc),  in hfi1_ipoib_tx_timeout()
849 		    atomic_read(&txq->tx_ring.ring_full));  in hfi1_ipoib_tx_timeout()
855 		    txq->tx_ring.sent_txreqs, txq->tx_ring.complete_txreqs,  in hfi1_ipoib_tx_timeout()
858 		    dev->tx_queue_len, txq->tx_ring.max_items);  in hfi1_ipoib_tx_timeout()
860 		    txq->tx_ring.head, txq->tx_ring.tail);  in hfi1_ipoib_tx_timeout()