480 static inline int t4_rqes_posted(struct t4_wq *wq)  in t4_rqes_posted()  argument
482 	return wq->rq.in_use;  in t4_rqes_posted()
485 static inline int t4_rq_empty(struct t4_wq *wq)  in t4_rq_empty()  argument
487 	return wq->rq.in_use == 0;  in t4_rq_empty()
490 static inline u32 t4_rq_avail(struct t4_wq *wq)  in t4_rq_avail()  argument
492 	return wq->rq.size - 1 - wq->rq.in_use;  in t4_rq_avail()
495 static inline void t4_rq_produce(struct t4_wq *wq, u8 len16)  in t4_rq_produce()  argument
497 	wq->rq.in_use++;  in t4_rq_produce()
498 	if (++wq->rq.pidx == wq->rq.size)  in t4_rq_produce()
499 		wq->rq.pidx = 0;  in t4_rq_produce()
500 	wq->rq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);  in t4_rq_produce()
501 	if (wq->rq.wq_pidx >= wq->rq.size * T4_RQ_NUM_SLOTS)  in t4_rq_produce()
502 		wq->rq.wq_pidx %= wq->rq.size * T4_RQ_NUM_SLOTS;  in t4_rq_produce()
505 static inline void t4_rq_consume(struct t4_wq *wq)  in t4_rq_consume()  argument
507 	wq->rq.in_use--;  in t4_rq_consume()
508 	if (++wq->rq.cidx == wq->rq.size)  in t4_rq_consume()
509 		wq->rq.cidx = 0;  in t4_rq_consume()
512 static inline u16 t4_rq_host_wq_pidx(struct t4_wq *wq)  in t4_rq_host_wq_pidx()  argument
514 	return wq->rq.queue[wq->rq.size].status.host_wq_pidx;  in t4_rq_host_wq_pidx()
517 static inline u16 t4_rq_wq_size(struct t4_wq *wq)  in t4_rq_wq_size()  argument
519 		return wq->rq.size * T4_RQ_NUM_SLOTS;  in t4_rq_wq_size()
527 static inline int t4_sq_empty(struct t4_wq *wq)  in t4_sq_empty()  argument
529 	return wq->sq.in_use == 0;  in t4_sq_empty()
532 static inline u32 t4_sq_avail(struct t4_wq *wq)  in t4_sq_avail()  argument
534 	return wq->sq.size - 1 - wq->sq.in_use;  in t4_sq_avail()
537 static inline void t4_sq_produce(struct t4_wq *wq, u8 len16)  in t4_sq_produce()  argument
539 	wq->sq.in_use++;  in t4_sq_produce()
540 	if (++wq->sq.pidx == wq->sq.size)  in t4_sq_produce()
541 		wq->sq.pidx = 0;  in t4_sq_produce()
542 	wq->sq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);  in t4_sq_produce()
543 	if (wq->sq.wq_pidx >= wq->sq.size * T4_SQ_NUM_SLOTS)  in t4_sq_produce()
544 		wq->sq.wq_pidx %= wq->sq.size * T4_SQ_NUM_SLOTS;  in t4_sq_produce()
547 static inline void t4_sq_consume(struct t4_wq *wq)  in t4_sq_consume()  argument
549 	if (wq->sq.cidx == wq->sq.flush_cidx)  in t4_sq_consume()
550 		wq->sq.flush_cidx = -1;  in t4_sq_consume()
551 	wq->sq.in_use--;  in t4_sq_consume()
552 	if (++wq->sq.cidx == wq->sq.size)  in t4_sq_consume()
553 		wq->sq.cidx = 0;  in t4_sq_consume()
556 static inline u16 t4_sq_host_wq_pidx(struct t4_wq *wq)  in t4_sq_host_wq_pidx()  argument
558 	return wq->sq.queue[wq->sq.size].status.host_wq_pidx;  in t4_sq_host_wq_pidx()
561 static inline u16 t4_sq_wq_size(struct t4_wq *wq)  in t4_sq_wq_size()  argument
563 		return wq->sq.size * T4_SQ_NUM_SLOTS;  in t4_sq_wq_size()
601 static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc, union t4_wr *wqe)  in t4_ring_sq_db()  argument
606 	if (wq->sq.bar2_va) {  in t4_ring_sq_db()
607 		if (inc == 1 && wq->sq.bar2_qid == 0 && wqe) {  in t4_ring_sq_db()
608 			pr_debug("WC wq->sq.pidx = %d\n", wq->sq.pidx);  in t4_ring_sq_db()
610 				 (wq->sq.bar2_va + SGE_UDB_WCDOORBELL),  in t4_ring_sq_db()
613 			pr_debug("DB wq->sq.pidx = %d\n", wq->sq.pidx);  in t4_ring_sq_db()
614 			writel(PIDX_T5_V(inc) | QID_V(wq->sq.bar2_qid),  in t4_ring_sq_db()
615 			       wq->sq.bar2_va + SGE_UDB_KDOORBELL);  in t4_ring_sq_db()
622 	writel(QID_V(wq->sq.qid) | PIDX_V(inc), wq->db);  in t4_ring_sq_db()
625 static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc,  in t4_ring_rq_db()  argument
631 	if (wq->rq.bar2_va) {  in t4_ring_rq_db()
632 		if (inc == 1 && wq->rq.bar2_qid == 0 && wqe) {  in t4_ring_rq_db()
633 			pr_debug("WC wq->rq.pidx = %d\n", wq->rq.pidx);  in t4_ring_rq_db()
635 				 (wq->rq.bar2_va + SGE_UDB_WCDOORBELL),  in t4_ring_rq_db()
638 			pr_debug("DB wq->rq.pidx = %d\n", wq->rq.pidx);  in t4_ring_rq_db()
639 			writel(PIDX_T5_V(inc) | QID_V(wq->rq.bar2_qid),  in t4_ring_rq_db()
640 			       wq->rq.bar2_va + SGE_UDB_KDOORBELL);  in t4_ring_rq_db()
647 	writel(QID_V(wq->rq.qid) | PIDX_V(inc), wq->db);  in t4_ring_rq_db()
650 static inline int t4_wq_in_error(struct t4_wq *wq)  in t4_wq_in_error()  argument
652 	return *wq->qp_errp;  in t4_wq_in_error()
655 static inline void t4_set_wq_in_error(struct t4_wq *wq, u32 srqidx)  in t4_set_wq_in_error()  argument
658 		*wq->srqidxp = srqidx;  in t4_set_wq_in_error()
659 	*wq->qp_errp = 1;  in t4_set_wq_in_error()
662 static inline void t4_disable_wq_db(struct t4_wq *wq)  in t4_disable_wq_db()  argument
664 	wq->rq.queue[wq->rq.size].status.db_off = 1;  in t4_disable_wq_db()
667 static inline void t4_enable_wq_db(struct t4_wq *wq)  in t4_enable_wq_db()  argument
669 	wq->rq.queue[wq->rq.size].status.db_off = 0;  in t4_enable_wq_db()