106 	u32 space, prod, cons;  in queue_has_space()  local
109 	cons = Q_IDX(q, q->cons);  in queue_has_space()
111 	if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))  in queue_has_space()
112 		space = (1 << q->max_n_shift) - (prod - cons);  in queue_has_space()
114 		space = cons - prod;  in queue_has_space()
121 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&  in queue_full()
122 	       Q_WRP(q, q->prod) != Q_WRP(q, q->cons);  in queue_full()
127 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&  in queue_empty()
128 	       Q_WRP(q, q->prod) == Q_WRP(q, q->cons);  in queue_empty()
133 	return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&  in queue_consumed()
134 		(Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||  in queue_consumed()
135 	       ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&  in queue_consumed()
136 		(Q_IDX(q, q->cons) <= Q_IDX(q, prod)));  in queue_consumed()
146 	writel_relaxed(q->llq.cons, q->cons_reg);  in queue_sync_cons_out()
151 	u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;  in queue_inc_cons()  local
152 	q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);  in queue_inc_cons()
228 	queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);  in queue_remove_raw()
372 	u32 cons = readl_relaxed(q->cons_reg);  in __arm_smmu_cmdq_skip_err()  local
373 	u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);  in __arm_smmu_cmdq_skip_err()
378 	dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,  in __arm_smmu_cmdq_skip_err()
404 	queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);  in __arm_smmu_cmdq_skip_err()
412 	queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);  in __arm_smmu_cmdq_skip_err()
594 		WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));  in arm_smmu_cmdq_poll_until_not_full()
632 	llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);  in __arm_smmu_cmdq_poll_until_msi()
684 		llq->cons = readl(cmdq->q.cons_reg);  in __arm_smmu_cmdq_poll_until_consumed()
758 		head.cons = llq.cons;  in arm_smmu_cmdq_issue_cmdlist()
843 			WRITE_ONCE(cmdq->q.llq.cons, llq.cons);  in arm_smmu_cmdq_issue_cmdlist()
1580 	llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |  in arm_smmu_evtq_thread()
1581 		    Q_IDX(llq, llq->cons);  in arm_smmu_evtq_thread()
1639 	llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |  in arm_smmu_priq_thread()
1640 		      Q_IDX(llq, llq->cons);  in arm_smmu_priq_thread()
2910 	q->llq.prod = q->llq.cons = 0;  in arm_smmu_init_one_queue()
3322 	writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);  in arm_smmu_device_reset()
3348 	writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS);  in arm_smmu_device_reset()
3364 		writel_relaxed(smmu->priq.q.llq.cons,  in arm_smmu_device_reset()