11 void notify_eq(struct erdma_eq *eq)  in notify_eq()  argument
13 	u64 db_data = FIELD_PREP(ERDMA_EQDB_CI_MASK, eq->ci) |  in notify_eq()
16 	*eq->db_record = db_data;  in notify_eq()
17 	writeq(db_data, eq->db_addr);  in notify_eq()
19 	atomic64_inc(&eq->notify_num);  in notify_eq()
22 void *get_next_valid_eqe(struct erdma_eq *eq)  in get_next_valid_eqe()  argument
24 	u64 *eqe = get_queue_entry(eq->qbuf, eq->ci, eq->depth, EQE_SHIFT);  in get_next_valid_eqe()
27 	return owner ^ !!(eq->ci & eq->depth) ? eqe : NULL;  in get_next_valid_eqe()
85 	struct erdma_eq *eq = &dev->aeq;  in erdma_aeq_init()  local
88 	eq->depth = ERDMA_DEFAULT_EQ_DEPTH;  in erdma_aeq_init()
89 	buf_size = eq->depth << EQE_SHIFT;  in erdma_aeq_init()
91 	eq->qbuf =  in erdma_aeq_init()
93 				   &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO);  in erdma_aeq_init()
94 	if (!eq->qbuf)  in erdma_aeq_init()
97 	spin_lock_init(&eq->lock);  in erdma_aeq_init()
98 	atomic64_set(&eq->event_num, 0);  in erdma_aeq_init()
99 	atomic64_set(&eq->notify_num, 0);  in erdma_aeq_init()
101 	eq->db_addr = (u64 __iomem *)(dev->func_bar + ERDMA_REGS_AEQ_DB_REG);  in erdma_aeq_init()
102 	eq->db_record = (u64 *)(eq->qbuf + buf_size);  in erdma_aeq_init()
105 			  upper_32_bits(eq->qbuf_dma_addr));  in erdma_aeq_init()
107 			  lower_32_bits(eq->qbuf_dma_addr));  in erdma_aeq_init()
108 	erdma_reg_write32(dev, ERDMA_REGS_AEQ_DEPTH_REG, eq->depth);  in erdma_aeq_init()
110 			  eq->qbuf_dma_addr + buf_size);  in erdma_aeq_init()
117 	struct erdma_eq *eq = &dev->aeq;  in erdma_aeq_destroy()  local
120 			  WARPPED_BUFSIZE(eq->depth << EQE_SHIFT), eq->qbuf,  in erdma_aeq_destroy()
121 			  eq->qbuf_dma_addr);  in erdma_aeq_destroy()
136 		ceqe = get_next_valid_eqe(&ceq_cb->eq);  in erdma_ceq_completion_handler()
141 		ceq_cb->eq.ci++;  in erdma_ceq_completion_handler()
156 	notify_eq(&ceq_cb->eq);  in erdma_ceq_completion_handler()
209 static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq)  in create_eq_cmd()  argument
217 	req.depth = ilog2(eq->depth);  in create_eq_cmd()
218 	req.qbuf_addr = eq->qbuf_dma_addr;  in create_eq_cmd()
222 	db_info_dma_addr = eq->qbuf_dma_addr + (eq->depth << EQE_SHIFT);  in create_eq_cmd()
231 	struct erdma_eq *eq = &dev->ceqs[ceqn].eq;  in erdma_ceq_init_one()  local
235 	eq->qbuf =  in erdma_ceq_init_one()
237 				   &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO);  in erdma_ceq_init_one()
238 	if (!eq->qbuf)  in erdma_ceq_init_one()
241 	spin_lock_init(&eq->lock);  in erdma_ceq_init_one()
242 	atomic64_set(&eq->event_num, 0);  in erdma_ceq_init_one()
243 	atomic64_set(&eq->notify_num, 0);  in erdma_ceq_init_one()
245 	eq->depth = ERDMA_DEFAULT_EQ_DEPTH;  in erdma_ceq_init_one()
246 	eq->db_addr =  in erdma_ceq_init_one()
249 	eq->db_record = (u64 *)(eq->qbuf + buf_size);  in erdma_ceq_init_one()
250 	eq->ci = 0;  in erdma_ceq_init_one()
254 	ret = create_eq_cmd(dev, ceqn + 1, eq);  in erdma_ceq_init_one()
262 	struct erdma_eq *eq = &dev->ceqs[ceqn].eq;  in erdma_ceq_uninit_one()  local
280 	dma_free_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), eq->qbuf,  in erdma_ceq_uninit_one()
281 			  eq->qbuf_dma_addr);  in erdma_ceq_uninit_one()