126 	struct efa_com_admin_queue *aq = &edev->aq;  in efa_com_admin_init_sq()  local
127 	struct efa_com_admin_sq *sq = &aq->sq;  in efa_com_admin_init_sq()
128 	u16 size = aq->depth * sizeof(*sq->entries);  in efa_com_admin_init_sq()
134 		dma_alloc_coherent(aq->dmadev, size, &sq->dma_addr, GFP_KERNEL);  in efa_com_admin_init_sq()
152 	EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_DEPTH, aq->depth);  in efa_com_admin_init_sq()
163 	struct efa_com_admin_queue *aq = &edev->aq;  in efa_com_admin_init_cq()  local
164 	struct efa_com_admin_cq *cq = &aq->cq;  in efa_com_admin_init_cq()
165 	u16 size = aq->depth * sizeof(*cq->entries);  in efa_com_admin_init_cq()
171 		dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL);  in efa_com_admin_init_cq()
186 	EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_DEPTH, aq->depth);  in efa_com_admin_init_cq()
190 		aq->msix_vector_idx);  in efa_com_admin_init_cq()
244 static u16 efa_com_alloc_ctx_id(struct efa_com_admin_queue *aq)  in efa_com_alloc_ctx_id()  argument
248 	spin_lock(&aq->comp_ctx_lock);  in efa_com_alloc_ctx_id()
249 	ctx_id = aq->comp_ctx_pool[aq->comp_ctx_pool_next];  in efa_com_alloc_ctx_id()
250 	aq->comp_ctx_pool_next++;  in efa_com_alloc_ctx_id()
251 	spin_unlock(&aq->comp_ctx_lock);  in efa_com_alloc_ctx_id()
256 static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq,  in efa_com_dealloc_ctx_id()  argument
259 	spin_lock(&aq->comp_ctx_lock);  in efa_com_dealloc_ctx_id()
260 	aq->comp_ctx_pool_next--;  in efa_com_dealloc_ctx_id()
261 	aq->comp_ctx_pool[aq->comp_ctx_pool_next] = ctx_id;  in efa_com_dealloc_ctx_id()
262 	spin_unlock(&aq->comp_ctx_lock);  in efa_com_dealloc_ctx_id()
265 static inline void efa_com_put_comp_ctx(struct efa_com_admin_queue *aq,  in efa_com_put_comp_ctx()  argument
270 	u16 ctx_id = cmd_id & (aq->depth - 1);  in efa_com_put_comp_ctx()
272 	ibdev_dbg(aq->efa_dev, "Put completion command_id %#x\n", cmd_id);  in efa_com_put_comp_ctx()
274 	efa_com_dealloc_ctx_id(aq, ctx_id);  in efa_com_put_comp_ctx()
277 static struct efa_comp_ctx *efa_com_get_comp_ctx(struct efa_com_admin_queue *aq,  in efa_com_get_comp_ctx()  argument
280 	u16 ctx_id = cmd_id & (aq->depth - 1);  in efa_com_get_comp_ctx()
282 	if (aq->comp_ctx[ctx_id].occupied && capture) {  in efa_com_get_comp_ctx()
284 			aq->efa_dev,  in efa_com_get_comp_ctx()
291 		aq->comp_ctx[ctx_id].occupied = 1;  in efa_com_get_comp_ctx()
292 		ibdev_dbg(aq->efa_dev,  in efa_com_get_comp_ctx()
296 	return &aq->comp_ctx[ctx_id];  in efa_com_get_comp_ctx()
299 static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,  in __efa_com_submit_admin_cmd()  argument
312 	queue_size_mask = aq->depth - 1;  in __efa_com_submit_admin_cmd()
313 	pi = aq->sq.pc & queue_size_mask;  in __efa_com_submit_admin_cmd()
315 	ctx_id = efa_com_alloc_ctx_id(aq);  in __efa_com_submit_admin_cmd()
319 	cmd_id |= aq->sq.pc & ~queue_size_mask;  in __efa_com_submit_admin_cmd()
324 		EFA_ADMIN_AQ_COMMON_DESC_PHASE, aq->sq.phase);  in __efa_com_submit_admin_cmd()
326 	comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, true);  in __efa_com_submit_admin_cmd()
328 		efa_com_dealloc_ctx_id(aq, ctx_id);  in __efa_com_submit_admin_cmd()
339 	aqe = &aq->sq.entries[pi];  in __efa_com_submit_admin_cmd()
343 	aq->sq.pc++;  in __efa_com_submit_admin_cmd()
344 	atomic64_inc(&aq->stats.submitted_cmd);  in __efa_com_submit_admin_cmd()
346 	if ((aq->sq.pc & queue_size_mask) == 0)  in __efa_com_submit_admin_cmd()
347 		aq->sq.phase = !aq->sq.phase;  in __efa_com_submit_admin_cmd()
350 	writel(aq->sq.pc, aq->sq.db_addr);  in __efa_com_submit_admin_cmd()
355 static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq)  in efa_com_init_comp_ctxt()  argument
357 	size_t pool_size = aq->depth * sizeof(*aq->comp_ctx_pool);  in efa_com_init_comp_ctxt()
358 	size_t size = aq->depth * sizeof(struct efa_comp_ctx);  in efa_com_init_comp_ctxt()
362 	aq->comp_ctx = devm_kzalloc(aq->dmadev, size, GFP_KERNEL);  in efa_com_init_comp_ctxt()
363 	aq->comp_ctx_pool = devm_kzalloc(aq->dmadev, pool_size, GFP_KERNEL);  in efa_com_init_comp_ctxt()
364 	if (!aq->comp_ctx || !aq->comp_ctx_pool) {  in efa_com_init_comp_ctxt()
365 		devm_kfree(aq->dmadev, aq->comp_ctx_pool);  in efa_com_init_comp_ctxt()
366 		devm_kfree(aq->dmadev, aq->comp_ctx);  in efa_com_init_comp_ctxt()
370 	for (i = 0; i < aq->depth; i++) {  in efa_com_init_comp_ctxt()
371 		comp_ctx = efa_com_get_comp_ctx(aq, i, false);  in efa_com_init_comp_ctxt()
375 		aq->comp_ctx_pool[i] = i;  in efa_com_init_comp_ctxt()
378 	spin_lock_init(&aq->comp_ctx_lock);  in efa_com_init_comp_ctxt()
380 	aq->comp_ctx_pool_next = 0;  in efa_com_init_comp_ctxt()
385 static struct efa_comp_ctx *efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,  in efa_com_submit_admin_cmd()  argument
393 	spin_lock(&aq->sq.lock);  in efa_com_submit_admin_cmd()
394 	if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) {  in efa_com_submit_admin_cmd()
395 		ibdev_err_ratelimited(aq->efa_dev, "Admin queue is closed\n");  in efa_com_submit_admin_cmd()
396 		spin_unlock(&aq->sq.lock);  in efa_com_submit_admin_cmd()
400 	comp_ctx = __efa_com_submit_admin_cmd(aq, cmd, cmd_size_in_bytes, comp,  in efa_com_submit_admin_cmd()
402 	spin_unlock(&aq->sq.lock);  in efa_com_submit_admin_cmd()
404 		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);  in efa_com_submit_admin_cmd()
409 static void efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq,  in efa_com_handle_single_admin_completion()  argument
418 	comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, false);  in efa_com_handle_single_admin_completion()
420 		ibdev_err(aq->efa_dev,  in efa_com_handle_single_admin_completion()
422 		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);  in efa_com_handle_single_admin_completion()
429 	if (!test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state))  in efa_com_handle_single_admin_completion()
433 static void efa_com_handle_admin_completion(struct efa_com_admin_queue *aq)  in efa_com_handle_admin_completion()  argument
441 	queue_size_mask = aq->depth - 1;  in efa_com_handle_admin_completion()
443 	ci = aq->cq.cc & queue_size_mask;  in efa_com_handle_admin_completion()
444 	phase = aq->cq.phase;  in efa_com_handle_admin_completion()
446 	cqe = &aq->cq.entries[ci];  in efa_com_handle_admin_completion()
456 		efa_com_handle_single_admin_completion(aq, cqe);  in efa_com_handle_admin_completion()
460 		if (ci == aq->depth) {  in efa_com_handle_admin_completion()
465 		cqe = &aq->cq.entries[ci];  in efa_com_handle_admin_completion()
468 	aq->cq.cc += comp_num;  in efa_com_handle_admin_completion()
469 	aq->cq.phase = phase;  in efa_com_handle_admin_completion()
470 	aq->sq.cc += comp_num;  in efa_com_handle_admin_completion()
471 	atomic64_add(comp_num, &aq->stats.completed_cmd);  in efa_com_handle_admin_completion()
494 						     struct efa_com_admin_queue *aq)  in efa_com_wait_and_process_admin_cq_polling()  argument
500 	timeout = jiffies + usecs_to_jiffies(aq->completion_timeout);  in efa_com_wait_and_process_admin_cq_polling()
503 		spin_lock_irqsave(&aq->cq.lock, flags);  in efa_com_wait_and_process_admin_cq_polling()
504 		efa_com_handle_admin_completion(aq);  in efa_com_wait_and_process_admin_cq_polling()
505 		spin_unlock_irqrestore(&aq->cq.lock, flags);  in efa_com_wait_and_process_admin_cq_polling()
512 				aq->efa_dev,  in efa_com_wait_and_process_admin_cq_polling()
515 			atomic64_inc(&aq->stats.no_completion);  in efa_com_wait_and_process_admin_cq_polling()
517 			clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);  in efa_com_wait_and_process_admin_cq_polling()
522 		msleep(aq->poll_interval);  in efa_com_wait_and_process_admin_cq_polling()
527 	efa_com_put_comp_ctx(aq, comp_ctx);  in efa_com_wait_and_process_admin_cq_polling()
532 							struct efa_com_admin_queue *aq)  in efa_com_wait_and_process_admin_cq_interrupts()  argument
538 				    usecs_to_jiffies(aq->completion_timeout));  in efa_com_wait_and_process_admin_cq_interrupts()
547 		spin_lock_irqsave(&aq->cq.lock, flags);  in efa_com_wait_and_process_admin_cq_interrupts()
548 		efa_com_handle_admin_completion(aq);  in efa_com_wait_and_process_admin_cq_interrupts()
549 		spin_unlock_irqrestore(&aq->cq.lock, flags);  in efa_com_wait_and_process_admin_cq_interrupts()
551 		atomic64_inc(&aq->stats.no_completion);  in efa_com_wait_and_process_admin_cq_interrupts()
555 				aq->efa_dev,  in efa_com_wait_and_process_admin_cq_interrupts()
559 				comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc);  in efa_com_wait_and_process_admin_cq_interrupts()
562 				aq->efa_dev,  in efa_com_wait_and_process_admin_cq_interrupts()
566 				comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc);  in efa_com_wait_and_process_admin_cq_interrupts()
568 		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);  in efa_com_wait_and_process_admin_cq_interrupts()
575 	efa_com_put_comp_ctx(aq, comp_ctx);  in efa_com_wait_and_process_admin_cq_interrupts()
588 					     struct efa_com_admin_queue *aq)  in efa_com_wait_and_process_admin_cq()  argument
590 	if (test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state))  in efa_com_wait_and_process_admin_cq()
591 		return efa_com_wait_and_process_admin_cq_polling(comp_ctx, aq);  in efa_com_wait_and_process_admin_cq()
593 	return efa_com_wait_and_process_admin_cq_interrupts(comp_ctx, aq);  in efa_com_wait_and_process_admin_cq()
609 int efa_com_cmd_exec(struct efa_com_admin_queue *aq,  in efa_com_cmd_exec()  argument
621 	down(&aq->avail_cmds);  in efa_com_cmd_exec()
623 	ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n",  in efa_com_cmd_exec()
626 	comp_ctx = efa_com_submit_admin_cmd(aq, cmd, cmd_size, comp, comp_size);  in efa_com_cmd_exec()
629 			aq->efa_dev,  in efa_com_cmd_exec()
634 		up(&aq->avail_cmds);  in efa_com_cmd_exec()
635 		atomic64_inc(&aq->stats.cmd_err);  in efa_com_cmd_exec()
639 	err = efa_com_wait_and_process_admin_cq(comp_ctx, aq);  in efa_com_cmd_exec()
642 			aq->efa_dev,  in efa_com_cmd_exec()
647 		atomic64_inc(&aq->stats.cmd_err);  in efa_com_cmd_exec()
650 	up(&aq->avail_cmds);  in efa_com_cmd_exec()
661 	struct efa_com_admin_queue *aq = &edev->aq;  in efa_com_admin_destroy()  local
663 	struct efa_com_admin_cq *cq = &aq->cq;  in efa_com_admin_destroy()
664 	struct efa_com_admin_sq *sq = &aq->sq;  in efa_com_admin_destroy()
667 	clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);  in efa_com_admin_destroy()
669 	devm_kfree(edev->dmadev, aq->comp_ctx_pool);  in efa_com_admin_destroy()
670 	devm_kfree(edev->dmadev, aq->comp_ctx);  in efa_com_admin_destroy()
672 	size = aq->depth * sizeof(*sq->entries);  in efa_com_admin_destroy()
675 	size = aq->depth * sizeof(*cq->entries);  in efa_com_admin_destroy()
698 		set_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state);  in efa_com_set_admin_polling_mode()
700 		clear_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state);  in efa_com_set_admin_polling_mode()
705 	atomic64_t *s = (atomic64_t *)&edev->aq.stats;  in efa_com_stats_init()
708 	for (i = 0; i < sizeof(edev->aq.stats) / sizeof(*s); i++, s++)  in efa_com_stats_init()
725 	struct efa_com_admin_queue *aq = &edev->aq;  in efa_com_admin_init()  local
738 	aq->depth = EFA_ADMIN_QUEUE_DEPTH;  in efa_com_admin_init()
740 	aq->dmadev = edev->dmadev;  in efa_com_admin_init()
741 	aq->efa_dev = edev->efa_dev;  in efa_com_admin_init()
742 	set_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state);  in efa_com_admin_init()
744 	sema_init(&aq->avail_cmds, aq->depth);  in efa_com_admin_init()
748 	err = efa_com_init_comp_ctxt(aq);  in efa_com_admin_init()
770 		aq->completion_timeout = timeout * 100000;  in efa_com_admin_init()
772 		aq->completion_timeout = ADMIN_CMD_TIMEOUT_US;  in efa_com_admin_init()
774 	aq->poll_interval = EFA_POLL_INTERVAL_MS;  in efa_com_admin_init()
776 	set_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);  in efa_com_admin_init()
781 	dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->cq.entries),  in efa_com_admin_init()
782 			  aq->cq.entries, aq->cq.dma_addr);  in efa_com_admin_init()
784 	dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->sq.entries),  in efa_com_admin_init()
785 			  aq->sq.entries, aq->sq.dma_addr);  in efa_com_admin_init()
787 	devm_kfree(edev->dmadev, aq->comp_ctx);  in efa_com_admin_init()
805 	spin_lock_irqsave(&edev->aq.cq.lock, flags);  in efa_com_admin_q_comp_intr_handler()
806 	efa_com_handle_admin_completion(&edev->aq);  in efa_com_admin_q_comp_intr_handler()
807 	spin_unlock_irqrestore(&edev->aq.cq.lock, flags);  in efa_com_admin_q_comp_intr_handler()
1086 		edev->aq.completion_timeout = timeout * 100000;  in efa_com_dev_reset()
1088 		edev->aq.completion_timeout = ADMIN_CMD_TIMEOUT_US;  in efa_com_dev_reset()
1097 	struct efa_com_admin_queue *aq = &edev->aq;  in efa_com_create_eq()  local
1112 	err = efa_com_cmd_exec(aq,  in efa_com_create_eq()
1131 	struct efa_com_admin_queue *aq = &edev->aq;  in efa_com_destroy_eq()  local
1139 	err = efa_com_cmd_exec(aq,  in efa_com_destroy_eq()